From 3d1968d013b0b2a0d10b0a6b30dd5f1022779150 Mon Sep 17 00:00:00 2001 From: gumbarros Date: Tue, 29 Sep 2026 13:18:47 -0300 Subject: [PATCH 1/4] Fix progress memory usage and 1xn problem as xls and text formats --- .../Exportation/ExportProgressReporter.cs | 19 ++++++++++++++++++ .../Exportation/Formats/CsvExportFormat.cs | 7 ++----- .../Formats/DelimitedTextExportFormat.cs | 20 +++++++++---------- .../Formats/ExcelXlsExportFormat.cs | 13 ++++++------ .../Formats/ExcelXlsxDataReader.cs | 10 ++++------ .../Formats/ExcelXlsxExportFormat.cs | 1 - .../Exportation/Formats/TextExportFormat.cs | 7 ++----- .../Services/FieldFormattingService.cs | 6 +++--- 8 files changed, 45 insertions(+), 38 deletions(-) create mode 100644 src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs diff --git a/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs new file mode 100644 index 000000000..82ac99249 --- /dev/null +++ b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs @@ -0,0 +1,19 @@ +namespace JJMasterData.Core.DataManager.Exportation; + +internal sealed class ExportProgressReporter(ExportContext context) +{ + private const long ReportInterval = 50; + private long _lastReported; + + public void Report(long processed, bool completed = false) + { + if (!completed && processed - _lastReported < ReportInterval) + return; + + if (processed == _lastReported) + return; + + _lastReported = processed; + context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + } +} diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs index f3ab43f98..9a083cfc2 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs @@ -1,16 +1,13 @@ using System.IO; -using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal sealed class CsvExportFormat(FieldFormattingService fieldFormattingService) - : DelimitedTextExportFormat(fieldFormattingService) +internal sealed class CsvExportFormat : DelimitedTextExportFormat { public override string Id => "csv"; public override string DisplayName => "CSV"; public override string FileExtension => "csv"; - public override string ContentType => "text/csv"; - + protected override string GetDelimiter(CsvExportOptions options) => options.Delimiter switch { CsvExportDelimiter.Semicolon => ";", diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs index 36294131f..5066a3ab0 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs @@ -1,24 +1,22 @@ +using System.Globalization; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; using CsvHelper; using CsvHelper.Configuration; -using JJMasterData.Core.DataDictionary.Models; using JJMasterData.Core.DataManager.Exportation.Abstractions; -using JJMasterData.Core.DataManager.Models; using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal abstract class DelimitedTextExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat where TOptions : ExportFormatOptions, new() +internal abstract class DelimitedTextExportFormat : IExportFormat where TOptions : ExportFormatOptions, new() { protected abstract string GetDelimiter(TOptions options); public abstract string Id { get; } public abstract string DisplayName { get; } public abstract string FileExtension { get; } - public abstract string ContentType { get; } public async Task WriteAsync( ExportContext context, @@ -28,7 +26,7 @@ public async Task WriteAsync( { await using var textWriter = new StreamWriter(output, new UTF8Encoding(true), leaveOpen: true); await using var csv = new CsvWriter(textWriter, - new CsvConfiguration(System.Globalization.CultureInfo.CurrentCulture) + new CsvConfiguration(CultureInfo.CurrentCulture) { Delimiter = GetDelimiter(options), HasHeaderRecord = false @@ -42,21 +40,21 @@ public async Task WriteAsync( } long processed = 0; + var progress = new ExportProgressReporter(context); await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { - var formState = new FormStateData(row, context.UserValues, PageState.List); foreach (var field in context.Columns) { - var value = await fieldFormattingService.FormatGridValueAsync( - new FormElementFieldSelector(context.FormElement, field.Name), formState); + row.TryGetValue(field.Name, out var rawValue); + var value = FieldFormattingService.FormatValue(field, rawValue); csv.WriteField(value); } await csv.NextRecordAsync(); processed++; - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + progress.Report(processed); } + progress.Report(processed, completed: true); await textWriter.FlushAsync(cancellationToken); } -} \ No newline at end of file +} diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs index 87dd45b66..209491842 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs @@ -3,14 +3,12 @@ using System.Threading; using System.Threading.Tasks; using System.Web; -using JJMasterData.Core.DataDictionary.Models; using JJMasterData.Core.DataManager.Exportation.Abstractions; -using JJMasterData.Core.DataManager.Models; using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -public sealed class ExcelXlsExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat +public sealed class ExcelXlsExportFormat : IExportFormat { public string Id => "excel"; public string DisplayName => "Excel (.xls)"; @@ -30,20 +28,21 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option } await writer.WriteAsync(""); long processed = 0; + var progress = new ExportProgressReporter(context); await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { await writer.WriteAsync(""); - var formState = new FormStateData(row, context.UserValues, PageState.List); foreach (var column in context.Columns) { - var value = await fieldFormattingService.FormatGridValueAsync( - new FormElementFieldSelector(context.FormElement, column.Name), formState); + row.TryGetValue(column.Name, out var rawValue); + var value = FieldFormattingService.FormatValue(column, rawValue); await writer.WriteAsync($"{HttpUtility.HtmlEncode(value)}"); } await writer.WriteAsync(""); processed++; - context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + progress.Report(processed); } + progress.Report(processed, completed: true); await writer.WriteAsync(""); await writer.FlushAsync(cancellationToken); } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs index 963f7d847..8d908f559 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using JJMasterData.Core.DataManager.Exportation; using MiniExcelLibs; namespace JJMasterData.Core.DataManager.Exportation.Formats; @@ -14,6 +13,7 @@ internal sealed class ExcelXlsxDataReader( private IAsyncEnumerator>? _enumerator; private Dictionary? _current; private long _processed; + private readonly ExportProgressReporter _progress = new(context); private bool _disposed; public override int FieldCount => context.Columns.Count; @@ -54,15 +54,13 @@ public override async Task ReadAsync(CancellationToken token = default) if (!await _enumerator.MoveNextAsync()) { _current = null; + _progress.Report(_processed, completed: true); return false; } _current = _enumerator.Current; _processed++; - context.Progress.Report(new ExportProgress( - _processed, - context.TotalRecords, - $"Exporting {_processed:N0} records...")); + _progress.Report(_processed); return true; } @@ -78,8 +76,8 @@ public override async ValueTask DisposeAsync() _disposed = true; _current = null; + if (_enumerator is not null) await _enumerator.DisposeAsync(); - GC.SuppressFinalize(this); } } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs index 39d22296d..c41c23426 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs @@ -1,7 +1,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using JJMasterData.Core.DataManager.Exportation; using JJMasterData.Core.DataManager.Exportation.Abstractions; using MiniExcelLibs; using MiniExcelLibs.OpenXml; diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs index 18d046c11..cd961fad7 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs @@ -1,16 +1,13 @@ using System.IO; -using JJMasterData.Core.DataManager.Services; namespace JJMasterData.Core.DataManager.Exportation.Formats; -internal sealed class TextExportFormat(FieldFormattingService fieldFormattingService) - : DelimitedTextExportFormat(fieldFormattingService) +internal sealed class TextExportFormat : DelimitedTextExportFormat { public override string Id => "txt"; public override string DisplayName => "Text"; public override string FileExtension => "txt"; - public override string ContentType => "text/plain"; - + protected override string GetDelimiter(TextExportOptions options) => options.Delimiter switch { TextExportDelimiter.Tab => "\t", diff --git a/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs b/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs index 9933803e4..68f6088dd 100644 --- a/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs +++ b/src/MasterData.Core/DataManager/Services/FieldFormattingService.cs @@ -39,7 +39,7 @@ public async ValueTask FormatGridValueAsync( break; case FormComponent.Number: case FormComponent.Slider: - stringValue = GetNumericValueAsString(field, value,$"N{field.NumberOfDecimalPlaces}"); + stringValue = GetNumericValueAsString(field, value, $"N{field.NumberOfDecimalPlaces}"); break; case FormComponent.Currency: stringValue = GetNumericValueAsString(field, value,$"C{field.NumberOfDecimalPlaces}"); @@ -63,7 +63,7 @@ public async ValueTask FormatGridValueAsync( if (field.EncodeHtml) stringValue = HttpUtility.HtmlEncode(stringValue); - + return stringValue.Trim(); } @@ -166,4 +166,4 @@ private static string GetNumericValueAsString(FormElementField field, object? va return stringValue ?? string.Empty; } -} \ No newline at end of file +} From 6674fbbed40929f80d2fe1059671f63c2c8cb06c Mon Sep 17 00:00:00 2001 From: gumbarros Date: Tue, 29 Sep 2026 13:32:57 -0300 Subject: [PATCH 2/4] Remove ExportSource --- .../Background/ExportJobHandler.cs | 84 ++++++++----------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs index be8a22bcb..2032fc51b 100644 --- a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs +++ b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs @@ -31,7 +31,7 @@ internal sealed class ExportJobHandler( IStringLocalizer localizer, IMasterDataUser masterDataUser) : BackgroundJobHandler { - private const int RecordsPerPage = 100_000; + private const int RecordsPerPage = 10_000; public override async Task ExecuteAsync( ExportRequest request, @@ -43,15 +43,36 @@ internal sealed class ExportJobHandler( var formElement = request.FormElement; var format = formats.GetRequired(request.FormatId); var columns = GetColumns(request.FormElement, request.ExportAllFields); - var source = await CreateSourceAsync(formElement, request, cancellationToken); + List> firstPage; + long totalRecords; + EntityParameters? parameters = null; + if (request.Rows is not null) + { + firstPage = request.Rows; + totalRecords = request.Rows.Count; + } + else + { + parameters = new EntityParameters + { + Filters = new Dictionary(request.Filters), + RecordsPerPage = RecordsPerPage, + OrderBy = OrderByData.FromString(request.OrderBy), + CurrentPage = 1 + }; + var result = await entityRepository.GetDictionaryListResultAsync(formElement, parameters); + cancellationToken.ThrowIfCancellationRequested(); + firstPage = result.Data; + totalRecords = result.TotalOfRecords; + } var context = new ExportContext { FormElement = formElement, Columns = columns, - Rows = GetRowsAsync(formElement, source, cancellationToken), + Rows = GetRowsAsync(formElement, firstPage, totalRecords, parameters, cancellationToken), UserValues = new Dictionary(request.UserValues), - TotalRecords = source.Total, + TotalRecords = totalRecords, Progress = new Progress(current => progress.Report( new BackgroundJobProgress(current.Percentage, current.Message, current))) }; @@ -99,60 +120,39 @@ private List GetColumns(FormElement formElement, bool exportAl .ToList(); } - private async Task CreateSourceAsync( - FormElement formElement, - ExportRequest request, - CancellationToken cancellationToken) - { - if (request.Rows is not null) - return new ExportSource(request.Rows, request.Rows.Count, null); - - var parameters = new EntityParameters - { - Filters = new Dictionary(request.Filters), - RecordsPerPage = RecordsPerPage, - OrderBy = OrderByData.FromString(request.OrderBy), - CurrentPage = 1 - }; - var firstPage = await entityRepository.GetDictionaryListResultAsync(formElement, parameters); - cancellationToken.ThrowIfCancellationRequested(); - return new ExportSource( - firstPage.Data, - firstPage.TotalOfRecords, - parameters); - } - private async IAsyncEnumerable> GetRowsAsync( FormElement formElement, - ExportSource source, + List> firstPage, + long totalOfRecords, + EntityParameters? parameters, [EnumeratorCancellation] CancellationToken cancellationToken) { - var totalPages = source.Parameters is null ? 1 : - (int)Math.Ceiling((source.Total ?? 0) / (double)RecordsPerPage); + var totalPages = parameters is null ? 1 : + (int)Math.Ceiling(totalOfRecords / (double)RecordsPerPage); for (var page = 1; page <= Math.Max(1, totalPages); page++) { List> rows; if (page == 1) - rows = source.FirstPage; + rows = firstPage; else { var pageParameters = new EntityParameters { - Filters = source.Parameters!.Filters, - RecordsPerPage = source.Parameters.RecordsPerPage, - OrderBy = source.Parameters.OrderBy, + Filters = parameters!.Filters, + RecordsPerPage = parameters.RecordsPerPage, + OrderBy = parameters.OrderBy, CurrentPage = page }; - var result = await entityRepository.GetDictionaryListResultAsync(formElement, pageParameters); + var result = await entityRepository.GetDictionaryListResultAsync( + formElement, pageParameters, recoverTotalOfRecords: false); rows = result.Data; } foreach (var sourceRow in rows) { cancellationToken.ThrowIfCancellationRequested(); - var row = new Dictionary(sourceRow, StringComparer.OrdinalIgnoreCase); - yield return row; + yield return sourceRow; } } } @@ -170,14 +170,4 @@ private string GetFileName(FormElement formElement, string extension) name = HttpUtility.UrlEncode(name, Encoding.UTF8); return $"{name}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.{extension.TrimStart('.').ToLowerInvariant()}"; } - - private sealed class ExportSource( - List> firstPage, - long? total, - EntityParameters? parameters) - { - public List> FirstPage { get; init; } = firstPage; - public long? Total { get; init; } = total; - public EntityParameters? Parameters { get; init; } = parameters; - } } From 461dfb251f2a6efa41faccc0e52c5ed7166cc119 Mon Sep 17 00:00:00 2001 From: gumbarros Date: Thu, 1 Oct 2026 11:50:04 -0300 Subject: [PATCH 3/4] Fix expressions "{Value}" = "" behavior --- Directory.Build.props | 2 +- .../Providers/DefaultExpressionProvider.cs | 3 +- .../DefaultExpressionProviderTests.cs | 54 +++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs diff --git a/Directory.Build.props b/Directory.Build.props index e337e46e6..a86731c04 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -3,7 +3,7 @@ https://www.github.com/JJConsulting/JJMasterData https://www.github.com/JJConsulting/JJMasterData JJMasterData.png - 5.0.4 + 5.0.5 $(VersionPrefix) $(VersionPrefix)-$(VersionSuffix) $(Version) diff --git a/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs b/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs index 0cc0308e0..03f26a04f 100644 --- a/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs +++ b/src/MasterData.Core/DataManager/Expressions/Providers/DefaultExpressionProvider.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Options; using NCalc; using NCalc.Factories; -using NCalc.Handlers; namespace JJMasterData.Core.DataManager.Expressions.Providers; @@ -63,7 +62,7 @@ private static string PrepareExpressionWithParameters( if (expression.Contains(quotedToken, StringComparison.InvariantCultureIgnoreCase)) { expression = expression.Replace(quotedToken, kvp.Key); - parameters[kvp.Key] = value?.ToString(); + parameters[kvp.Key] = value?.ToString() ?? string.Empty; } else { diff --git a/test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs b/test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs new file mode 100644 index 000000000..940612f45 --- /dev/null +++ b/test/MasterData.Core.Test/DataManager/Expressions/DefaultExpressionProviderTests.cs @@ -0,0 +1,54 @@ +using JJMasterData.Core.Configuration.Options; +using JJMasterData.Core.DataManager.Expressions.Providers; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NCalc.DependencyInjection; +using NCalc.Factories; + +namespace JJMasterData.Core.Test.DataManager.Expressions; + +public class DefaultExpressionProviderTests +{ + [Theory] + [InlineData(null, true)] + [InlineData("", true)] + [InlineData("filled", false)] + public void QuotedParameterEqualsEmptyStringOnlyWhenEmpty(string? value, bool expected) + { + var provider = CreateProvider(); + + var result = provider.Evaluate("'{VALOR}' = ''", new Dictionary + { + ["VALOR"] = value + }); + + Assert.Equal(expected, result); + } + + [Fact] + public void UnquotedNullParameterRemainsNull() + { + var provider = CreateProvider(); + + var result = provider.Evaluate("{VALOR} = null", new Dictionary + { + ["VALOR"] = null + }); + + Assert.Equal(true, result); + } + + private static DefaultExpressionProvider CreateProvider() + { + var services = new ServiceCollection(); + services.AddNCalc(); + var serviceProvider = services.BuildServiceProvider(); + + return new DefaultExpressionProvider( + serviceProvider.GetRequiredService(), + serviceProvider, + Options.Create(new MasterDataCoreOptions()), + NullLogger.Instance); + } +} From dee6af8a6083751e8c6b9a4d11ad32ba4bd005db Mon Sep 17 00:00:00 2001 From: gumbarros Date: Thu, 22 Oct 2026 16:27:35 -0300 Subject: [PATCH 4/4] Make progress easier --- .../articles/developer-guide/export-formats.md | 8 +++----- .../Exportation/Background/ExportJobHandler.cs | 5 +++-- .../DataManager/Exportation/ExportContext.cs | 3 +-- .../Exportation/ExportProgressReporter.cs | 12 +++++++++--- .../Exportation/Formats/DelimitedTextExportFormat.cs | 5 ++--- .../Exportation/Formats/ExcelXlsExportFormat.cs | 5 ++--- .../Exportation/Formats/ExcelXlsxDataReader.cs | 5 ++--- src/Plugins/MasterData.Pdf/PdfExportFormat.cs | 4 ++-- .../Exports/ExcelXlsxExportFormatTests.cs | 2 +- .../DataManager/Exports/ExportFormatCatalogTests.cs | 2 +- 10 files changed, 26 insertions(+), 25 deletions(-) diff --git a/doc/MasterData.Docs/articles/developer-guide/export-formats.md b/doc/MasterData.Docs/articles/developer-guide/export-formats.md index 098712433..bb16fd6b1 100644 --- a/doc/MasterData.Docs/articles/developer-guide/export-formats.md +++ b/doc/MasterData.Docs/articles/developer-guide/export-formats.md @@ -46,18 +46,16 @@ public sealed class JsonExportFormat : IExportFormat } writer.WriteEndObject(); processed++; + context.Progress.Report(processed); if (processed % 1000 == 0) { await writer.FlushAsync(cancellationToken); - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exported {processed} records.")); } } writer.WriteEndArray(); await writer.FlushAsync(cancellationToken); - context.Progress.Report(new ExportProgress( - processed, context.TotalRecords, $"Exported {processed} records.")); + context.Progress.Complete(processed); } } ``` @@ -87,7 +85,7 @@ Add `WithExportFormat` to the application's existing registration chain. The for | `Rows` | Asynchronous sequence of repository rows. | | `UserValues` | Values supplied with the export request. | | `TotalRecords` | Expected count, which can be null. | -| `Progress` | Report `ExportProgress` with processed count, total and message. | +| `Progress` | Call `Report(processed)` for each row and `Complete(processed)` at the end. The reporter sends updates every 50 records and sends the final count if it has not already been reported, creating the progress object and message only when sending an update. | Enumerate rows once, honor cancellation and write incrementally for large exports. Keep the supplied stream open for the pipeline. The pipeline handles repository paging, naming, temporary-file cleanup and saving to `IFileStorage`; formats should not duplicate those steps. diff --git a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs index 2032fc51b..92b892431 100644 --- a/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs +++ b/src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs @@ -73,8 +73,9 @@ internal sealed class ExportJobHandler( Rows = GetRowsAsync(formElement, firstPage, totalRecords, parameters, cancellationToken), UserValues = new Dictionary(request.UserValues), TotalRecords = totalRecords, - Progress = new Progress(current => progress.Report( - new BackgroundJobProgress(current.Percentage, current.Message, current))) + Progress = new ExportProgressReporter(totalRecords, + new Progress(current => progress.Report( + new BackgroundJobProgress(current.Percentage, current.Message, current)))) }; progress.Report(new BackgroundJobProgress(0, localizer["Retrieving records..."])); diff --git a/src/MasterData.Core/DataManager/Exportation/ExportContext.cs b/src/MasterData.Core/DataManager/Exportation/ExportContext.cs index dc0d0bed9..3fba4f736 100644 --- a/src/MasterData.Core/DataManager/Exportation/ExportContext.cs +++ b/src/MasterData.Core/DataManager/Exportation/ExportContext.cs @@ -1,4 +1,3 @@ -using System; using System.Collections.Generic; using JJMasterData.Core.DataDictionary.Models; @@ -11,5 +10,5 @@ public sealed class ExportContext public required IAsyncEnumerable> Rows { get; init; } public required Dictionary UserValues { get; init; } public required long? TotalRecords { get; init; } - public required IProgress Progress { get; init; } + public required ExportProgressReporter Progress { get; init; } } diff --git a/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs index 82ac99249..76aac4ad6 100644 --- a/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs +++ b/src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs @@ -1,11 +1,17 @@ +using System; + namespace JJMasterData.Core.DataManager.Exportation; -internal sealed class ExportProgressReporter(ExportContext context) +public sealed class ExportProgressReporter(long? totalRecords, IProgress progress) { private const long ReportInterval = 50; private long _lastReported; - public void Report(long processed, bool completed = false) + public void Report(long processed) => Report(processed, completed: false); + + public void Complete(long processed) => Report(processed, completed: true); + + private void Report(long processed, bool completed) { if (!completed && processed - _lastReported < ReportInterval) return; @@ -14,6 +20,6 @@ public void Report(long processed, bool completed = false) return; _lastReported = processed; - context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records...")); + progress.Report(new ExportProgress(processed, totalRecords, $"Exporting {processed:N0} records...")); } } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs index 5066a3ab0..ee9f7218d 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs @@ -40,7 +40,6 @@ public async Task WriteAsync( } long processed = 0; - var progress = new ExportProgressReporter(context); await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { foreach (var field in context.Columns) @@ -51,10 +50,10 @@ public async Task WriteAsync( } await csv.NextRecordAsync(); processed++; - progress.Report(processed); + context.Progress.Report(processed); } - progress.Report(processed, completed: true); + context.Progress.Complete(processed); await textWriter.FlushAsync(cancellationToken); } } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs index 209491842..6f81503f1 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs @@ -28,7 +28,6 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option } await writer.WriteAsync(""); long processed = 0; - var progress = new ExportProgressReporter(context); await foreach (var row in context.Rows.WithCancellation(cancellationToken)) { await writer.WriteAsync(""); @@ -40,9 +39,9 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option } await writer.WriteAsync(""); processed++; - progress.Report(processed); + context.Progress.Report(processed); } - progress.Report(processed, completed: true); + context.Progress.Complete(processed); await writer.WriteAsync(""); await writer.FlushAsync(cancellationToken); } diff --git a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs index a0f628684..9fa1818e9 100644 --- a/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs +++ b/src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs @@ -13,7 +13,6 @@ internal sealed class ExcelXlsxDataReader( private IAsyncEnumerator>? _enumerator; private Dictionary? _current; private long _processed; - private readonly ExportProgressReporter _progress = new(context); private bool _disposed; public override int FieldCount => context.Columns.Count; @@ -55,13 +54,13 @@ public override async Task ReadAsync(CancellationToken token = default) if (!await _enumerator.MoveNextAsync()) { _current = null; - _progress.Report(_processed, completed: true); + context.Progress.Complete(_processed); return false; } _current = _enumerator.Current; _processed++; - _progress.Report(_processed); + context.Progress.Report(_processed); return true; } diff --git a/src/Plugins/MasterData.Pdf/PdfExportFormat.cs b/src/Plugins/MasterData.Pdf/PdfExportFormat.cs index 4b0b9bd1a..8255b03ce 100644 --- a/src/Plugins/MasterData.Pdf/PdfExportFormat.cs +++ b/src/Plugins/MasterData.Pdf/PdfExportFormat.cs @@ -62,11 +62,11 @@ public async Task WriteAsync( table.AddCell(Style(new Cell().Add(new Paragraph(value)), options, false, striped)); } processed++; - context.Progress.Report(new ExportProgress(processed, context.TotalRecords, - $"Exporting {processed:N0} records...")); + context.Progress.Report(processed); table.Flush(); } table.Complete(); + context.Progress.Complete(processed); } private static Cell Style(Cell cell, PdfExportOptions options, bool header, bool striped) diff --git a/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs b/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs index c091df3b4..321a97786 100644 --- a/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs +++ b/test/MasterData.Core.Test/DataManager/Exports/ExcelXlsxExportFormatTests.cs @@ -270,7 +270,7 @@ private static ExportContext CreateContext( Rows = rows, UserValues = [], TotalRecords = totalRecords, - Progress = progress ?? new Progress() + Progress = new ExportProgressReporter(totalRecords, progress ?? new Progress()) }; } diff --git a/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs b/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs index 9f03d6f61..825ba31c1 100644 --- a/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs +++ b/test/MasterData.Core.Test/DataManager/Exports/ExportFormatCatalogTests.cs @@ -20,7 +20,7 @@ public async Task CustomJsonFormatIsDiscoveredAndWritesWithoutCoreChanges() Rows = Rows(TestContext.Current.CancellationToken), UserValues = [], TotalRecords = 1, - Progress = new Progress() + Progress = new ExportProgressReporter(1, new Progress()) }; await using var output = new MemoryStream();