Bugfixes export

doc/MasterData.Docs/articles/developer-guide/export-formats.md

@@ -46,18 +46,16 @@ public sealed class JsonExportFormat : IExportFormat<JsonExportOptions>
             }
             writer.WriteEndObject();
             processed++;
+            context.Progress.Report(processed);
             if (processed % 1000 == 0)
             {
                 await writer.FlushAsync(cancellationToken);
-                context.Progress.Report(new ExportProgress(
-                    processed, context.TotalRecords, $"Exported {processed} records."));
             }
         }
 
         writer.WriteEndArray();
         await writer.FlushAsync(cancellationToken);
-        context.Progress.Report(new ExportProgress(
-            processed, context.TotalRecords, $"Exported {processed} records."));
+        context.Progress.Complete(processed);
     }
 }
 ```
@@ -87,7 +85,7 @@ Add `WithExportFormat` to the application's existing registration chain. The for
 | `Rows` | Asynchronous sequence of repository rows. |
 | `UserValues` | Values supplied with the export request. |
 | `TotalRecords` | Expected count, which can be null. |
-| `Progress` | Report `ExportProgress` with processed count, total and message. |
+| `Progress` | Call `Report(processed)` for each row and `Complete(processed)` at the end. The reporter sends updates every 50 records and sends the final count if it has not already been reported, creating the progress object and message only when sending an update. |
 
 Enumerate rows once, honor cancellation and write incrementally for large exports. Keep the supplied stream open for the pipeline. The pipeline handles repository paging, naming, temporary-file cleanup and saving to `IFileStorage`; formats should not duplicate those steps.
 

src/MasterData.Core/DataManager/Exportation/Background/ExportJobHandler.cs

@@ -31,7 +31,7 @@ internal sealed class ExportJobHandler(
     IStringLocalizer<MasterDataResources> localizer,
     IMasterDataUser masterDataUser) : BackgroundJobHandler<ExportRequest>
 {
-    private const int RecordsPerPage = 100_000;
+    private const int RecordsPerPage = 10_000;
 
     public override async Task<object?> ExecuteAsync(
         ExportRequest request,
@@ -43,17 +43,39 @@ internal sealed class ExportJobHandler(
         var formElement = request.FormElement;
         var format = formats.GetRequired(request.FormatId);
         var columns = GetColumns(request.FormElement, request.ExportAllFields);
-        var source = await CreateSourceAsync(formElement, request, cancellationToken);
+        List<Dictionary<string, object?>> firstPage;
+        long totalRecords;
+        EntityParameters? parameters = null;
+        if (request.Rows is not null)
+        {
+            firstPage = request.Rows;
+            totalRecords = request.Rows.Count;
+        }
+        else
+        {
+            parameters = new EntityParameters
+            {
+                Filters = new Dictionary<string, object?>(request.Filters),
+                RecordsPerPage = RecordsPerPage,
+                OrderBy = OrderByData.FromString(request.OrderBy),
+                CurrentPage = 1
+            };
+            var result = await entityRepository.GetDictionaryListResultAsync(formElement, parameters);
+            cancellationToken.ThrowIfCancellationRequested();
+            firstPage = result.Data;
+            totalRecords = result.TotalOfRecords;
+        }
 
         var context = new ExportContext
         {
             FormElement = formElement,
             Columns = columns,
-            Rows = GetRowsAsync(formElement, source, cancellationToken),
+            Rows = GetRowsAsync(formElement, firstPage, totalRecords, parameters, cancellationToken),
             UserValues = new Dictionary<string, object?>(request.UserValues),
-            TotalRecords = source.Total,
-            Progress = new Progress<ExportProgress>(current => progress.Report(
-                new BackgroundJobProgress(current.Percentage, current.Message, current)))
+            TotalRecords = totalRecords,
+            Progress = new ExportProgressReporter(totalRecords,
+                new Progress<ExportProgress>(current => progress.Report(
+                    new BackgroundJobProgress(current.Percentage, current.Message, current))))
         };
 
         progress.Report(new BackgroundJobProgress(0, localizer["Retrieving records..."]));
@@ -99,60 +121,39 @@ private List<FormElementField> GetColumns(FormElement formElement, bool exportAl
             .ToList();
     }
 
-    private async Task<ExportSource> CreateSourceAsync(
-        FormElement formElement,
-        ExportRequest request,
-        CancellationToken cancellationToken)
-    {
-        if (request.Rows is not null)
-            return new ExportSource(request.Rows, request.Rows.Count, null);
-
-        var parameters = new EntityParameters
-        {
-            Filters = new Dictionary<string, object?>(request.Filters),
-            RecordsPerPage = RecordsPerPage,
-            OrderBy = OrderByData.FromString(request.OrderBy),
-            CurrentPage = 1
-        };
-        var firstPage = await entityRepository.GetDictionaryListResultAsync(formElement, parameters);
-        cancellationToken.ThrowIfCancellationRequested();
-        return new ExportSource(
-            firstPage.Data,
-            firstPage.TotalOfRecords,
-            parameters);
-    }
-
     private async IAsyncEnumerable<Dictionary<string, object?>> GetRowsAsync(
         FormElement formElement,
-        ExportSource source,
+        List<Dictionary<string, object?>> firstPage,
+        long totalOfRecords,
+        EntityParameters? parameters,
         [EnumeratorCancellation] CancellationToken cancellationToken)
     {
-        var totalPages = source.Parameters is null ? 1 :
-            (int)Math.Ceiling((source.Total ?? 0) / (double)RecordsPerPage);
+        var totalPages = parameters is null ? 1 :
+            (int)Math.Ceiling(totalOfRecords / (double)RecordsPerPage);
 
         for (var page = 1; page <= Math.Max(1, totalPages); page++)
         {
             List<Dictionary<string, object?>> rows;
             if (page == 1)
-                rows = source.FirstPage;
+                rows = firstPage;
             else
             {
                 var pageParameters = new EntityParameters
                 {
-                    Filters = source.Parameters!.Filters,
-                    RecordsPerPage = source.Parameters.RecordsPerPage,
-                    OrderBy = source.Parameters.OrderBy,
+                    Filters = parameters!.Filters,
+                    RecordsPerPage = parameters.RecordsPerPage,
+                    OrderBy = parameters.OrderBy,
                     CurrentPage = page
                 };
-                var result = await entityRepository.GetDictionaryListResultAsync(formElement, pageParameters);
+                var result = await entityRepository.GetDictionaryListResultAsync(
+                    formElement, pageParameters, recoverTotalOfRecords: false);
                 rows = result.Data;
             }
 
             foreach (var sourceRow in rows)
             {
                 cancellationToken.ThrowIfCancellationRequested();
-                var row = new Dictionary<string, object?>(sourceRow, StringComparer.OrdinalIgnoreCase);
-                yield return row;
+                yield return sourceRow;
             }
         }
     }
@@ -170,14 +171,4 @@ private string GetFileName(FormElement formElement, string extension)
         name = HttpUtility.UrlEncode(name, Encoding.UTF8);
         return $"{name}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.{extension.TrimStart('.').ToLowerInvariant()}";
     }
-
-    private sealed class ExportSource(
-        List<Dictionary<string, object?>> firstPage,
-        long? total,
-        EntityParameters? parameters)
-    {
-        public List<Dictionary<string, object?>> FirstPage { get; init; } = firstPage;
-        public long? Total { get; init; } = total;
-        public EntityParameters? Parameters { get; init; } = parameters;
-    }
 }

src/MasterData.Core/DataManager/Exportation/ExportContext.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using JJMasterData.Core.DataDictionary.Models;
 
@@ -11,5 +10,5 @@ public sealed class ExportContext
     public required IAsyncEnumerable<Dictionary<string, object?>> Rows { get; init; }
     public required Dictionary<string, object?> UserValues { get; init; }
     public required long? TotalRecords { get; init; }
-    public required IProgress<ExportProgress> Progress { get; init; }
+    public required ExportProgressReporter Progress { get; init; }
 }

src/MasterData.Core/DataManager/Exportation/ExportProgressReporter.cs

@@ -0,0 +1,25 @@
+using System;
+
+namespace JJMasterData.Core.DataManager.Exportation;
+
+public sealed class ExportProgressReporter(long? totalRecords, IProgress<ExportProgress> progress)
+{
+    private const long ReportInterval = 50;
+    private long _lastReported;
+
+    public void Report(long processed) => Report(processed, completed: false);
+
+    public void Complete(long processed) => Report(processed, completed: true);
+
+    private void Report(long processed, bool completed)
+    {
+        if (!completed && processed - _lastReported < ReportInterval)
+            return;
+
+        if (processed == _lastReported)
+            return;
+
+        _lastReported = processed;
+        progress.Report(new ExportProgress(processed, totalRecords, $"Exporting {processed:N0} records..."));
+    }
+}

src/MasterData.Core/DataManager/Exportation/Formats/CsvExportFormat.cs

@@ -1,16 +1,13 @@
 using System.IO;
-using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-internal sealed class CsvExportFormat(FieldFormattingService fieldFormattingService)
-    : DelimitedTextExportFormat<CsvExportOptions>(fieldFormattingService)
+internal sealed class CsvExportFormat : DelimitedTextExportFormat<CsvExportOptions>
 {
     public override string Id => "csv";
     public override string DisplayName => "CSV";
     public override string FileExtension => "csv";
-    public override string ContentType => "text/csv";
-
+
     protected override string GetDelimiter(CsvExportOptions options) => options.Delimiter switch
     {
         CsvExportDelimiter.Semicolon => ";",

src/MasterData.Core/DataManager/Exportation/Formats/DelimitedTextExportFormat.cs

@@ -1,24 +1,22 @@
+using System.Globalization;
 using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using CsvHelper;
 using CsvHelper.Configuration;
-using JJMasterData.Core.DataDictionary.Models;
 using JJMasterData.Core.DataManager.Exportation.Abstractions;
-using JJMasterData.Core.DataManager.Models;
 using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-internal abstract class DelimitedTextExportFormat<TOptions>(FieldFormattingService fieldFormattingService) : IExportFormat<TOptions> where TOptions : ExportFormatOptions, new()
+internal abstract class DelimitedTextExportFormat<TOptions> : IExportFormat<TOptions> where TOptions : ExportFormatOptions, new()
 {
     protected abstract string GetDelimiter(TOptions options);
 
     public abstract string Id { get; }
     public abstract string DisplayName { get; }
     public abstract string FileExtension { get; }
-    public abstract string ContentType { get; }
 
     public async Task WriteAsync(
         ExportContext context,
@@ -28,7 +26,7 @@ public async Task WriteAsync(
     {
         await using var textWriter = new StreamWriter(output, new UTF8Encoding(true), leaveOpen: true);
         await using var csv = new CsvWriter(textWriter,
-            new CsvConfiguration(System.Globalization.CultureInfo.CurrentCulture)
+            new CsvConfiguration(CultureInfo.CurrentCulture)
             {
                 Delimiter = GetDelimiter(options),
                 HasHeaderRecord = false
@@ -44,19 +42,18 @@ public async Task WriteAsync(
         long processed = 0;
         await foreach (var row in context.Rows.WithCancellation(cancellationToken))
         {
-            var formState = new FormStateData(row, context.UserValues, PageState.List);
             foreach (var field in context.Columns)
             {
-                var value = await fieldFormattingService.FormatGridValueAsync(
-                    new FormElementFieldSelector(context.FormElement, field.Name), formState);
+                row.TryGetValue(field.Name, out var rawValue);
+                var value = FieldFormattingService.FormatValue(field, rawValue);
                 csv.WriteField(value);
             }
             await csv.NextRecordAsync();
             processed++;
-            context.Progress.Report(new ExportProgress(
-                processed, context.TotalRecords, $"Exporting {processed:N0} records..."));
+            context.Progress.Report(processed);
         }
 
+        context.Progress.Complete(processed);
         await textWriter.FlushAsync(cancellationToken);
     }
-}
+}

src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsExportFormat.cs

@@ -3,14 +3,12 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
-using JJMasterData.Core.DataDictionary.Models;
 using JJMasterData.Core.DataManager.Exportation.Abstractions;
-using JJMasterData.Core.DataManager.Models;
 using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-public sealed class ExcelXlsExportFormat(FieldFormattingService fieldFormattingService) : IExportFormat<ExcelXlsExportOptions>
+public sealed class ExcelXlsExportFormat : IExportFormat<ExcelXlsExportOptions>
 {
     public string Id => "excel";
     public string DisplayName => "Excel (.xls)";
@@ -33,17 +31,17 @@ public async Task WriteAsync(ExportContext context, ExcelXlsExportOptions option
         await foreach (var row in context.Rows.WithCancellation(cancellationToken))
         {
             await writer.WriteAsync("<tr>");
-            var formState = new FormStateData(row, context.UserValues, PageState.List);
             foreach (var column in context.Columns)
             {
-                var value = await fieldFormattingService.FormatGridValueAsync(
-                    new FormElementFieldSelector(context.FormElement, column.Name), formState);
+                row.TryGetValue(column.Name, out var rawValue);
+                var value = FieldFormattingService.FormatValue(column, rawValue);
                 await writer.WriteAsync($"<td>{HttpUtility.HtmlEncode(value)}</td>");
             }
             await writer.WriteAsync("</tr>");
             processed++;
-            context.Progress.Report(new ExportProgress(processed, context.TotalRecords, $"Exporting {processed:N0} records..."));
+            context.Progress.Report(processed);
         }
+        context.Progress.Complete(processed);
         await writer.WriteAsync("</tbody></table></body></html>");
         await writer.FlushAsync(cancellationToken);
     }

src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxDataReader.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using JJMasterData.Core.DataManager.Exportation;
 using MiniExcelLibs;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
@@ -55,15 +54,13 @@ public override async Task<bool> ReadAsync(CancellationToken token = default)
         if (!await _enumerator.MoveNextAsync())
         {
             _current = null;
+            context.Progress.Complete(_processed);
             return false;
         }
 
         _current = _enumerator.Current;
         _processed++;
-        context.Progress.Report(new ExportProgress(
-            _processed,
-            context.TotalRecords,
-            $"Exporting {_processed:N0} records..."));
+        context.Progress.Report(_processed);
         return true;
     }
 
@@ -79,8 +76,8 @@ public override async ValueTask DisposeAsync()
 
         _disposed = true;
         _current = null;
+
         if (_enumerator is not null)
             await _enumerator.DisposeAsync();
-        GC.SuppressFinalize(this);
     }
 }

src/MasterData.Core/DataManager/Exportation/Formats/ExcelXlsxExportFormat.cs

@@ -1,7 +1,6 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using JJMasterData.Core.DataManager.Exportation;
 using JJMasterData.Core.DataManager.Exportation.Abstractions;
 using MiniExcelLibs;
 using MiniExcelLibs.OpenXml;

src/MasterData.Core/DataManager/Exportation/Formats/TextExportFormat.cs

@@ -1,16 +1,13 @@
 using System.IO;
-using JJMasterData.Core.DataManager.Services;
 
 namespace JJMasterData.Core.DataManager.Exportation.Formats;
 
-internal sealed class TextExportFormat(FieldFormattingService fieldFormattingService)
-    : DelimitedTextExportFormat<TextExportOptions>(fieldFormattingService)
+internal sealed class TextExportFormat : DelimitedTextExportFormat<TextExportOptions>
 {
     public override string Id => "txt";
     public override string DisplayName => "Text";
     public override string FileExtension => "txt";
-    public override string ContentType => "text/plain";
-
+
     protected override string GetDelimiter(TextExportOptions options) => options.Delimiter switch
     {
         TextExportDelimiter.Tab => "\t",