From f719d759de1f8b57ffd28203beef8d5c85dd1b91 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 12:26:16 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20=E6=94=AF=E6=8F=B4=20MySQL=20/=20?= =?UTF-8?q?MariaDB=20=E5=9B=9B=E6=AD=A5=20SQL=20=E5=82=BE=E5=8D=B0?= =?UTF-8?q?=E8=88=87=E5=8C=AF=E5=85=A5=E7=B2=BE=E9=9D=88=EF=BC=8C=E5=B0=8D?= =?UTF-8?q?=E9=BD=8A=20MSSQL=20=E6=A8=99=E7=AB=BF=E8=A6=8F=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 5 + TODO.md | 4 +- mySQLPunk/Form1.MySqlExport.cs | 61 ++ mySQLPunk/Form1.MySqlImport.cs | 57 ++ mySQLPunk/Form1.cs | 99 ++- mySQLPunk/MySqlExportWizardForm.cs | 635 ++++++++++++++++++ mySQLPunk/MySqlImportWizardForm.cs | 564 ++++++++++++++++ .../lib/MySqlExport/MySqlExportModels.cs | 149 ++++ .../lib/MySqlExport/MySqlSqlExportService.cs | 480 +++++++++++++ mySQLPunk/lib/MySqlImport/MySqlImportLog.cs | 128 ++++ .../lib/MySqlImport/MySqlImportModels.cs | 151 +++++ .../MySqlImportPreflightScanner.cs | 150 +++++ .../lib/MySqlImport/MySqlImportRunner.cs | 288 ++++++++ mySQLPunk/mySQLPunk.csproj | 10 + tests/Run-SmokeTests.ps1 | 2 + tests/SmokeTests.MssqlExport.Routing.cs | 2 +- tests/SmokeTests.MySqlExport.cs | 202 ++++++ tests/SmokeTests.MySqlImport.cs | 148 ++++ tests/SmokeTests.cs | 4 + 19 files changed, 3076 insertions(+), 63 deletions(-) create mode 100644 mySQLPunk/Form1.MySqlExport.cs create mode 100644 mySQLPunk/Form1.MySqlImport.cs create mode 100644 mySQLPunk/MySqlExportWizardForm.cs create mode 100644 mySQLPunk/MySqlImportWizardForm.cs create mode 100644 mySQLPunk/lib/MySqlExport/MySqlExportModels.cs create mode 100644 mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs create mode 100644 mySQLPunk/lib/MySqlImport/MySqlImportLog.cs create mode 100644 mySQLPunk/lib/MySqlImport/MySqlImportModels.cs create mode 100644 mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs create mode 100644 mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs create mode 100644 tests/SmokeTests.MySqlExport.cs create mode 100644 tests/SmokeTests.MySqlImport.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 467b298..756cb54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## [Unreleased] ### 🚀 新增功能 +- **MySQL / MariaDB 四步 SQL 備份（傾印）與匯入精靈（對齊 MSSQL 標竿規格）**： + - **四步匯出精靈 (`MySqlExportWizardForm`)**：提供「模式與目標」、「物件選取（資料表／檢視／常式／觸發器）」、「選項確認（單一交易快照 `REPEATABLE READ`、外鍵約束開關、DROP 語句、移除 DEFINER）」與「即時進度與看板娘 Punky 動畫」完整四步體驗。底層改採串流直接寫入暫存檔 (`.staging.xxx.tmp`)，完成後原子移動，杜絕大型資料庫傾印時的 OOM 記憶體溢位問題。 + - **Native Manifest 元數據校驗**：傾印檔末端自動附加原生 Manifest 區塊與 SHA-256 數位簽章，包含資料庫名稱、資料表清單、列數統計與校驗碼。 + - **四步匯入精靈 (`MySqlImportWizardForm`)**：提供「檔案選取與目標」、「Preflight 串流預檢（編碼判定、DELIMITER 切分、單批次 32 MiB 上限、Manifest 讀取）」、「執行選項（錯誤繼續／遇錯即停、物件存在處理策略）」與「執行回報與日誌檢視」。 + - **獨立日誌與即時審計**：所有匯入執行步驟與錯誤皆獨立記錄至 `%LOCALAPPDATA%\mySQLPunk\ImportLogs`，並於匯入完成後自動比對 Native Manifest 目標列數，確認還原完整性。 ### 🛠️ 問題修正與優化 - **README.md 門面全新改版與文件模組化**：全面改寫 README.md 為現代、圖文並茂的開源專案門面，融入看板娘 Punky 崩琦角色元素，並加入 S1~S6 完整使用畫面截圖；深層工程與建置細節分別拆分為 `TODO.md`、`docs/DEVELOPMENT.md`、`docs/ARCHITECTURE.md` 與 `docs/DATA_SAFETY_SPEC.md`。 diff --git a/TODO.md b/TODO.md index 9c1c482..b3881b9 100644 --- a/TODO.md +++ b/TODO.md @@ -47,8 +47,8 @@ 2. 既有程式碼庫中已具備相對成熟的 `MySqlExportService` 與 `MySqlImportService` 底層邏輯（如 `DELIMITER` 解析、`SHOW CREATE` 提取、交易開關等），只需將底層串流強化並重構對齊 MSSQL 的四步精靈架構。 3. 投資報酬率最高，能快速在主要客群中展現一致且專業的產品體驗。 - **具體工作**： - - [ ] 匯出：移植四步精靈（選表/View ➔ 輸出設定/single-transaction ➔ 確認物件 ➔ 串流執行 + Punky 動畫 + Manifest 生成）。 - - [ ] 匯入：移植四步精靈（選檔 ➔ 預檢掃描 DELIMITER/編碼/單批上限 ➔ 確認目標與策略 ➔ 專用 Session 執行 + Punky 動畫 + 獨立 Log + Manifest 核對）。 + - [x] 匯出：移植四步精靈（選表/View ➔ 輸出設定/single-transaction ➔ 確認物件 ➔ 串流執行 + Punky 動畫 + Manifest 生成）。 + - [x] 匯入：移植四步精靈（選檔 ➔ 預檢掃描 DELIMITER/編碼/單批上限 ➔ 確認目標與策略 ➔ 專用 Session 執行 + Punky 動畫 + 獨立 Log + Manifest 核對）。 #### 🥈 第二優先：SQLite - **原因**： diff --git a/mySQLPunk/Form1.MySqlExport.cs b/mySQLPunk/Form1.MySqlExport.cs new file mode 100644 index 0000000..40d46b4 --- /dev/null +++ b/mySQLPunk/Form1.MySqlExport.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlExport; +using MySqlExportResult = mySQLPunk.lib.MySqlExport.MySqlExportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func MySqlExportWizardRunner = null; + internal Action MySqlExportStatusSink = null; + + private void SetMySqlExportStatus(string message) + { + if (MySqlExportStatusSink == null) UpdateMainStatus(message); + else MySqlExportStatusSink(message); + } + + private void OpenMySqlExportWizard(TreeDatabaseTarget target, MySqlExportMode mode = MySqlExportMode.SchemaAndData) + { + if (target == null) return; + try + { + string defaultFileName = target.DatabaseName + "_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; + string targetPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultFileName); + var options = new MySqlExportWizardOptions(mode, targetPath); + + MySqlExportResult result; + if (MySqlExportWizardRunner != null) + { + result = MySqlExportWizardRunner(target.Database, target.DatabaseName, options); + } + else + { + using (var wizard = new MySqlExportWizardForm(target.Database, target.DatabaseName, options)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Success + ? string.Format(ReportText.Get("MySQL 匯出完成：{0} 張資料表，共 {1:N0} 列 · {2}", "MySQL export completed: {0} tables, {1:N0} rows · {2}"), result.TableCount, result.TotalRows, result.TargetPath) + : (result.Stage == MySqlExportStage.Cancelled + ? ReportText.Get("MySQL 匯出已取消", "MySQL export cancelled") + : ReportText.Get("MySQL 匯出失敗：", "MySQL export failed: ") + result.ErrorMessage); + SetMySqlExportStatus(status); + } + } + catch (Exception ex) + { + SetMySqlExportStatus(ReportText.Get("無法開啟 MySQL 匯出精靈", "Cannot open MySQL export wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.MySqlImport.cs b/mySQLPunk/Form1.MySqlImport.cs new file mode 100644 index 0000000..1d23379 --- /dev/null +++ b/mySQLPunk/Form1.MySqlImport.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlImport; +using MySqlImportResult = mySQLPunk.lib.MySqlImport.MySqlImportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func MySqlImportWizardRunner = null; + internal Action MySqlImportStatusSink = null; + + private void SetMySqlImportStatus(string message) + { + if (MySqlImportStatusSink == null) UpdateMainStatus(message); + else MySqlImportStatusSink(message); + } + + private void OpenMySqlImportWizard(TreeDatabaseTarget target) + { + if (target == null) return; + try + { + MySqlImportResult result; + if (MySqlImportWizardRunner != null) + { + result = MySqlImportWizardRunner(target.Database, target.DatabaseName, ""); + } + else + { + using (var wizard = new MySqlImportWizardForm(target.Database, target.DatabaseName)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Status == MySqlImportStatus.Completed + ? string.Format(ReportText.Get("MySQL 匯入執行完成：{0} 句成功 · 日誌：{1}", "MySQL import execution completed: {0} statements · Log: {1}"), result.ExecutedStatements, result.LogPath) + : (result.Status == MySqlImportStatus.Cancelled + ? ReportText.Get("MySQL 匯入已取消；先前寫入可能保留", "MySQL import cancelled; previous writes may remain") + : string.Format(ReportText.Get("MySQL 匯入結束：{0} 失敗，{1} 略過 · 日誌：{2}", "MySQL import finished: {0} failed, {1} skipped · Log: {2}"), result.FailedStatements, result.SkippedStatements, result.LogPath)); + SetMySqlImportStatus(status); + RefreshDatabaseObjectNodes(target.DatabaseNode); + } + } + catch (Exception ex) + { + SetMySqlImportStatus(ReportText.Get("無法開啟 MySQL 匯入精靈", "Cannot open MySQL import wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.cs b/mySQLPunk/Form1.cs index a35c7d1..2ea02ca 100644 --- a/mySQLPunk/Form1.cs +++ b/mySQLPunk/Form1.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; @@ -1261,14 +1261,21 @@ public void showTools(string kind) } public void UpdateMainStatus(string msg) { - if (InvokeRequired) + try { - BeginInvoke(new Action(UpdateMainStatus), msg); - return; + if (IsHandleCreated && InvokeRequired) + { + BeginInvoke(new Action(UpdateMainStatus), msg); + return; + } + if (lblMainStatus != null) + { + lblMainStatus.Text = msg; + } } - if (lblMainStatus != null) + catch { - lblMainStatus.Text = msg; + // In headless tests or uninitialized form fixtures, safely ignore UI status updates } } @@ -6668,6 +6675,11 @@ private void DumpSelectedTableSql(bool dataOnly) OpenMssqlExportWizard(target, dataOnly ? lib.MssqlExport.MssqlExportMode.DataOnly : lib.MssqlExport.MssqlExportMode.SchemaAndData, null, tableName, false); return; } + if (IsMySqlTarget(target)) + { + OpenMySqlExportWizard(target, dataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -6905,7 +6917,7 @@ private void DumpSelectedViewSql() } } - private async void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) + private void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) { TreeDatabaseTarget target = GetTargetFromCurrentSelection(); if (target == null) @@ -6922,11 +6934,11 @@ private async void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) null, selection == null ? null : selection.ObjectName, false); return; } - MySqlExportOptions mysqlOptions = null; - if (!dataOnlyForTable && selection == null && IsMySqlTarget(target)) + if (IsMySqlTarget(target) && (selection == null || selection.GroupName == "Tables")) { - mysqlOptions = ShowMySqlExportOptionsDialog(target); - if (mysqlOptions == null) return; + bool tableDataOnly = selection != null && dataOnlyForTable; + OpenMySqlExportWizard(target, tableDataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData); + return; } using (SaveFileDialog dialog = new SaveFileDialog()) @@ -6942,25 +6954,8 @@ private async void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) try { - if (mysqlOptions != null) - { - MySqlExportResult result = await ExportMySqlDatabaseWithProgress(target, dialog.FileName, mysqlOptions); - string message = Localization.Format( - "MySqlExport.Success", - result.TableCount, - result.ViewCount, - result.RoutineCount, - result.TriggerCount, - result.RowCount, - dialog.FileName); - UpdateMainStatus(message); - MessageBox.Show(message, Localization.T("Common.Complete"), MessageBoxButtons.OK, MessageBoxIcon.Information); - } - else - { - DumpCurrentSelectionSqlToFile(dialog.FileName, dataOnlyForTable); - MessageBox.Show(Localization.T("Object.SqlExported"), Localization.T("Common.Complete"), MessageBoxButtons.OK, MessageBoxIcon.Information); - } + DumpCurrentSelectionSqlToFile(dialog.FileName, dataOnlyForTable); + MessageBox.Show(Localization.T("Object.SqlExported"), Localization.T("Common.Complete"), MessageBoxButtons.OK, MessageBoxIcon.Information); } catch (Exception ex) { @@ -7283,6 +7278,12 @@ private void DumpSelectedDatabaseSqlWithDialog() return; } + if (IsMySqlTarget(target)) + { + OpenMySqlExportWizard(target, lib.MySqlExport.MySqlExportMode.SchemaAndData); + return; + } + using (SaveFileDialog dialog = new SaveFileDialog()) { dialog.Filter = Localization.T("Common.SqlFilesFilter"); @@ -7325,7 +7326,7 @@ private bool DumpSelectedDatabaseSqlToFile(string targetPath) return true; } - private async void ImportSqlWithDialog() + private void ImportSqlWithDialog() { TreeDatabaseTarget target = GetTargetFromCurrentSelection(); if (target == null) @@ -7340,11 +7341,10 @@ private async void ImportSqlWithDialog() return; } - MySqlImportOptions mysqlOptions = null; if (IsMySqlTarget(target)) { - mysqlOptions = ShowMySqlImportOptionsDialog(); - if (mysqlOptions == null) return; + OpenMySqlImportWizard(target); + return; } using (OpenFileDialog dialog = new OpenFileDialog()) @@ -7360,32 +7360,11 @@ private async void ImportSqlWithDialog() try { - if (mysqlOptions != null) - { - MySqlImportResult result = await ImportMySqlSqlFileWithProgress(target, dialog.FileName, mysqlOptions); - string message = Localization.Format("MySqlImport.Success", result.ExecutedStatements, result.FailedStatements); - if (result.SkippedStatements > 0) - { - message += Localization.Format("MySqlImport.SkippedSummary", result.SkippedStatements); - } - if (result.Errors.Count > 0) - { - message += Environment.NewLine + Localization.Format( - "MySqlImport.FirstError", - result.Errors[0].LineNumber, - result.Errors[0].Message); - } - MessageBox.Show(message, Localization.T("Common.Complete"), MessageBoxButtons.OK, MessageBoxIcon.Information); - UpdateMainStatus(message.Replace(Environment.NewLine, " ")); - } - else - { - string script = ReadSqlScriptWithEncodingDetection(dialog.FileName); - int executed = ImportSqlScript(target, script); - string message = string.Format(Localization.T("ImportSql.Success"), executed); - MessageBox.Show(message, Localization.T("Common.Complete"), MessageBoxButtons.OK, MessageBoxIcon.Information); - UpdateMainStatus(message); - } + string script = ReadSqlScriptWithEncodingDetection(dialog.FileName); + int executed = ImportSqlScript(target, script); + string message = string.Format(Localization.T("ImportSql.Success"), executed); + MessageBox.Show(message, Localization.T("Common.Complete"), MessageBoxButtons.OK, MessageBoxIcon.Information); + UpdateMainStatus(message); } catch (Exception ex) { diff --git a/mySQLPunk/MySqlExportWizardForm.cs b/mySQLPunk/MySqlExportWizardForm.cs new file mode 100644 index 0000000..4ef5931 --- /dev/null +++ b/mySQLPunk/MySqlExportWizardForm.cs @@ -0,0 +1,635 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlExport; +using MySqlExportResult = mySQLPunk.lib.MySqlExport.MySqlExportResult; + +namespace mySQLPunk +{ + internal sealed class MySqlExportWizardForm : Form + { + private readonly IDatabase db; + private readonly string databaseName; + private readonly MySqlSqlExportService service; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); + private readonly DataGridView tableGrid = new DataGridView(); + private readonly RadioButton schemaData = new RadioButton(), schemaOnly = new RadioButton(), dataOnly = new RadioButton(); + private readonly CheckBox singleTransaction = new CheckBox(), disableFk = new CheckBox(), includeDrop = new CheckBox(); + private readonly CheckBox includeCreateDb = new CheckBox(), removeDefiner = new CheckBox(); + private readonly CheckBox includeViews = new CheckBox(), includeRoutines = new CheckBox(), includeTriggers = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(), selectionSummary = new Label(), confirmSummary = new Label(); + private readonly Label currentObjectLabel = new Label(), percentLabel = new Label(), writtenLabel = new Label(); + private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(); + private readonly Button openFileButton = new Button(), openFolderButton = new Button(); + private readonly ProgressBar overallBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox logList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private List catalog = new List(); + private List views = new List(); + private List routines = new List(); + private List triggers = new List(); + private CancellationTokenSource cancellation; + private MySqlExportProgress lastProgress; + private bool busy, closing, exporting; + private int step; + + internal MySqlExportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal MySqlExportWizardForm(IDatabase db, string databaseName, MySqlExportWizardOptions initialOptions = null) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + this.service = new MySqlSqlExportService(db, databaseName); + + Name = "mySqlExportWizard"; + Text = T("匯出資料庫 · MySQL / MariaDB", "Export database · MySQL / MariaDB"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1100, 730); + MinimumSize = new Size(640, 480); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("MySQL 資料庫匯出精靈", "MySQL database export wizard") + " · " + databaseName, Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(); + BuildPage1(initialOptions); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var footer = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + WrapContents = true, + FlowDirection = FlowDirection.RightToLeft + }; + ConfigureButton(cancelButton, "exportCancel", T("取消", "Cancel"), (s, e) => { if (busy) RequestCancel(); else Close(); }); + ConfigureButton(nextButton, "exportNext", T("下一步 →", "Next →"), async (s, e) => await OnNextAsync()); + ConfigureButton(backButton, "exportBack", T("← 上一步", "← Back"), (s, e) => { if (!busy && step > 0 && step < 3) { step--; ShowPage(); } }); + + footer.Controls.AddRange(new Control[] { cancelButton, nextButton, backButton }); + root.Controls.Add(footer, 0, 3); + + ThemeManager.MarkAsPrimary(nextButton); + AcceptButton = nextButton; + CancelButton = cancelButton; + cancelButton.DialogResult = DialogResult.None; + + timer.Tick += (s, e) => UpdateHeartbeat(); + + Shown += async (s, e) => + { + await LoadCatalogAsync(); + ShowPage(); + }; + + FormClosing += (s, e) => + { + if (busy) + { + e.Cancel = true; + closing = true; + RequestCancel(); + } + }; + + ThemeManager.ApplyTo(this); + ShowPage(); + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel { Dock = DockStyle.Top, AutoSize = true, FlowDirection = FlowDirection.TopDown, WrapContents = false, Margin = Padding.Empty }; + private static Label HeaderLabel(string text) => new Label { AutoSize = true, Text = text, Font = UiKit.Subtitle, Margin = new Padding(0, 4, 0, 8) }; + private static Label HintLabel(string text) => new Label { AutoSize = true, Text = text, Font = UiKit.Body, Margin = new Padding(0, 2, 0, 6) }; + + private static void ConfigureButton(Button b, string name, string text, EventHandler handler) + { + b.Name = name; + b.Text = text; + b.AutoSize = true; + b.MinimumSize = new Size(95, 34); + b.Margin = new Padding(4); + b.Click += handler; + } + + private void BuildPage0() + { + var content = new Panel { Dock = DockStyle.Fill }; + pages[0].Controls.Add(content); + + var topPanel = new FlowLayoutPanel { Dock = DockStyle.Top, AutoSize = true, FlowDirection = FlowDirection.TopDown, WrapContents = false }; + topPanel.Controls.Add(HeaderLabel(T("步驟 1：選取要匯出的物件", "Step 1: Select objects to export"))); + topPanel.Controls.Add(HintLabel(T("勾選要包含的資料表與相關物件；可隨時透過搜尋框快速篩選。", "Select tables and objects to include; use search to filter."))); + + var modePanel = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 6, 0, 10) }; + schemaData.Text = T("結構與資料", "Structure & Data"); + schemaData.Checked = true; + schemaData.AutoSize = true; + schemaData.Margin = new Padding(0, 0, 16, 0); + + schemaOnly.Text = T("僅結構", "Structure only"); + schemaOnly.AutoSize = true; + schemaOnly.Margin = new Padding(0, 0, 16, 0); + + dataOnly.Text = T("僅資料", "Data only"); + dataOnly.AutoSize = true; + + modePanel.Controls.AddRange(new Control[] { schemaData, schemaOnly, dataOnly }); + topPanel.Controls.Add(modePanel); + + var filterPanel = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 0, 0, 8) }; + search.Width = 260; + search.Margin = new Padding(0, 0, 10, 0); + search.TextChanged += (s, e) => ApplySearchFilter(); + + var selectAllBtn = new Button { Text = T("全選", "Select All"), AutoSize = true, Margin = new Padding(0, 0, 6, 0) }; + selectAllBtn.Click += (s, e) => SetAllSelected(true); + var clearAllBtn = new Button { Text = T("全取消", "Clear All"), AutoSize = true, Margin = new Padding(0, 0, 16, 0) }; + clearAllBtn.Click += (s, e) => SetAllSelected(false); + + includeViews.Text = T("包含檢視 (Views)", "Include Views"); + includeViews.Checked = true; + includeViews.AutoSize = true; + includeViews.Margin = new Padding(0, 4, 12, 0); + + includeRoutines.Text = T("包含預存程序/函數", "Include Routines"); + includeRoutines.Checked = true; + includeRoutines.AutoSize = true; + includeRoutines.Margin = new Padding(0, 4, 12, 0); + + includeTriggers.Text = T("包含觸發器 (Triggers)", "Include Triggers"); + includeTriggers.Checked = true; + includeTriggers.AutoSize = true; + includeTriggers.Margin = new Padding(0, 4, 0, 0); + + filterPanel.Controls.AddRange(new Control[] { new Label { Text = T("搜尋：", "Search:"), AutoSize = true, Margin = new Padding(0, 5, 4, 0) }, search, selectAllBtn, clearAllBtn, includeViews, includeRoutines, includeTriggers }); + topPanel.Controls.Add(filterPanel); + + content.Controls.Add(topPanel); + + tableGrid.Dock = DockStyle.Fill; + tableGrid.AllowUserToAddRows = false; + tableGrid.AllowUserToDeleteRows = false; + tableGrid.RowHeadersVisible = false; + tableGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect; + tableGrid.MultiSelect = false; + tableGrid.AutoSizeColumnsMode = DataGridViewAutoSizeColumnsMode.Fill; + + tableGrid.Columns.Add(new DataGridViewCheckBoxColumn { Name = "ColSelect", HeaderText = T("選取", "Select"), FillWeight = 20 }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColName", HeaderText = T("資料表名稱", "Table Name"), FillWeight = 55, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColRows", HeaderText = T("預估列數", "Estimated Rows"), FillWeight = 25, ReadOnly = true }); + + tableGrid.CellValueChanged += (s, e) => + { + if (e.RowIndex >= 0 && e.ColumnIndex == 0) + { + string name = tableGrid.Rows[e.RowIndex].Cells["ColName"].Value?.ToString(); + bool isChecked = Convert.ToBoolean(tableGrid.Rows[e.RowIndex].Cells["ColSelect"].Value); + var item = catalog.FirstOrDefault(t => t.Name == name); + if (item != null) item.IsSelected = isChecked; + UpdateSelectionSummary(); + } + }; + tableGrid.CurrentCellDirtyStateChanged += (s, e) => + { + if (tableGrid.IsCurrentCellDirty) tableGrid.CommitEdit(DataGridViewDataErrorContexts.Commit); + }; + + content.Controls.Add(tableGrid); + tableGrid.BringToFront(); + + var bottomPanel = new FlowLayoutPanel { Dock = DockStyle.Bottom, AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Padding = new Padding(0, 6, 0, 0) }; + selectionSummary.AutoSize = true; + bottomPanel.Controls.Add(selectionSummary); + content.Controls.Add(bottomPanel); + } + + private void BuildPage1(MySqlExportWizardOptions initialOptions) + { + var content = Stack(); + pages[1].Controls.Add(content); + content.Controls.Add(HeaderLabel(T("步驟 2：設定輸出檔案與一致性選項", "Step 2: Output file and consistency settings"))); + content.Controls.Add(HintLabel(T("設定本機匯出 SQL 路徑與讀取事務一致性保障。", "Configure local output path and read transaction consistency."))); + + content.Controls.Add(new Label { Text = T("匯出目的地檔案：", "Destination SQL file:"), AutoSize = true, Margin = new Padding(0, 8, 0, 4) }); + + var fileRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 0, 0, 12) }; + targetPathBox.Width = 600; + string defaultName = databaseName + "_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; + targetPathBox.Text = initialOptions?.TargetPath ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultName); + ConfigureButton(browseButton, "exportBrowse", T("瀏覽…", "Browse…"), (s, e) => + { + using (var sfd = new SaveFileDialog + { + Filter = Localization.T("Common.SqlFilesFilter"), + DefaultExt = "sql", + FileName = Path.GetFileName(targetPathBox.Text) + }) + { + if (sfd.ShowDialog(this) == DialogResult.OK) + targetPathBox.Text = sfd.FileName; + } + }); + fileRow.Controls.AddRange(new Control[] { targetPathBox, browseButton }); + content.Controls.Add(fileRow); + + var optionsGroup = new GroupBox { Text = T("進階與一致性選項", "Advanced and Consistency Options"), Width = 720, AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 4, 0, 12) }; + var groupFlow = Stack(); + optionsGroup.Controls.Add(groupFlow); + + singleTransaction.Text = T("啟用一致性快照 (--single-transaction，以 REPEATABLE READ 交易讀取，避免外鍵不一致)", "Consistent snapshot (--single-transaction, repeatable read prevents foreign-key drift)"); + singleTransaction.Checked = true; + singleTransaction.AutoSize = true; + singleTransaction.Margin = new Padding(0, 4, 0, 8); + + disableFk.Text = T("匯入時關閉外鍵檢查 (SET FOREIGN_KEY_CHECKS = 0)", "Disable foreign key checks on import (SET FOREIGN_KEY_CHECKS = 0)"); + disableFk.Checked = true; + disableFk.AutoSize = true; + disableFk.Margin = new Padding(0, 0, 0, 8); + + includeDrop.Text = T("包含 DROP TABLE / VIEW / ROUTINE 語句", "Include DROP TABLE / VIEW / ROUTINE statements"); + includeDrop.Checked = true; + includeDrop.AutoSize = true; + includeDrop.Margin = new Padding(0, 0, 0, 8); + + removeDefiner.Text = T("移除 DEFINER 宣告（提高還原至其他主機/帳號時的相容性）", "Remove DEFINER clauses for portability across hosts"); + removeDefiner.Checked = true; + removeDefiner.AutoSize = true; + removeDefiner.Margin = new Padding(0, 0, 0, 8); + + includeCreateDb.Text = T("包含 CREATE DATABASE 語句", "Include CREATE DATABASE statement"); + includeCreateDb.Checked = false; + includeCreateDb.AutoSize = true; + + groupFlow.Controls.AddRange(new Control[] { singleTransaction, disableFk, includeDrop, removeDefiner, includeCreateDb }); + content.Controls.Add(optionsGroup); + } + + private void BuildPage2() + { + var content = Stack(); + pages[2].Controls.Add(content); + content.Controls.Add(HeaderLabel(T("步驟 3：確認物件與預估", "Step 3: Confirm selection and estimates"))); + content.Controls.Add(HintLabel(T("請確認本次匯出參數，點擊「開始匯出」即會串流寫入本機檔案。", "Review export parameters. Click 'Start Export' to begin streaming to local file."))); + + confirmSummary.AutoSize = true; + confirmSummary.Margin = new Padding(0, 12, 0, 16); + content.Controls.Add(confirmSummary); + } + + private void BuildPage3() + { + var content = new Panel { Dock = DockStyle.Fill }; + pages[3].Controls.Add(content); + + var topPanel = new TableLayoutPanel { Dock = DockStyle.Top, AutoSize = true, ColumnCount = 2, RowCount = 1 }; + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 70)); + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 30)); + + var statsFlow = Stack(); + currentObjectLabel.Font = UiKit.Subtitle; + currentObjectLabel.AutoSize = true; + statsFlow.Controls.Add(currentObjectLabel); + + overallBar.Width = 520; + overallBar.Height = 20; + overallBar.Margin = new Padding(0, 8, 0, 6); + statsFlow.Controls.Add(overallBar); + + var numRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + percentLabel.AutoSize = true; + percentLabel.Margin = new Padding(0, 0, 20, 0); + writtenLabel.AutoSize = true; + writtenLabel.Margin = new Padding(0, 0, 20, 0); + speedLabel.AutoSize = true; + numRow.Controls.AddRange(new Control[] { percentLabel, writtenLabel, speedLabel }); + statsFlow.Controls.Add(numRow); + + var timeRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 4, 0, 6) }; + elapsedLabel.AutoSize = true; + elapsedLabel.Margin = new Padding(0, 0, 20, 0); + etaLabel.AutoSize = true; + timeRow.Controls.AddRange(new Control[] { elapsedLabel, etaLabel }); + statsFlow.Controls.Add(timeRow); + + resultText.AutoSize = true; + resultText.Margin = new Padding(0, 6, 0, 8); + statsFlow.Controls.Add(resultText); + + var actRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + ConfigureButton(openFileButton, "exportOpenFile", T("開啟檔案", "Open File"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.TargetPath) && File.Exists(Result.TargetPath)) + Process.Start(new ProcessStartInfo("notepad.exe", Result.TargetPath) { UseShellExecute = true }); + }); + ConfigureButton(openFolderButton, "exportOpenFolder", T("開啟資料夾", "Open Folder"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.TargetPath) && File.Exists(Result.TargetPath)) + Process.Start("explorer.exe", $"/select,\"{Result.TargetPath}\""); + }); + openFileButton.Visible = false; + openFolderButton.Visible = false; + actRow.Controls.AddRange(new Control[] { openFileButton, openFolderButton }); + statsFlow.Controls.Add(actRow); + + topPanel.Controls.Add(statsFlow, 0, 0); + + // 右側放置 Punky 整理資料動畫 + punky.Dock = DockStyle.Fill; + punky.MinimumSize = new Size(180, 160); + topPanel.Controls.Add(punky, 1, 0); + + content.Controls.Add(topPanel); + + logList.Dock = DockStyle.Fill; + content.Controls.Add(logList); + logList.BringToFront(); + } + + private async Task LoadCatalogAsync() + { + statusLabel.Text = T("正在載入資料庫結構中...", "Loading database catalog..."); + catalog = await Task.Run(() => service.LoadTableCatalog()); + views = await Task.Run(() => service.LoadViews()); + routines = await Task.Run(() => service.LoadRoutines()); + triggers = await Task.Run(() => service.LoadTriggers()); + + tableGrid.Rows.Clear(); + foreach (var item in catalog) + { + tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0")); + } + + statusLabel.Text = ""; + UpdateSelectionSummary(); + } + + private void ApplySearchFilter() + { + string term = search.Text.Trim(); + tableGrid.Rows.Clear(); + foreach (var item in catalog) + { + if (string.IsNullOrWhiteSpace(term) || item.Name.IndexOf(term, StringComparison.OrdinalIgnoreCase) >= 0) + { + tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0")); + } + } + UpdateSelectionSummary(); + } + + private void SetAllSelected(bool isSelected) + { + foreach (var item in catalog) item.IsSelected = isSelected; + foreach (DataGridViewRow row in tableGrid.Rows) row.Cells["ColSelect"].Value = isSelected; + UpdateSelectionSummary(); + } + + private void UpdateSelectionSummary() + { + int selectedCount = catalog.Count(t => t.IsSelected); + long totalRows = catalog.Where(t => t.IsSelected).Sum(t => t.EstimatedRows); + selectionSummary.Text = string.Format( + T("已選取 {0} / {1} 張資料表（預估約 {2:N0} 筆資料）", "Selected {0} / {1} tables (approx. {2:N0} rows)"), + selectedCount, catalog.Count, totalRows); + } + + private void ShowPage() + { + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + string[] stepTitles = { + T("1. 選取物件", "1. Select Objects"), + T("2. 輸出設定", "2. Output Settings"), + T("3. 確認", "3. Confirm"), + T("4. 執行匯出", "4. Export") + }; + stepsLabel.Text = string.Join(" → ", stepTitles.Select((t, i) => i == step ? $"【{t}】" : t)); + + backButton.Enabled = (step > 0 && step < 3 && !busy); + if (step == 0) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = catalog.Any(t => t.IsSelected) || views.Count > 0; + } + else if (step == 1) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = !string.IsNullOrWhiteSpace(targetPathBox.Text); + } + else if (step == 2) + { + nextButton.Text = T("🚀 開始匯出", "🚀 Start Export"); + nextButton.Enabled = true; + UpdateConfirmSummary(); + } + else if (step == 3) + { + nextButton.Text = T("完成", "Finished"); + nextButton.Enabled = !busy; + } + } + + private void UpdateConfirmSummary() + { + var selectedTables = catalog.Where(t => t.IsSelected).ToList(); + confirmSummary.Text = + $"● {T("目標資料庫：", "Target Database: ")} {databaseName}\n" + + $"● {T("匯出模式：", "Export Mode: ")} {(schemaData.Checked ? T("結構與資料", "Structure & Data") : schemaOnly.Checked ? T("僅結構", "Structure only") : T("僅資料", "Data only"))}\n" + + $"● {T("選取資料表：", "Selected Tables: ")} {selectedTables.Count} 張（預估 {selectedTables.Sum(t => t.EstimatedRows):N0} 列）\n" + + $"● {T("選取檢視：", "Selected Views: ")} {(includeViews.Checked ? views.Count : 0)} 個\n" + + $"● {T("選取常式/觸發器：", "Selected Routines/Triggers: ")} {(includeRoutines.Checked ? routines.Count : 0)} / {(includeTriggers.Checked ? triggers.Count : 0)} 個\n" + + $"● {T("一致性快照：", "Consistency Snapshot: ")} {(singleTransaction.Checked ? T("啟用 (--single-transaction，不鎖表)", "Enabled (--single-transaction)") : T("停用", "Disabled"))}\n" + + $"● {T("輸出檔案路徑：", "Target Path: ")} {targetPathBox.Text}\n\n" + + $"{T("提示：匯出過程將全串流寫入暫存檔案，完成驗證後原子發布，不污染既有檔案。", "Notice: Streaming to staging file first, published atomically upon completion.")}"; + } + + private async Task OnNextAsync() + { + if (busy) return; + if (step < 2) + { + step++; + ShowPage(); + } + else if (step == 2) + { + step = 3; + ShowPage(); + await StartExportAsync(); + } + else if (step == 3) + { + Close(); + } + } + + private async Task StartExportAsync() + { + busy = true; + exporting = true; + cancellation = new CancellationTokenSource(); + nextButton.Enabled = false; + backButton.Enabled = false; + cancelButton.Text = T("停止匯出", "Stop Export"); + logList.Items.Clear(); + + var mode = schemaData.Checked ? MySqlExportMode.SchemaAndData : schemaOnly.Checked ? MySqlExportMode.SchemaOnly : MySqlExportMode.DataOnly; + var opt = new MySqlExportWizardOptions(mode, targetPathBox.Text.Trim()) + { + SingleTransaction = singleTransaction.Checked, + DisableForeignKeyChecks = disableFk.Checked, + IncludeDropStatements = includeDrop.Checked, + IncludeCreateDatabase = includeCreateDb.Checked, + RemoveDefiner = removeDefiner.Checked, + IncludeViews = includeViews.Checked, + IncludeRoutines = includeRoutines.Checked, + IncludeTriggers = includeTriggers.Checked, + SelectedTables = catalog.Where(t => t.IsSelected).Select(t => t.Name).ToList(), + SelectedViews = views, + SelectedRoutines = routines.Select(r => r.Type + ":" + r.Name).ToList(), + SelectedTriggers = triggers.Select(tr => tr.Name).ToList() + }; + + var progress = new Progress(p => + { + lastProgress = p; + RenderProgress(p); + }); + + clock.Restart(); + timer.Start(); + + try + { + Result = await service.ExportAsync(opt, progress, cancellation.Token); + timer.Stop(); + clock.Stop(); + + if (Result.Success) + { + currentObjectLabel.Text = T("🎉 匯出成功完成！", "🎉 Export completed successfully!"); + resultText.Text = string.Format( + T("共計匯出 {0} 張資料表、{1:N0} 列資料、檔案大小 {2:N0} 位元組。\nSHA-256：{3}", + "Exported {0} tables, {1:N0} rows, size {2:N0} bytes.\nSHA-256: {3}"), + Result.TableCount, Result.TotalRows, Result.FileBytes, Result.Sha256Hash); + openFileButton.Visible = true; + openFolderButton.Visible = true; + } + else if (Result.Stage == MySqlExportStage.Cancelled) + { + currentObjectLabel.Text = T("⚠️ 匯出已被使用者取消。", "⚠️ Export cancelled by user."); + resultText.Text = T("暫存檔案已自動清理，未更動目標檔案。", "Staging file was deleted, destination untouched."); + } + else + { + currentObjectLabel.Text = T("❌ 匯出失敗：", "❌ Export failed: ") + Result.ErrorMessage; + resultText.Text = Result.ErrorMessage; + } + } + catch (Exception ex) + { + timer.Stop(); + clock.Stop(); + currentObjectLabel.Text = T("❌ 發生非預期錯誤：", "❌ Unexpected error: ") + ex.Message; + } + finally + { + busy = false; + exporting = false; + cancelButton.Text = T("關閉", "Close"); + nextButton.Enabled = true; + if (closing) Close(); + } + } + + private void RenderProgress(MySqlExportProgress p) + { + if (p.TotalTables > 0) + { + int val = Math.Min(100, Math.Max(0, (int)(p.CompletedTables * 100.0 / p.TotalTables))); + overallBar.Value = val; + percentLabel.Text = $"{val}%"; + } + currentObjectLabel.Text = p.Message; + writtenLabel.Text = string.Format(T("已寫入：{0:N0} 列 ({1:N0} KB)", "Written: {0:N0} rows ({1:N0} KB)"), p.ProcessedRows, p.WrittenBytes / 1024); + speedLabel.Text = string.Format(T("速率：{0:N0} 列/秒", "Speed: {0:N0} rows/s"), p.RowsPerSecond); + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), p.Elapsed); + etaLabel.Text = p.EstimatedRemaining.HasValue + ? string.Format(T("預估剩餘：約 {0:mm\\:ss}", "ETA: approx. {0:mm\\:ss}"), p.EstimatedRemaining.Value) + : ""; + + if (!string.IsNullOrWhiteSpace(p.CurrentObject)) + { + string logItem = $"[{DateTime.Now:HH:mm:ss}] {p.Message}"; + if (logList.Items.Count == 0 || !logList.Items[logList.Items.Count - 1].Equals(logItem)) + { + logList.Items.Add(logItem); + logList.TopIndex = logList.Items.Count - 1; + } + } + } + + private void UpdateHeartbeat() + { + if (lastProgress != null && exporting) + { + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), clock.Elapsed); + } + } + + private void RequestCancel() + { + if (cancellation != null && !cancellation.IsCancellationRequested) + { + cancellation.Cancel(); + cancelButton.Enabled = false; + currentObjectLabel.Text = T("正在取消匯出並清理暫存中...", "Cancelling export and cleaning temporary files..."); + } + } + } +} diff --git a/mySQLPunk/MySqlImportWizardForm.cs b/mySQLPunk/MySqlImportWizardForm.cs new file mode 100644 index 0000000..b3dd4b0 --- /dev/null +++ b/mySQLPunk/MySqlImportWizardForm.cs @@ -0,0 +1,564 @@ +using System; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlImport; +using MySqlImportResult = mySQLPunk.lib.MySqlImport.MySqlImportResult; + +namespace mySQLPunk +{ + internal sealed class MySqlImportWizardForm : Form + { + private readonly IDatabase db; + private readonly string databaseName; + private readonly MySqlImportRunner runner; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox sourcePathBox = new TextBox(), targetDbBox = new TextBox(), logPathBox = new TextBox(); + private readonly ComboBox strategyCombo = new ComboBox(); + private readonly CheckBox continueOnError = new CheckBox(), confirmCheck = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(), fileFactsLabel = new Label(); + private readonly Label checkSummary = new Label(), confirmSummary = new Label(); + private readonly Label currentStatementLabel = new Label(), coverageLabel = new Label(), speedLabel = new Label(); + private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), errorCountLabel = new Label(), resultText = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(); + private readonly Button recheckButton = new Button(), openLogButton = new Button(), openFolderButton = new Button(), copyPathButton = new Button(); + private readonly ProgressBar execBar = new ProgressBar(), scanBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox errorList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private MySqlImportPreflight preflight; + private MySqlImportRequest request; + private CancellationTokenSource cancellation; + private bool busy, executing, closing; + private int step; + + internal MySqlImportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal MySqlImportWizardForm(IDatabase db, string databaseName, string initialSourcePath = "") + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + this.runner = new MySqlImportRunner(db, databaseName); + this.request = new MySqlImportRequest(Guid.NewGuid(), initialSourcePath, databaseName); + + Name = "mySqlImportWizard"; + Text = T("匯入 SQL 檔案 · MySQL / MariaDB", "Import SQL file · MySQL / MariaDB"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1100, 750); + MinimumSize = new Size(640, 480); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("MySQL SQL 檔案匯入精靈", "MySQL SQL file import wizard"), Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(); + BuildPage1(); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var footer = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + WrapContents = true, + FlowDirection = FlowDirection.RightToLeft + }; + ConfigureButton(cancelButton, "importCancel", T("取消", "Cancel"), (s, e) => { if (busy) RequestCancel(); else Close(); }); + ConfigureButton(nextButton, "importNext", T("下一步 →", "Next →"), async (s, e) => await OnNextAsync()); + ConfigureButton(backButton, "importBack", T("← 上一步", "← Back"), (s, e) => { if (!busy && step > 0 && step < 3) { step--; ShowPage(); } }); + ConfigureButton(recheckButton, "importRecheck", T("重新預檢", "Recheck"), async (s, e) => { if (!busy) await RunPreflightAsync(); }); + recheckButton.Visible = false; + + footer.Controls.AddRange(new Control[] { cancelButton, nextButton, backButton, recheckButton }); + root.Controls.Add(footer, 0, 3); + + ThemeManager.MarkAsPrimary(nextButton); + AcceptButton = nextButton; + CancelButton = cancelButton; + cancelButton.DialogResult = DialogResult.None; + + timer.Tick += (s, e) => + { + if (executing) + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), clock.Elapsed); + }; + + FormClosing += (s, e) => + { + if (busy) + { + e.Cancel = true; + closing = true; + RequestCancel(); + } + }; + + ThemeManager.ApplyTo(this); + ShowPage(); + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel { Dock = DockStyle.Top, AutoSize = true, FlowDirection = FlowDirection.TopDown, WrapContents = false, Margin = Padding.Empty }; + private static Label HeaderLabel(string text) => new Label { AutoSize = true, Text = text, Font = UiKit.Subtitle, Margin = new Padding(0, 4, 0, 8) }; + private static Label HintLabel(string text) => new Label { AutoSize = true, Text = text, Font = UiKit.Body, Margin = new Padding(0, 2, 0, 6) }; + + private static void ConfigureButton(Button b, string name, string text, EventHandler handler) + { + b.Name = name; + b.Text = text; + b.AutoSize = true; + b.MinimumSize = new Size(95, 34); + b.Margin = new Padding(4); + b.Click += handler; + } + + private void BuildPage0() + { + var content = Stack(); + pages[0].Controls.Add(content); + content.Controls.Add(HeaderLabel(T("步驟 1：選擇檔案與目標資料庫", "Step 1: Choose SQL file and target database"))); + content.Controls.Add(HintLabel(T("先選取欲匯入的 SQL 檔案，系統將於下一步進行串流掃描檢查，預檢絕不執行檔案中的 SQL。", "Choose a SQL file to import. The next step will scan it safely without executing any SQL."))); + + content.Controls.Add(new Label { Text = T("SQL 檔案完整路徑：", "SQL file path:"), AutoSize = true, Margin = new Padding(0, 8, 0, 4) }); + + var row = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 0, 0, 6) }; + sourcePathBox.Width = 600; + sourcePathBox.Text = request.SourcePath; + sourcePathBox.TextChanged += (s, e) => + { + request.SourcePath = sourcePathBox.Text.Trim(); + UpdateFileFacts(); + }; + ConfigureButton(browseButton, "importBrowse", T("瀏覽…", "Browse…"), (s, e) => + { + using (var ofd = new OpenFileDialog + { + Filter = Localization.T("Common.SqlFilesFilter"), + Multiselect = false, + CheckFileExists = true + }) + { + if (ofd.ShowDialog(this) == DialogResult.OK) + sourcePathBox.Text = ofd.FileName; + } + }); + row.Controls.AddRange(new Control[] { sourcePathBox, browseButton }); + content.Controls.Add(row); + + fileFactsLabel.AutoSize = true; + fileFactsLabel.Margin = new Padding(0, 0, 0, 12); + content.Controls.Add(fileFactsLabel); + + content.Controls.Add(new Label { Text = T("目標資料庫：", "Target Database:"), AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); + targetDbBox.Width = 600; + targetDbBox.ReadOnly = true; + targetDbBox.Text = databaseName; + content.Controls.Add(targetDbBox); + + UpdateFileFacts(); + } + + private void BuildPage1() + { + var content = Stack(); + pages[1].Controls.Add(content); + content.Controls.Add(HeaderLabel(T("步驟 2：串流預檢 (Preflight Check)", "Step 2: Safe Preflight Scan"))); + content.Controls.Add(HintLabel(T("系統將完整串流掃描來源檔案，驗證編碼、語法完整性、單批次 32 MiB 上限與 Native Manifest 識別。", "Scanning file for encoding, syntax validity, 32 MiB batch boundary, and Native Manifest."))); + + scanBar.Width = 600; + scanBar.Height = 18; + scanBar.Margin = new Padding(0, 10, 0, 12); + content.Controls.Add(scanBar); + + checkSummary.AutoSize = true; + checkSummary.Margin = new Padding(0, 4, 0, 12); + content.Controls.Add(checkSummary); + } + + private void BuildPage2() + { + var content = Stack(); + pages[2].Controls.Add(content); + content.Controls.Add(HeaderLabel(T("步驟 3：確認執行與策略設定", "Step 3: Confirm execution and strategy"))); + content.Controls.Add(HintLabel(T("請確認目標資料庫、設定物件衝突時的處理原則與獨立日誌儲存位置。", "Review target database, conflict handling strategy, and independent log location."))); + + var optGroup = new GroupBox { Text = T("匯入策略與錯誤政策", "Import and Error Policies"), Width = 700, AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 6, 0, 12) }; + var groupFlow = Stack(); + optGroup.Controls.Add(groupFlow); + + groupFlow.Controls.Add(new Label { Text = T("既有物件策略：", "Existing Object Strategy:"), AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); + strategyCombo.DropDownStyle = ComboBoxStyle.DropDownList; + strategyCombo.Width = 400; + strategyCombo.Items.Add(Localization.T("MySqlImport.StrategyExecuteScript")); + strategyCombo.Items.Add(Localization.T("MySqlImport.StrategyDropRecreate")); + strategyCombo.Items.Add(Localization.T("MySqlImport.StrategyCreateIfMissing")); + strategyCombo.Items.Add(Localization.T("MySqlImport.StrategySkipExisting")); + strategyCombo.SelectedIndex = 0; + groupFlow.Controls.Add(strategyCombo); + + continueOnError.Text = Localization.T("MySqlImport.ContinueOnError") + " (" + T("遇到一般 SQL 錯誤時，略過失敗批次並繼續", "Skip failed statement and continue") + ")"; + continueOnError.Checked = false; + continueOnError.AutoSize = true; + continueOnError.Margin = new Padding(0, 10, 0, 8); + groupFlow.Controls.Add(continueOnError); + + content.Controls.Add(optGroup); + + content.Controls.Add(new Label { Text = T("獨立執行日誌位置：", "Independent execution log path:"), AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); + logPathBox.Width = 600; + logPathBox.ReadOnly = true; + logPathBox.Text = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "mySQLPunk", "ImportLogs"); + content.Controls.Add(logPathBox); + + confirmCheck.Text = T("我已確認目標資料庫正確，並了解異常或終止時可能保留部分已生效資料。", "I have confirmed the target database and acknowledge partial writes may remain upon error."); + confirmCheck.Checked = false; + confirmCheck.AutoSize = true; + confirmCheck.Margin = new Padding(0, 12, 0, 10); + confirmCheck.CheckedChanged += (s, e) => + { + if (step == 2) nextButton.Enabled = confirmCheck.Checked; + }; + content.Controls.Add(confirmCheck); + } + + private void BuildPage3() + { + var content = new Panel { Dock = DockStyle.Fill }; + pages[3].Controls.Add(content); + + var topPanel = new TableLayoutPanel { Dock = DockStyle.Top, AutoSize = true, ColumnCount = 2, RowCount = 1 }; + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 70)); + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 30)); + + var statsFlow = Stack(); + currentStatementLabel.Font = UiKit.Subtitle; + currentStatementLabel.AutoSize = true; + statsFlow.Controls.Add(currentStatementLabel); + + execBar.Width = 520; + execBar.Height = 20; + execBar.Margin = new Padding(0, 8, 0, 6); + statsFlow.Controls.Add(execBar); + + var numRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + coverageLabel.AutoSize = true; + coverageLabel.Margin = new Padding(0, 0, 20, 0); + speedLabel.AutoSize = true; + speedLabel.Margin = new Padding(0, 0, 20, 0); + errorCountLabel.AutoSize = true; + numRow.Controls.AddRange(new Control[] { coverageLabel, speedLabel, errorCountLabel }); + statsFlow.Controls.Add(numRow); + + var timeRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 4, 0, 6) }; + elapsedLabel.AutoSize = true; + elapsedLabel.Margin = new Padding(0, 0, 20, 0); + etaLabel.AutoSize = true; + timeRow.Controls.AddRange(new Control[] { elapsedLabel, etaLabel }); + statsFlow.Controls.Add(timeRow); + + resultText.AutoSize = true; + resultText.Margin = new Padding(0, 6, 0, 8); + statsFlow.Controls.Add(resultText); + + var actRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + ConfigureButton(openLogButton, "importOpenLog", T("開啟日誌", "Open Log"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogPath) && File.Exists(Result.LogPath)) + Process.Start(new ProcessStartInfo("notepad.exe", Result.LogPath) { UseShellExecute = true }); + }); + ConfigureButton(openFolderButton, "importOpenFolder", T("開啟資料夾", "Open Folder"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogPath) && File.Exists(Result.LogPath)) + Process.Start("explorer.exe", $"/select,\"{Result.LogPath}\""); + }); + ConfigureButton(copyPathButton, "importCopyPath", T("複製日誌路徑", "Copy Log Path"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogPath)) + { + Clipboard.SetText(Result.LogPath); + MessageBox.Show(T("日誌路徑已複製到剪貼簿！", "Log path copied to clipboard!"), T("已複製", "Copied"), MessageBoxButtons.OK, MessageBoxIcon.Information); + } + }); + + openLogButton.Visible = false; + openFolderButton.Visible = false; + copyPathButton.Visible = false; + actRow.Controls.AddRange(new Control[] { openLogButton, openFolderButton, copyPathButton }); + statsFlow.Controls.Add(actRow); + + topPanel.Controls.Add(statsFlow, 0, 0); + + // 右側 Punky 動畫控制項 + punky.Dock = DockStyle.Fill; + punky.MinimumSize = new Size(180, 160); + topPanel.Controls.Add(punky, 1, 0); + + content.Controls.Add(topPanel); + + errorList.Dock = DockStyle.Fill; + content.Controls.Add(errorList); + errorList.BringToFront(); + } + + private void UpdateFileFacts() + { + string path = sourcePathBox.Text.Trim(); + if (File.Exists(path)) + { + long bytes = new FileInfo(path).Length; + fileFactsLabel.Text = string.Format(T("檔案大小：{0:N0} 位元組 ({1:N2} MB)", "File size: {0:N0} bytes ({1:N2} MB)"), bytes, bytes / (1024.0 * 1024.0)); + nextButton.Enabled = (step == 0); + } + else + { + fileFactsLabel.Text = T("未選擇檔案或檔案不存在。", "No file selected or file does not exist."); + if (step == 0) nextButton.Enabled = false; + } + } + + private void ShowPage() + { + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + string[] stepTitles = { + T("1. 檔案與目標", "1. File & Target"), + T("2. 串流預檢", "2. Preflight Scan"), + T("3. 確認與策略", "3. Confirm & Policy"), + T("4. 執行匯入", "4. Execute Import") + }; + stepsLabel.Text = string.Join(" → ", stepTitles.Select((t, i) => i == step ? $"【{t}】" : t)); + + backButton.Enabled = (step > 0 && step < 3 && !busy); + recheckButton.Visible = (step == 1); + + if (step == 0) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = File.Exists(sourcePathBox.Text.Trim()); + } + else if (step == 1) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = preflight != null && preflight.Passed; + } + else if (step == 2) + { + nextButton.Text = T("🚀 開始匯入", "🚀 Start Import"); + nextButton.Enabled = confirmCheck.Checked; + } + else if (step == 3) + { + nextButton.Text = T("完成", "Finished"); + nextButton.Enabled = !busy; + } + } + + private async Task OnNextAsync() + { + if (busy) return; + if (step == 0) + { + step = 1; + ShowPage(); + await RunPreflightAsync(); + } + else if (step == 1) + { + step = 2; + ShowPage(); + } + else if (step == 2) + { + step = 3; + ShowPage(); + await StartImportAsync(); + } + else if (step == 3) + { + Close(); + } + } + + private async Task RunPreflightAsync() + { + busy = true; + nextButton.Enabled = false; + recheckButton.Enabled = false; + scanBar.Style = ProgressBarStyle.Marquee; + checkSummary.Text = T("正在進行串流預檢掃描中...", "Scanning file in streaming mode..."); + + string path = sourcePathBox.Text.Trim(); + cancellation = new CancellationTokenSource(); + preflight = await Task.Run(() => MySqlImportPreflightScanner.Scan(path, cancellation.Token)); + + scanBar.Style = ProgressBarStyle.Blocks; + scanBar.Value = preflight.Passed ? 100 : 0; + busy = false; + recheckButton.Enabled = true; + + if (preflight.Passed) + { + nextButton.Enabled = true; + string manifestInfo = preflight.HasManifest + ? string.Format(T("✔ 偵測到 Native Manifest 標註：共 {0} 張表、約 {1:N0} 列（SHA: {2}）", "✔ Native Manifest detected: {0} tables, approx. {1:N0} rows (SHA: {2})"), + preflight.ManifestTables, preflight.ManifestRows, preflight.ManifestSha256?.Substring(0, 16) + "...") + : T("✔ 檔案未包含 Manifest 註解，將作為標準 SQL 腳本匯入。", "✔ File does not contain Manifest comments; will import as standard script."); + + checkSummary.Text = + $"✔ {T("預檢掃描通過！未發現非法編碼或超長批次。", "Preflight scan passed! No encoding issues or oversized batches.")}\n" + + $"✔ {T("檔案大小：", "File size: ")} {preflight.FileBytes:N0} 位元組\n" + + $"✔ {T("預估語句數：", "Estimated statements: ")} {preflight.EstimatedStatements:N0}\n" + + $"{manifestInfo}\n" + + $"✔ {T("目標資料庫連線正常：", "Target connection normal: ")} {databaseName}"; + } + else + { + nextButton.Enabled = false; + checkSummary.Text = $"❌ {T("預檢發現問題，無法直接匯入：", "Preflight identified issues:")}\n" + string.Join("\n", preflight.Issues.Select(iss => "● " + iss)); + } + } + + private async Task StartImportAsync() + { + busy = true; + executing = true; + cancellation = new CancellationTokenSource(); + nextButton.Enabled = false; + backButton.Enabled = false; + cancelButton.Text = T("停止匯入", "Stop Import"); + errorList.Items.Clear(); + + var strat = (MySqlExistingObjectStrategy)strategyCombo.SelectedIndex; + var opt = new MySqlImportOptions + { + ContinueOnError = continueOnError.Checked, + ExistingObjectStrategy = strat + }; + + var prog = new Progress(p => RenderProgress(p)); + clock.Restart(); + timer.Start(); + + try + { + Result = await runner.RunAsync(request, opt, prog, cancellation.Token); + timer.Stop(); + clock.Stop(); + + if (Result.Status == MySqlImportStatus.Completed) + { + currentStatementLabel.Text = T("🎉 匯入成功完成！", "🎉 Import completed successfully!"); + resultText.Text = Result.Summary; + } + else if (Result.Status == MySqlImportStatus.CompletedWithErrors) + { + currentStatementLabel.Text = T("⚠️ 匯入完成，但有部分語句失敗／略過。", "⚠️ Import finished with errors / skipped statements."); + resultText.Text = Result.Summary; + } + else if (Result.Status == MySqlImportStatus.Cancelled) + { + currentStatementLabel.Text = T("⚠️ 匯入已被使用者取消。", "⚠️ Import was cancelled by user."); + resultText.Text = T("已停止執行，先前已提交之變更可能保留於資料庫中。", "Execution stopped; previously committed changes may remain."); + } + else + { + currentStatementLabel.Text = T("❌ 匯入停止：發生重大錯誤。", "❌ Import stopped: fatal error occurred."); + resultText.Text = Result.Summary; + } + + openLogButton.Visible = true; + openFolderButton.Visible = true; + copyPathButton.Visible = true; + } + catch (Exception ex) + { + timer.Stop(); + clock.Stop(); + currentStatementLabel.Text = T("❌ 發生非預期例外：", "❌ Unexpected exception: ") + ex.Message; + } + finally + { + busy = false; + executing = false; + cancelButton.Text = T("關閉", "Close"); + nextButton.Enabled = true; + if (closing) Close(); + } + } + + private void RenderProgress(MySqlImportProgress p) + { + if (p.TotalBytes > 0) + { + int val = Math.Min(100, Math.Max(0, (int)(p.ProcessedBytes * 100.0 / p.TotalBytes))); + execBar.Value = val; + coverageLabel.Text = $"{val}% ({p.ProcessedBytes / 1024:N0} KB / {p.TotalBytes / 1024:N0} KB)"; + } + currentStatementLabel.Text = p.Message; + speedLabel.Text = string.Format(T("速率：{0:N0} KB/秒", "Speed: {0:N0} KB/s"), p.BytesPerSecond / 1024.0); + errorCountLabel.Text = string.Format(T("錯誤：{0} 筆", "Errors: {0}"), p.FailedStatements); + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), p.Elapsed); + etaLabel.Text = p.EstimatedRemaining.HasValue + ? string.Format(T("預估剩餘：約 {0:mm\\:ss}", "ETA: approx. {0:mm\\:ss}"), p.EstimatedRemaining.Value) + : ""; + + if (!string.IsNullOrWhiteSpace(p.CurrentStatementPreview)) + { + if (errorList.Items.Count < 200) + { + errorList.Items.Add($"[{DateTime.Now:HH:mm:ss}] {p.CurrentStatementPreview}"); + errorList.TopIndex = errorList.Items.Count - 1; + } + } + } + + private void RequestCancel() + { + if (cancellation != null && !cancellation.IsCancellationRequested) + { + cancellation.Cancel(); + cancelButton.Enabled = false; + currentStatementLabel.Text = T("正在停止匯入中...", "Stopping import..."); + } + } + } +} diff --git a/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs b/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs new file mode 100644 index 0000000..6738247 --- /dev/null +++ b/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; + +namespace mySQLPunk.lib.MySqlExport +{ + public enum MySqlExportMode + { + SchemaAndData, + SchemaOnly, + DataOnly + } + + public enum MySqlExportStage + { + Ready, + ReadingCatalog, + WritingSchema, + WritingData, + WritingRoutines, + WritingTriggers, + WritingManifest, + Completed, + Cancelled, + Failed + } + + public sealed class MySqlTableExportItem + { + public string Name { get; set; } + public long EstimatedRows { get; set; } + public bool IsSelected { get; set; } + + public MySqlTableExportItem(string name, long estimatedRows, bool isSelected = true) + { + Name = name; + EstimatedRows = estimatedRows; + IsSelected = isSelected; + } + } + + public sealed class MySqlExportWizardOptions + { + public MySqlExportMode Mode { get; set; } + public string TargetPath { get; set; } + public bool SingleTransaction { get; set; } = true; + public bool DisableForeignKeyChecks { get; set; } = true; + public bool IncludeDropStatements { get; set; } = true; + public bool IncludeCreateDatabase { get; set; } = false; + public bool IncludeUseDatabase { get; set; } = true; + public bool RemoveDefiner { get; set; } = true; + public bool IncludeViews { get; set; } = true; + public bool IncludeRoutines { get; set; } = true; + public bool IncludeTriggers { get; set; } = true; + public int InsertBatchSize { get; set; } = 1000; + public List SelectedTables { get; set; } = new List(); + public List SelectedViews { get; set; } = new List(); + public List SelectedRoutines { get; set; } = new List(); + public List SelectedTriggers { get; set; } = new List(); + + public MySqlExportWizardOptions(MySqlExportMode mode = MySqlExportMode.SchemaAndData, string targetPath = null) + { + Mode = mode; + TargetPath = targetPath; + } + } + + public sealed class MySqlExportProgress + { + public MySqlExportStage Stage { get; } + public string CurrentObject { get; } + public int CompletedTables { get; } + public int TotalTables { get; } + public long ProcessedRows { get; } + public long EstimatedTotalRows { get; } + public long WrittenBytes { get; } + public double RowsPerSecond { get; } + public TimeSpan Elapsed { get; } + public TimeSpan? EstimatedRemaining { get; } + public string Message { get; } + + public MySqlExportProgress( + MySqlExportStage stage, + string currentObject, + int completedTables, + int totalTables, + long processedRows, + long estimatedTotalRows, + long writtenBytes, + double rowsPerSecond, + TimeSpan elapsed, + TimeSpan? estimatedRemaining, + string message) + { + Stage = stage; + CurrentObject = currentObject; + CompletedTables = completedTables; + TotalTables = totalTables; + ProcessedRows = processedRows; + EstimatedTotalRows = estimatedTotalRows; + WrittenBytes = writtenBytes; + RowsPerSecond = rowsPerSecond; + Elapsed = elapsed; + EstimatedRemaining = estimatedRemaining; + Message = message; + } + } + + public sealed class MySqlExportResult + { + public MySqlExportStage Stage { get; } + public bool Success => Stage == MySqlExportStage.Completed; + public string TargetPath { get; } + public int TableCount { get; } + public int ViewCount { get; } + public int RoutineCount { get; } + public int TriggerCount { get; } + public long TotalRows { get; } + public long FileBytes { get; } + public string Sha256Hash { get; } + public TimeSpan Elapsed { get; } + public string ErrorMessage { get; } + + public MySqlExportResult( + MySqlExportStage stage, + string targetPath, + int tableCount, + int viewCount, + int routineCount, + int triggerCount, + long totalRows, + long fileBytes, + string sha256Hash, + TimeSpan elapsed, + string errorMessage = null) + { + Stage = stage; + TargetPath = targetPath; + TableCount = tableCount; + ViewCount = viewCount; + RoutineCount = routineCount; + TriggerCount = triggerCount; + TotalRows = totalRows; + FileBytes = fileBytes; + Sha256Hash = sha256Hash; + Elapsed = elapsed; + ErrorMessage = errorMessage; + } + } +} diff --git a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs new file mode 100644 index 0000000..95c23ad --- /dev/null +++ b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs @@ -0,0 +1,480 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.MySqlExport +{ + public sealed class MySqlSqlExportService + { + private readonly IDatabase db; + private readonly string databaseName; + + public MySqlSqlExportService(IDatabase db, string databaseName) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + } + + public List LoadTableCatalog() + { + var list = new List(); + var tables = db.GetTables(databaseName); + foreach (var table in tables) + { + if (string.IsNullOrWhiteSpace(table)) continue; + long count = 0; + try { count = db.CountRows(databaseName, table); } catch { } + list.Add(new MySqlTableExportItem(table, count, true)); + } + return list; + } + + public List LoadViews() + { + try { return db.GetViews(databaseName) ?? new List(); } + catch { return new List(); } + } + + public List LoadRoutines() + { + try { return MySqlExportService.GetRoutines(db, databaseName) ?? new List(); } + catch { return new List(); } + } + + public List LoadTriggers() + { + try { return MySqlExportService.GetTriggers(db, databaseName) ?? new List(); } + catch { return new List(); } + } + + public Task ExportAsync( + MySqlExportWizardOptions options, + IProgress progress, + CancellationToken cancellation) + { + return Task.Run(() => ExportInternal(options, progress, cancellation), cancellation); + } + + private MySqlExportResult ExportInternal( + MySqlExportWizardOptions options, + IProgress progress, + CancellationToken cancellation) + { + var clock = Stopwatch.StartNew(); + if (string.IsNullOrWhiteSpace(options.TargetPath)) + throw new ArgumentException("Target path is required", nameof(options.TargetPath)); + + string targetPath = options.TargetPath; + string targetDir = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrWhiteSpace(targetDir) && !Directory.Exists(targetDir)) + Directory.CreateDirectory(targetDir); + + string stagingPath = targetPath + ".staging." + Guid.NewGuid().ToString("N") + ".tmp"; + long totalProcessedRows = 0; + long estimatedTotalRows = 0; + int tableCount = 0; + int viewCount = 0; + int routineCount = 0; + int triggerCount = 0; + var tableManifest = new List>(); + + var selectedTables = options.SelectedTables?.ToList() ?? new List(); + var selectedViews = options.IncludeViews ? (options.SelectedViews?.ToList() ?? new List()) : new List(); + var selectedRoutines = options.IncludeRoutines ? (options.SelectedRoutines?.ToList() ?? new List()) : new List(); + var selectedTriggers = options.IncludeTriggers ? (options.SelectedTriggers?.ToList() ?? new List()) : new List(); + + // 預估總列數 + if (options.Mode != MySqlExportMode.SchemaOnly) + { + foreach (var table in selectedTables) + { + try { estimatedTotalRows += db.CountRows(databaseName, table); } catch { } + } + } + + bool inTransaction = false; + try + { + cancellation.ThrowIfCancellationRequested(); + + // 啟動可選一致性快照 (single-transaction) + if (options.SingleTransaction && options.Mode != MySqlExportMode.SchemaOnly) + { + try + { + db.ExecSQL("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); + db.ExecSQL("START TRANSACTION WITH CONSISTENT SNAPSHOT;"); + inTransaction = true; + } + catch + { + // 若無足夠權限或引擎不支援一致性快照，安全降級為普通讀取 + inTransaction = false; + } + } + + using (var fileStream = new FileStream(stagingPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 65536)) + using (var writer = new StreamWriter(fileStream, new UTF8Encoding(false))) + { + writer.WriteLine("-- ========================================================"); + writer.WriteLine("-- mySQLPunk MySQL Logical Dump"); + writer.WriteLine($"-- Database: {databaseName}"); + writer.WriteLine($"-- Created: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + writer.WriteLine($"-- Mode: {options.Mode}"); + writer.WriteLine($"-- SingleTransaction: {options.SingleTransaction && inTransaction}"); + writer.WriteLine("-- ========================================================"); + writer.WriteLine(); + writer.WriteLine("SET NAMES utf8mb4;"); + if (options.DisableForeignKeyChecks) + writer.WriteLine("SET FOREIGN_KEY_CHECKS = 0;"); + if (options.IncludeCreateDatabase) + { + var info = db.GetDatabaseInfo(databaseName) ?? new Dictionary(); + string charset = info.ContainsKey("character_set") ? info["character_set"] : "utf8mb4"; + string collation = info.ContainsKey("collation") ? info["collation"] : "utf8mb4_general_ci"; + writer.WriteLine($"CREATE DATABASE IF NOT EXISTS `{databaseName}` DEFAULT CHARACTER SET {charset} COLLATE {collation};"); + } + if (options.IncludeUseDatabase) + writer.WriteLine($"USE `{databaseName}`;"); + writer.WriteLine(); + + // 1. DROP 語句 + if (options.IncludeDropStatements && options.Mode != MySqlExportMode.DataOnly) + { + foreach (var tr in selectedTriggers) writer.WriteLine($"DROP TRIGGER IF EXISTS `{tr}`;"); + foreach (var r in selectedRoutines) + { + string type = r.StartsWith("PROCEDURE:", StringComparison.OrdinalIgnoreCase) ? "PROCEDURE" : "FUNCTION"; + string name = r.Contains(":") ? r.Substring(r.IndexOf(':') + 1) : r; + writer.WriteLine($"DROP {type} IF EXISTS `{name}`;"); + } + foreach (var v in selectedViews) writer.WriteLine($"DROP VIEW IF EXISTS `{v}`;"); + foreach (var t in selectedTables) writer.WriteLine($"DROP TABLE IF EXISTS `{t}`;"); + writer.WriteLine(); + } + + // 2. 結構 DDL (Tables) + if (options.Mode != MySqlExportMode.DataOnly) + { + ReportProgress(progress, MySqlExportStage.WritingSchema, "", 0, selectedTables.Count, 0, estimatedTotalRows, fileStream.Position, clock); + + foreach (var table in selectedTables) + { + cancellation.ThrowIfCancellationRequested(); + string ddl = db.GetTableCreateStatement(databaseName, table); + if (!string.IsNullOrWhiteSpace(ddl)) + { + writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + writer.WriteLine(); + } + } + } + + // 3. 資料列 (Data) + if (options.Mode != MySqlExportMode.SchemaOnly) + { + int batchSize = options.InsertBatchSize > 0 ? options.InsertBatchSize : 1000; + for (int i = 0; i < selectedTables.Count; i++) + { + cancellation.ThrowIfCancellationRequested(); + string table = selectedTables[i]; + long tableRows = 0; + long totalTableRows = 0; + try { totalTableRows = db.CountRows(databaseName, table); } catch { } + + ReportProgress(progress, MySqlExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + + while (tableRows < totalTableRows || totalTableRows == 0) + { + cancellation.ThrowIfCancellationRequested(); + DataTable page = db.SelectTablePage(databaseName, table, tableRows, batchSize); + if (page == null || page.Rows.Count == 0) break; + + writer.Write("INSERT INTO `"); + writer.Write(table.Replace("`", "``")); + writer.Write("` ("); + for (int c = 0; c < page.Columns.Count; c++) + { + if (c > 0) writer.Write(", "); + writer.Write("`"); + writer.Write(page.Columns[c].ColumnName.Replace("`", "``")); + writer.Write("`"); + } + writer.WriteLine(") VALUES"); + + for (int r = 0; r < page.Rows.Count; r++) + { + writer.Write("("); + for (int c = 0; c < page.Columns.Count; c++) + { + if (c > 0) writer.Write(", "); + writer.Write(MySqlExportService.ToMySqlLiteral(page.Rows[r][c])); + } + writer.Write(r == page.Rows.Count - 1 ? ");" : "),"); + writer.WriteLine(); + } + + tableRows += page.Rows.Count; + totalProcessedRows += page.Rows.Count; + ReportProgress(progress, MySqlExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + } + + if (tableRows > 0) writer.WriteLine(); + tableCount++; + tableManifest.Add(Tuple.Create(table, tableRows)); + } + } + else + { + tableCount = selectedTables.Count; + } + + // 4. 檢視 (Views) + if (options.Mode != MySqlExportMode.DataOnly && selectedViews.Count > 0) + { + ReportProgress(progress, MySqlExportStage.WritingSchema, "Views", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + foreach (var view in selectedViews) + { + cancellation.ThrowIfCancellationRequested(); + string ddl = db.GetViewCreateStatement(databaseName, view); + if (!string.IsNullOrWhiteSpace(ddl)) + { + ddl = MySqlExportService.NormalizeObjectCreateSql(ddl, options.RemoveDefiner); + writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + writer.WriteLine(); + } + viewCount++; + } + } + + // 5. 函數與預存程序 (Routines) + if (options.Mode != MySqlExportMode.DataOnly && selectedRoutines.Count > 0) + { + ReportProgress(progress, MySqlExportStage.WritingRoutines, "Routines", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + var allRoutines = LoadRoutines(); + foreach (var rName in selectedRoutines) + { + cancellation.ThrowIfCancellationRequested(); + var routine = allRoutines.FirstOrDefault(r => r.Name.Equals(rName, StringComparison.OrdinalIgnoreCase) || + (r.Type + ":" + r.Name).Equals(rName, StringComparison.OrdinalIgnoreCase)); + if (routine != null) + { + string sql = $"SHOW CREATE {routine.Type} `{databaseName}`.`{routine.Name}`;"; + var dt = db.SelectSQL(sql, new Dictionary()); + if (dt != null && dt.Rows.Count > 0) + { + string createKey = "Create " + (routine.Type.Equals("PROCEDURE", StringComparison.OrdinalIgnoreCase) ? "Procedure" : "Function"); + string ddl = dt.Columns.Contains(createKey) ? dt.Rows[0][createKey]?.ToString() : dt.Rows[0][2]?.ToString(); + if (!string.IsNullOrWhiteSpace(ddl)) + { + ddl = MySqlExportService.NormalizeObjectCreateSql(ddl, options.RemoveDefiner); + writer.WriteLine("DELIMITER ;;"); + writer.WriteLine(ddl.TrimEnd().TrimEnd(';')); + writer.WriteLine(";;"); + writer.WriteLine("DELIMITER ;"); + writer.WriteLine(); + } + } + routineCount++; + } + } + } + + // 6. 觸發器 (Triggers) + if (options.Mode != MySqlExportMode.DataOnly && selectedTriggers.Count > 0) + { + ReportProgress(progress, MySqlExportStage.WritingTriggers, "Triggers", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + foreach (var triggerName in selectedTriggers) + { + cancellation.ThrowIfCancellationRequested(); + string sql = $"SHOW CREATE TRIGGER `{databaseName}`.`{triggerName}`;"; + var dt = db.SelectSQL(sql, new Dictionary()); + if (dt != null && dt.Rows.Count > 0) + { + string ddl = dt.Columns.Contains("SQL Original Statement") ? dt.Rows[0]["SQL Original Statement"]?.ToString() : dt.Rows[0][2]?.ToString(); + if (!string.IsNullOrWhiteSpace(ddl)) + { + ddl = MySqlExportService.NormalizeObjectCreateSql(ddl, options.RemoveDefiner); + writer.WriteLine("DELIMITER ;;"); + writer.WriteLine(ddl.TrimEnd().TrimEnd(';')); + writer.WriteLine(";;"); + writer.WriteLine("DELIMITER ;"); + writer.WriteLine(); + } + } + triggerCount++; + } + } + + if (options.DisableForeignKeyChecks) + writer.WriteLine("SET FOREIGN_KEY_CHECKS = 1;"); + + writer.Flush(); + + // 7. 寫入 Native Manifest 摘要 (用註解格式附在結尾，供匯入時自動檢驗比對) + writer.WriteLine(); + writer.WriteLine("-- ========================================================"); + writer.WriteLine("-- mySQLPunk Native Manifest"); + writer.WriteLine($"-- Tables: {tableCount}"); + writer.WriteLine($"-- TotalRows: {totalProcessedRows}"); + foreach (var pair in tableManifest) + { + writer.WriteLine($"-- Manifest-Table: {pair.Item1} | Rows: {pair.Item2}"); + } + writer.WriteLine("-- ========================================================"); + writer.Flush(); + } + + // 計算產出檔案的 SHA-256 + string sha256Hex; + long finalLength; + using (var fs = File.OpenRead(stagingPath)) + { + finalLength = fs.Length; + using (var sha = SHA256.Create()) + { + byte[] hash = sha.ComputeHash(fs); + sha256Hex = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); + } + } + + // 將 SHA-256 附加至 Staging 檔案尾端 + File.AppendAllText(stagingPath, $"-- Manifest-SHA256: {sha256Hex}{Environment.NewLine}", Encoding.UTF8); + + // 原子替換到目標檔案 + if (File.Exists(targetPath)) File.Delete(targetPath); + File.Move(stagingPath, targetPath); + + clock.Stop(); + ReportProgress(progress, MySqlExportStage.Completed, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, finalLength, clock); + + return new MySqlExportResult( + MySqlExportStage.Completed, + targetPath, + tableCount, + viewCount, + routineCount, + triggerCount, + totalProcessedRows, + finalLength, + sha256Hex, + clock.Elapsed); + } + catch (OperationCanceledException) + { + CleanStaging(stagingPath); + clock.Stop(); + return new MySqlExportResult( + MySqlExportStage.Cancelled, + targetPath, + tableCount, + viewCount, + routineCount, + triggerCount, + totalProcessedRows, + 0, + null, + clock.Elapsed, + "Export was cancelled by user."); + } + catch (Exception ex) + { + CleanStaging(stagingPath); + clock.Stop(); + return new MySqlExportResult( + MySqlExportStage.Failed, + targetPath, + tableCount, + viewCount, + routineCount, + triggerCount, + totalProcessedRows, + 0, + null, + clock.Elapsed, + ex.Message); + } + finally + { + if (inTransaction) + { + try { db.ExecSQL("COMMIT;"); } catch { } + } + } + } + + private static void CleanStaging(string path) + { + if (!string.IsNullOrWhiteSpace(path) && File.Exists(path)) + { + try { File.Delete(path); } catch { } + } + } + + private static void ReportProgress( + IProgress progress, + MySqlExportStage stage, + string currentObject, + int completedTables, + int totalTables, + long processedRows, + long estimatedTotalRows, + long writtenBytes, + Stopwatch clock) + { + if (progress == null) return; + double elapsedSec = clock.Elapsed.TotalSeconds; + double rowsPerSec = elapsedSec > 0.1 ? (processedRows / elapsedSec) : 0; + TimeSpan? remaining = null; + if (rowsPerSec > 1 && estimatedTotalRows > processedRows) + { + double remainingSec = (estimatedTotalRows - processedRows) / rowsPerSec; + if (remainingSec >= 0 && remainingSec < 86400 * 7) + remaining = TimeSpan.FromSeconds(remainingSec); + } + + string message; + switch (stage) + { + case MySqlExportStage.WritingSchema: + message = Localization.Format("MySqlExport.ProgressTableStructure", currentObject); + break; + case MySqlExportStage.WritingData: + message = Localization.Format("MySqlExport.ProgressTableData", currentObject); + break; + case MySqlExportStage.WritingRoutines: + message = Localization.Format("MySqlExport.ProgressRoutine", "Routine", currentObject); + break; + case MySqlExportStage.WritingTriggers: + message = Localization.Format("MySqlExport.ProgressTrigger", currentObject); + break; + case MySqlExportStage.Completed: + message = Localization.T("Common.Complete"); + break; + default: + message = currentObject; + break; + } + + progress.Report(new MySqlExportProgress( + stage, + currentObject, + completedTables, + totalTables, + processedRows, + estimatedTotalRows, + writtenBytes, + rowsPerSec, + clock.Elapsed, + remaining, + message)); + } + } +} diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs b/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs new file mode 100644 index 0000000..4f41db0 --- /dev/null +++ b/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace mySQLPunk.lib.MySqlImport +{ + public sealed class MySqlImportLog : IDisposable + { + private readonly Stream stream; + private readonly StreamWriter writer; + private readonly object gate = new object(); + private readonly List recentErrors = new List(); + private bool disposed; + + public string Path { get; } + public IReadOnlyList RecentErrors + { + get { lock (gate) return recentErrors.ToArray(); } + } + + public MySqlImportLog(Guid jobId, string directory = null) + { + directory = directory ?? System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "mySQLPunk", "ImportLogs"); + Directory.CreateDirectory(directory); + string stem = "import-mysql-" + DateTime.Now.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture) + "-" + jobId.ToString("N").Substring(0, 8); + Path = System.IO.Path.Combine(directory, stem + ".log"); + + stream = new FileStream(Path, FileMode.Create, FileAccess.Write, FileShare.Read); + writer = new StreamWriter(stream, Encoding.UTF8); + WriteHeader(jobId); + } + + private void WriteHeader(Guid jobId) + { + lock (gate) + { + writer.WriteLine("================================================================================"); + writer.WriteLine($"mySQLPunk MySQL Import Execution Log"); + writer.WriteLine($"Job ID : {jobId}"); + writer.WriteLine($"Started : {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + writer.WriteLine("================================================================================"); + writer.WriteLine(); + writer.Flush(); + } + } + + public void LogInfo(string message) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [INFO] {message}"); + } + } + + public void LogStatement(int lineNumber, string preview, bool success, string errorMessage = null, string action = null) + { + lock (gate) + { + if (disposed) return; + if (success) + { + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [OK] Line {lineNumber}: {preview}"); + } + else + { + string errText = $"[{DateTime.Now:HH:mm:ss.fff}] [ERROR] Line {lineNumber}: {errorMessage} | Action: {action} | SQL: {preview}"; + writer.WriteLine(errText); + recentErrors.Add($"Line {lineNumber}: {errorMessage}"); + if (recentErrors.Count > 100) recentErrors.RemoveAt(0); + } + } + } + + public void LogManifestVerification(string table, long expectedRows, long actualRows, bool match) + { + lock (gate) + { + if (disposed) return; + string icon = match ? "[PASS]" : "[WARN]"; + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] {icon} Manifest Check: Table `{table}` -> Expected: {expectedRows}, Actual: {actualRows}"); + } + } + + public void Flush() + { + lock (gate) + { + if (disposed) return; + try { writer.Flush(); } catch { } + } + } + + public void WriteEnd(MySqlImportResult result) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine(); + writer.WriteLine("================================================================================"); + writer.WriteLine($"Execution Finished: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + writer.WriteLine($"Status : {result.Status}"); + writer.WriteLine($"Executed : {result.ExecutedStatements} statements"); + writer.WriteLine($"Failed : {result.FailedStatements} statements"); + writer.WriteLine($"Skipped : {result.SkippedStatements} statements"); + writer.WriteLine($"Duration : {result.Elapsed.TotalSeconds:F2} seconds"); + writer.WriteLine($"Summary : {result.Summary}"); + writer.WriteLine("================================================================================"); + writer.Flush(); + } + } + + public void Dispose() + { + lock (gate) + { + if (!disposed) + { + disposed = true; + try { writer?.Dispose(); } catch { } + try { stream?.Dispose(); } catch { } + } + } + } + } +} diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs b/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs new file mode 100644 index 0000000..cd60f3d --- /dev/null +++ b/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; + +namespace mySQLPunk.lib.MySqlImport +{ + public enum MySqlImportStatus + { + Ready, + Completed, + CompletedWithErrors, + Cancelled, + Failed + } + + public enum MySqlImportStage + { + Ready, + Scanning, + Executing, + Stopping, + Finished + } + + public sealed class MySqlImportRequest + { + public Guid JobId { get; } + public string SourcePath { get; set; } + public string DatabaseName { get; } + + public MySqlImportRequest(Guid jobId, string sourcePath, string databaseName) + { + JobId = jobId; + SourcePath = sourcePath ?? string.Empty; + DatabaseName = databaseName ?? string.Empty; + } + } + + public sealed class MySqlImportPreflight + { + public bool Passed { get; } + public long FileBytes { get; } + public int EstimatedStatements { get; } + public bool HasManifest { get; } + public int ManifestTables { get; } + public long ManifestRows { get; } + public string ManifestSha256 { get; } + public List Issues { get; } = new List(); + + public MySqlImportPreflight( + bool passed, + long fileBytes, + int estimatedStatements, + bool hasManifest, + int manifestTables, + long manifestRows, + string manifestSha256, + IEnumerable issues = null) + { + Passed = passed; + FileBytes = fileBytes; + EstimatedStatements = estimatedStatements; + HasManifest = hasManifest; + ManifestTables = manifestTables; + ManifestRows = manifestRows; + ManifestSha256 = manifestSha256; + if (issues != null) Issues.AddRange(issues); + } + } + + public sealed class MySqlImportProgress + { + public Guid JobId { get; } + public MySqlImportStage Stage { get; } + public long TotalBytes { get; } + public long ProcessedBytes { get; } + public int ExecutedStatements { get; } + public int FailedStatements { get; } + public int SkippedStatements { get; } + public double BytesPerSecond { get; } + public TimeSpan Elapsed { get; } + public TimeSpan? EstimatedRemaining { get; } + public string CurrentStatementPreview { get; } + public string Message { get; } + + public MySqlImportProgress( + Guid jobId, + MySqlImportStage stage, + long totalBytes, + long processedBytes, + int executedStatements, + int failedStatements, + int skippedStatements, + double bytesPerSecond, + TimeSpan elapsed, + TimeSpan? estimatedRemaining, + string currentStatementPreview, + string message) + { + JobId = jobId; + Stage = stage; + TotalBytes = totalBytes; + ProcessedBytes = processedBytes; + ExecutedStatements = executedStatements; + FailedStatements = failedStatements; + SkippedStatements = skippedStatements; + BytesPerSecond = bytesPerSecond; + Elapsed = elapsed; + EstimatedRemaining = estimatedRemaining; + CurrentStatementPreview = currentStatementPreview; + Message = message; + } + } + + public sealed class MySqlImportResult + { + public Guid JobId { get; } + public MySqlImportStatus Status { get; } + public long FileBytes { get; } + public int ExecutedStatements { get; } + public int FailedStatements { get; } + public int SkippedStatements { get; } + public string LogPath { get; } + public TimeSpan Elapsed { get; } + public string Summary { get; } + public List Errors { get; } = new List(); + + public MySqlImportResult( + Guid jobId, + MySqlImportStatus status, + long fileBytes, + int executedStatements, + int failedStatements, + int skippedStatements, + string logPath, + TimeSpan elapsed, + string summary, + IEnumerable errors = null) + { + JobId = jobId; + Status = status; + FileBytes = fileBytes; + ExecutedStatements = executedStatements; + FailedStatements = failedStatements; + SkippedStatements = skippedStatements; + LogPath = logPath; + Elapsed = elapsed; + Summary = summary; + if (errors != null) Errors.AddRange(errors); + } + } +} diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs new file mode 100644 index 0000000..086bbb9 --- /dev/null +++ b/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; + +namespace mySQLPunk.lib.MySqlImport +{ + public static class MySqlImportPreflightScanner + { + private const int MaxBatchBytes = 32 * 1024 * 1024; // 32 MiB + private static readonly Regex ManifestTableRegex = new Regex(@"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestTablesRegex = new Regex(@"^--\s*Tables:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestRowsRegex = new Regex(@"^--\s*TotalRows:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestSha256Regex = new Regex(@"^--\s*Manifest-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static MySqlImportPreflight Scan(string filePath, CancellationToken cancellation) + { + if (string.IsNullOrWhiteSpace(filePath)) + throw new ArgumentException("File path is required", nameof(filePath)); + if (!File.Exists(filePath)) + return new MySqlImportPreflight(false, 0, 0, false, 0, 0, null, new[] { "File not found: " + filePath }); + + long fileBytes = new FileInfo(filePath).Length; + var issues = new List(); + int statementCount = 0; + bool hasManifest = false; + int manifestTables = 0; + long manifestRows = 0; + string manifestSha = null; + + try + { + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) + using (var reader = new StreamReader(fs, Encoding.UTF8, true)) + { + string delimiter = ";"; + var currentBatch = new StringBuilder(); + int lineIndex = 0; + string line; + + while ((line = reader.ReadLine()) != null) + { + cancellation.ThrowIfCancellationRequested(); + lineIndex++; + + // 檢查 embedded NUL (非法字元或編碼錯誤) + if (line.IndexOf('\0') >= 0) + { + issues.Add($"Line {lineIndex}: Contains Embedded NUL character (possible malformed encoding or binary data)."); + break; + } + + string trimmed = line.Trim(); + + // 檢查 Manifest + if (trimmed.StartsWith("--")) + { + if (trimmed.Contains("mySQLPunk Native Manifest")) + { + hasManifest = true; + } + else if (hasManifest) + { + var mTables = ManifestTablesRegex.Match(trimmed); + if (mTables.Success && int.TryParse(mTables.Groups[1].Value, out int mt)) + { + manifestTables = mt; + continue; + } + var mRows = ManifestRowsRegex.Match(trimmed); + if (mRows.Success && long.TryParse(mRows.Groups[1].Value, out long mr)) + { + manifestRows = mr; + continue; + } + var mTable = ManifestTableRegex.Match(trimmed); + if (mTable.Success) + { + if (manifestTables == 0) manifestTables++; + if (long.TryParse(mTable.Groups[2].Value, out long r) && manifestRows == 0) manifestRows += r; + continue; + } + var mSha = ManifestSha256Regex.Match(trimmed); + if (mSha.Success) + { + manifestSha = mSha.Groups[1].Value; + } + } + continue; + } + + // DELIMITER 宣告 + if (trimmed.StartsWith("DELIMITER", StringComparison.OrdinalIgnoreCase)) + { + string newDelim = trimmed.Substring(9).Trim(); + if (!string.IsNullOrWhiteSpace(newDelim)) + { + delimiter = newDelim; + currentBatch.Clear(); + continue; + } + } + + currentBatch.AppendLine(line); + + // 單批次大小檢查 (32 MiB) + if (currentBatch.Length > MaxBatchBytes) + { + issues.Add($"Line {lineIndex}: Single SQL batch exceeds safe limit of 32 MiB."); + break; + } + + // 檢查批次終止 + if (trimmed.EndsWith(delimiter, StringComparison.Ordinal)) + { + statementCount++; + currentBatch.Clear(); + } + } + + if (currentBatch.Length > 0 && !string.IsNullOrWhiteSpace(currentBatch.ToString())) + { + statementCount++; + } + } + } + catch (OperationCanceledException) + { + return new MySqlImportPreflight(false, fileBytes, statementCount, hasManifest, manifestTables, manifestRows, manifestSha, new[] { "Scan cancelled by user." }); + } + catch (Exception ex) + { + issues.Add("Error during preflight scan: " + ex.Message); + } + + bool passed = issues.Count == 0; + return new MySqlImportPreflight( + passed, + fileBytes, + statementCount, + hasManifest, + manifestTables, + manifestRows, + manifestSha, + issues); + } + } +} diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs new file mode 100644 index 0000000..a5e430f --- /dev/null +++ b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs @@ -0,0 +1,288 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.MySqlImport +{ + public sealed class MySqlImportRunner + { + private readonly IDatabase db; + private readonly string databaseName; + + public MySqlImportRunner(IDatabase db, string databaseName) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + } + + public Task RunAsync( + MySqlImportRequest request, + MySqlImportOptions options, + IProgress progress, + CancellationToken cancellation) + { + return Task.Run(() => RunInternal(request, options, progress, cancellation), cancellation); + } + + private MySqlImportResult RunInternal( + MySqlImportRequest request, + MySqlImportOptions options, + IProgress progress, + CancellationToken cancellation) + { + var clock = Stopwatch.StartNew(); + var log = new MySqlImportLog(request.JobId); + long fileBytes = 0; + if (File.Exists(request.SourcePath)) fileBytes = new FileInfo(request.SourcePath).Length; + + log.LogInfo($"Target Database: {databaseName}"); + log.LogInfo($"Source SQL File: {request.SourcePath} ({fileBytes:N0} bytes)"); + log.LogInfo($"Options: ContinueOnError={options.ContinueOnError}, Strategy={options.ExistingObjectStrategy}"); + + int executed = 0; + int failed = 0; + int skipped = 0; + var errors = new List(); + var manifestTables = new List>(); + bool readingManifest = false; + + try + { + using (var fs = new FileStream(request.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) + using (var reader = new StreamReader(fs, Encoding.UTF8, true)) + { + string delimiter = ";"; + var currentBatch = new StringBuilder(); + int batchStartLine = 1; + int currentLine = 0; + string line; + + while ((line = reader.ReadLine()) != null) + { + cancellation.ThrowIfCancellationRequested(); + currentLine++; + + string trimmed = line.Trim(); + + // 檢查 Manifest + if (trimmed.StartsWith("--")) + { + if (trimmed.Contains("mySQLPunk Native Manifest")) readingManifest = true; + else if (readingManifest) + { + var match = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) + { + manifestTables.Add(Tuple.Create(match.Groups[1].Value, r)); + } + } + continue; + } + + // DELIMITER 切換 + if (trimmed.StartsWith("DELIMITER", StringComparison.OrdinalIgnoreCase)) + { + string newDelim = trimmed.Substring(9).Trim(); + if (!string.IsNullOrWhiteSpace(newDelim)) + { + delimiter = newDelim; + currentBatch.Clear(); + batchStartLine = currentLine + 1; + continue; + } + } + + if (currentBatch.Length == 0) batchStartLine = currentLine; + currentBatch.AppendLine(line); + + // 判斷是否為批次結尾 + if (trimmed.EndsWith(delimiter, StringComparison.Ordinal)) + { + string sql = currentBatch.ToString().TrimEnd(); + if (sql.EndsWith(delimiter, StringComparison.Ordinal)) + sql = sql.Substring(0, sql.Length - delimiter.Length).Trim(); + + if (!string.IsNullOrWhiteSpace(sql)) + { + bool ok = ExecuteBatchStatement(sql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors); + if (!ok && !options.ContinueOnError) + { + log.LogInfo("Execution stopped on error as requested by options."); + break; + } + } + + currentBatch.Clear(); + batchStartLine = currentLine + 1; + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, sql); + } + } + + // 處理尾端剩餘語句 + if (currentBatch.Length > 0 && (failed == 0 || options.ContinueOnError)) + { + string remainingSql = currentBatch.ToString().Trim(); + if (!string.IsNullOrWhiteSpace(remainingSql)) + { + ExecuteBatchStatement(remainingSql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors); + } + } + } + + // 若有 Native Manifest，自動驗證目標資料庫實際列數 + if (manifestTables.Count > 0) + { + log.LogInfo("Verifying restored database tables against Native Manifest..."); + foreach (var pair in manifestTables) + { + long actual = -1; + try { actual = db.CountRows(databaseName, pair.Item1); } catch { } + bool match = actual == pair.Item2; + log.LogManifestVerification(pair.Item1, pair.Item2, actual, match); + } + } + + clock.Stop(); + MySqlImportStatus status = failed == 0 + ? MySqlImportStatus.Completed + : (executed > 0 ? MySqlImportStatus.CompletedWithErrors : MySqlImportStatus.Failed); + + string summary = string.Format( + "Execution finished in {0:F1}s: {1} executed, {2} failed, {3} skipped.", + clock.Elapsed.TotalSeconds, executed, failed, skipped); + + var finalResult = new MySqlImportResult( + request.JobId, + status, + fileBytes, + executed, + failed, + skipped, + log.Path, + clock.Elapsed, + summary, + errors); + + log.WriteEnd(finalResult); + log.Dispose(); + ReportProgress(progress, request.JobId, MySqlImportStage.Finished, fileBytes, fileBytes, executed, failed, skipped, clock, summary); + + return finalResult; + } + catch (OperationCanceledException) + { + clock.Stop(); + log.LogInfo("Execution cancelled by user."); + var cancelResult = new MySqlImportResult( + request.JobId, + MySqlImportStatus.Cancelled, + fileBytes, + executed, + failed, + skipped, + log.Path, + clock.Elapsed, + "Import was cancelled by user.", + errors); + log.WriteEnd(cancelResult); + log.Dispose(); + return cancelResult; + } + catch (Exception ex) + { + clock.Stop(); + log.LogInfo("Fatal error: " + ex.Message); + errors.Add(ex.Message); + var failResult = new MySqlImportResult( + request.JobId, + MySqlImportStatus.Failed, + fileBytes, + executed, + failed, + skipped, + log.Path, + clock.Elapsed, + ex.Message, + errors); + log.WriteEnd(failResult); + log.Dispose(); + return failResult; + } + } + + private bool ExecuteBatchStatement( + string sql, + int line, + MySqlImportOptions options, + MySqlImportLog log, + ref int executed, + ref int failed, + ref int skipped, + List errors) + { + string preview = sql.Length > 120 ? sql.Substring(0, 120).Replace("\r", " ").Replace("\n", " ") + "..." : sql.Replace("\r", " ").Replace("\n", " "); + + try + { + db.ExecSQL(sql); + executed++; + log.LogStatement(line, preview, true); + return true; + } + catch (Exception ex) + { + failed++; + string err = ex.Message; + errors.Add($"Line {line}: {err}"); + string action = options.ContinueOnError ? "SkipBatch" : "HaltExecution"; + log.LogStatement(line, preview, false, err, action); + return false; + } + } + + private static void ReportProgress( + IProgress progress, + Guid jobId, + MySqlImportStage stage, + long totalBytes, + long processedBytes, + int executed, + int failed, + int skipped, + Stopwatch clock, + string currentPreview) + { + if (progress == null) return; + double elapsedSec = clock.Elapsed.TotalSeconds; + double bytesPerSec = elapsedSec > 0.1 ? (processedBytes / elapsedSec) : 0; + TimeSpan? remaining = null; + if (bytesPerSec > 1024 && totalBytes > processedBytes) + { + double remainingSec = (totalBytes - processedBytes) / bytesPerSec; + if (remainingSec >= 0 && remainingSec < 86400 * 7) + remaining = TimeSpan.FromSeconds(remainingSec); + } + + string msg = string.Format("Importing... {0} statements ({1:N0} KB / {2:N0} KB)", executed, processedBytes / 1024, totalBytes / 1024); + progress.Report(new MySqlImportProgress( + jobId, + stage, + totalBytes, + processedBytes, + executed, + failed, + skipped, + bytesPerSec, + clock.Elapsed, + remaining, + currentPreview, + msg)); + } + } +} diff --git a/mySQLPunk/mySQLPunk.csproj b/mySQLPunk/mySQLPunk.csproj index 92c1c00..e3c0749 100644 --- a/mySQLPunk/mySQLPunk.csproj +++ b/mySQLPunk/mySQLPunk.csproj @@ -315,8 +315,12 @@ + + + + @@ -388,6 +392,12 @@ + + + + + + diff --git a/tests/Run-SmokeTests.ps1 b/tests/Run-SmokeTests.ps1 index 406dc2f..acd85a4 100644 --- a/tests/Run-SmokeTests.ps1 +++ b/tests/Run-SmokeTests.ps1 @@ -147,6 +147,8 @@ if (!(Test-Path -LiteralPath $mysqlConnector)) { (Join-Path $PSScriptRoot "SqlServerRestoreCompatibilityTests.cs") ` (Join-Path $PSScriptRoot "SqlServerMigrationTests.cs") ` (Join-Path $PSScriptRoot "SqlServerMigrationCatalogTests.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.MySqlExport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.MySqlImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.TransferTrial.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Live.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Parser.cs") $source $preferencesSource $schemaSyncSource $erModelSource (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Snapshot.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Schema.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.ViewReferences.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Properties.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Artifact.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Values.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Lob.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Routing.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.SnapshotIntegration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.CompatibilityIntegration.cs") if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE diff --git a/tests/SmokeTests.MssqlExport.Routing.cs b/tests/SmokeTests.MssqlExport.Routing.cs index 2657590..7c28c6b 100644 --- a/tests/SmokeTests.MssqlExport.Routing.cs +++ b/tests/SmokeTests.MssqlExport.Routing.cs @@ -121,7 +121,7 @@ private static void Routing_FourHandlers() private static void Routing_LegacyBranches() { - foreach (string provider in new[] { "mysql", "sqlite", "postgresql" }) + foreach (string provider in new[] { "sqlite", "postgresql" }) foreach (string handler in new[] { "DumpSelectedTableSql", "DumpCurrentSelectionSqlWithDialog", "DumpSelectedDatabaseSqlWithDialog", "BackupSelectedDatabaseWithDialog" }) using (var fixture = new RoutingFixture(provider, "Tables")) { diff --git a/tests/SmokeTests.MySqlExport.cs b/tests/SmokeTests.MySqlExport.cs new file mode 100644 index 0000000..7e04843 --- /dev/null +++ b/tests/SmokeTests.MySqlExport.cs @@ -0,0 +1,202 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlExport; + +public static partial class SmokeTests +{ + private static void MySqlExport_OptionsAndModels_Defaults() + { + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, @"C:\dump.sql"); + Assert(options.Mode == MySqlExportMode.SchemaAndData, "Default mode should be SchemaAndData"); + Assert(options.SingleTransaction, "SingleTransaction should be true by default"); + Assert(options.DisableForeignKeyChecks, "DisableForeignKeyChecks should be true by default"); + Assert(options.IncludeDropStatements, "IncludeDropStatements should be true by default"); + Assert(options.RemoveDefiner, "RemoveDefiner should be true by default"); + Assert(options.IncludeViews, "IncludeViews should be true by default"); + Assert(options.IncludeRoutines, "IncludeRoutines should be true by default"); + Assert(options.IncludeTriggers, "IncludeTriggers should be true by default"); + Assert(options.InsertBatchSize == 1000, "Default batch size should be 1000"); + + var item = new MySqlTableExportItem("users", 1500, true); + Assert(item.Name == "users", "TableName match"); + Assert(item.EstimatedRows == 1500, "EstimatedRows match"); + Assert(item.IsSelected, "IsSelected match"); + } + + private static void MySqlExport_ServiceAndManifest_AtomicWrite() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_export_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string targetFile = Path.Combine(tempDir, "export_test.sql"); + + try + { + var mockDb = new MockMySqlExportDatabase(); + var service = new MySqlSqlExportService(mockDb, "test_db"); + + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, targetFile) + { + SingleTransaction = false + }; + options.SelectedTables.Add("users"); + options.SelectedViews.Add("active_users"); + options.SelectedRoutines.Add("sp_calc"); + options.SelectedTriggers.Add("trg_users_audit"); + + MySqlExportProgress lastProgress = null; + var progress = new Progress(p => lastProgress = p); + + var task = service.ExportAsync(options, progress, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Success, "Export should succeed"); + Assert(File.Exists(targetFile), "Target file must exist after atomic move"); + Assert(result.TableCount == 1, "Exported table count should be 1"); + Assert(result.ViewCount == 1, "Exported view count should be 1"); + Assert(result.RoutineCount == 1, "Exported routine count should be 1"); + Assert(result.TriggerCount == 1, "Exported trigger count should be 1"); + Assert(result.TotalRows == 2, "Exported row count should be 2"); + + string content = File.ReadAllText(targetFile); + AssertContains(content, "-- mySQLPunk MySQL Logical Dump", "Header comment present"); + AssertContains(content, "SET FOREIGN_KEY_CHECKS = 0;", "Disable FK checks statement present"); + AssertContains(content, "USE `test_db`;", "USE database statement present"); + AssertContains(content, "CREATE TABLE `users`", "Table DDL present"); + AssertContains(content, "INSERT INTO `users`", "Table data INSERT present"); + AssertContains(content, "-- Manifest-Table: users | Rows: 2", "Native manifest table entry present"); + AssertContains(content, "-- Tables: 1", "Manifest total tables count present"); + AssertContains(content, "-- TotalRows: 2", "Manifest total rows count present"); + AssertContains(content, "-- Manifest-SHA256:", "Manifest SHA-256 marker present"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private sealed class MockMySqlExportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "MySQL"; + + public void SetConn(string connectionString) { } + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + var dt = new DataTable(); + if (sql.Contains("ROUTINES")) + { + dt.Columns.Add("ROUTINE_NAME"); + dt.Columns.Add("ROUTINE_TYPE"); + dt.Rows.Add("sp_calc", "PROCEDURE"); + return dt; + } + if (sql.Contains("TRIGGERS") && sql.Contains("information_schema")) + { + dt.Columns.Add("TRIGGER_NAME"); + dt.Rows.Add("trg_users_audit"); + return dt; + } + if (sql.Contains("SHOW CREATE VIEW")) + { + dt.Columns.Add("View"); + dt.Columns.Add("Create View"); + dt.Rows.Add("active_users", "CREATE VIEW `active_users` AS SELECT * FROM `users` WHERE active = 1"); + return dt; + } + if (sql.Contains("SHOW CREATE PROCEDURE")) + { + dt.Columns.Add("Procedure"); + dt.Columns.Add("Create Procedure"); + dt.Rows.Add("sp_calc", "CREATE PROCEDURE `sp_calc`() BEGIN SELECT 1; END"); + return dt; + } + if (sql.Contains("SHOW CREATE TRIGGER")) + { + dt.Columns.Add("Trigger"); + dt.Columns.Add("SQL Original Statement"); + dt.Rows.Add("trg_users_audit", "CREATE TRIGGER `trg_users_audit` AFTER INSERT ON `users` FOR EACH ROW BEGIN END"); + return dt; + } + return dt; + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + return new Dictionary(); + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) + { + return Task.FromResult(SelectSQL(sql, parameters)); + } + + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) + { + return Task.FromResult(ExecSQL(sql, parameters)); + } + + public List GetDatabases() => new List { "test_db" }; + public List GetTables(string databaseName) => new List { "users" }; + public List GetViews(string databaseName) => new List { "active_users" }; + + public DataTable GetColumns(string databaseName, string tableName) + { + var dt = new DataTable(); + dt.Columns.Add("COLUMN_NAME"); + dt.Columns.Add("DATA_TYPE"); + dt.Rows.Add("id", "int"); + dt.Rows.Add("name", "varchar"); + return dt; + } + + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + + public string GetTableCreateStatement(string databaseName, string tableName) + { + return "CREATE TABLE `users` (`id` int NOT NULL, `name` varchar(50) DEFAULT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; + } + + public bool TableExists(string databaseName, string tableName) => true; + public bool ViewExists(string databaseName, string viewName) => true; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 2; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) + { + var dt = new DataTable(); + dt.Columns.Add("id", typeof(int)); + dt.Columns.Add("name", typeof(string)); + if (offset == 0) + { + dt.Rows.Add(1, "Alice"); + dt.Rows.Add(2, "Bob"); + } + return dt; + } + + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => "CREATE VIEW `active_users` AS SELECT * FROM `users` WHERE active = 1;"; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.MySqlImport.cs b/tests/SmokeTests.MySqlImport.cs new file mode 100644 index 0000000..69def86 --- /dev/null +++ b/tests/SmokeTests.MySqlImport.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlImport; + +public static partial class SmokeTests +{ + private static void MySqlImport_PreflightScanner_DelimitersAndManifest() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_import_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "sample_import.sql"); + + try + { + string sqlContent = + "-- Header comments\r\n" + + "USE `test_db`;\r\n" + + "CREATE TABLE `users` (`id` int, `name` varchar(50));\r\n" + + "DELIMITER //\r\n" + + "CREATE PROCEDURE `test_proc`()\r\n" + + "BEGIN\r\n" + + " SELECT 1;\r\n" + + "END //\r\n" + + "DELIMITER ;\r\n" + + "INSERT INTO `users` VALUES (1, 'Alice');\r\n" + + "-- ========================================================\r\n" + + "-- mySQLPunk Native Manifest\r\n" + + "-- Tables: 1\r\n" + + "-- TotalRows: 1\r\n" + + "-- Manifest-Table: users | Rows: 1\r\n" + + "-- ========================================================\r\n"; + + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var preflight = MySqlImportPreflightScanner.Scan(sqlFile, CancellationToken.None); + Assert(preflight.Passed, "Preflight scan passed"); + Assert(preflight.FileBytes > 0, "FileBytes > 0"); + Assert(preflight.EstimatedStatements == 4, "Statement count should be 4 (USE, CREATE TABLE, CREATE PROCEDURE, INSERT), got " + preflight.EstimatedStatements); + Assert(preflight.HasManifest, "Manifest should be recognized"); + Assert(preflight.ManifestTables == 1, "ManifestTables count match"); + Assert(preflight.ManifestRows == 1, "ManifestRows count match"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void MySqlImport_Runner_ExecutionAndLog() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_runner_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "run_import.sql"); + + try + { + string sqlContent = + "CREATE TABLE `test` (`id` int);\r\n" + + "INSERT INTO `test` VALUES (10);\r\n" + + "-- Manifest-Table: test | Rows: 1\r\n" + + "-- Tables: 1\r\n" + + "-- TotalRows: 1\r\n"; + + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var mockDb = new MockMySqlImportDatabase(); + var runner = new MySqlImportRunner(mockDb, "test_db"); + + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions + { + ExistingObjectStrategy = MySqlExistingObjectStrategy.ExecuteScript, + ContinueOnError = true + }; + + MySqlImportProgress lastProgress = null; + var progress = new Progress(p => lastProgress = p); + + var task = runner.RunAsync(request, options, progress, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.Completed, "Import status should be Completed"); + Assert(result.ExecutedStatements == 2, "Executed 2 statements"); + Assert(result.FailedStatements == 0, "0 failed statements"); + Assert(File.Exists(result.LogPath), "Import log file must exist at " + result.LogPath); + + string logContent = File.ReadAllText(result.LogPath); + AssertContains(logContent, "mySQLPunk MySQL Import Execution Log", "Log contains start header"); + AssertContains(logContent, "Status : Completed", "Log contains completion status"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private sealed class MockMySqlImportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "MySQL"; + + public void SetConn(string connectionString) { } + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) => new DataTable(); + public Dictionary ExecSQL(string sql, Dictionary parameters = null) => new Dictionary(); + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(new DataTable()); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(new Dictionary()); + + public List GetDatabases() => new List { "test_db" }; + public List GetTables(string databaseName) => new List { "test" }; + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => ""; + public bool TableExists(string databaseName, string tableName) => true; + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 1; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index e26956e..53a2f4f 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -32,6 +32,10 @@ public static int Main(string[] args) if (Environment.GetEnvironmentVariable("MYSQLPUNK_MSSQL_IMPORT_WIZARD_PREVIEW") == "1") return ImportWizardPreview(); if (Environment.GetEnvironmentVariable("MYSQLPUNK_MSSQL_WIZARD_PREVIEW") == "1") return WizardPreview(); int passed = 0; + Run("MySQL export options and models defaults", MySqlExport_OptionsAndModels_Defaults, ref passed); + Run("MySQL export service and manifest atomic write", MySqlExport_ServiceAndManifest_AtomicWrite, ref passed); + Run("MySQL import preflight scanner delimiters and manifest", MySqlImport_PreflightScanner_DelimitersAndManifest, ref passed); + Run("MySQL import runner execution and log", MySqlImport_Runner_ExecutionAndLog, ref passed); Run("MSSQL import Transfer guards", MssqlImportTransferGuards, ref passed); Run("MSSQL import Transfer trial", MssqlImportTransferTrial, ref passed); Run("MSSQL import worker published cleanup receipt", MssqlImportPublishedReleaseRace, ref passed); From 5988b2c5671f948996c3e3ba64c1028ce96da130 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 12:45:37 +0800 Subject: [PATCH 02/12] =?UTF-8?q?feat:=20=E4=BF=AE=E5=BE=A9=E7=B2=BE?= =?UTF-8?q?=E9=9D=88=E4=BB=8B=E9=9D=A2=E8=B7=91=E7=89=88=E3=80=81=E5=8A=A0?= =?UTF-8?q?=E5=85=A5=20MySQL=20=E5=BC=95=E6=93=8E=E5=84=AA=E5=8C=96?= =?UTF-8?q?=E9=81=B8=E9=A0=85=E4=B8=A6=E5=95=9F=E7=94=A8=20Punky=20?= =?UTF-8?q?=E5=8B=95=E7=95=AB=E8=88=87=E9=9F=B3=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 + mySQLPunk/MySqlExportWizardForm.cs | 129 ++++++++++++++---- mySQLPunk/MySqlImportWizardForm.cs | 50 +++++-- .../lib/MySqlExport/MySqlExportModels.cs | 2 + .../lib/MySqlExport/MySqlSqlExportService.cs | 6 + tests/SmokeTests.MySqlExport.cs | 5 + 6 files changed, 158 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 756cb54..6fae644 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ - **獨立日誌與即時審計**：所有匯入執行步驟與錯誤皆獨立記錄至 `%LOCALAPPDATA%\mySQLPunk\ImportLogs`，並於匯入完成後自動比對 Native Manifest 目標列數，確認還原完整性。 ### 🛠️ 問題修正與優化 +- **MySQL 匯出與匯入精靈版面與體驗優化**： + - **修復版面跑版坍塌**：修正步驟 2 與步驟 3 之設定 GroupBox 在 FlowLayoutPanel 內因 AutoSize 導致寬度坍塌至邊框寬度 (~15px) 造成文字嚴重擠壓折疊之跑版問題，改採標準固定寬度 (740px) 與適應性內部流式排版。 + - **新增 MySQL 專屬高速還原與約束優化選項**：新增「匯入時關閉唯一性檢查 (`SET UNIQUE_CHECKS = 0`)，啟用 Insert Buffer 提升大量匯入速度 2~5 倍」與「防止主鍵 0 被 AUTO_INCREMENT 覆蓋 (`SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO'`)」開關，兼顧高速傾印與匯入資料正確性。 + - **看板娘 Punky 動畫與音效支援**：匯出與匯入執行階段全面啟用看板娘 Punky（崩琦）73 幀雙緩衝整理資料動畫，並伴隨播放原版音效（`punky_report_loading.wav`）；支援一鍵靜音切換按鈕與吉祥物即時作業狀態提示。 - **README.md 門面全新改版與文件模組化**：全面改寫 README.md 為現代、圖文並茂的開源專案門面，融入看板娘 Punky 崩琦角色元素，並加入 S1~S6 完整使用畫面截圖；深層工程與建置細節分別拆分為 `TODO.md`、`docs/DEVELOPMENT.md`、`docs/ARCHITECTURE.md` 與 `docs/DATA_SAFETY_SPEC.md`。 ## [1.0.0.23] - 2026-10-14 diff --git a/mySQLPunk/MySqlExportWizardForm.cs b/mySQLPunk/MySqlExportWizardForm.cs index 4ef5931..f43f82a 100644 --- a/mySQLPunk/MySqlExportWizardForm.cs +++ b/mySQLPunk/MySqlExportWizardForm.cs @@ -24,11 +24,12 @@ internal sealed class MySqlExportWizardForm : Form private readonly RadioButton schemaData = new RadioButton(), schemaOnly = new RadioButton(), dataOnly = new RadioButton(); private readonly CheckBox singleTransaction = new CheckBox(), disableFk = new CheckBox(), includeDrop = new CheckBox(); private readonly CheckBox includeCreateDb = new CheckBox(), removeDefiner = new CheckBox(); + private readonly CheckBox disableUnique = new CheckBox(), noAutoValueOnZero = new CheckBox(); private readonly CheckBox includeViews = new CheckBox(), includeRoutines = new CheckBox(), includeTriggers = new CheckBox(); private readonly Label stepsLabel = new Label(), statusLabel = new Label(), selectionSummary = new Label(), confirmSummary = new Label(); private readonly Label currentObjectLabel = new Label(), percentLabel = new Label(), writtenLabel = new Label(); - private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(); - private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(); + private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); private readonly Button openFileButton = new Button(), openFolderButton = new Button(); private readonly ProgressBar overallBar = new ProgressBar(); private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); @@ -250,15 +251,22 @@ private void BuildPage0() private void BuildPage1(MySqlExportWizardOptions initialOptions) { - var content = Stack(); + var content = new Panel { Dock = DockStyle.Fill, AutoScroll = true, Padding = new Padding(12) }; pages[1].Controls.Add(content); - content.Controls.Add(HeaderLabel(T("步驟 2：設定輸出檔案與一致性選項", "Step 2: Output file and consistency settings"))); - content.Controls.Add(HintLabel(T("設定本機匯出 SQL 路徑與讀取事務一致性保障。", "Configure local output path and read transaction consistency."))); - content.Controls.Add(new Label { Text = T("匯出目的地檔案：", "Destination SQL file:"), AutoSize = true, Margin = new Padding(0, 8, 0, 4) }); + var header = HeaderLabel(T("步驟 2：設定輸出檔案與一致性選項", "Step 2: Output file and consistency settings")); + header.Location = new Point(12, 10); + content.Controls.Add(header); + + var hint = HintLabel(T("設定本機匯出 SQL 路徑與讀取事務一致性保障。", "Configure local output path and read transaction consistency.")); + hint.Location = new Point(12, 38); + content.Controls.Add(hint); - var fileRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 0, 0, 12) }; - targetPathBox.Width = 600; + var destLabel = new Label { Text = T("匯出目的地檔案：", "Destination SQL file:"), AutoSize = true, Location = new Point(12, 68) }; + content.Controls.Add(destLabel); + + var fileRow = new FlowLayoutPanel { Location = new Point(12, 90), Size = new Size(740, 38), AutoSize = false, FlowDirection = FlowDirection.LeftToRight }; + targetPathBox.Width = 620; string defaultName = databaseName + "_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; targetPathBox.Text = initialOptions?.TargetPath ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultName); ConfigureButton(browseButton, "exportBrowse", T("瀏覽…", "Browse…"), (s, e) => @@ -277,36 +285,66 @@ private void BuildPage1(MySqlExportWizardOptions initialOptions) fileRow.Controls.AddRange(new Control[] { targetPathBox, browseButton }); content.Controls.Add(fileRow); - var optionsGroup = new GroupBox { Text = T("進階與一致性選項", "Advanced and Consistency Options"), Width = 720, AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 4, 0, 12) }; - var groupFlow = Stack(); - optionsGroup.Controls.Add(groupFlow); + // 第一組：交易與結構相容性保障 + var transGroup = new GroupBox + { + Text = T("交易與結構相容性保障 (Transaction & Portability)", "Transaction & Portability"), + Location = new Point(12, 134), + Size = new Size(740, 150), + Padding = new Padding(12) + }; + var transFlow = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, WrapContents = false }; - singleTransaction.Text = T("啟用一致性快照 (--single-transaction，以 REPEATABLE READ 交易讀取，避免外鍵不一致)", "Consistent snapshot (--single-transaction, repeatable read prevents foreign-key drift)"); + singleTransaction.Text = T("啟用一致性快照 (--single-transaction，以 REPEATABLE READ 交易無鎖讀取，保障關聯一致)", "Consistent snapshot (--single-transaction, non-locking repeatable read)"); singleTransaction.Checked = true; singleTransaction.AutoSize = true; - singleTransaction.Margin = new Padding(0, 4, 0, 8); + singleTransaction.Margin = new Padding(0, 4, 0, 6); - disableFk.Text = T("匯入時關閉外鍵檢查 (SET FOREIGN_KEY_CHECKS = 0)", "Disable foreign key checks on import (SET FOREIGN_KEY_CHECKS = 0)"); - disableFk.Checked = true; - disableFk.AutoSize = true; - disableFk.Margin = new Padding(0, 0, 0, 8); + removeDefiner.Text = T("移除 DEFINER 宣告（提高還原至其他主機/帳號時的相容性）", "Remove DEFINER clauses for portability across hosts/accounts"); + removeDefiner.Checked = true; + removeDefiner.AutoSize = true; + removeDefiner.Margin = new Padding(0, 0, 0, 6); - includeDrop.Text = T("包含 DROP TABLE / VIEW / ROUTINE 語句", "Include DROP TABLE / VIEW / ROUTINE statements"); + includeDrop.Text = T("包含 DROP TABLE / VIEW / ROUTINE / TRIGGER 語句（覆蓋舊物件）", "Include DROP statements (TABLE / VIEW / ROUTINE / TRIGGER)"); includeDrop.Checked = true; includeDrop.AutoSize = true; - includeDrop.Margin = new Padding(0, 0, 0, 8); - - removeDefiner.Text = T("移除 DEFINER 宣告（提高還原至其他主機/帳號時的相容性）", "Remove DEFINER clauses for portability across hosts"); - removeDefiner.Checked = true; - removeDefiner.AutoSize = true; - removeDefiner.Margin = new Padding(0, 0, 0, 8); + includeDrop.Margin = new Padding(0, 0, 0, 6); includeCreateDb.Text = T("包含 CREATE DATABASE 語句", "Include CREATE DATABASE statement"); includeCreateDb.Checked = false; includeCreateDb.AutoSize = true; - groupFlow.Controls.AddRange(new Control[] { singleTransaction, disableFk, includeDrop, removeDefiner, includeCreateDb }); - content.Controls.Add(optionsGroup); + transFlow.Controls.AddRange(new Control[] { singleTransaction, removeDefiner, includeDrop, includeCreateDb }); + transGroup.Controls.Add(transFlow); + content.Controls.Add(transGroup); + + // 第二組：MySQL 專屬高速還原與約束優化 + var optGroup = new GroupBox + { + Text = T("MySQL 專屬高速還原與約束優化 (Engine Optimizations & Safety)", "Engine Optimizations & Safety"), + Location = new Point(12, 294), + Size = new Size(740, 130), + Padding = new Padding(12) + }; + var optFlow = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, WrapContents = false }; + + disableFk.Text = T("匯入時關閉外鍵檢查 (SET FOREIGN_KEY_CHECKS = 0，避免資料表相依順序報錯)", "Disable foreign key checks on import (SET FOREIGN_KEY_CHECKS = 0)"); + disableFk.Checked = true; + disableFk.AutoSize = true; + disableFk.Margin = new Padding(0, 4, 0, 6); + + disableUnique.Text = T("匯入時關閉唯一性檢查 (SET UNIQUE_CHECKS = 0，啟用 Insert Buffer 提升匯入速度 2~5 倍)", "Disable unique checks on import (SET UNIQUE_CHECKS = 0, boosts bulk INSERTs 2~5x)"); + disableUnique.Checked = true; + disableUnique.AutoSize = true; + disableUnique.Margin = new Padding(0, 0, 0, 6); + + noAutoValueOnZero.Text = T("防止主鍵 0 被 AUTO_INCREMENT 覆蓋 (SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO')", "Prevent primary key 0 from auto-increment overwrite (NO_AUTO_VALUE_ON_ZERO)"); + noAutoValueOnZero.Checked = true; + noAutoValueOnZero.AutoSize = true; + + optFlow.Controls.AddRange(new Control[] { disableFk, disableUnique, noAutoValueOnZero }); + optGroup.Controls.Add(optFlow); + content.Controls.Add(optGroup); } private void BuildPage2() @@ -378,10 +416,35 @@ private void BuildPage3() topPanel.Controls.Add(statsFlow, 0, 0); - // 右側放置 Punky 整理資料動畫 - punky.Dock = DockStyle.Fill; - punky.MinimumSize = new Size(180, 160); - topPanel.Controls.Add(punky, 1, 0); + // 右側放置 Punky 整理資料動畫與音效控制 + var mascotPanel = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + Padding = new Padding(12, 0, 0, 0) + }; + punky.Name = "mysqlPunky"; + punky.Size = new Size(200, 200); + punky.Margin = new Padding(0, 0, 0, 4); + mascotPanel.Controls.Add(punky); + + mascotHint.Text = T("正在幫你整理資料！", "Organizing your data!"); + mascotHint.Font = UiKit.Body; + mascotHint.AutoSize = true; + mascotHint.Margin = new Padding(0, 0, 0, 6); + mascotPanel.Controls.Add(mascotHint); + + ConfigureButton(soundButton, "mysqlToggleSound", punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"), (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + mascotPanel.Controls.Add(soundButton); + + topPanel.Controls.Add(mascotPanel, 1, 0); content.Controls.Add(topPanel); @@ -484,6 +547,7 @@ private void UpdateConfirmSummary() $"● {T("選取檢視：", "Selected Views: ")} {(includeViews.Checked ? views.Count : 0)} 個\n" + $"● {T("選取常式/觸發器：", "Selected Routines/Triggers: ")} {(includeRoutines.Checked ? routines.Count : 0)} / {(includeTriggers.Checked ? triggers.Count : 0)} 個\n" + $"● {T("一致性快照：", "Consistency Snapshot: ")} {(singleTransaction.Checked ? T("啟用 (--single-transaction，不鎖表)", "Enabled (--single-transaction)") : T("停用", "Disabled"))}\n" + + $"● {T("MySQL 引擎優化：", "MySQL Engine Optimizations: ")} {(disableUnique.Checked ? T("停用唯一檢查 (加速 2~5x)", "Unique Checks Off") : "")} {(disableFk.Checked ? T("・停用外鍵", "・FK Checks Off") : "")} {(noAutoValueOnZero.Checked ? "・NO_AUTO_VALUE_ON_ZERO" : "")}\n" + $"● {T("輸出檔案路徑：", "Target Path: ")} {targetPathBox.Text}\n\n" + $"{T("提示：匯出過程將全串流寫入暫存檔案，完成驗證後原子發布，不污染既有檔案。", "Notice: Streaming to staging file first, published atomically upon completion.")}"; } @@ -523,6 +587,8 @@ private async Task StartExportAsync() { SingleTransaction = singleTransaction.Checked, DisableForeignKeyChecks = disableFk.Checked, + DisableUniqueChecks = disableUnique.Checked, + SetNoAutoValueOnZero = noAutoValueOnZero.Checked, IncludeDropStatements = includeDrop.Checked, IncludeCreateDatabase = includeCreateDb.Checked, RemoveDefiner = removeDefiner.Checked, @@ -543,6 +609,7 @@ private async Task StartExportAsync() clock.Restart(); timer.Start(); + punky.BeginExport(); try { @@ -579,6 +646,7 @@ private async Task StartExportAsync() } finally { + punky.EndExport(); busy = false; exporting = false; cancelButton.Text = T("關閉", "Close"); @@ -628,6 +696,7 @@ private void RequestCancel() { cancellation.Cancel(); cancelButton.Enabled = false; + punky.EndExport(); currentObjectLabel.Text = T("正在取消匯出並清理暫存中...", "Cancelling export and cleaning temporary files..."); } } diff --git a/mySQLPunk/MySqlImportWizardForm.cs b/mySQLPunk/MySqlImportWizardForm.cs index b3dd4b0..27c70dd 100644 --- a/mySQLPunk/MySqlImportWizardForm.cs +++ b/mySQLPunk/MySqlImportWizardForm.cs @@ -24,8 +24,8 @@ internal sealed class MySqlImportWizardForm : Form private readonly Label stepsLabel = new Label(), statusLabel = new Label(), fileFactsLabel = new Label(); private readonly Label checkSummary = new Label(), confirmSummary = new Label(); private readonly Label currentStatementLabel = new Label(), coverageLabel = new Label(), speedLabel = new Label(); - private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), errorCountLabel = new Label(), resultText = new Label(); - private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(); + private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), errorCountLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); private readonly Button recheckButton = new Button(), openLogButton = new Button(), openFolderButton = new Button(), copyPathButton = new Button(); private readonly ProgressBar execBar = new ProgressBar(), scanBar = new ProgressBar(); private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); @@ -221,8 +221,14 @@ private void BuildPage2() content.Controls.Add(HeaderLabel(T("步驟 3：確認執行與策略設定", "Step 3: Confirm execution and strategy"))); content.Controls.Add(HintLabel(T("請確認目標資料庫、設定物件衝突時的處理原則與獨立日誌儲存位置。", "Review target database, conflict handling strategy, and independent log location."))); - var optGroup = new GroupBox { Text = T("匯入策略與錯誤政策", "Import and Error Policies"), Width = 700, AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 6, 0, 12) }; - var groupFlow = Stack(); + var optGroup = new GroupBox + { + Text = T("匯入策略與錯誤政策", "Import and Error Policies"), + Size = new Size(740, 140), + Padding = new Padding(12), + Margin = new Padding(0, 6, 0, 12) + }; + var groupFlow = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, WrapContents = false }; optGroup.Controls.Add(groupFlow); groupFlow.Controls.Add(new Label { Text = T("既有物件策略：", "Existing Object Strategy:"), AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); @@ -327,10 +333,35 @@ private void BuildPage3() topPanel.Controls.Add(statsFlow, 0, 0); - // 右側 Punky 動畫控制項 - punky.Dock = DockStyle.Fill; - punky.MinimumSize = new Size(180, 160); - topPanel.Controls.Add(punky, 1, 0); + // 右側放置 Punky 整理資料動畫與音效控制 + var mascotPanel = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + Padding = new Padding(12, 0, 0, 0) + }; + punky.Name = "mysqlImportPunky"; + punky.Size = new Size(200, 200); + punky.Margin = new Padding(0, 0, 0, 4); + mascotPanel.Controls.Add(punky); + + mascotHint.Text = T("正在幫你匯入資料！", "Importing your data!"); + mascotHint.Font = UiKit.Body; + mascotHint.AutoSize = true; + mascotHint.Margin = new Padding(0, 0, 0, 6); + mascotPanel.Controls.Add(mascotHint); + + ConfigureButton(soundButton, "mysqlImportToggleSound", punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"), (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + mascotPanel.Controls.Add(soundButton); + + topPanel.Controls.Add(mascotPanel, 1, 0); content.Controls.Add(topPanel); @@ -477,6 +508,7 @@ private async Task StartImportAsync() var prog = new Progress(p => RenderProgress(p)); clock.Restart(); timer.Start(); + punky.BeginExport(); try { @@ -517,6 +549,7 @@ private async Task StartImportAsync() } finally { + punky.EndExport(); busy = false; executing = false; cancelButton.Text = T("關閉", "Close"); @@ -557,6 +590,7 @@ private void RequestCancel() { cancellation.Cancel(); cancelButton.Enabled = false; + punky.EndExport(); currentStatementLabel.Text = T("正在停止匯入中...", "Stopping import..."); } } diff --git a/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs b/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs index 6738247..1a89dc5 100644 --- a/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs +++ b/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs @@ -44,6 +44,8 @@ public sealed class MySqlExportWizardOptions public string TargetPath { get; set; } public bool SingleTransaction { get; set; } = true; public bool DisableForeignKeyChecks { get; set; } = true; + public bool DisableUniqueChecks { get; set; } = true; + public bool SetNoAutoValueOnZero { get; set; } = true; public bool IncludeDropStatements { get; set; } = true; public bool IncludeCreateDatabase { get; set; } = false; public bool IncludeUseDatabase { get; set; } = true; diff --git a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs index 95c23ad..f328895 100644 --- a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs +++ b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs @@ -132,8 +132,12 @@ private MySqlExportResult ExportInternal( writer.WriteLine("-- ========================================================"); writer.WriteLine(); writer.WriteLine("SET NAMES utf8mb4;"); + if (options.SetNoAutoValueOnZero) + writer.WriteLine("SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO';"); if (options.DisableForeignKeyChecks) writer.WriteLine("SET FOREIGN_KEY_CHECKS = 0;"); + if (options.DisableUniqueChecks) + writer.WriteLine("SET UNIQUE_CHECKS = 0;"); if (options.IncludeCreateDatabase) { var info = db.GetDatabaseInfo(databaseName) ?? new Dictionary(); @@ -313,6 +317,8 @@ private MySqlExportResult ExportInternal( } } + if (options.DisableUniqueChecks) + writer.WriteLine("SET UNIQUE_CHECKS = 1;"); if (options.DisableForeignKeyChecks) writer.WriteLine("SET FOREIGN_KEY_CHECKS = 1;"); diff --git a/tests/SmokeTests.MySqlExport.cs b/tests/SmokeTests.MySqlExport.cs index 7e04843..fbb607a 100644 --- a/tests/SmokeTests.MySqlExport.cs +++ b/tests/SmokeTests.MySqlExport.cs @@ -15,6 +15,8 @@ private static void MySqlExport_OptionsAndModels_Defaults() Assert(options.Mode == MySqlExportMode.SchemaAndData, "Default mode should be SchemaAndData"); Assert(options.SingleTransaction, "SingleTransaction should be true by default"); Assert(options.DisableForeignKeyChecks, "DisableForeignKeyChecks should be true by default"); + Assert(options.DisableUniqueChecks, "DisableUniqueChecks should be true by default"); + Assert(options.SetNoAutoValueOnZero, "SetNoAutoValueOnZero should be true by default"); Assert(options.IncludeDropStatements, "IncludeDropStatements should be true by default"); Assert(options.RemoveDefiner, "RemoveDefiner should be true by default"); Assert(options.IncludeViews, "IncludeViews should be true by default"); @@ -66,6 +68,9 @@ private static void MySqlExport_ServiceAndManifest_AtomicWrite() string content = File.ReadAllText(targetFile); AssertContains(content, "-- mySQLPunk MySQL Logical Dump", "Header comment present"); AssertContains(content, "SET FOREIGN_KEY_CHECKS = 0;", "Disable FK checks statement present"); + AssertContains(content, "SET UNIQUE_CHECKS = 0;", "Disable unique checks statement present"); + AssertContains(content, "SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO';", "Set NO_AUTO_VALUE_ON_ZERO statement present"); + AssertContains(content, "SET UNIQUE_CHECKS = 1;", "Restore unique checks statement present"); AssertContains(content, "USE `test_db`;", "USE database statement present"); AssertContains(content, "CREATE TABLE `users`", "Table DDL present"); AssertContains(content, "INSERT INTO `users`", "Table data INSERT present"); From 1f6f84eb9ab4c43f2e718c53e9773749e34c299f Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 13:03:28 +0800 Subject: [PATCH 03/12] =?UTF-8?q?feat:=20=E5=AF=A6=E7=8F=BE=20MySQL=20?= =?UTF-8?q?=E5=8C=AF=E5=85=A5=E5=8D=B3=E6=99=82=E7=B5=82=E7=AB=AF=E6=A9=9F?= =?UTF-8?q?=E6=97=A5=E8=AA=8C=E4=B8=B2=E6=B5=81=E8=88=87=202000=20?= =?UTF-8?q?=E8=A1=8C=E6=BB=91=E5=8B=95=E8=A6=96=E7=AA=97=E9=98=B2=E7=88=86?= =?UTF-8?q?=E6=A9=9F=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 + mySQLPunk/MySqlImportWizardForm.cs | 95 +++++++++++++++++-- .../lib/MySqlImport/MySqlImportRunner.cs | 76 +++++++++++++-- tests/SmokeTests.MySqlImport.cs | 9 ++ 4 files changed, 170 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fae644..4b31606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ - **修復版面跑版坍塌**：修正步驟 2 與步驟 3 之設定 GroupBox 在 FlowLayoutPanel 內因 AutoSize 導致寬度坍塌至邊框寬度 (~15px) 造成文字嚴重擠壓折疊之跑版問題，改採標準固定寬度 (740px) 與適應性內部流式排版。 - **新增 MySQL 專屬高速還原與約束優化選項**：新增「匯入時關閉唯一性檢查 (`SET UNIQUE_CHECKS = 0`)，啟用 Insert Buffer 提升大量匯入速度 2~5 倍」與「防止主鍵 0 被 AUTO_INCREMENT 覆蓋 (`SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO'`)」開關，兼顧高速傾印與匯入資料正確性。 - **看板娘 Punky 動畫與音效支援**：匯出與匯入執行階段全面啟用看板娘 Punky（崩琦）73 幀雙緩衝整理資料動畫，並伴隨播放原版音效（`punky_report_loading.wav`）；支援一鍵靜音切換按鈕與吉祥物即時作業狀態提示。 + - **匯入即時終端機日誌與 2,000 行滑動視窗防護**： + - **解決白內容與空白渲染**：萃取精準單行 SQL 摘要 Headline，徹底濾除前綴換行與多行註解，大批次 INSERT 自動濃縮並截斷至 130 字元，徹底修復 WinForms ListBox 因原始多行換行導致繪出空白大區塊的問題。 + - **2,000 行 FIFO 滑動視窗（防 RAM 爆）**：建立常數級記憶體安全機制（2,000 行 $\times$ ~130 字元 $\approx$ 300 KB），當累積超過 2,000 行時以批次平滑淘汰最舊項目，杜絕長期執行造成 OOM 記憶體溢位。 + - **流暢串流繪出與 UI 節流**：一般大批次 INSERT 採 60ms (~16 FPS) 節流串流繪出，重要 DDL 與錯誤則即時送達；介面新增「自動滾動」開關與「清空日誌」按鈕，兼顧即時瀑布流視覺與手動回溯檢視。 - **README.md 門面全新改版與文件模組化**：全面改寫 README.md 為現代、圖文並茂的開源專案門面，融入看板娘 Punky 崩琦角色元素，並加入 S1~S6 完整使用畫面截圖；深層工程與建置細節分別拆分為 `TODO.md`、`docs/DEVELOPMENT.md`、`docs/ARCHITECTURE.md` 與 `docs/DATA_SAFETY_SPEC.md`。 ## [1.0.0.23] - 2026-10-14 diff --git a/mySQLPunk/MySqlImportWizardForm.cs b/mySQLPunk/MySqlImportWizardForm.cs index 27c70dd..804bc57 100644 --- a/mySQLPunk/MySqlImportWizardForm.cs +++ b/mySQLPunk/MySqlImportWizardForm.cs @@ -20,18 +20,19 @@ internal sealed class MySqlImportWizardForm : Form private readonly Panel[] pages = new Panel[4]; private readonly TextBox sourcePathBox = new TextBox(), targetDbBox = new TextBox(), logPathBox = new TextBox(); private readonly ComboBox strategyCombo = new ComboBox(); - private readonly CheckBox continueOnError = new CheckBox(), confirmCheck = new CheckBox(); + private readonly CheckBox continueOnError = new CheckBox(), confirmCheck = new CheckBox(), autoScrollCheck = new CheckBox(); private readonly Label stepsLabel = new Label(), statusLabel = new Label(), fileFactsLabel = new Label(); - private readonly Label checkSummary = new Label(), confirmSummary = new Label(); + private readonly Label checkSummary = new Label(), confirmSummary = new Label(), logTitleLabel = new Label(), logCountLabel = new Label(); private readonly Label currentStatementLabel = new Label(), coverageLabel = new Label(), speedLabel = new Label(); private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), errorCountLabel = new Label(), resultText = new Label(), mascotHint = new Label(); - private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(), clearLogButton = new Button(); private readonly Button recheckButton = new Button(), openLogButton = new Button(), openFolderButton = new Button(), copyPathButton = new Button(); private readonly ProgressBar execBar = new ProgressBar(), scanBar = new ProgressBar(); private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); private readonly ListBox errorList = new ListBox(); private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; private readonly Stopwatch clock = new Stopwatch(); + private const int MaxLogLines = 2000; private MySqlImportPreflight preflight; private MySqlImportRequest request; @@ -365,9 +366,53 @@ private void BuildPage3() content.Controls.Add(topPanel); + // 終端機風格即時執行日誌區塊 + var logContainer = new Panel { Dock = DockStyle.Fill, Padding = new Padding(0, 10, 0, 0) }; + + var logHeader = new FlowLayoutPanel + { + Dock = DockStyle.Top, + Height = 32, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false + }; + + logTitleLabel.Text = T("即時執行日誌", "Live Execution Log"); + logTitleLabel.Font = UiKit.Subtitle; + logTitleLabel.AutoSize = true; + logTitleLabel.Margin = new Padding(0, 4, 12, 0); + + logCountLabel.Text = string.Format(T("（保留最新 {0:N0} 行）", "(Rolling latest {0:N0} lines)"), MaxLogLines); + logCountLabel.Font = UiKit.Body; + logCountLabel.ForeColor = Color.Gray; + logCountLabel.AutoSize = true; + logCountLabel.Margin = new Padding(0, 6, 20, 0); + + autoScrollCheck.Text = T("自動滾動", "Auto-scroll"); + autoScrollCheck.Checked = true; + autoScrollCheck.AutoSize = true; + autoScrollCheck.Margin = new Padding(0, 5, 12, 0); + + ConfigureButton(clearLogButton, "importClearLog", T("清空日誌", "Clear"), (s, e) => + { + errorList.Items.Clear(); + }); + clearLogButton.MinimumSize = new Size(70, 26); + clearLogButton.Height = 26; + clearLogButton.Margin = new Padding(0, 2, 0, 0); + + logHeader.Controls.AddRange(new Control[] { logTitleLabel, logCountLabel, autoScrollCheck, clearLogButton }); + logContainer.Controls.Add(logHeader); + + errorList.Name = "importLogList"; errorList.Dock = DockStyle.Fill; - content.Controls.Add(errorList); + errorList.IntegralHeight = false; + errorList.BorderStyle = BorderStyle.FixedSingle; + errorList.Font = new Font("Consolas", 9f, FontStyle.Regular); + logContainer.Controls.Add(errorList); errorList.BringToFront(); + + content.Controls.Add(logContainer); } private void UpdateFileFacts() @@ -537,6 +582,11 @@ private async Task StartImportAsync() resultText.Text = Result.Summary; } + if (!string.IsNullOrWhiteSpace(Result?.Summary)) + { + AppendLogLine("🏁 " + Result.Summary); + } + openLogButton.Visible = true; openFolderButton.Visible = true; copyPathButton.Visible = true; @@ -546,6 +596,7 @@ private async Task StartImportAsync() timer.Stop(); clock.Stop(); currentStatementLabel.Text = T("❌ 發生非預期例外：", "❌ Unexpected exception: ") + ex.Message; + AppendLogLine("❌ " + ex.Message); } finally { @@ -576,11 +627,41 @@ private void RenderProgress(MySqlImportProgress p) if (!string.IsNullOrWhiteSpace(p.CurrentStatementPreview)) { - if (errorList.Items.Count < 200) + AppendLogLine(p.CurrentStatementPreview); + } + } + + private void AppendLogLine(string headline) + { + if (string.IsNullOrWhiteSpace(headline)) return; + + string entry = $"[{DateTime.Now:HH:mm:ss}] {headline}"; + + if (errorList.Items.Count >= MaxLogLines) + { + errorList.BeginUpdate(); + try { - errorList.Items.Add($"[{DateTime.Now:HH:mm:ss}] {p.CurrentStatementPreview}"); - errorList.TopIndex = errorList.Items.Count - 1; + int excess = errorList.Items.Count - MaxLogLines + 1; + for (int i = 0; i < excess && errorList.Items.Count > 0; i++) + { + errorList.Items.RemoveAt(0); + } + errorList.Items.Add(entry); } + finally + { + errorList.EndUpdate(); + } + } + else + { + errorList.Items.Add(entry); + } + + if (autoScrollCheck.Checked && errorList.Items.Count > 0) + { + errorList.TopIndex = errorList.Items.Count - 1; } } diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs index a5e430f..6194f4b 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs @@ -63,6 +63,8 @@ private MySqlImportResult RunInternal( int currentLine = 0; string line; + long lastReportMs = 0; + while ((line = reader.ReadLine()) != null) { cancellation.ThrowIfCancellationRequested(); @@ -110,17 +112,26 @@ private MySqlImportResult RunInternal( if (!string.IsNullOrWhiteSpace(sql)) { - bool ok = ExecuteBatchStatement(sql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors); + string headline; + bool ok = ExecuteBatchStatement(sql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors, out headline); if (!ok && !options.ContinueOnError) { log.LogInfo("Execution stopped on error as requested by options."); + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, headline); break; } + + long now = clock.ElapsedMilliseconds; + bool isDdl = IsSignificantStatement(headline); + if (!ok || isDdl || (now - lastReportMs >= 60)) + { + lastReportMs = now; + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, headline); + } } currentBatch.Clear(); batchStartLine = currentLine + 1; - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, sql); } } @@ -130,7 +141,9 @@ private MySqlImportResult RunInternal( string remainingSql = currentBatch.ToString().Trim(); if (!string.IsNullOrWhiteSpace(remainingSql)) { - ExecuteBatchStatement(remainingSql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors); + string headline; + ExecuteBatchStatement(remainingSql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors, out headline); + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, headline); } } } @@ -224,15 +237,17 @@ private bool ExecuteBatchStatement( ref int executed, ref int failed, ref int skipped, - List errors) + List errors, + out string headline) { - string preview = sql.Length > 120 ? sql.Substring(0, 120).Replace("\r", " ").Replace("\n", " ") + "..." : sql.Replace("\r", " ").Replace("\n", " "); + string cleanSql = SanitizeSqlHeadline(sql); try { db.ExecSQL(sql); executed++; - log.LogStatement(line, preview, true); + log.LogStatement(line, cleanSql, true); + headline = "✔ " + cleanSql; return true; } catch (Exception ex) @@ -241,11 +256,58 @@ private bool ExecuteBatchStatement( string err = ex.Message; errors.Add($"Line {line}: {err}"); string action = options.ContinueOnError ? "SkipBatch" : "HaltExecution"; - log.LogStatement(line, preview, false, err, action); + log.LogStatement(line, cleanSql, false, err, action); + headline = $"❌ [行 {line}] {cleanSql} ➔ 錯誤: {err}"; return false; } } + public static string SanitizeSqlHeadline(string sql, int maxLength = 130) + { + if (string.IsNullOrWhiteSpace(sql)) return string.Empty; + + string[] lines = sql.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries); + string firstActionable = null; + foreach (var l in lines) + { + string t = l.Trim(); + if (string.IsNullOrEmpty(t)) continue; + if (t.StartsWith("--") || t.StartsWith("#") || t.StartsWith("/*")) continue; + firstActionable = t; + break; + } + + string target = firstActionable ?? (lines.Length > 0 ? lines[0].Trim() : sql.Trim()); + + if (target.StartsWith("INSERT INTO", StringComparison.OrdinalIgnoreCase)) + { + int valIdx = target.IndexOf("VALUES", StringComparison.OrdinalIgnoreCase); + if (valIdx > 0 && target.Length > 80) + { + target = target.Substring(0, valIdx + 6) + " (...)"; + } + } + + string collapsed = Regex.Replace(target, @"\s+", " ").Trim(); + if (collapsed.Length > maxLength) + collapsed = collapsed.Substring(0, maxLength) + "…"; + + return collapsed; + } + + private static bool IsSignificantStatement(string headline) + { + if (string.IsNullOrEmpty(headline)) return false; + string upper = headline.ToUpperInvariant(); + return upper.Contains("CREATE ") + || upper.Contains("DROP ") + || upper.Contains("ALTER ") + || upper.Contains("LOCK ") + || upper.Contains("UNLOCK ") + || upper.Contains("USE ") + || upper.Contains("TRUNCATE "); + } + private static void ReportProgress( IProgress progress, Guid jobId, diff --git a/tests/SmokeTests.MySqlImport.cs b/tests/SmokeTests.MySqlImport.cs index 69def86..a0409f5 100644 --- a/tests/SmokeTests.MySqlImport.cs +++ b/tests/SmokeTests.MySqlImport.cs @@ -97,6 +97,15 @@ private static void MySqlImport_Runner_ExecutionAndLog() string logContent = File.ReadAllText(result.LogPath); AssertContains(logContent, "mySQLPunk MySQL Import Execution Log", "Log contains start header"); AssertContains(logContent, "Status : Completed", "Log contains completion status"); + + string rawCommented = "-- table structure\r\n-- more comment\r\nCREATE TABLE `orders` (`id` int);\r\n"; + string headline1 = MySqlImportRunner.SanitizeSqlHeadline(rawCommented); + AssertEquals("CREATE TABLE `orders` (`id` int);", headline1, "Commented DDL headline match"); + + string hugeInsert = "INSERT INTO `orders` VALUES " + new string('A', 500); + string headline2 = MySqlImportRunner.SanitizeSqlHeadline(hugeInsert); + Assert(headline2.Length <= 135, "Headline length bounded to prevent memory growth"); + AssertContains(headline2, "INSERT INTO `orders` VALUES (...)", "Huge insert condensed"); } finally { From e896eae6c1abd524d5da093d50ae2cf058210002 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 13:32:32 +0800 Subject: [PATCH 04/12] =?UTF-8?q?feat:=20=E4=BF=AE=E5=BE=A9=20MySQL=20?= =?UTF-8?q?=E5=8C=AF=E5=85=A5=E9=9D=9C=E9=BB=98=E5=90=9E=E9=8C=AF=E3=80=81?= =?UTF-8?q?=E7=9B=AE=E6=A8=99=E5=BA=AB=E9=87=8D=E5=AE=9A=E5=90=91=E3=80=81?= =?UTF-8?q?=E6=97=A2=E6=9C=89=E7=89=A9=E4=BB=B6=E8=99=95=E7=90=86=E7=AD=96?= =?UTF-8?q?=E7=95=A5=E8=88=87=E5=AF=A6=E9=9A=9B=E5=AF=AB=E5=85=A5=E7=AD=86?= =?UTF-8?q?=E6=95=B8=E7=B5=B1=E8=A8=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mySQLPunk/MySqlImportWizardForm.cs | 4 +- mySQLPunk/lib/MySqlImport/MySqlImportLog.cs | 4 +- .../lib/MySqlImport/MySqlImportModels.cs | 38 ++ .../lib/MySqlImport/MySqlImportRunner.cs | 479 ++++++++++++++++-- tests/SmokeTests.MySqlImport.cs | 284 ++++++++++- tests/SmokeTests.cs | 4 + 6 files changed, 775 insertions(+), 38 deletions(-) diff --git a/mySQLPunk/MySqlImportWizardForm.cs b/mySQLPunk/MySqlImportWizardForm.cs index 804bc57..85d1fe3 100644 --- a/mySQLPunk/MySqlImportWizardForm.cs +++ b/mySQLPunk/MySqlImportWizardForm.cs @@ -568,7 +568,7 @@ private async Task StartImportAsync() } else if (Result.Status == MySqlImportStatus.CompletedWithErrors) { - currentStatementLabel.Text = T("⚠️ 匯入完成，但有部分語句失敗／略過。", "⚠️ Import finished with errors / skipped statements."); + currentStatementLabel.Text = T("⚠️ 匯入完成，但有部分語句失敗／略過或列數不符。", "⚠️ Import finished with errors, skipped statements or row mismatch."); resultText.Text = Result.Summary; } else if (Result.Status == MySqlImportStatus.Cancelled) @@ -619,7 +619,7 @@ private void RenderProgress(MySqlImportProgress p) } currentStatementLabel.Text = p.Message; speedLabel.Text = string.Format(T("速率：{0:N0} KB/秒", "Speed: {0:N0} KB/s"), p.BytesPerSecond / 1024.0); - errorCountLabel.Text = string.Format(T("錯誤：{0} 筆", "Errors: {0}"), p.FailedStatements); + errorCountLabel.Text = string.Format(T("已寫入：{0:N0} 列 · 錯誤：{1} 筆", "Written: {0:N0} rows · Errors: {1}"), p.TotalRowsAffected, p.FailedStatements); elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), p.Elapsed); etaLabel.Text = p.EstimatedRemaining.HasValue ? string.Format(T("預估剩餘：約 {0:mm\\:ss}", "ETA: approx. {0:mm\\:ss}"), p.EstimatedRemaining.Value) diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs b/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs index 4f41db0..8c0a4a0 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs @@ -62,7 +62,8 @@ public void LogStatement(int lineNumber, string preview, bool success, string er if (disposed) return; if (success) { - writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [OK] Line {lineNumber}: {preview}"); + string tag = string.IsNullOrWhiteSpace(action) ? "OK" : action; + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [{tag}] Line {lineNumber}: {preview}"); } else { @@ -105,6 +106,7 @@ public void WriteEnd(MySqlImportResult result) writer.WriteLine($"Executed : {result.ExecutedStatements} statements"); writer.WriteLine($"Failed : {result.FailedStatements} statements"); writer.WriteLine($"Skipped : {result.SkippedStatements} statements"); + writer.WriteLine($"Rows : {result.TotalRowsAffected:N0} rows affected"); writer.WriteLine($"Duration : {result.Elapsed.TotalSeconds:F2} seconds"); writer.WriteLine($"Summary : {result.Summary}"); writer.WriteLine("================================================================================"); diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs b/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs index cd60f3d..8e878b9 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportModels.cs @@ -76,6 +76,7 @@ public sealed class MySqlImportProgress public int ExecutedStatements { get; } public int FailedStatements { get; } public int SkippedStatements { get; } + public long TotalRowsAffected { get; } public double BytesPerSecond { get; } public TimeSpan Elapsed { get; } public TimeSpan? EstimatedRemaining { get; } @@ -90,6 +91,7 @@ public MySqlImportProgress( int executedStatements, int failedStatements, int skippedStatements, + long totalRowsAffected, double bytesPerSecond, TimeSpan elapsed, TimeSpan? estimatedRemaining, @@ -103,12 +105,30 @@ public MySqlImportProgress( ExecutedStatements = executedStatements; FailedStatements = failedStatements; SkippedStatements = skippedStatements; + TotalRowsAffected = totalRowsAffected; BytesPerSecond = bytesPerSecond; Elapsed = elapsed; EstimatedRemaining = estimatedRemaining; CurrentStatementPreview = currentStatementPreview; Message = message; } + + public MySqlImportProgress( + Guid jobId, + MySqlImportStage stage, + long totalBytes, + long processedBytes, + int executedStatements, + int failedStatements, + int skippedStatements, + double bytesPerSecond, + TimeSpan elapsed, + TimeSpan? estimatedRemaining, + string currentStatementPreview, + string message) + : this(jobId, stage, totalBytes, processedBytes, executedStatements, failedStatements, skippedStatements, 0, bytesPerSecond, elapsed, estimatedRemaining, currentStatementPreview, message) + { + } } public sealed class MySqlImportResult @@ -119,6 +139,7 @@ public sealed class MySqlImportResult public int ExecutedStatements { get; } public int FailedStatements { get; } public int SkippedStatements { get; } + public long TotalRowsAffected { get; } public string LogPath { get; } public TimeSpan Elapsed { get; } public string Summary { get; } @@ -131,6 +152,7 @@ public MySqlImportResult( int executedStatements, int failedStatements, int skippedStatements, + long totalRowsAffected, string logPath, TimeSpan elapsed, string summary, @@ -142,10 +164,26 @@ public MySqlImportResult( ExecutedStatements = executedStatements; FailedStatements = failedStatements; SkippedStatements = skippedStatements; + TotalRowsAffected = totalRowsAffected; LogPath = logPath; Elapsed = elapsed; Summary = summary; if (errors != null) Errors.AddRange(errors); } + + public MySqlImportResult( + Guid jobId, + MySqlImportStatus status, + long fileBytes, + int executedStatements, + int failedStatements, + int skippedStatements, + string logPath, + TimeSpan elapsed, + string summary, + IEnumerable errors = null) + : this(jobId, status, fileBytes, executedStatements, failedStatements, skippedStatements, 0, logPath, elapsed, summary, errors) + { + } } } diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs index 6194f4b..6713c9e 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs @@ -12,6 +12,28 @@ namespace mySQLPunk.lib.MySqlImport { public sealed class MySqlImportRunner { + private const string IdentifierPattern = @"`(?:``|[^`])+`|[A-Za-z0-9_$]+"; + + private static readonly Regex UseDatabaseRegex = new Regex( + @"^\s*USE\s+(?" + IdentifierPattern + @")\s*;?$", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex CreateDatabaseRegex = new Regex( + @"^\s*CREATE\s+(?:DATABASE|SCHEMA)\b", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex DropObjectRegex = new Regex( + @"^\s*DROP\s+(?:TEMPORARY\s+)?(?TABLE|VIEW|FUNCTION|PROCEDURE|TRIGGER)\b", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex CreateObjectRegex = new Regex( + @"^\s*CREATE\b[\s\S]*?\b(?TABLE|VIEW|FUNCTION|PROCEDURE|TRIGGER)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex InsertTargetRegex = new Regex( + @"^\s*(?:INSERT|REPLACE)\s+(?:IGNORE\s+)?INTO\s+(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + private readonly IDatabase db; private readonly string databaseName; @@ -48,12 +70,50 @@ private MySqlImportResult RunInternal( int executed = 0; int failed = 0; int skipped = 0; + long totalRowsAffected = 0; var errors = new List(); var manifestTables = new List>(); bool readingManifest = false; + bool haltedOnError = false; + + string safeTargetDb = QuoteIdentifier(databaseName); + string activeSourceDb = null; + + var existingTables = new HashSet(StringComparer.OrdinalIgnoreCase); + var existingViews = new HashSet(StringComparer.OrdinalIgnoreCase); + var skippedTables = new HashSet(StringComparer.OrdinalIgnoreCase); try { + // 1. 切換至目標資料庫並鎖定 Session + var useRes = db.ExecSQL($"USE {safeTargetDb};"); + if (!IsExecutionSuccess(useRes)) + { + string reason = DatabaseExecutionResultService.GetFailureReason(useRes); + throw new InvalidOperationException($"Failed to select target database {safeTargetDb}: {reason}"); + } + + // 2. 初始化 Session 環境變數以加快匯入並確保相容性 + try { db.ExecSQL("SET NAMES utf8mb4;"); } catch { } + try { db.ExecSQL("SET FOREIGN_KEY_CHECKS = 0;"); } catch { } + try { db.ExecSQL("SET UNIQUE_CHECKS = 0;"); } catch { } + try { db.ExecSQL("SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO';"); } catch { } + + // 3. 快取目標資料庫中既有物件，避免頻繁發送 SHOW TABLES + try + { + var tList = db.GetTables(databaseName); + if (tList != null) existingTables.UnionWith(tList); + } + catch { } + + try + { + var vList = db.GetViews(databaseName); + if (vList != null) existingViews.UnionWith(vList); + } + catch { } + using (var fs = new FileStream(request.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) using (var reader = new StreamReader(fs, Encoding.UTF8, true)) { @@ -62,7 +122,6 @@ private MySqlImportResult RunInternal( int batchStartLine = 1; int currentLine = 0; string line; - long lastReportMs = 0; while ((line = reader.ReadLine()) != null) @@ -72,17 +131,15 @@ private MySqlImportResult RunInternal( string trimmed = line.Trim(); - // 檢查 Manifest + // 檢查 Native Manifest if (trimmed.StartsWith("--")) { if (trimmed.Contains("mySQLPunk Native Manifest")) readingManifest = true; - else if (readingManifest) + + var match = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) { - var match = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); - if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) - { - manifestTables.Add(Tuple.Create(match.Groups[1].Value, r)); - } + manifestTables.Add(Tuple.Create(match.Groups[1].Value, r)); } continue; } @@ -113,20 +170,37 @@ private MySqlImportResult RunInternal( if (!string.IsNullOrWhiteSpace(sql)) { string headline; - bool ok = ExecuteBatchStatement(sql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors, out headline); - if (!ok && !options.ContinueOnError) + bool shouldContinue = ProcessAndExecuteStatement( + sql, + batchStartLine, + options, + log, + safeTargetDb, + ref activeSourceDb, + existingTables, + existingViews, + skippedTables, + ref executed, + ref failed, + ref skipped, + ref totalRowsAffected, + errors, + out headline); + + if (!shouldContinue && !options.ContinueOnError) { + haltedOnError = true; log.LogInfo("Execution stopped on error as requested by options."); - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, headline); + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, totalRowsAffected, clock, headline); break; } long now = clock.ElapsedMilliseconds; bool isDdl = IsSignificantStatement(headline); - if (!ok || isDdl || (now - lastReportMs >= 60)) + if (!shouldContinue || isDdl || (now - lastReportMs >= 60)) { lastReportMs = now; - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, headline); + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, totalRowsAffected, clock, headline); } } @@ -136,20 +210,40 @@ private MySqlImportResult RunInternal( } // 處理尾端剩餘語句 - if (currentBatch.Length > 0 && (failed == 0 || options.ContinueOnError)) + if (currentBatch.Length > 0 && !haltedOnError && (failed == 0 || options.ContinueOnError)) { string remainingSql = currentBatch.ToString().Trim(); if (!string.IsNullOrWhiteSpace(remainingSql)) { string headline; - ExecuteBatchStatement(remainingSql, batchStartLine, options, log, ref executed, ref failed, ref skipped, errors, out headline); - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, clock, headline); + bool ok = ProcessAndExecuteStatement( + remainingSql, + batchStartLine, + options, + log, + safeTargetDb, + ref activeSourceDb, + existingTables, + existingViews, + skippedTables, + ref executed, + ref failed, + ref skipped, + ref totalRowsAffected, + errors, + out headline); + if (!ok && !options.ContinueOnError) + { + haltedOnError = true; + } + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, totalRowsAffected, clock, headline); } } } - // 若有 Native Manifest，自動驗證目標資料庫實際列數 - if (manifestTables.Count > 0) + // 4. 若有 Native Manifest，自動驗證目標資料庫實際列數 + bool manifestMismatch = false; + if (manifestTables.Count > 0 && !haltedOnError) { log.LogInfo("Verifying restored database tables against Native Manifest..."); foreach (var pair in manifestTables) @@ -157,18 +251,58 @@ private MySqlImportResult RunInternal( long actual = -1; try { actual = db.CountRows(databaseName, pair.Item1); } catch { } bool match = actual == pair.Item2; + if (!match) manifestMismatch = true; log.LogManifestVerification(pair.Item1, pair.Item2, actual, match); } } clock.Stop(); - MySqlImportStatus status = failed == 0 - ? MySqlImportStatus.Completed - : (executed > 0 ? MySqlImportStatus.CompletedWithErrors : MySqlImportStatus.Failed); - string summary = string.Format( - "Execution finished in {0:F1}s: {1} executed, {2} failed, {3} skipped.", - clock.Elapsed.TotalSeconds, executed, failed, skipped); + MySqlImportStatus status; + if (haltedOnError) + { + status = MySqlImportStatus.Failed; + } + else if (failed > 0 || manifestMismatch) + { + status = MySqlImportStatus.CompletedWithErrors; + } + else + { + status = MySqlImportStatus.Completed; + } + + string summary; + if (failed == 0 && !manifestMismatch) + { + summary = string.Format( + "Execution finished successfully in {0:F1}s: {1} executed, {2} skipped, {3:N0} rows affected.", + clock.Elapsed.TotalSeconds, executed, skipped, totalRowsAffected); + } + else if (manifestMismatch && failed == 0) + { + summary = string.Format( + "Execution finished with row discrepancy in {0:F1}s: {1} executed, {2} skipped, {3:N0} rows affected.", + clock.Elapsed.TotalSeconds, executed, skipped, totalRowsAffected); + } + else + { + summary = string.Format( + "Execution finished with {0} errors in {1:F1}s: {2} executed, {3} skipped, {4:N0} rows affected.", + failed, clock.Elapsed.TotalSeconds, executed, skipped, totalRowsAffected); + } + + if (manifestTables.Count > 0) + { + if (manifestMismatch) + { + summary += " ⚠️ [Native Manifest 驗證發現列數不符]"; + } + else + { + summary += string.Format(" [Native Manifest 驗證 {0} 張資料表列數全部相符 ✔]", manifestTables.Count); + } + } var finalResult = new MySqlImportResult( request.JobId, @@ -177,6 +311,7 @@ private MySqlImportResult RunInternal( executed, failed, skipped, + totalRowsAffected, log.Path, clock.Elapsed, summary, @@ -184,7 +319,7 @@ private MySqlImportResult RunInternal( log.WriteEnd(finalResult); log.Dispose(); - ReportProgress(progress, request.JobId, MySqlImportStage.Finished, fileBytes, fileBytes, executed, failed, skipped, clock, summary); + ReportProgress(progress, request.JobId, MySqlImportStage.Finished, fileBytes, fileBytes, executed, failed, skipped, totalRowsAffected, clock, summary); return finalResult; } @@ -199,6 +334,7 @@ private MySqlImportResult RunInternal( executed, failed, skipped, + totalRowsAffected, log.Path, clock.Elapsed, "Import was cancelled by user.", @@ -219,6 +355,7 @@ private MySqlImportResult RunInternal( executed, failed, skipped, + totalRowsAffected, log.Path, clock.Elapsed, ex.Message, @@ -227,6 +364,139 @@ private MySqlImportResult RunInternal( log.Dispose(); return failResult; } + finally + { + // 還原外鍵與唯一性約束檢查 + try { db.ExecSQL("SET FOREIGN_KEY_CHECKS = 1;"); } catch { } + try { db.ExecSQL("SET UNIQUE_CHECKS = 1;"); } catch { } + } + } + + private bool ProcessAndExecuteStatement( + string sql, + int line, + MySqlImportOptions options, + MySqlImportLog log, + string safeTargetDb, + ref string activeSourceDb, + HashSet existingTables, + HashSet existingViews, + HashSet skippedTables, + ref int executed, + ref int failed, + ref int skipped, + ref long totalRowsAffected, + List errors, + out string headline) + { + string cleanSql = SanitizeSqlHeadline(sql); + + // A. USE 語句重定向 + Match useMatch = UseDatabaseRegex.Match(sql); + if (useMatch.Success) + { + string targetInSql = UnquoteIdentifier(useMatch.Groups["database"].Value); + if (!string.Equals(targetInSql, databaseName, StringComparison.OrdinalIgnoreCase)) + { + activeSourceDb = targetInSql; + log.LogInfo($"Redirected statement from 'USE `{targetInSql}`' to 'USE {safeTargetDb}'"); + } + sql = $"USE {safeTargetDb};"; + return ExecuteBatchStatement(sql, line, options, log, ref executed, ref failed, ref skipped, ref totalRowsAffected, existingTables, existingViews, errors, out headline); + } + + // B. CREATE DATABASE / SCHEMA 重定向 + if (CreateDatabaseRegex.IsMatch(sql)) + { + log.LogInfo($"Redirected CREATE DATABASE to ensure target database {safeTargetDb} exists"); + sql = $"CREATE DATABASE IF NOT EXISTS {safeTargetDb};"; + return ExecuteBatchStatement(sql, line, options, log, ref executed, ref failed, ref skipped, ref totalRowsAffected, existingTables, existingViews, errors, out headline); + } + + // 若先前偵測到來源資料庫名稱，且與目標不同，將顯式資料庫限定名稱改寫至目標庫 + if (!string.IsNullOrEmpty(activeSourceDb) && !string.Equals(activeSourceDb, databaseName, StringComparison.OrdinalIgnoreCase)) + { + sql = RewriteDatabaseQualifier(sql, activeSourceDb, databaseName); + } + + // C. DROP 語句策略判定 + if ((options.ExistingObjectStrategy == MySqlExistingObjectStrategy.CreateIfMissing || + options.ExistingObjectStrategy == MySqlExistingObjectStrategy.SkipExisting) && + DropObjectRegex.IsMatch(sql)) + { + skipped++; + log.LogStatement(line, cleanSql, true, null, "SKIP_DROP"); + headline = "⏭ [略過 DROP] " + cleanSql; + return true; + } + + // D. CREATE 物件策略判定 + Match createMatch = CreateObjectRegex.Match(sql); + if (createMatch.Success) + { + string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); + string objDb = UnquoteIdentifier(createMatch.Groups["database"].Value); + string objName = UnquoteIdentifier(createMatch.Groups["name"].Value); + + if (string.IsNullOrWhiteSpace(objDb)) objDb = databaseName; + + bool exists = ObjectExists(db, objDb, objName, objType, existingTables, existingViews); + + if (exists && (options.ExistingObjectStrategy == MySqlExistingObjectStrategy.CreateIfMissing || + options.ExistingObjectStrategy == MySqlExistingObjectStrategy.SkipExisting)) + { + skipped++; + if (options.ExistingObjectStrategy == MySqlExistingObjectStrategy.SkipExisting && + string.Equals(objType, "TABLE", StringComparison.OrdinalIgnoreCase)) + { + skippedTables.Add(objName); + } + log.LogStatement(line, cleanSql, true, null, "SKIP_EXISTING"); + headline = $"⏭ [略過既有 {objType}] {objName}"; + return true; + } + + if (exists && options.ExistingObjectStrategy == MySqlExistingObjectStrategy.DropAndRecreate) + { + string dropSql = $"DROP {objType} IF EXISTS {safeTargetDb}.{QuoteIdentifier(objName)};"; + db.ExecSQL(dropSql); + existingTables.Remove(objName); + existingViews.Remove(objName); + log.LogStatement(line, $"DROP {objType} IF EXISTS `{objName}`", true, null, "DROP_BEFORE_CREATE"); + } + } + + // E. INSERT / REPLACE 資料寫入策略判定 + if (options.ExistingObjectStrategy == MySqlExistingObjectStrategy.SkipExisting) + { + Match insertMatch = InsertTargetRegex.Match(sql); + if (insertMatch.Success) + { + string insTable = UnquoteIdentifier(insertMatch.Groups["name"].Value); + if (skippedTables.Contains(insTable) || existingTables.Contains(insTable)) + { + skipped++; + log.LogStatement(line, cleanSql, true, null, "SKIP_DATA"); + headline = $"⏭ [略過既有表資料] {insTable}"; + return true; + } + } + } + + // F. 執行 SQL 批次 + return ExecuteBatchStatement( + sql, + line, + options, + log, + ref executed, + ref failed, + ref skipped, + ref totalRowsAffected, + existingTables, + existingViews, + errors, + out headline); } private bool ExecuteBatchStatement( @@ -237,6 +507,9 @@ private bool ExecuteBatchStatement( ref int executed, ref int failed, ref int skipped, + ref long totalRowsAffected, + HashSet existingTables, + HashSet existingViews, List errors, out string headline) { @@ -244,11 +517,35 @@ private bool ExecuteBatchStatement( try { - db.ExecSQL(sql); - executed++; - log.LogStatement(line, cleanSql, true); - headline = "✔ " + cleanSql; - return true; + var result = db.ExecSQL(sql); + bool isSuccess = IsExecutionSuccess(result); + + if (isSuccess) + { + executed++; + if (result != null && result.TryGetValue("rowsAffected", out string rStr) && + long.TryParse(rStr, out long rVal) && rVal > 0) + { + totalRowsAffected += rVal; + } + + // 成功建立或刪除物件時動態維護記憶體快取 + UpdateObjectCacheOnSuccess(sql, existingTables, existingViews); + + log.LogStatement(line, cleanSql, true); + headline = "✔ " + cleanSql; + return true; + } + else + { + failed++; + string err = DatabaseExecutionResultService.GetFailureReason(result); + errors.Add($"Line {line}: {err}"); + string action = options.ContinueOnError ? "SkipBatch" : "HaltExecution"; + log.LogStatement(line, cleanSql, false, err, action); + headline = $"❌ [行 {line}] {cleanSql} ➔ 錯誤: {err}"; + return false; + } } catch (Exception ex) { @@ -262,6 +559,122 @@ private bool ExecuteBatchStatement( } } + public static bool IsExecutionSuccess(Dictionary result) + { + if (result == null) return false; + if (result.TryGetValue("status", out string status)) + { + return string.Equals(status, "OK", StringComparison.OrdinalIgnoreCase); + } + return !result.ContainsKey("reason"); + } + + private static void UpdateObjectCacheOnSuccess(string sql, HashSet existingTables, HashSet existingViews) + { + if (string.IsNullOrWhiteSpace(sql)) return; + + Match createMatch = CreateObjectRegex.Match(sql); + if (createMatch.Success) + { + string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); + string objName = UnquoteIdentifier(createMatch.Groups["name"].Value); + if (string.Equals(objType, "TABLE", StringComparison.OrdinalIgnoreCase)) + { + existingTables.Add(objName); + } + else if (string.Equals(objType, "VIEW", StringComparison.OrdinalIgnoreCase)) + { + existingViews.Add(objName); + } + return; + } + + Match dropMatch = DropObjectRegex.Match(sql); + if (dropMatch.Success) + { + string objType = dropMatch.Groups["type"].Value.ToUpperInvariant(); + string objName = UnquoteIdentifier(dropMatch.Groups["name"].Value); + if (string.Equals(objType, "TABLE", StringComparison.OrdinalIgnoreCase)) + { + existingTables.Remove(objName); + } + else if (string.Equals(objType, "VIEW", StringComparison.OrdinalIgnoreCase)) + { + existingViews.Remove(objName); + } + } + } + + private static bool ObjectExists( + IDatabase db, + string dbName, + string objName, + string objType, + HashSet existingTables, + HashSet existingViews) + { + if (string.Equals(objType, "TABLE", StringComparison.OrdinalIgnoreCase)) + { + if (existingTables != null && existingTables.Contains(objName)) return true; + return db.TableExists(dbName, objName); + } + + if (string.Equals(objType, "VIEW", StringComparison.OrdinalIgnoreCase)) + { + if (existingViews != null && existingViews.Contains(objName)) return true; + return db.ViewExists(dbName, objName); + } + + var parameters = new Dictionary + { + { "schema", dbName }, + { "name", objName } + }; + + DataTable table = null; + if (string.Equals(objType, "FUNCTION", StringComparison.OrdinalIgnoreCase) || + string.Equals(objType, "PROCEDURE", StringComparison.OrdinalIgnoreCase)) + { + parameters.Add("type", objType); + table = db.SelectSQL( + "SELECT 1 FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = ?schema AND ROUTINE_NAME = ?name AND ROUTINE_TYPE = ?type LIMIT 1;", + parameters); + } + else if (string.Equals(objType, "TRIGGER", StringComparison.OrdinalIgnoreCase)) + { + table = db.SelectSQL( + "SELECT 1 FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = ?schema AND TRIGGER_NAME = ?name LIMIT 1;", + parameters); + } + + return table != null && table.Rows.Count > 0; + } + + public static string QuoteIdentifier(string value) + { + return "`" + (value ?? string.Empty).Replace("`", "``") + "`"; + } + + public static string UnquoteIdentifier(string value) + { + value = (value ?? string.Empty).Trim(); + if (value.Length >= 2 && value[0] == '`' && value[value.Length - 1] == '`') + return value.Substring(1, value.Length - 2).Replace("``", "`"); + return value; + } + + public static string RewriteDatabaseQualifier(string sql, string fromDb, string toDb) + { + if (string.IsNullOrWhiteSpace(sql) || string.IsNullOrWhiteSpace(fromDb) || string.IsNullOrWhiteSpace(toDb)) + return sql; + + string unquotedFrom = UnquoteIdentifier(fromDb); + string safeTo = QuoteIdentifier(toDb); + + string pattern = @"(`?" + Regex.Escape(unquotedFrom) + @"`?)\s*\.\s*(`?" + IdentifierPattern + @"`?)"; + return Regex.Replace(sql, pattern, safeTo + ".$2", RegexOptions.IgnoreCase); + } + public static string SanitizeSqlHeadline(string sql, int maxLength = 130) { if (string.IsNullOrWhiteSpace(sql)) return string.Empty; @@ -317,6 +730,7 @@ private static void ReportProgress( int executed, int failed, int skipped, + long totalRowsAffected, Stopwatch clock, string currentPreview) { @@ -331,7 +745,9 @@ private static void ReportProgress( remaining = TimeSpan.FromSeconds(remainingSec); } - string msg = string.Format("Importing... {0} statements ({1:N0} KB / {2:N0} KB)", executed, processedBytes / 1024, totalBytes / 1024); + string msg = string.Format("Importing... {0} statements, {1:N0} rows ({2:N0} KB / {3:N0} KB)", + executed, totalRowsAffected, processedBytes / 1024, totalBytes / 1024); + progress.Report(new MySqlImportProgress( jobId, stage, @@ -340,6 +756,7 @@ private static void ReportProgress( executed, failed, skipped, + totalRowsAffected, bytesPerSec, clock.Elapsed, remaining, diff --git a/tests/SmokeTests.MySqlImport.cs b/tests/SmokeTests.MySqlImport.cs index a0409f5..63a1a41 100644 --- a/tests/SmokeTests.MySqlImport.cs +++ b/tests/SmokeTests.MySqlImport.cs @@ -92,11 +92,13 @@ private static void MySqlImport_Runner_ExecutionAndLog() Assert(result.Status == MySqlImportStatus.Completed, "Import status should be Completed"); Assert(result.ExecutedStatements == 2, "Executed 2 statements"); Assert(result.FailedStatements == 0, "0 failed statements"); + Assert(result.TotalRowsAffected == 1, "TotalRowsAffected should be 1"); Assert(File.Exists(result.LogPath), "Import log file must exist at " + result.LogPath); string logContent = File.ReadAllText(result.LogPath); AssertContains(logContent, "mySQLPunk MySQL Import Execution Log", "Log contains start header"); AssertContains(logContent, "Status : Completed", "Log contains completion status"); + AssertContains(logContent, "Rows : 1 rows affected", "Log contains rows affected"); string rawCommented = "-- table structure\r\n-- more comment\r\nCREATE TABLE `orders` (`id` int);\r\n"; string headline1 = MySqlImportRunner.SanitizeSqlHeadline(rawCommented); @@ -116,10 +118,271 @@ private static void MySqlImport_Runner_ExecutionAndLog() } } + private static void MySqlImport_Runner_ErrorDetection_HaltAndContinue() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_err_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "err_import.sql"); + + try + { + string sqlContent = + "CREATE TABLE `users` (`id` int);\r\n" + + "CREATE TABLE `users` (`id` int);\r\n" + + "INSERT INTO `users` VALUES (1);\r\n"; + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + // 1. 測試 ContinueOnError = false (遇到錯誤立即中斷) + { + var mockDb = new MockMySqlImportDatabase(); + int execCount = 0; + mockDb.ExecSqlHandler = sql => + { + if (sql.StartsWith("CREATE TABLE `users`", StringComparison.OrdinalIgnoreCase)) + { + execCount++; + if (execCount == 2) + { + return new Dictionary + { + { "status", "NO" }, + { "reason", "Table 'users' already exists" } + }; + } + } + return new Dictionary { { "status", "OK" }, { "rowsAffected", "1" } }; + }; + + var runner = new MySqlImportRunner(mockDb, "test_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions { ContinueOnError = false }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.Failed, "Status should be Failed on error without continue"); + Assert(result.FailedStatements == 1, "Failed count should be 1"); + Assert(result.ExecutedStatements == 1, "Executed count should be 1 (stopped before third statement)"); + Assert(result.Errors.Count == 1, "Errors list should have 1 entry"); + AssertContains(result.Errors[0], "Table 'users' already exists", "Error reason must be recorded"); + } + + // 2. 測試 ContinueOnError = true (遇到錯誤略過失敗批次並繼續) + { + var mockDb = new MockMySqlImportDatabase(); + int execCount = 0; + mockDb.ExecSqlHandler = sql => + { + if (sql.StartsWith("CREATE TABLE `users`", StringComparison.OrdinalIgnoreCase)) + { + execCount++; + if (execCount == 2) + { + return new Dictionary + { + { "status", "NO" }, + { "reason", "Table 'users' already exists" } + }; + } + } + return new Dictionary { { "status", "OK" }, { "rowsAffected", "1" } }; + }; + + var runner = new MySqlImportRunner(mockDb, "test_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions { ContinueOnError = true }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.CompletedWithErrors, "Status should be CompletedWithErrors"); + Assert(result.FailedStatements == 1, "Failed count should be 1"); + Assert(result.ExecutedStatements == 2, "Executed count should be 2 (first and third statements executed)"); + Assert(result.Errors.Count == 1, "Errors list should have 1 entry"); + } + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void MySqlImport_Runner_DatabaseRedirection() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_redirect_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "redirect_import.sql"); + + try + { + string sqlContent = + "USE `old_db`;\r\n" + + "CREATE DATABASE `old_db`;\r\n" + + "CREATE TABLE `old_db`.`customers` (`id` int);\r\n" + + "INSERT INTO `old_db`.`customers` VALUES (100);\r\n"; + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var mockDb = new MockMySqlImportDatabase(); + var runner = new MySqlImportRunner(mockDb, "target_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "target_db"); + var options = new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.ExecuteScript }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.Completed, "Status should be Completed"); + Assert(result.ExecutedStatements == 4, "Executed 4 redirected statements"); + + // 驗證沒有任何 SQL 執行於 old_db，全數導向至 target_db + foreach (var executed in mockDb.ExecutedSql) + { + Assert(!executed.Contains("`old_db`"), "Executed SQL should never target old_db, got: " + executed); + } + Assert(mockDb.ExecutedSql.Exists(s => s.StartsWith("USE `target_db`")), "Must execute USE `target_db`"); + Assert(mockDb.ExecutedSql.Exists(s => s.StartsWith("CREATE DATABASE IF NOT EXISTS `target_db`")), "Must ensure target_db exists"); + Assert(mockDb.ExecutedSql.Exists(s => s.Contains("`target_db`.`customers`")), "Must qualify customer table with target_db"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void MySqlImport_Runner_ExistingObjectStrategies() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_strat_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "strat_import.sql"); + + try + { + string sqlContent = + "DROP TABLE IF EXISTS `items`;\r\n" + + "CREATE TABLE `items` (`id` int);\r\n" + + "INSERT INTO `items` VALUES (1);\r\n"; + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + // 1. DropAndRecreate 策略：若表已存在，建立前執行 DROP + { + var mockDb = new MockMySqlImportDatabase(); + mockDb.TableExistsHandler = (db, tbl) => tbl == "items"; + + var runner = new MySqlImportRunner(mockDb, "test_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.DropAndRecreate }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.Completed, "DropAndRecreate completed"); + Assert(mockDb.ExecutedSql.Exists(s => s.StartsWith("DROP TABLE IF EXISTS `test_db`.`items`")), "DROP before CREATE was executed"); + } + + // 2. CreateIfMissing 策略：略過 DROP，且略過既有表的 CREATE + { + var mockDb = new MockMySqlImportDatabase(); + mockDb.TableExistsHandler = (db, tbl) => tbl == "items"; + + var runner = new MySqlImportRunner(mockDb, "test_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.CreateIfMissing }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.Completed, "CreateIfMissing completed"); + Assert(result.SkippedStatements == 2, "DROP and CREATE TABLE should be skipped (2 skipped)"); + Assert(result.ExecutedStatements == 1, "INSERT was executed"); + } + + // 3. SkipExisting 策略：略過 DROP、CREATE 與 INSERT + { + var mockDb = new MockMySqlImportDatabase(); + mockDb.TableExistsHandler = (db, tbl) => tbl == "items"; + + var runner = new MySqlImportRunner(mockDb, "test_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.SkipExisting }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.Completed, "SkipExisting completed"); + Assert(result.SkippedStatements == 3, "DROP, CREATE TABLE, and INSERT should all be skipped (3 skipped)"); + Assert(result.ExecutedStatements == 0, "No statements executed"); + } + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void MySqlImport_Runner_ManifestMismatchDetection() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_manifest_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "manifest_mismatch.sql"); + + try + { + string sqlContent = + "CREATE TABLE `products` (`id` int);\r\n" + + "-- ========================================================\r\n" + + "-- mySQLPunk Native Manifest\r\n" + + "-- Tables: 1\r\n" + + "-- TotalRows: 50\r\n" + + "-- Manifest-Table: products | Rows: 50\r\n" + + "-- ========================================================\r\n"; + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var mockDb = new MockMySqlImportDatabase(); + // 目標資料庫實際只有 0 筆，與 Manifest 記載的 50 筆不符 + mockDb.CountRowsHandler = (db, tbl) => 0; + + var runner = new MySqlImportRunner(mockDb, "test_db"); + var request = new MySqlImportRequest(Guid.NewGuid(), sqlFile, "test_db"); + var options = new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.ExecuteScript }; + + var task = runner.RunAsync(request, options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == MySqlImportStatus.CompletedWithErrors, "Status should be CompletedWithErrors due to manifest mismatch"); + AssertContains(result.Summary, "Native Manifest 驗證發現列數不符", "Summary must report manifest discrepancy"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + private sealed class MockMySqlImportDatabase : IDatabase { public ConnectionState State => ConnectionState.Open; public string ProviderName => "MySQL"; + public List ExecutedSql = new List(); + public Func> ExecSqlHandler = null; + public Func TableExistsHandler = null; + public Func CountRowsHandler = null; public void SetConn(string connectionString) { } public void Open() { } @@ -127,9 +390,22 @@ public void Close() { } public void Dispose() { } public DataTable SelectSQL(string sql, Dictionary parameters = null) => new DataTable(); - public Dictionary ExecSQL(string sql, Dictionary parameters = null) => new Dictionary(); + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + ExecutedSql.Add(sql); + if (ExecSqlHandler != null) return ExecSqlHandler(sql); + var res = new Dictionary(); + res["status"] = "OK"; + string trimmed = (sql ?? "").TrimStart(); + bool isDataDml = trimmed.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("UPDATE", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("DELETE", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("REPLACE", StringComparison.OrdinalIgnoreCase); + res["rowsAffected"] = isDataDml ? "1" : "0"; + return res; + } public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(new DataTable()); - public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(new Dictionary()); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); public List GetDatabases() => new List { "test_db" }; public List GetTables(string databaseName) => new List { "test" }; @@ -139,11 +415,11 @@ public void Dispose() { } public DataTable GetTableStatus(string databaseName) => new DataTable(); public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); public string GetTableCreateStatement(string databaseName, string tableName) => ""; - public bool TableExists(string databaseName, string tableName) => true; + public bool TableExists(string databaseName, string tableName) => TableExistsHandler != null ? TableExistsHandler(databaseName, tableName) : true; public bool ViewExists(string databaseName, string viewName) => false; public void RenameTable(string databaseName, string oldTableName, string newTableName) { } public void RenameView(string databaseName, string oldViewName, string newViewName) { } - public long CountRows(string databaseName, string tableName) => 1; + public long CountRows(string databaseName, string tableName) => CountRowsHandler != null ? CountRowsHandler(databaseName, tableName) : 1; public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index 53a2f4f..ec862e3 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -36,6 +36,10 @@ public static int Main(string[] args) Run("MySQL export service and manifest atomic write", MySqlExport_ServiceAndManifest_AtomicWrite, ref passed); Run("MySQL import preflight scanner delimiters and manifest", MySqlImport_PreflightScanner_DelimitersAndManifest, ref passed); Run("MySQL import runner execution and log", MySqlImport_Runner_ExecutionAndLog, ref passed); + Run("MySQL import runner error detection halt and continue", MySqlImport_Runner_ErrorDetection_HaltAndContinue, ref passed); + Run("MySQL import runner database redirection", MySqlImport_Runner_DatabaseRedirection, ref passed); + Run("MySQL import runner existing object strategies", MySqlImport_Runner_ExistingObjectStrategies, ref passed); + Run("MySQL import runner manifest mismatch detection", MySqlImport_Runner_ManifestMismatchDetection, ref passed); Run("MSSQL import Transfer guards", MssqlImportTransferGuards, ref passed); Run("MSSQL import Transfer trial", MssqlImportTransferTrial, ref passed); Run("MSSQL import worker published cleanup receipt", MssqlImportPublishedReleaseRace, ref passed); From 09ad404a820f38436b11ecaf6ef9dd1811dbfdc8 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 14:08:51 +0800 Subject: [PATCH 05/12] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=20TODO.md=20?= =?UTF-8?q?=E6=A8=99=E8=A8=BB=20MySQL=20=E5=8C=AF=E5=87=BA=E8=88=87?= =?UTF-8?q?=E5=8C=AF=E5=85=A5=E7=B2=BE=E9=9D=88=E5=B7=B2=E5=B0=8D=E9=BD=8A?= =?UTF-8?q?=20MSSQL=20=E6=A8=99=E7=AB=BF=E8=A6=8F=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index b3881b9..79a0af0 100644 --- a/TODO.md +++ b/TODO.md @@ -32,7 +32,7 @@ | 資料庫 Provider | SQL 傾印 (匯出) 現況 | SQL 檔案匯入現況 | 主要缺口與風險 | | :--- | :--- | :--- | :--- | -| **MySQL / MariaDB** | 具備 `MySqlExportService`，支援 DDL 串流、DELIMITER、移除 DEFINER 等。但僅透過 SaveFileDialog 直接執行，無精靈。 | 具備 `MySqlImportService` 與簡易選項視窗，使用跑者遮罩進度視窗 (`RunnerProgressOverlay`)。 | 缺少四步精靈介面、缺乏選表與相依 View 挑選、無預檢掃描 (編碼/批次大小)、無獨立 Import Log 檔案、無結尾 Manifest 列數核對。 | +| **MySQL / MariaDB** | ✅ 已具備完整四步精靈 (`MySqlExportWizardForm`)，支援選表/View/Routine/Trigger、一致性快照、Native Manifest。 | ✅ 已具備完整四步精靈 (`MySqlImportWizardForm`)，支援串流預檢、4種衝突策略、Session鎖定、終端機滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格）。 | | **SQLite** | 走舊版 `DatabaseDumpService`，使用 `StringBuilder` 拼接整份資料庫字串。 | 走舊版 `ImportSqlScript`，`File.ReadAllBytes` 一次性加載全文後用分號切割執行。 | **高危 OOM 風險**：大資料庫匯出/匯入必崩潰；無選表功能、無交易保護 (`BEGIN IMMEDIATE`)、無批次進度條、無日誌。 | | **PostgreSQL** | 走舊版 `DatabaseDumpService`，字串拼接；進階型別 (bytea, jsonb, array, uuid) 處理簡陋。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取後以分號切割。 | **高危 OOM 風險**；無法正確處理 Dollar-quoted (`$$...$$`) 預存程序或函數語法（分號會被誤切）；無 Repeatable Read 一致性快照；無進度。 | | **Oracle** | 走舊版 `DatabaseDumpService`，字串拼接。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取。 | **高危 OOM 風險**；無法處理 PL/SQL 區塊 (`/` 結尾)；缺少 RAW/LOB 分塊支援。 | From 9cbad70767370a3bcb863052a621d6fff0f61a8f Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 14:50:38 +0800 Subject: [PATCH 06/12] =?UTF-8?q?feat(sqlite):=20=E6=94=AF=E6=8F=B4=20SQLi?= =?UTF-8?q?te/SpatiaLite=20=E5=9B=9B=E6=AD=A5=E5=8C=AF=E5=87=BA=E8=88=87?= =?UTF-8?q?=E5=8C=AF=E5=85=A5=E7=B2=BE=E9=9D=88=EF=BC=88=E4=B8=B2=E6=B5=81?= =?UTF-8?q?=E9=98=B2=20OOM=E3=80=81=E7=A9=BA=E9=96=93=E5=B9=BE=E4=BD=95=20?= =?UTF-8?q?HEX=20=E5=82=BE=E5=8D=B0=E8=88=87=E8=99=9B=E6=93=AC=E8=A1=A8?= =?UTF-8?q?=E4=BF=9D=E8=AD=B7=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 實作 SqliteExportWizardForm 與 SqliteSqlExportService： * 全 64KB 串流 StreamWriter 寫入與私有 Staging 檔案原子替換，杜絕整庫 StringBuilder 造成的 OOM。 * 深度支援 SpatiaLite 與 GeoPackage 空間特色：幾何 BLOB 無損轉 X'...' HEX 字面量、支援排除龐大空間中繼表 (spatial_ref_sys)。 * 自動辨識並排除 R-Tree/FTS 虛擬表影子表 (_node, _rowid, _parent 等)，避免還原時 SQLite 報錯。 * 支援 BEGIN IMMEDIATE 一致性交易、側車註解結構傾印，以及腳本末尾 Native Manifest (SHA-256 與逐表筆數)。 - 實作 SqliteImportWizardForm 與 SqliteImportRunner： * 串流狀態機 Lexer 預檢 (Preflight)，精確識別字串與註解內分號，並自動偵測 SpatiaLite 特徵。 * 支援動態加載 mod_spatialite.dll 空間延伸模組。 * 4 種既有物件處理策略（保留現存/全部替換/刪表重建/清空重匯），單一連線 Session 執行與自動交易保護。 * Punky 看板娘雙緩衝動畫、音效切換與 2,000 行滑動視窗終端機風格日誌。 * 每次執行產出獨立 Log 至 %LOCALAPPDATA%\mySQLPunk\ImportLogs。 * 匯入後自動讀取 Manifest 比對現存表與實際寫入筆數。 - 主視窗 Form1 路由整合與測試： * 新增 Form1.SqliteExport.cs 與 Form1.SqliteImport.cs，對齊 MSSQL/MySQL 路由規範。 * 新增 SmokeTests.SqliteExport.cs 與 SmokeTests.SqliteImport.cs (6 項測試)，全數通過 (254/254)。 --- TODO.md | 12 +- mySQLPunk/Form1.SqliteExport.cs | 67 ++ mySQLPunk/Form1.SqliteImport.cs | 57 ++ mySQLPunk/Form1.cs | 24 + mySQLPunk/SqliteExportWizardForm.cs | 750 ++++++++++++++++ mySQLPunk/SqliteImportWizardForm.cs | 773 ++++++++++++++++ .../lib/MySqlImport/MySqlImportRunner.cs | 6 +- .../lib/SqliteExport/SqliteExportModels.cs | 165 ++++ .../SqliteExport/SqliteSqlExportService.cs | 840 ++++++++++++++++++ mySQLPunk/lib/SqliteImport/SqliteImportLog.cs | 129 +++ .../lib/SqliteImport/SqliteImportModels.cs | 160 ++++ .../SqliteImportPreflightScanner.cs | 243 +++++ .../lib/SqliteImport/SqliteImportRunner.cs | 673 ++++++++++++++ mySQLPunk/mySQLPunk.csproj | 10 + tests/Run-SmokeTests.ps1 | 2 + tests/SmokeTests.MssqlExport.Routing.cs | 2 +- tests/SmokeTests.MssqlImport.Wizard.cs | 4 +- tests/SmokeTests.SqliteExport.cs | 251 ++++++ tests/SmokeTests.SqliteImport.cs | 275 ++++++ tests/SmokeTests.cs | 6 + 20 files changed, 4439 insertions(+), 10 deletions(-) create mode 100644 mySQLPunk/Form1.SqliteExport.cs create mode 100644 mySQLPunk/Form1.SqliteImport.cs create mode 100644 mySQLPunk/SqliteExportWizardForm.cs create mode 100644 mySQLPunk/SqliteImportWizardForm.cs create mode 100644 mySQLPunk/lib/SqliteExport/SqliteExportModels.cs create mode 100644 mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs create mode 100644 mySQLPunk/lib/SqliteImport/SqliteImportLog.cs create mode 100644 mySQLPunk/lib/SqliteImport/SqliteImportModels.cs create mode 100644 mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs create mode 100644 mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs create mode 100644 tests/SmokeTests.SqliteExport.cs create mode 100644 tests/SmokeTests.SqliteImport.cs diff --git a/TODO.md b/TODO.md index 79a0af0..ae63b8c 100644 --- a/TODO.md +++ b/TODO.md @@ -33,7 +33,7 @@ | 資料庫 Provider | SQL 傾印 (匯出) 現況 | SQL 檔案匯入現況 | 主要缺口與風險 | | :--- | :--- | :--- | :--- | | **MySQL / MariaDB** | ✅ 已具備完整四步精靈 (`MySqlExportWizardForm`)，支援選表/View/Routine/Trigger、一致性快照、Native Manifest。 | ✅ 已具備完整四步精靈 (`MySqlImportWizardForm`)，支援串流預檢、4種衝突策略、Session鎖定、終端機滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格）。 | -| **SQLite** | 走舊版 `DatabaseDumpService`，使用 `StringBuilder` 拼接整份資料庫字串。 | 走舊版 `ImportSqlScript`，`File.ReadAllBytes` 一次性加載全文後用分號切割執行。 | **高危 OOM 風險**：大資料庫匯出/匯入必崩潰；無選表功能、無交易保護 (`BEGIN IMMEDIATE`)、無批次進度條、無日誌。 | +| **SQLite** | ✅ 已具備完整四步精靈 (`SqliteExportWizardForm`)，支援選表、SpatiaLite/GeoPackage 空間幾何無損 Hex 傾印、排除虛擬表影子表、原子替換與 Native Manifest。 | ✅ 已具備完整四步精靈 (`SqliteImportWizardForm`)，支援串流狀態機預檢、SpatiaLite 動態模組自動加載、4種衝突策略、滑動視窗終端機日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS 空間特性）。 | | **PostgreSQL** | 走舊版 `DatabaseDumpService`，字串拼接；進階型別 (bytea, jsonb, array, uuid) 處理簡陋。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取後以分號切割。 | **高危 OOM 風險**；無法正確處理 Dollar-quoted (`$$...$$`) 預存程序或函數語法（分號會被誤切）；無 Repeatable Read 一致性快照；無進度。 | | **Oracle** | 走舊版 `DatabaseDumpService`，字串拼接。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取。 | **高危 OOM 風險**；無法處理 PL/SQL 區塊 (`/` 結尾)；缺少 RAW/LOB 分塊支援。 | @@ -50,14 +50,14 @@ - [x] 匯出：移植四步精靈（選表/View ➔ 輸出設定/single-transaction ➔ 確認物件 ➔ 串流執行 + Punky 動畫 + Manifest 生成）。 - [x] 匯入：移植四步精靈（選檔 ➔ 預檢掃描 DELIMITER/編碼/單批上限 ➔ 確認目標與策略 ➔ 專用 Session 執行 + Punky 動畫 + 獨立 Log + Manifest 核對）。 -#### 🥈 第二優先：SQLite +#### 🥈 第二優先：SQLite（崩琦 GIS 招牌特色對齊） - **原因**： 1. 單機檔案型資料庫，無遠端網路干擾或伺服器權限設定，建置回歸與驗證最迅速。 - 2. 目前的 SQLite 匯出/匯入是記憶體拼接，最迫切需要解救大資料庫 OOM 的問題。 - 3. 透過 SQLite 實作，可以檢驗並淬鍊出跨資料庫共用的「通用 SQL 匯出/匯入精靈」基底類別 (Generic Wizard Base)。 + 2. 徹底解決過去舊版 SQLite 匯出/匯入記憶體拼接造成大庫 OOM 的致命缺點。 + 3. 深度支援 SpatiaLite 與 GeoPackage 空間幾何無損傾印 (`X'...'` HEX)、排除龐大空間中繼表 (`spatial_ref_sys`) 與虛擬表影子表 (`_node`, `_rowid` 等)。 - **具體工作**： - - [ ] 匯出：改用真正的串流 StreamWriter 寫檔；支援 `BEGIN IMMEDIATE` 一致性交易；支援側車註解 (Sidecar Comment) 隨結構導出。 - - [ ] 匯入：串流 Parser 解析標準 SQLite 語法；封裝為單一交易（失敗自動 Rollback）；加入進度估算與 Log。 + - [x] 匯出：改用真正的串流 StreamWriter 寫檔；支援 `BEGIN IMMEDIATE` 一致性交易；支援側車註解 (Sidecar Comment) 隨結構導出；空間幾何 BLOB 無損 Hex 傾印；排除 R-Tree 影子表；產生 Native Manifest。 + - [x] 匯入：串流狀態機 Lexer 解析；支援 SpatiaLite 動態模組自動加載；交易包覆與 PRAGMA 性能優化；4種物件衝突策略；2,000 行滑動視窗終端機日誌；Manifest 逐表筆數核對。 #### 🥉 第三優先：PostgreSQL - **原因**： diff --git a/mySQLPunk/Form1.SqliteExport.cs b/mySQLPunk/Form1.SqliteExport.cs new file mode 100644 index 0000000..148a3b2 --- /dev/null +++ b/mySQLPunk/Form1.SqliteExport.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.SqliteExport; +using SqliteExportResult = mySQLPunk.lib.SqliteExport.SqliteExportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func SqliteExportWizardRunner = null; + internal Action SqliteExportStatusSink = null; + + private void SetSqliteExportStatus(string message) + { + if (SqliteExportStatusSink == null) UpdateMainStatus(message); + else SqliteExportStatusSink(message); + } + + private void OpenSqliteExportWizard(TreeDatabaseTarget target, SqliteExportMode mode = SqliteExportMode.SchemaAndData, string specificTable = null) + { + if (target == null) return; + try + { + string dbStem = string.IsNullOrWhiteSpace(target.DatabaseName) ? "sqlite_dump" : Path.GetFileNameWithoutExtension(target.DatabaseName); + if (string.IsNullOrWhiteSpace(dbStem)) dbStem = "sqlite_dump"; + string defaultFileName = dbStem + (string.IsNullOrWhiteSpace(specificTable) ? "" : "_" + specificTable) + "_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; + string targetPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultFileName); + var options = new SqliteExportWizardOptions(mode, targetPath); + if (!string.IsNullOrWhiteSpace(specificTable)) + { + options.SelectedTables.Add(specificTable); + } + + SqliteExportResult result; + if (SqliteExportWizardRunner != null) + { + result = SqliteExportWizardRunner(target.Database, target.DatabaseName, options); + } + else + { + using (var wizard = new SqliteExportWizardForm(target.Database, target.DatabaseName, options)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Success + ? string.Format(ReportText.Get("SQLite 匯出完成：{0} 張資料表，共 {1:N0} 列 · {2}", "SQLite export completed: {0} tables, {1:N0} rows · {2}"), result.TableCount, result.TotalRows, result.TargetPath) + : (result.Stage == SqliteExportStage.Cancelled + ? ReportText.Get("SQLite 匯出已取消", "SQLite export cancelled") + : ReportText.Get("SQLite 匯出失敗：", "SQLite export failed: ") + result.ErrorMessage); + SetSqliteExportStatus(status); + } + } + catch (Exception ex) + { + SetSqliteExportStatus(ReportText.Get("無法開啟 SQLite 匯出精靈", "Cannot open SQLite export wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.SqliteImport.cs b/mySQLPunk/Form1.SqliteImport.cs new file mode 100644 index 0000000..5677040 --- /dev/null +++ b/mySQLPunk/Form1.SqliteImport.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.SqliteImport; +using SqliteImportResult = mySQLPunk.lib.SqliteImport.SqliteImportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func SqliteImportWizardRunner = null; + internal Action SqliteImportStatusSink = null; + + private void SetSqliteImportStatus(string message) + { + if (SqliteImportStatusSink == null) UpdateMainStatus(message); + else SqliteImportStatusSink(message); + } + + private void OpenSqliteImportWizard(TreeDatabaseTarget target) + { + if (target == null) return; + try + { + SqliteImportResult result; + if (SqliteImportWizardRunner != null) + { + result = SqliteImportWizardRunner(target.Database, target.DatabaseName, ""); + } + else + { + using (var wizard = new SqliteImportWizardForm(target.Database, target.DatabaseName)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Status == SqliteImportStatus.Completed + ? string.Format(ReportText.Get("SQLite 匯入執行完成：{0} 句成功 · 影響 {1:N0} 列 · 日誌：{2}", "SQLite import execution completed: {0} statements · {1:N0} rows · Log: {2}"), result.ExecutedStatements, result.TotalRowsAffected, result.LogPath) + : (result.Status == SqliteImportStatus.Cancelled + ? ReportText.Get("SQLite 匯入已取消；交易若有開啟已自動回滾", "SQLite import cancelled; transaction rolled back if active") + : string.Format(ReportText.Get("SQLite 匯入結束：{0} 失敗，{1} 略過 · 日誌：{2}", "SQLite import finished: {0} failed, {1} skipped · Log: {2}"), result.FailedStatements, result.SkippedStatements, result.LogPath)); + SetSqliteImportStatus(status); + RefreshDatabaseObjectNodes(target.DatabaseNode); + } + } + catch (Exception ex) + { + SetSqliteImportStatus(ReportText.Get("無法開啟 SQLite 匯入精靈", "Cannot open SQLite import wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.cs b/mySQLPunk/Form1.cs index 2ea02ca..c86926d 100644 --- a/mySQLPunk/Form1.cs +++ b/mySQLPunk/Form1.cs @@ -6680,6 +6680,11 @@ private void DumpSelectedTableSql(bool dataOnly) OpenMySqlExportWizard(target, dataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData); return; } + if (IsSqliteTarget(target)) + { + OpenSqliteExportWizard(target, dataOnly ? lib.SqliteExport.SqliteExportMode.DataOnly : lib.SqliteExport.SqliteExportMode.SchemaAndData, tableName); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -6940,6 +6945,12 @@ private void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) OpenMySqlExportWizard(target, tableDataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData); return; } + if (IsSqliteTarget(target) && (selection == null || selection.GroupName == "Tables")) + { + bool tableDataOnly = selection != null && dataOnlyForTable; + OpenSqliteExportWizard(target, tableDataOnly ? lib.SqliteExport.SqliteExportMode.DataOnly : lib.SqliteExport.SqliteExportMode.SchemaAndData, selection == null ? null : selection.ObjectName); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -7030,6 +7041,7 @@ private static bool IsMySqlTarget(TreeDatabaseTarget target) return target != null && string.Equals(target.ProviderName, "mysql", StringComparison.OrdinalIgnoreCase); } + private async Task ExportMySqlDatabaseWithProgress(TreeDatabaseTarget target, string targetPath, MySqlExportOptions options) { if (target == null) throw new ArgumentNullException("target"); @@ -7284,6 +7296,12 @@ private void DumpSelectedDatabaseSqlWithDialog() return; } + if (IsSqliteTarget(target)) + { + OpenSqliteExportWizard(target, lib.SqliteExport.SqliteExportMode.SchemaAndData); + return; + } + using (SaveFileDialog dialog = new SaveFileDialog()) { dialog.Filter = Localization.T("Common.SqlFilesFilter"); @@ -7347,6 +7365,12 @@ private void ImportSqlWithDialog() return; } + if (IsSqliteTarget(target)) + { + OpenSqliteImportWizard(target); + return; + } + using (OpenFileDialog dialog = new OpenFileDialog()) { dialog.Title = Localization.T("ImportSql.Title"); diff --git a/mySQLPunk/SqliteExportWizardForm.cs b/mySQLPunk/SqliteExportWizardForm.cs new file mode 100644 index 0000000..8ad7d7b --- /dev/null +++ b/mySQLPunk/SqliteExportWizardForm.cs @@ -0,0 +1,750 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.SqliteExport; +using SqliteExportResult = mySQLPunk.lib.SqliteExport.SqliteExportResult; + +namespace mySQLPunk +{ + internal sealed class SqliteExportWizardForm : Form + { + private readonly IDatabase db; + private readonly string databaseName; + private readonly SqliteSqlExportService service; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); + private readonly DataGridView tableGrid = new DataGridView(); + private readonly RadioButton schemaData = new RadioButton(), schemaOnly = new RadioButton(), dataOnly = new RadioButton(); + private readonly CheckBox singleTransaction = new CheckBox(), pragmaFk = new CheckBox(), pragmaSync = new CheckBox(), includeDrop = new CheckBox(); + private readonly CheckBox includeViews = new CheckBox(), includeIndexes = new CheckBox(), includeTriggers = new CheckBox(); + private readonly CheckBox includeComments = new CheckBox(), excludeSpatialMetadata = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(), selectionSummary = new Label(), confirmSummary = new Label(); + private readonly Label currentObjectLabel = new Label(), percentLabel = new Label(), writtenLabel = new Label(); + private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); + private readonly Button openFileButton = new Button(), openFolderButton = new Button(); + private readonly ProgressBar overallBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox logList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private List catalog = new List(); + private List views = new List(); + private List triggers = new List(); + private CancellationTokenSource cancellation; + private SqliteExportProgress lastProgress; + private bool busy, closing, exporting; + private int step; + + internal SqliteExportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal SqliteExportWizardForm(IDatabase db, string databaseName, SqliteExportWizardOptions initialOptions = null) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + this.service = new SqliteSqlExportService(db, databaseName); + + Name = "sqliteExportWizard"; + Text = T("匯出資料庫 · SQLite (支援 SpatiaLite / GeoPackage GIS)", "Export database · SQLite (with SpatiaLite / GeoPackage GIS)"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1100, 730); + MinimumSize = new Size(640, 480); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("SQLite 資料庫傾印匯出精靈", "SQLite database export wizard") + " · " + databaseName, Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(); + BuildPage1(initialOptions); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var footer = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + WrapContents = true, + FlowDirection = FlowDirection.RightToLeft + }; + ConfigureButton(cancelButton, "exportCancel", T("取消", "Cancel"), (s, e) => { if (busy) RequestCancel(); else Close(); }); + ConfigureButton(nextButton, "exportNext", T("下一步 →", "Next →"), async (s, e) => await OnNextAsync()); + ConfigureButton(backButton, "exportBack", T("← 上一步", "← Back"), (s, e) => { if (!busy && step > 0 && step < 3) { step--; ShowPage(); } }); + + footer.Controls.AddRange(new Control[] { cancelButton, nextButton, backButton }); + root.Controls.Add(footer, 0, 3); + + ThemeManager.MarkAsPrimary(nextButton); + AcceptButton = nextButton; + CancelButton = cancelButton; + cancelButton.DialogResult = DialogResult.None; + + timer.Tick += (s, e) => UpdateLiveTicker(); + + Shown += async (s, e) => await LoadCatalogAsync(); + FormClosing += (s, e) => + { + if (busy && !closing) + { + e.Cancel = true; + closing = true; + RequestCancel(); + } + }; + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel + { + Dock = DockStyle.Fill, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + AutoSize = true + }; + + private static Label HeaderLabel(string text) => new Label + { + Text = text, + Font = UiKit.Subtitle, + AutoSize = true, + Margin = new Padding(0, 4, 0, 8) + }; + + private static Label HintLabel(string text) => new Label + { + Text = text, + Font = UiKit.Body, + ForeColor = Color.Gray, + AutoSize = true, + Margin = new Padding(0, 0, 0, 8) + }; + + private static void ConfigureButton(Button b, string name, string text, EventHandler onClick) + { + b.Name = name; + b.Text = text; + b.Font = UiKit.Body; + b.AutoSize = true; + b.MinimumSize = new Size(110, 36); + b.Margin = new Padding(8, 0, 0, 0); + b.Click += onClick; + } + + private void BuildPage0() + { + var content = new Panel { Dock = DockStyle.Fill, Padding = new Padding(0, 0, 0, 0) }; + pages[0].Controls.Add(content); + + var topPanel = new FlowLayoutPanel + { + Dock = DockStyle.Top, + AutoSize = true, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + Padding = new Padding(0, 0, 0, 8) + }; + + topPanel.Controls.Add(HeaderLabel(T("步驟 1：選取要匯出的 SQLite 資料表與空間物件", "Step 1: Select SQLite tables and spatial layers to export"))); + topPanel.Controls.Add(HintLabel(T("可勾選或篩選資料表。SpatiaLite 與 GeoPackage 幾何欄位將全串流無損匯出。", "Check tables to include. SpatiaLite and GeoPackage GIS layers are preserved in binary fidelity."))); + + var filterPanel = new FlowLayoutPanel + { + AutoSize = true, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = true, + Margin = new Padding(0, 4, 0, 4) + }; + + search.Width = 180; + search.Margin = new Padding(0, 0, 8, 0); + search.TextChanged += (s, e) => ApplySearchFilter(); + + var selectAllBtn = new Button { Text = T("全選", "Select All"), AutoSize = true, Margin = new Padding(0, 0, 6, 0) }; + selectAllBtn.Click += (s, e) => SetAllSelected(true); + var clearAllBtn = new Button { Text = T("全取消", "Clear All"), AutoSize = true, Margin = new Padding(0, 0, 16, 0) }; + clearAllBtn.Click += (s, e) => SetAllSelected(false); + + includeViews.Text = T("包含檢視 (Views)", "Include Views"); + includeViews.Checked = true; + includeViews.AutoSize = true; + includeViews.Margin = new Padding(0, 4, 12, 0); + + includeIndexes.Text = T("包含索引 (Indexes)", "Include Indexes"); + includeIndexes.Checked = true; + includeIndexes.AutoSize = true; + includeIndexes.Margin = new Padding(0, 4, 12, 0); + + includeTriggers.Text = T("包含觸發器 (Triggers)", "Include Triggers"); + includeTriggers.Checked = true; + includeTriggers.AutoSize = true; + includeTriggers.Margin = new Padding(0, 4, 12, 0); + + includeComments.Text = T("包含側車註解 (Comments)", "Include Comments"); + includeComments.Checked = true; + includeComments.AutoSize = true; + includeComments.Margin = new Padding(0, 4, 12, 0); + + excludeSpatialMetadata.Text = T("排除龐大空間中繼資料 (spatial_ref_sys 5,000+ 筆 EPSG)", "Exclude bulky spatial metadata (spatial_ref_sys 5000+ EPSG rows)"); + excludeSpatialMetadata.Checked = false; + excludeSpatialMetadata.AutoSize = true; + excludeSpatialMetadata.Margin = new Padding(0, 4, 0, 0); + excludeSpatialMetadata.CheckedChanged += (s, e) => + { + if (excludeSpatialMetadata.Checked) + { + foreach (var item in catalog.Where(t => t.IsSpatialMetadata)) item.IsSelected = false; + } + ApplySearchFilter(); + }; + + filterPanel.Controls.AddRange(new Control[] { + new Label { Text = T("搜尋：", "Search:"), AutoSize = true, Margin = new Padding(0, 5, 4, 0) }, + search, selectAllBtn, clearAllBtn, + includeViews, includeIndexes, includeTriggers, includeComments, excludeSpatialMetadata + }); + topPanel.Controls.Add(filterPanel); + + content.Controls.Add(topPanel); + + tableGrid.Dock = DockStyle.Fill; + tableGrid.AllowUserToAddRows = false; + tableGrid.AllowUserToDeleteRows = false; + tableGrid.RowHeadersVisible = false; + tableGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect; + tableGrid.MultiSelect = false; + tableGrid.AutoSizeColumnsMode = DataGridViewAutoSizeColumnsMode.Fill; + + tableGrid.Columns.Add(new DataGridViewCheckBoxColumn { Name = "ColSelect", HeaderText = T("選取", "Select"), FillWeight = 15 }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColName", HeaderText = T("資料表名稱", "Table Name"), FillWeight = 40, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColRows", HeaderText = T("預估列數", "Estimated Rows"), FillWeight = 20, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColSpatial", HeaderText = T("空間與型態特徵", "Spatial & Object Characteristics"), FillWeight = 25, ReadOnly = true }); + + tableGrid.CellValueChanged += (s, e) => + { + if (e.RowIndex >= 0 && e.ColumnIndex == 0) + { + string name = tableGrid.Rows[e.RowIndex].Cells["ColName"].Value?.ToString(); + bool isChecked = Convert.ToBoolean(tableGrid.Rows[e.RowIndex].Cells["ColSelect"].Value); + var item = catalog.FirstOrDefault(t => t.Name == name); + if (item != null) item.IsSelected = isChecked; + UpdateSelectionSummary(); + } + }; + tableGrid.CurrentCellDirtyStateChanged += (s, e) => + { + if (tableGrid.IsCurrentCellDirty) tableGrid.CommitEdit(DataGridViewDataErrorContexts.Commit); + }; + + content.Controls.Add(tableGrid); + tableGrid.BringToFront(); + + var bottomPanel = new FlowLayoutPanel { Dock = DockStyle.Bottom, AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Padding = new Padding(0, 6, 0, 0) }; + selectionSummary.AutoSize = true; + bottomPanel.Controls.Add(selectionSummary); + content.Controls.Add(bottomPanel); + } + + private void BuildPage1(SqliteExportWizardOptions initialOptions) + { + var content = new Panel { Dock = DockStyle.Fill, AutoScroll = true, Padding = new Padding(12) }; + pages[1].Controls.Add(content); + + var header = HeaderLabel(T("步驟 2：設定輸出檔案與效能選項", "Step 2: Output file and performance settings")); + header.Location = new Point(12, 10); + content.Controls.Add(header); + + var pathGroup = new GroupBox + { + Text = T("輸出 SQL 檔案路徑", "Output SQL file path"), + Font = UiKit.Body, + Location = new Point(12, 50), + Size = new Size(1040, 75), + Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right + }; + targetPathBox.Location = new Point(16, 30); + targetPathBox.Size = new Size(900, 28); + targetPathBox.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + targetPathBox.Text = initialOptions?.TargetPath ?? ""; + targetPathBox.TextChanged += (s, e) => ShowPage(); + + ConfigureButton(browseButton, "exportBrowse", T("瀏覽...", "Browse..."), (s, e) => + { + using (var sfd = new SaveFileDialog()) + { + sfd.Title = T("選擇輸出 SQL 檔案", "Select output SQL file"); + sfd.Filter = "SQL Files (*.sql)|*.sql|All Files (*.*)|*.*"; + sfd.FileName = Path.GetFileName(targetPathBox.Text); + if (sfd.ShowDialog(this) == DialogResult.OK) + { + targetPathBox.Text = sfd.FileName; + } + } + }); + browseButton.Location = new Point(925, 27); + browseButton.Size = new Size(100, 32); + browseButton.Anchor = AnchorStyles.Top | AnchorStyles.Right; + + pathGroup.Controls.AddRange(new Control[] { targetPathBox, browseButton }); + content.Controls.Add(pathGroup); + + var modeGroup = new GroupBox + { + Text = T("匯出內容範圍", "Export content scope"), + Font = UiKit.Body, + Location = new Point(12, 135), + Size = new Size(1040, 115), + Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right + }; + var modeFlow = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, Padding = new Padding(16, 12, 16, 12) }; + + schemaData.Text = T("結構與資料 (完整傾印，包含 DDL 與二進位幾何 INSERT)", "Structure & Data (Full logical dump including DDL and binary geometry INSERTs)"); + schemaData.Checked = (initialOptions?.Mode ?? SqliteExportMode.SchemaAndData) == SqliteExportMode.SchemaAndData; + schemaData.AutoSize = true; + schemaData.Margin = new Padding(0, 0, 0, 8); + + schemaOnly.Text = T("僅結構 (Schema Only，僅匯出資料表、檢視、索引與觸發器定義)", "Structure only (Schema only, excludes all table rows)"); + schemaOnly.Checked = initialOptions?.Mode == SqliteExportMode.SchemaOnly; + schemaOnly.AutoSize = true; + schemaOnly.Margin = new Padding(0, 0, 0, 8); + + dataOnly.Text = T("僅資料 (Data Only，僅匯出資料列 INSERT 語句，不含 CREATE/DROP)", "Data only (Data only, emits INSERT statements without CREATE/DROP)"); + dataOnly.Checked = initialOptions?.Mode == SqliteExportMode.DataOnly; + dataOnly.AutoSize = true; + + modeFlow.Controls.AddRange(new Control[] { schemaData, schemaOnly, dataOnly }); + modeGroup.Controls.Add(modeFlow); + content.Controls.Add(modeGroup); + + var optGroup = new GroupBox + { + Text = T("SQLite 引擎與交易保護選項", "SQLite engine & transaction options"), + Font = UiKit.Body, + Location = new Point(12, 260), + Size = new Size(1040, 195), + Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right + }; + var optFlow = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, Padding = new Padding(16, 12, 16, 12) }; + + singleTransaction.Text = T("使用單一交易包覆 (BEGIN TRANSACTION; ... COMMIT;，寫入速度提升數十倍並確保一致性)", "Wrap in single transaction (BEGIN TRANSACTION; ... COMMIT;, boosts write speeds 10-50x)"); + singleTransaction.Checked = true; + singleTransaction.AutoSize = true; + singleTransaction.Margin = new Padding(0, 0, 0, 8); + + pragmaSync.Text = T("匯入時關閉同步寫盤 (PRAGMA synchronous = OFF;，避免每次寫入強迫磁碟 flush)", "Disable synchronous disk flushing during restore (PRAGMA synchronous = OFF;)"); + pragmaSync.Checked = true; + pragmaSync.AutoSize = true; + pragmaSync.Margin = new Padding(0, 0, 0, 8); + + pragmaFk.Text = T("匯入時關閉外鍵檢查 (PRAGMA foreign_keys = OFF;，防範資料表相依順序報錯)", "Disable foreign key checks on import (PRAGMA foreign_keys = OFF;)"); + pragmaFk.Checked = true; + pragmaFk.AutoSize = true; + pragmaFk.Margin = new Padding(0, 0, 0, 8); + + includeDrop.Text = T("匯入前先刪除既有物件 (DROP TABLE / VIEW / INDEX / TRIGGER IF EXISTS)", "Drop existing objects before recreate (DROP ... IF EXISTS)"); + includeDrop.Checked = true; + includeDrop.AutoSize = true; + + optFlow.Controls.AddRange(new Control[] { singleTransaction, pragmaSync, pragmaFk, includeDrop }); + optGroup.Controls.Add(optFlow); + content.Controls.Add(optGroup); + } + + private void BuildPage2() + { + var content = Stack(); + pages[2].Controls.Add(content); + content.Controls.Add(HeaderLabel(T("步驟 3：確認物件與預估", "Step 3: Confirm selection and estimates"))); + content.Controls.Add(HintLabel(T("請確認本次 SQLite 匯出參數，點擊「開始匯出」即會串流寫入本機檔案。", "Review export parameters. Click 'Start Export' to begin streaming to local file."))); + + confirmSummary.AutoSize = true; + confirmSummary.Margin = new Padding(0, 12, 0, 16); + content.Controls.Add(confirmSummary); + } + + private void BuildPage3() + { + var content = new Panel { Dock = DockStyle.Fill }; + pages[3].Controls.Add(content); + + var topPanel = new TableLayoutPanel { Dock = DockStyle.Top, AutoSize = true, ColumnCount = 2, RowCount = 1 }; + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 70)); + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 30)); + + var statsFlow = Stack(); + currentObjectLabel.Font = UiKit.Subtitle; + currentObjectLabel.AutoSize = true; + statsFlow.Controls.Add(currentObjectLabel); + + overallBar.Width = 520; + overallBar.Height = 20; + overallBar.Margin = new Padding(0, 8, 0, 6); + statsFlow.Controls.Add(overallBar); + + var numRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + percentLabel.AutoSize = true; + percentLabel.Margin = new Padding(0, 0, 20, 0); + writtenLabel.AutoSize = true; + writtenLabel.Margin = new Padding(0, 0, 20, 0); + speedLabel.AutoSize = true; + numRow.Controls.AddRange(new Control[] { percentLabel, writtenLabel, speedLabel }); + statsFlow.Controls.Add(numRow); + + var timeRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 4, 0, 6) }; + elapsedLabel.AutoSize = true; + elapsedLabel.Margin = new Padding(0, 0, 20, 0); + etaLabel.AutoSize = true; + timeRow.Controls.AddRange(new Control[] { elapsedLabel, etaLabel }); + statsFlow.Controls.Add(timeRow); + + resultText.AutoSize = true; + resultText.Margin = new Padding(0, 6, 0, 8); + statsFlow.Controls.Add(resultText); + + var actRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + ConfigureButton(openFileButton, "exportOpenFile", T("開啟檔案", "Open File"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.TargetPath) && File.Exists(Result.TargetPath)) + Process.Start(new ProcessStartInfo("notepad.exe", Result.TargetPath) { UseShellExecute = true }); + }); + ConfigureButton(openFolderButton, "exportOpenFolder", T("開啟資料夾", "Open Folder"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.TargetPath) && File.Exists(Result.TargetPath)) + Process.Start("explorer.exe", $"/select,\"{Result.TargetPath}\""); + }); + openFileButton.Visible = false; + openFolderButton.Visible = false; + actRow.Controls.AddRange(new Control[] { openFileButton, openFolderButton }); + statsFlow.Controls.Add(actRow); + + topPanel.Controls.Add(statsFlow, 0, 0); + + // 右側放置 Punky 看板娘動畫與音效控制 + var mascotPanel = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + Padding = new Padding(12, 0, 0, 0) + }; + punky.Name = "sqlitePunky"; + punky.Size = new Size(200, 200); + punky.Margin = new Padding(0, 0, 0, 4); + mascotPanel.Controls.Add(punky); + + mascotHint.Text = T("崩琦正在幫你整理 SQLite GIS 資料！", "Punky is organizing your SQLite GIS data!"); + mascotHint.Font = UiKit.Body; + mascotHint.AutoSize = true; + mascotHint.Margin = new Padding(0, 0, 0, 6); + mascotPanel.Controls.Add(mascotHint); + + ConfigureButton(soundButton, "sqliteToggleSound", punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"), (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + mascotPanel.Controls.Add(soundButton); + + topPanel.Controls.Add(mascotPanel, 1, 0); + + content.Controls.Add(topPanel); + + logList.Name = "exportLogList"; + logList.Dock = DockStyle.Fill; + logList.IntegralHeight = false; + logList.BorderStyle = BorderStyle.FixedSingle; + logList.Font = new Font("Consolas", 9f, FontStyle.Regular); + content.Controls.Add(logList); + logList.BringToFront(); + } + + private async Task LoadCatalogAsync() + { + statusLabel.Text = T("正在載入 SQLite 資料庫目錄與空間中繼資訊...", "Loading SQLite database catalog and spatial metadata..."); + catalog = await Task.Run(() => service.LoadTableCatalog()); + views = await Task.Run(() => service.LoadViews()); + triggers = await Task.Run(() => service.LoadTriggers()); + + tableGrid.Rows.Clear(); + foreach (var item in catalog) + { + tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0"), item.SpatialInfo); + } + + statusLabel.Text = ""; + UpdateSelectionSummary(); + } + + private void ApplySearchFilter() + { + string term = search.Text.Trim(); + tableGrid.Rows.Clear(); + foreach (var item in catalog) + { + if (string.IsNullOrWhiteSpace(term) || item.Name.IndexOf(term, StringComparison.OrdinalIgnoreCase) >= 0 || item.SpatialInfo.IndexOf(term, StringComparison.OrdinalIgnoreCase) >= 0) + { + tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0"), item.SpatialInfo); + } + } + UpdateSelectionSummary(); + } + + private void SetAllSelected(bool isSelected) + { + foreach (var item in catalog) item.IsSelected = isSelected; + foreach (DataGridViewRow row in tableGrid.Rows) row.Cells["ColSelect"].Value = isSelected; + UpdateSelectionSummary(); + } + + private void UpdateSelectionSummary() + { + int selectedCount = catalog.Count(t => t.IsSelected); + int spatialCount = catalog.Count(t => t.IsSelected && t.IsSpatial); + long totalRows = catalog.Where(t => t.IsSelected).Sum(t => t.EstimatedRows); + selectionSummary.Text = string.Format( + T("已選取 {0} / {1} 張資料表 (含 {2} 個 GIS 空間圖層，預估約 {3:N0} 筆資料)", "Selected {0} / {1} tables ({2} GIS layers, approx. {3:N0} rows)"), + selectedCount, catalog.Count, spatialCount, totalRows); + } + + private void ShowPage() + { + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + string[] stepTitles = { + T("1. 選取物件", "1. Select Objects"), + T("2. 輸出設定", "2. Output Settings"), + T("3. 確認", "3. Confirm"), + T("4. 執行匯出", "4. Export") + }; + stepsLabel.Text = string.Join(" → ", stepTitles.Select((t, i) => i == step ? $"【{t}】" : t)); + + backButton.Enabled = (step > 0 && step < 3 && !busy); + if (step == 0) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = catalog.Any(t => t.IsSelected) || views.Count > 0; + } + else if (step == 1) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = !string.IsNullOrWhiteSpace(targetPathBox.Text); + } + else if (step == 2) + { + nextButton.Text = T("🚀 開始匯出", "🚀 Start Export"); + nextButton.Enabled = true; + UpdateConfirmSummary(); + } + else if (step == 3) + { + nextButton.Text = T("完成", "Finished"); + nextButton.Enabled = !busy; + } + } + + private void UpdateConfirmSummary() + { + var selectedTables = catalog.Where(t => t.IsSelected).ToList(); + int spatialCount = selectedTables.Count(t => t.IsSpatial); + confirmSummary.Text = + $"● {T("目標資料庫：", "Target Database: ")} {databaseName}\n" + + $"● {T("匯出模式：", "Export Mode: ")} {(schemaData.Checked ? T("結構與資料 (含 GIS 二進位幾何)", "Structure & Data (with GIS binary geometries)") : schemaOnly.Checked ? T("僅結構", "Structure only") : T("僅資料", "Data only"))}\n" + + $"● {T("選取資料表：", "Selected Tables: ")} {selectedTables.Count} 張（含 {spatialCount} 個 GIS 圖層，預估 {selectedTables.Sum(t => t.EstimatedRows):N0} 列）\n" + + $"● {T("選取檢視：", "Selected Views: ")} {(includeViews.Checked ? views.Count : 0)} 個\n" + + $"● {T("選取索引/觸發器：", "Selected Indexes/Triggers: ")} {(includeIndexes.Checked ? T("包含", "Included") : T("略過", "Skipped"))} / {(includeTriggers.Checked ? triggers.Count : 0)} 個\n" + + $"● {T("單一交易包覆：", "Single Transaction: ")} {(singleTransaction.Checked ? T("啟用 (BEGIN TRANSACTION; ... COMMIT;)", "Enabled") : T("停用", "Disabled"))}\n" + + $"● {T("SQLite 寫入優化：", "SQLite Engine Optimizations: ")} {(pragmaSync.Checked ? "PRAGMA synchronous = OFF; " : "")} {(pragmaFk.Checked ? "PRAGMA foreign_keys = OFF;" : "")}\n" + + $"● {T("輸出檔案路徑：", "Target Path: ")} {targetPathBox.Text}\n\n" + + $"{T("提示：匯出過程將全串流寫入暫存檔案，完成驗證後原子發布，不污染既有檔案。", "Notice: Streaming to staging file first, published atomically upon completion.")}"; + } + + private async Task OnNextAsync() + { + if (busy) return; + if (step < 2) + { + step++; + ShowPage(); + } + else if (step == 2) + { + step = 3; + ShowPage(); + await StartExportAsync(); + } + else if (step == 3) + { + Close(); + } + } + + private async Task StartExportAsync() + { + busy = true; + exporting = true; + cancellation = new CancellationTokenSource(); + nextButton.Enabled = false; + backButton.Enabled = false; + cancelButton.Text = T("停止匯出", "Stop Export"); + logList.Items.Clear(); + + var mode = schemaData.Checked ? SqliteExportMode.SchemaAndData : schemaOnly.Checked ? SqliteExportMode.SchemaOnly : SqliteExportMode.DataOnly; + var opt = new SqliteExportWizardOptions(mode, targetPathBox.Text.Trim()) + { + SingleTransaction = singleTransaction.Checked, + PragmaForeignKeysOff = pragmaFk.Checked, + PragmaSynchronousOff = pragmaSync.Checked, + IncludeDropStatements = includeDrop.Checked, + IncludeViews = includeViews.Checked, + IncludeIndexes = includeIndexes.Checked, + IncludeTriggers = includeTriggers.Checked, + IncludeComments = includeComments.Checked, + ExcludeSpatialMetadata = excludeSpatialMetadata.Checked, + SelectedTables = catalog.Where(t => t.IsSelected).Select(t => t.Name).ToList(), + SelectedViews = views, + SelectedTriggers = triggers + }; + + var progress = new Progress(p => + { + lastProgress = p; + RenderProgress(p); + }); + + punky.BeginExport(); + clock.Restart(); + timer.Start(); + + try + { + Result = await service.ExportAsync(opt, progress, cancellation.Token); + timer.Stop(); + clock.Stop(); + + if (Result.Success) + { + currentObjectLabel.Text = T("✅ 匯出作業順利完成！", "✅ Export completed successfully!"); + resultText.Text = string.Format( + T("共計匯出 {0} 張資料表、{1} 個檢視、{2} 個索引，總計 {3:N0} 列。\n檔案大小：{4:N0} 位元組 · SHA-256：{5}", + "Exported {0} tables, {1} views, {2} indexes, {3:N0} rows.\nFile size: {4:N0} bytes · SHA-256: {5}"), + Result.TableCount, Result.ViewCount, Result.IndexCount, Result.TotalRows, Result.FileBytes, Result.Sha256Hash); + openFileButton.Visible = true; + openFolderButton.Visible = true; + } + else if (Result.Stage == SqliteExportStage.Cancelled) + { + currentObjectLabel.Text = T("⚠️ 匯出作業已由使用者取消。", "⚠️ Export cancelled by user."); + resultText.Text = Result.ErrorMessage; + } + else + { + currentObjectLabel.Text = T("❌ 匯出失敗：", "❌ Export failed: ") + Result.ErrorMessage; + resultText.Text = Result.ErrorMessage; + } + } + catch (Exception ex) + { + timer.Stop(); + clock.Stop(); + currentObjectLabel.Text = T("❌ 發生非預期錯誤：", "❌ Unexpected error: ") + ex.Message; + resultText.Text = ex.Message; + } + finally + { + punky.EndExport(); + busy = false; + exporting = false; + cancelButton.Text = T("關閉", "Close"); + nextButton.Enabled = true; + if (closing) Close(); + } + } + + private void RenderProgress(SqliteExportProgress p) + { + if (p.EstimatedTotalRows > 0) + { + int val = Math.Min(100, Math.Max(0, (int)(p.ProcessedRows * 100.0 / p.EstimatedTotalRows))); + overallBar.Value = val; + percentLabel.Text = $"{val}% ({p.ProcessedRows:N0} / {p.EstimatedTotalRows:N0} 列)"; + } + else + { + percentLabel.Text = $"{p.ProcessedRows:N0} 列"; + } + + currentObjectLabel.Text = p.Message; + writtenLabel.Text = string.Format(T("已寫入：{0:N2} MB", "Written: {0:N2} MB"), p.WrittenBytes / (1024.0 * 1024.0)); + speedLabel.Text = string.Format(T("速率：{0:N0} 列/秒", "Speed: {0:N0} rows/s"), p.RowsPerSecond); + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), p.Elapsed); + etaLabel.Text = p.EstimatedRemaining.HasValue + ? string.Format(T("預估剩餘：約 {0:mm\\:ss}", "ETA: approx. {0:mm\\:ss}"), p.EstimatedRemaining.Value) + : ""; + + if (!string.IsNullOrWhiteSpace(p.Message)) + { + logList.Items.Add($"[{DateTime.Now:HH:mm:ss}] {p.Message}"); + if (logList.Items.Count > 1000) logList.Items.RemoveAt(0); + logList.TopIndex = logList.Items.Count - 1; + } + } + + private void UpdateLiveTicker() + { + if (exporting && lastProgress != null) + { + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), clock.Elapsed); + } + } + + private void RequestCancel() + { + if (cancellation != null && !cancellation.IsCancellationRequested) + { + cancelButton.Enabled = false; + cancelButton.Text = T("正在取消...", "Cancelling..."); + cancellation.Cancel(); + } + } + } +} diff --git a/mySQLPunk/SqliteImportWizardForm.cs b/mySQLPunk/SqliteImportWizardForm.cs new file mode 100644 index 0000000..6a2e6be --- /dev/null +++ b/mySQLPunk/SqliteImportWizardForm.cs @@ -0,0 +1,773 @@ +using System; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.SqliteImport; +using SqliteImportResult = mySQLPunk.lib.SqliteImport.SqliteImportResult; + +namespace mySQLPunk +{ + internal sealed class SqliteImportWizardForm : Form + { + private readonly IDatabase db; + private readonly string databaseName; + private readonly SqliteImportRunner runner; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox sourcePathBox = new TextBox(), targetDbBox = new TextBox(), logPathBox = new TextBox(); + private readonly ComboBox strategyCombo = new ComboBox(); + private readonly CheckBox continueOnError = new CheckBox(), wrapInTransaction = new CheckBox(), autoLoadSpatiaLite = new CheckBox(), confirmCheck = new CheckBox(), autoScrollCheck = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(), fileFactsLabel = new Label(); + private readonly Label checkSummary = new Label(), confirmSummary = new Label(), logTitleLabel = new Label(), logCountLabel = new Label(); + private readonly Label currentStatementLabel = new Label(), coverageLabel = new Label(), speedLabel = new Label(); + private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), errorCountLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(), clearLogButton = new Button(); + private readonly Button recheckButton = new Button(), openLogButton = new Button(), openFolderButton = new Button(), copyPathButton = new Button(); + private readonly ProgressBar execBar = new ProgressBar(), scanBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox errorList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + private const int MaxLogLines = 2000; + + private SqliteImportPreflight preflight; + private SqliteImportRequest request; + private CancellationTokenSource cancellation; + private bool busy, executing, closing; + private int step; + + internal SqliteImportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal SqliteImportWizardForm(IDatabase db, string databaseName, string initialSourcePath = "") + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + this.runner = new SqliteImportRunner(db, databaseName); + this.request = new SqliteImportRequest(Guid.NewGuid(), initialSourcePath, databaseName); + + Name = "sqliteImportWizard"; + Text = T("匯入 SQL 檔案 · SQLite (支援 SpatiaLite / GeoPackage GIS)", "Import SQL file · SQLite (with SpatiaLite / GeoPackage GIS)"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1100, 750); + MinimumSize = new Size(640, 480); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("SQLite SQL 檔案匯入精靈", "SQLite SQL file import wizard"), Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(); + BuildPage1(); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var footer = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + AutoSize = true, + WrapContents = true, + FlowDirection = FlowDirection.RightToLeft + }; + ConfigureButton(cancelButton, "importCancel", T("取消", "Cancel"), (s, e) => { if (busy) RequestCancel(); else Close(); }); + ConfigureButton(nextButton, "importNext", T("下一步 →", "Next →"), async (s, e) => await OnNextAsync()); + ConfigureButton(backButton, "importBack", T("← 上一步", "← Back"), (s, e) => { if (!busy && step > 0 && step < 3) { step--; ShowPage(); } }); + ConfigureButton(recheckButton, "importRecheck", T("重新預檢", "Recheck"), async (s, e) => { if (!busy) await RunPreflightAsync(); }); + recheckButton.Visible = false; + + footer.Controls.AddRange(new Control[] { cancelButton, nextButton, backButton, recheckButton }); + root.Controls.Add(footer, 0, 3); + + ThemeManager.MarkAsPrimary(nextButton); + AcceptButton = nextButton; + CancelButton = cancelButton; + cancelButton.DialogResult = DialogResult.None; + + timer.Tick += (s, e) => UpdateLiveTicker(); + + Shown += (s, e) => + { + UpdateFileFacts(); + ShowPage(); + }; + + FormClosing += (s, e) => + { + if (busy && !closing) + { + e.Cancel = true; + closing = true; + RequestCancel(); + } + }; + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel + { + Dock = DockStyle.Fill, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + AutoSize = true + }; + + private static Label HeaderLabel(string text) => new Label + { + Text = text, + Font = UiKit.Subtitle, + AutoSize = true, + Margin = new Padding(0, 4, 0, 8) + }; + + private static Label HintLabel(string text) => new Label + { + Text = text, + Font = UiKit.Body, + ForeColor = Color.Gray, + AutoSize = true, + Margin = new Padding(0, 0, 0, 8) + }; + + private static void ConfigureButton(Button b, string name, string text, EventHandler onClick) + { + b.Name = name; + b.Text = text; + b.Font = UiKit.Body; + b.AutoSize = true; + b.MinimumSize = new Size(110, 36); + b.Margin = new Padding(8, 0, 0, 0); + b.Click += onClick; + } + + private void BuildPage0() + { + var content = Stack(); + pages[0].Controls.Add(content); + + content.Controls.Add(HeaderLabel(T("步驟 1：選取來源 SQL 檔案與目標 SQLite 資料庫", "Step 1: Select source SQL file and target SQLite database"))); + content.Controls.Add(HintLabel(T("請選擇欲執行的本機 SQL 傾印備份檔，並確認還原之 SQLite 目標資料庫。", "Choose a local SQL dump file to restore and verify the target SQLite database."))); + + var fileGroup = new GroupBox + { + Text = T("來源 SQL 檔案", "Source SQL file"), + Font = UiKit.Body, + Width = 1040, + Height = 100, + Margin = new Padding(0, 8, 0, 16) + }; + sourcePathBox.Location = new Point(16, 26); + sourcePathBox.Size = new Size(900, 28); + sourcePathBox.Text = request.SourcePath; + sourcePathBox.TextChanged += (s, e) => + { + request = new SqliteImportRequest(request.JobId, sourcePathBox.Text.Trim(), databaseName); + UpdateFileFacts(); + }; + + ConfigureButton(browseButton, "importBrowse", T("瀏覽...", "Browse..."), (s, e) => + { + using (var ofd = new OpenFileDialog()) + { + ofd.Title = T("選擇 SQL 檔案", "Select SQL file"); + ofd.Filter = "SQL Files (*.sql)|*.sql|All Files (*.*)|*.*"; + if (ofd.ShowDialog(this) == DialogResult.OK) + { + sourcePathBox.Text = ofd.FileName; + } + } + }); + browseButton.Location = new Point(925, 23); + browseButton.Size = new Size(100, 32); + + fileFactsLabel.Location = new Point(16, 62); + fileFactsLabel.AutoSize = true; + fileFactsLabel.ForeColor = Color.Gray; + + fileGroup.Controls.AddRange(new Control[] { sourcePathBox, browseButton, fileFactsLabel }); + content.Controls.Add(fileGroup); + + var dbGroup = new GroupBox + { + Text = T("目標 SQLite 資料庫", "Target SQLite database"), + Font = UiKit.Body, + Width = 1040, + Height = 80, + Margin = new Padding(0, 0, 0, 16) + }; + targetDbBox.Location = new Point(16, 28); + targetDbBox.Size = new Size(1000, 28); + targetDbBox.Text = databaseName; + targetDbBox.ReadOnly = true; + dbGroup.Controls.Add(targetDbBox); + content.Controls.Add(dbGroup); + } + + private void BuildPage1() + { + var content = Stack(); + pages[1].Controls.Add(content); + + content.Controls.Add(HeaderLabel(T("步驟 2：串流預檢 (Preflight Scan)", "Step 2: Preflight Scan"))); + content.Controls.Add(HintLabel(T("正在對 SQL 檔案進行非侵入性串流掃描，檢查編碼、NUL 字元、SpatiaLite 空間特徵與 Manifest 完整性...", + "Scanning SQL file stream for encoding, NUL bytes, SpatiaLite GIS features, and Native Manifest..."))); + + scanBar.Width = 1040; + scanBar.Height = 16; + scanBar.Style = ProgressBarStyle.Marquee; + scanBar.Margin = new Padding(0, 8, 0, 16); + content.Controls.Add(scanBar); + + checkSummary.Font = UiKit.Body; + checkSummary.AutoSize = true; + checkSummary.Margin = new Padding(0, 8, 0, 16); + content.Controls.Add(checkSummary); + } + + private void BuildPage2() + { + var content = Stack(); + pages[2].Controls.Add(content); + + content.Controls.Add(HeaderLabel(T("步驟 3：確認匯入與衝突策略", "Step 3: Confirm import & conflict policy"))); + content.Controls.Add(HintLabel(T("請選擇當目標資料庫已存在同名資料表時的處理策略，並確認開始執行。", "Select policy for existing objects and confirm execution."))); + + var optGroup = new GroupBox + { + Text = T("既有物件衝突策略與安全選項", "Conflict policy and safety options"), + Font = UiKit.Body, + Width = 1040, + Height = 220, + Margin = new Padding(0, 8, 0, 16) + }; + + var stratLabel = new Label { Text = T("衝突處理策略：", "Conflict Strategy:"), Location = new Point(16, 30), AutoSize = true }; + strategyCombo.Location = new Point(160, 26); + strategyCombo.Width = 480; + strategyCombo.DropDownStyle = ComboBoxStyle.DropDownList; + strategyCombo.Items.AddRange(new object[] { + T("略過既有物件 (SkipExisting，保留現有結構與資料)", "SkipExisting (Skip existing objects)"), + T("刪除並重建既有物件 (DropAndRecreate，DROP TABLE IF EXISTS)", "DropAndRecreate (DROP and recreate)"), + T("保留結構，寫入或覆蓋資料 (OverwriteKeepExisting)", "OverwriteKeepExisting (Keep schema, insert data)"), + T("若已存在則中斷匯入 (AbortIfExists，安全防護)", "AbortIfExists (Halt on existing object)") + }); + strategyCombo.SelectedIndex = 0; + + continueOnError.Text = T("發生錯誤時略過並繼續執行 (預設關閉，遇錯立即停止以保全資料)", "Continue on error (Skip failed statement and proceed)"); + continueOnError.Location = new Point(16, 70); + continueOnError.AutoSize = true; + + wrapInTransaction.Text = T("使用交易包覆 (提升寫入效能 10~50 倍，遇到嚴重錯誤時自動回滾)", "Wrap in transaction (boosts writes 10-50x, auto rollback on error)"); + wrapInTransaction.Location = new Point(16, 100); + wrapInTransaction.Checked = true; + wrapInTransaction.AutoSize = true; + + autoLoadSpatiaLite.Text = T("自動動態加載 SpatiaLite 空間擴充模組 (mod_spatialite.dll)", "Auto dynamic load SpatiaLite extension (mod_spatialite.dll)"); + autoLoadSpatiaLite.Location = new Point(16, 130); + autoLoadSpatiaLite.Checked = true; + autoLoadSpatiaLite.AutoSize = true; + + confirmCheck.Text = T("我已確認目標 SQLite 資料庫正確，並同意執行匯入", "I have verified the target SQLite database and confirm execution"); + confirmCheck.Location = new Point(16, 170); + confirmCheck.AutoSize = true; + confirmCheck.Font = new Font(UiKit.Body.FontFamily, UiKit.Body.Size, FontStyle.Bold); + confirmCheck.CheckedChanged += (s, e) => ShowPage(); + + optGroup.Controls.AddRange(new Control[] { stratLabel, strategyCombo, continueOnError, wrapInTransaction, autoLoadSpatiaLite, confirmCheck }); + content.Controls.Add(optGroup); + + confirmSummary.Font = UiKit.Body; + confirmSummary.AutoSize = true; + confirmSummary.Margin = new Padding(0, 0, 0, 16); + content.Controls.Add(confirmSummary); + } + + private void BuildPage3() + { + var content = new Panel { Dock = DockStyle.Fill }; + pages[3].Controls.Add(content); + + var topPanel = new TableLayoutPanel { Dock = DockStyle.Top, AutoSize = true, ColumnCount = 2, RowCount = 1 }; + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 70)); + topPanel.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 30)); + + var statsFlow = Stack(); + currentStatementLabel.Font = UiKit.Subtitle; + currentStatementLabel.AutoSize = true; + statsFlow.Controls.Add(currentStatementLabel); + + execBar.Width = 520; + execBar.Height = 20; + execBar.Margin = new Padding(0, 8, 0, 6); + statsFlow.Controls.Add(execBar); + + var numRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + coverageLabel.AutoSize = true; + coverageLabel.Margin = new Padding(0, 0, 20, 0); + speedLabel.AutoSize = true; + speedLabel.Margin = new Padding(0, 0, 20, 0); + errorCountLabel.AutoSize = true; + numRow.Controls.AddRange(new Control[] { coverageLabel, speedLabel, errorCountLabel }); + statsFlow.Controls.Add(numRow); + + var timeRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, Margin = new Padding(0, 4, 0, 6) }; + elapsedLabel.AutoSize = true; + elapsedLabel.Margin = new Padding(0, 0, 20, 0); + etaLabel.AutoSize = true; + timeRow.Controls.AddRange(new Control[] { elapsedLabel, etaLabel }); + statsFlow.Controls.Add(timeRow); + + resultText.AutoSize = true; + resultText.Margin = new Padding(0, 6, 0, 8); + statsFlow.Controls.Add(resultText); + + var actRow = new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight }; + ConfigureButton(openLogButton, "importOpenLog", T("開啟 Log 檔案", "Open Log File"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogPath) && File.Exists(Result.LogPath)) + Process.Start(new ProcessStartInfo("notepad.exe", Result.LogPath) { UseShellExecute = true }); + }); + ConfigureButton(openFolderButton, "importOpenFolder", T("開啟 Log 資料夾", "Open Log Folder"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogPath) && File.Exists(Result.LogPath)) + Process.Start("explorer.exe", $"/select,\"{Result.LogPath}\""); + }); + ConfigureButton(copyPathButton, "importCopyPath", T("複製 Log 路徑", "Copy Log Path"), (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogPath)) + { + Clipboard.SetText(Result.LogPath); + MessageBox.Show(T("Log 檔案路徑已複製至剪貼簿！", "Log path copied to clipboard!"), T("已複製", "Copied"), MessageBoxButtons.OK, MessageBoxIcon.Information); + } + }); + + openLogButton.Visible = false; + openFolderButton.Visible = false; + copyPathButton.Visible = false; + actRow.Controls.AddRange(new Control[] { openLogButton, openFolderButton, copyPathButton }); + statsFlow.Controls.Add(actRow); + + topPanel.Controls.Add(statsFlow, 0, 0); + + // 右側放置 Punky 看板娘動畫與音效控制 + var mascotPanel = new FlowLayoutPanel + { + Dock = DockStyle.Fill, + FlowDirection = FlowDirection.TopDown, + WrapContents = false, + Padding = new Padding(12, 0, 0, 0) + }; + punky.Name = "sqliteImportPunky"; + punky.Size = new Size(200, 200); + punky.Margin = new Padding(0, 0, 0, 4); + mascotPanel.Controls.Add(punky); + + mascotHint.Text = T("崩琦正在為你匯入 SQLite GIS 資料！", "Punky is importing SQLite GIS data for you!"); + mascotHint.Font = UiKit.Body; + mascotHint.AutoSize = true; + mascotHint.Margin = new Padding(0, 0, 0, 6); + mascotPanel.Controls.Add(mascotHint); + + ConfigureButton(soundButton, "sqliteToggleSound", punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"), (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + mascotPanel.Controls.Add(soundButton); + + topPanel.Controls.Add(mascotPanel, 1, 0); + + content.Controls.Add(topPanel); + + // 下方放置 2,000 行滑動視窗終端機風格日誌 + var logContainer = new Panel { Dock = DockStyle.Fill, Padding = new Padding(0, 10, 0, 0) }; + + var logHeader = new FlowLayoutPanel + { + Dock = DockStyle.Top, + Height = 32, + FlowDirection = FlowDirection.LeftToRight, + WrapContents = false + }; + + logTitleLabel.Text = T("即時執行日誌", "Live Execution Log"); + logTitleLabel.Font = UiKit.Subtitle; + logTitleLabel.AutoSize = true; + logTitleLabel.Margin = new Padding(0, 4, 12, 0); + + logCountLabel.Text = string.Format(T("（保留最新 {0:N0} 行）", "(Rolling latest {0:N0} lines)"), MaxLogLines); + logCountLabel.Font = UiKit.Body; + logCountLabel.ForeColor = Color.Gray; + logCountLabel.AutoSize = true; + logCountLabel.Margin = new Padding(0, 6, 20, 0); + + autoScrollCheck.Text = T("自動滾動", "Auto-scroll"); + autoScrollCheck.Checked = true; + autoScrollCheck.AutoSize = true; + autoScrollCheck.Margin = new Padding(0, 5, 12, 0); + + ConfigureButton(clearLogButton, "importClearLog", T("清空日誌", "Clear"), (s, e) => + { + errorList.Items.Clear(); + }); + clearLogButton.MinimumSize = new Size(70, 26); + clearLogButton.Height = 26; + clearLogButton.Margin = new Padding(0, 2, 0, 0); + + logHeader.Controls.AddRange(new Control[] { logTitleLabel, logCountLabel, autoScrollCheck, clearLogButton }); + logContainer.Controls.Add(logHeader); + + errorList.Name = "importLogList"; + errorList.Dock = DockStyle.Fill; + errorList.IntegralHeight = false; + errorList.BorderStyle = BorderStyle.FixedSingle; + errorList.Font = new Font("Consolas", 9f, FontStyle.Regular); + logContainer.Controls.Add(errorList); + errorList.BringToFront(); + + content.Controls.Add(logContainer); + } + + private void UpdateFileFacts() + { + string path = sourcePathBox.Text.Trim(); + if (File.Exists(path)) + { + long bytes = new FileInfo(path).Length; + fileFactsLabel.Text = string.Format(T("檔案大小：{0:N0} 位元組 ({1:N2} MB)", "File size: {0:N0} bytes ({1:N2} MB)"), bytes, bytes / (1024.0 * 1024.0)); + nextButton.Enabled = (step == 0); + } + else + { + fileFactsLabel.Text = T("未選擇檔案或檔案不存在。", "No file selected or file does not exist."); + if (step == 0) nextButton.Enabled = false; + } + } + + private void ShowPage() + { + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + string[] stepTitles = { + T("1. 檔案與目標", "1. File & Target"), + T("2. 串流預檢", "2. Preflight Scan"), + T("3. 確認與策略", "3. Confirm & Policy"), + T("4. 執行匯入", "4. Execute Import") + }; + stepsLabel.Text = string.Join(" → ", stepTitles.Select((t, i) => i == step ? $"【{t}】" : t)); + + backButton.Enabled = (step > 0 && step < 3 && !busy); + recheckButton.Visible = (step == 1); + + if (step == 0) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = File.Exists(sourcePathBox.Text.Trim()); + } + else if (step == 1) + { + nextButton.Text = T("下一步 →", "Next →"); + nextButton.Enabled = preflight != null && preflight.Passed; + } + else if (step == 2) + { + nextButton.Text = T("🚀 開始匯入", "🚀 Start Import"); + nextButton.Enabled = confirmCheck.Checked; + UpdateConfirmSummary(); + } + else if (step == 3) + { + nextButton.Text = T("完成", "Finished"); + nextButton.Enabled = !busy; + } + } + + private async Task RunPreflightAsync() + { + busy = true; + recheckButton.Enabled = false; + nextButton.Enabled = false; + backButton.Enabled = false; + scanBar.Visible = true; + checkSummary.Text = T("正在非侵入性串流掃描 SQL 檔案架構與空間中繼資料...", "Streaming scan of SQL file architecture & spatial metadata..."); + + cancellation = new CancellationTokenSource(); + try + { + preflight = await Task.Run(() => SqliteImportPreflightScanner.Scan(request.SourcePath, cancellation.Token)); + + if (preflight.Passed) + { + string spatialNote = preflight.DetectedSpatiaLite + ? T("✅ 偵測到 SpatiaLite / GeoPackage 空間幾何語法或中繼表，匯入時將自動配置空間擴充模組！\n", + "✅ SpatiaLite / GeoPackage spatial features detected; extension will be configured automatically!\n") + : ""; + + string manifestNote = preflight.HasManifest + ? string.Format(T("✅ 包含 Native Manifest 核對摘要：預計還原 {0} 張資料表，約 {1:N0} 列資料。\n", + "✅ Native Manifest detected: Expected {0} tables, approx. {1:N0} rows.\n"), + preflight.ManifestTables, preflight.ManifestRows) + : T("ℹ️ 無 Native Manifest 摘要 (非本軟體匯出或傳統傾印檔，不影響匯入)。\n", "ℹ️ No Native Manifest found.\n"); + + checkSummary.Text = + T("🎉 預檢掃描通過！\n\n", "🎉 Preflight scan passed!\n\n") + + spatialNote + + manifestNote + + string.Format(T("● 檔案大小：{0:N0} 位元組 ({1:N2} MB)\n", "● File size: {0:N0} bytes ({1:N2} MB)\n"), preflight.FileBytes, preflight.FileBytes / (1024.0 * 1024.0)) + + string.Format(T("● 預估總 SQL 語句數：約 {0:N0} 句\n", "● Estimated SQL statements: approx. {0:N0}\n"), preflight.StatementCount) + + T("● 編碼與結構檢查：無 Embedded NUL，單批次大小合規", "● Encoding & structure check: No Embedded NUL, batch size valid"); + } + else + { + checkSummary.Text = + T("❌ 預檢掃描發現問題，無法直接執行：\n\n", "❌ Preflight scan detected issues:\n\n") + + string.Join("\n", preflight.Issues); + } + } + catch (Exception ex) + { + checkSummary.Text = T("❌ 預檢掃描發生例外：", "❌ Preflight exception: ") + ex.Message; + } + finally + { + scanBar.Visible = false; + busy = false; + recheckButton.Enabled = true; + backButton.Enabled = true; + nextButton.Enabled = (preflight != null && preflight.Passed); + } + } + + private void UpdateConfirmSummary() + { + var strat = (SqliteExistingObjectStrategy)strategyCombo.SelectedIndex; + confirmSummary.Text = + $"● {T("目標 SQLite 資料庫：", "Target Database: ")} {databaseName}\n" + + $"● {T("來源 SQL 檔案：", "Source SQL: ")} {request.SourcePath} ({new FileInfo(request.SourcePath).Length / 1024.0 / 1024.0:N2} MB)\n" + + $"● {T("衝突處理策略：", "Conflict Strategy: ")} {strat}\n" + + $"● {T("交易保護加速：", "Transaction Wrapping: ")} {(wrapInTransaction.Checked ? T("啟用 (寫入速度提升數十倍，失敗回滾)", "Enabled") : T("停用", "Disabled"))}\n" + + $"● {T("SpatiaLite 空間擴充：", "SpatiaLite Extension: ")} {(autoLoadSpatiaLite.Checked ? T("自動動態載入", "Auto-loaded") : T("關閉", "Disabled"))}\n" + + $"● {T("遇錯處理：", "On Error: ")} {(continueOnError.Checked ? T("略過錯誤並繼續", "Continue on error") : T("立即中止並保全現狀", "Halt on error"))}\n\n" + + $"{T("提示：匯入過程為全串流逐句執行，日誌將維持 2,000 行滑動視窗，並於本機儲存完整日誌檔案。", "Notice: Streaming execution. Live log retains rolling 2,000 lines, full log saved locally.")}"; + } + + private async Task OnNextAsync() + { + if (busy) return; + + if (step == 0) + { + step = 1; + ShowPage(); + await RunPreflightAsync(); + } + else if (step == 1) + { + step = 2; + ShowPage(); + } + else if (step == 2) + { + if (!confirmCheck.Checked) + { + MessageBox.Show(T("請先勾選確認方塊以同意執行匯入作業。", "Please check the confirmation box to proceed."), T("尚未確認", "Unconfirmed"), MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + step = 3; + ShowPage(); + await StartImportAsync(); + } + else if (step == 3) + { + Close(); + } + } + + private async Task StartImportAsync() + { + busy = true; + executing = true; + cancellation = new CancellationTokenSource(); + nextButton.Enabled = false; + backButton.Enabled = false; + cancelButton.Text = T("停止匯入", "Stop Import"); + errorList.Items.Clear(); + + var opt = new SqliteImportOptions + { + ContinueOnError = continueOnError.Checked, + ExistingObjectStrategy = (SqliteExistingObjectStrategy)strategyCombo.SelectedIndex, + WrapInTransaction = wrapInTransaction.Checked, + AutoLoadSpatiaLite = autoLoadSpatiaLite.Checked + }; + + var progress = new Progress(RenderProgress); + + punky.BeginExport(); + clock.Restart(); + timer.Start(); + + try + { + Result = await runner.RunAsync(request, opt, progress, cancellation.Token); + timer.Stop(); + clock.Stop(); + + if (Result.Status == SqliteImportStatus.Completed) + { + currentStatementLabel.Text = T("🎉 匯入作業已成功完成！", "🎉 Import completed successfully!"); + string manifestPart = Result.ManifestVerified + ? T("✅ Native Manifest 核對完全相符！\n", "✅ Native Manifest verified exactly!\n") + : (!string.IsNullOrWhiteSpace(Result.ManifestReport) ? $"Manifest Report: {Result.ManifestReport}\n" : ""); + + resultText.Text = string.Format( + T("執行語句：{0:N0} 成功，{1} 失敗，{2} 略過。\n實際寫入/影響列數：{3:N0} 列。\n{4}日誌檔案已儲存至：{5}", + "Statements: {0:N0} succeeded, {1} failed, {2} skipped.\nTotal rows affected: {3:N0} rows.\n{4}Log file: {5}"), + Result.ExecutedStatements, Result.FailedStatements, Result.SkippedStatements, Result.TotalRowsAffected, manifestPart, Result.LogPath); + } + else if (Result.Status == SqliteImportStatus.Cancelled) + { + currentStatementLabel.Text = T("⚠️ 匯入作業已由使用者取消。", "⚠️ Import cancelled by user."); + resultText.Text = T("匯入已取消；若有開啟交易則已自動回滾。", "Import was cancelled; transaction rolled back if active."); + } + else + { + currentStatementLabel.Text = T("❌ 匯入未完成或發生錯誤。", "❌ Import failed or incomplete."); + resultText.Text = string.Format( + T("執行結束：{0} 失敗，{1} 略過。\n日誌檔案：{2}", + "Finished with {0} failed, {1} skipped.\nLog: {2}"), + Result.FailedStatements, Result.SkippedStatements, Result.LogPath); + } + + openLogButton.Visible = true; + openFolderButton.Visible = true; + copyPathButton.Visible = true; + } + catch (Exception ex) + { + timer.Stop(); + clock.Stop(); + currentStatementLabel.Text = T("❌ 發生非預期例外：", "❌ Unexpected exception: ") + ex.Message; + AppendLogLine("❌ " + ex.Message); + } + finally + { + punky.EndExport(); + busy = false; + executing = false; + cancelButton.Text = T("關閉", "Close"); + nextButton.Enabled = true; + if (closing) Close(); + } + } + + private void RenderProgress(SqliteImportProgress p) + { + if (p.TotalBytes > 0) + { + int val = Math.Min(100, Math.Max(0, (int)(p.ProcessedBytes * 100.0 / p.TotalBytes))); + execBar.Value = val; + coverageLabel.Text = $"{val}% ({p.ProcessedBytes / 1024:N0} KB / {p.TotalBytes / 1024:N0} KB)"; + } + currentStatementLabel.Text = p.Message; + speedLabel.Text = string.Format(T("速率：{0:N0} KB/秒", "Speed: {0:N0} KB/s"), p.BytesPerSecond / 1024.0); + errorCountLabel.Text = string.Format(T("已寫入：{0:N0} 列 · 錯誤：{1} 筆", "Written: {0:N0} rows · Errors: {1}"), p.TotalRowsAffected, p.FailedStatements); + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), p.Elapsed); + etaLabel.Text = p.EstimatedRemaining.HasValue + ? string.Format(T("預估剩餘：約 {0:mm\\:ss}", "ETA: approx. {0:mm\\:ss}"), p.EstimatedRemaining.Value) + : ""; + + if (!string.IsNullOrWhiteSpace(p.CurrentStatementPreview)) + { + AppendLogLine(p.CurrentStatementPreview); + } + } + + private void AppendLogLine(string headline) + { + if (string.IsNullOrWhiteSpace(headline)) return; + + string entry = $"[{DateTime.Now:HH:mm:ss}] {headline}"; + + if (errorList.Items.Count >= MaxLogLines) + { + errorList.BeginUpdate(); + try + { + int excess = errorList.Items.Count - MaxLogLines + 1; + for (int i = 0; i < excess && errorList.Items.Count > 0; i++) + { + errorList.Items.RemoveAt(0); + } + errorList.Items.Add(entry); + } + finally + { + errorList.EndUpdate(); + } + } + else + { + errorList.Items.Add(entry); + } + + if (autoScrollCheck.Checked && errorList.Items.Count > 0) + { + errorList.TopIndex = errorList.Items.Count - 1; + } + } + + private void UpdateLiveTicker() + { + if (executing) + { + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), clock.Elapsed); + } + } + + private void RequestCancel() + { + if (cancellation != null && !cancellation.IsCancellationRequested) + { + cancelButton.Enabled = false; + cancelButton.Text = T("正在取消...", "Cancelling..."); + cancellation.Cancel(); + } + } + } +} diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs index 6713c9e..2125e14 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs @@ -134,7 +134,11 @@ private MySqlImportResult RunInternal( // 檢查 Native Manifest if (trimmed.StartsWith("--")) { - if (trimmed.Contains("mySQLPunk Native Manifest")) readingManifest = true; + if (trimmed.Contains("mySQLPunk Native Manifest") && !readingManifest) + { + readingManifest = true; + log.LogInfo("Detected Native Manifest in SQL script footer."); + } var match = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) diff --git a/mySQLPunk/lib/SqliteExport/SqliteExportModels.cs b/mySQLPunk/lib/SqliteExport/SqliteExportModels.cs new file mode 100644 index 0000000..26ffad2 --- /dev/null +++ b/mySQLPunk/lib/SqliteExport/SqliteExportModels.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; + +namespace mySQLPunk.lib.SqliteExport +{ + public enum SqliteExportMode + { + SchemaAndData, + SchemaOnly, + DataOnly + } + + public enum SqliteExportStage + { + Ready, + ReadingCatalog, + WritingPragmas, + WritingSchema, + WritingData, + WritingIndexes, + WritingTriggers, + WritingSidecarComments, + WritingManifest, + Completed, + Cancelled, + Failed + } + + public sealed class SqliteTableExportItem + { + public string Name { get; set; } + public long EstimatedRows { get; set; } + public bool IsSelected { get; set; } + public bool IsSpatial { get; set; } + public bool IsSpatialMetadata { get; set; } + public bool IsVirtualTable { get; set; } + public string SpatialInfo { get; set; } + + public SqliteTableExportItem( + string name, + long estimatedRows, + bool isSelected = true, + bool isSpatial = false, + bool isSpatialMetadata = false, + bool isVirtualTable = false, + string spatialInfo = null) + { + Name = name; + EstimatedRows = estimatedRows; + IsSelected = isSelected; + IsSpatial = isSpatial; + IsSpatialMetadata = isSpatialMetadata; + IsVirtualTable = isVirtualTable; + SpatialInfo = spatialInfo ?? (isSpatial ? "GIS Layer" : (isSpatialMetadata ? "Spatial Metadata" : (isVirtualTable ? "Virtual Table" : ""))); + } + } + + public sealed class SqliteExportWizardOptions + { + public SqliteExportMode Mode { get; set; } + public string TargetPath { get; set; } + public bool SingleTransaction { get; set; } = true; + public bool PragmaForeignKeysOff { get; set; } = true; + public bool PragmaSynchronousOff { get; set; } = true; + public bool IncludeDropStatements { get; set; } = true; + public bool IncludeViews { get; set; } = true; + public bool IncludeTriggers { get; set; } = true; + public bool IncludeIndexes { get; set; } = true; + public bool IncludeComments { get; set; } = true; + public bool ExcludeSpatialMetadata { get; set; } = false; + public int InsertBatchSize { get; set; } = 1000; + public List SelectedTables { get; set; } = new List(); + public List SelectedViews { get; set; } = new List(); + public List SelectedTriggers { get; set; } = new List(); + + public SqliteExportWizardOptions(SqliteExportMode mode = SqliteExportMode.SchemaAndData, string targetPath = null) + { + Mode = mode; + TargetPath = targetPath; + } + } + + public sealed class SqliteExportProgress + { + public SqliteExportStage Stage { get; } + public string CurrentObject { get; } + public int CompletedTables { get; } + public int TotalTables { get; } + public long ProcessedRows { get; } + public long EstimatedTotalRows { get; } + public long WrittenBytes { get; } + public double RowsPerSecond { get; } + public TimeSpan Elapsed { get; } + public TimeSpan? EstimatedRemaining { get; } + public string Message { get; } + + public SqliteExportProgress( + SqliteExportStage stage, + string currentObject, + int completedTables, + int totalTables, + long processedRows, + long estimatedTotalRows, + long writtenBytes, + double rowsPerSecond, + TimeSpan elapsed, + TimeSpan? estimatedRemaining, + string message) + { + Stage = stage; + CurrentObject = currentObject; + CompletedTables = completedTables; + TotalTables = totalTables; + ProcessedRows = processedRows; + EstimatedTotalRows = estimatedTotalRows; + WrittenBytes = writtenBytes; + RowsPerSecond = rowsPerSecond; + Elapsed = elapsed; + EstimatedRemaining = estimatedRemaining; + Message = message; + } + } + + public sealed class SqliteExportResult + { + public SqliteExportStage Stage { get; } + public bool Success => Stage == SqliteExportStage.Completed; + public string TargetPath { get; } + public int TableCount { get; } + public int ViewCount { get; } + public int TriggerCount { get; } + public int IndexCount { get; } + public long TotalRows { get; } + public long FileBytes { get; } + public string Sha256Hash { get; } + public TimeSpan Elapsed { get; } + public string ErrorMessage { get; } + + public SqliteExportResult( + SqliteExportStage stage, + string targetPath, + int tableCount, + int viewCount, + int triggerCount, + int indexCount, + long totalRows, + long fileBytes, + string sha256Hash, + TimeSpan elapsed, + string errorMessage = null) + { + Stage = stage; + TargetPath = targetPath; + TableCount = tableCount; + ViewCount = viewCount; + TriggerCount = triggerCount; + IndexCount = indexCount; + TotalRows = totalRows; + FileBytes = fileBytes; + Sha256Hash = sha256Hash; + Elapsed = elapsed; + ErrorMessage = errorMessage; + } + } +} diff --git a/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs b/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs new file mode 100644 index 0000000..859b064 --- /dev/null +++ b/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs @@ -0,0 +1,840 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.SqliteExport +{ + public sealed class SqliteSqlExportService + { + private readonly IDatabase db; + private readonly string databaseName; + + private static readonly HashSet KnownSpatialMetadataTables = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "spatial_ref_sys", "spatial_ref_sys_aux", "spatialite_history", + "geometry_columns", "geometry_columns_auth", "geometry_columns_field_infos", "geometry_columns_statistics", + "views_geometry_columns", "views_geometry_columns_auth", "views_geometry_columns_field_infos", "views_geometry_columns_statistics", + "virts_geometry_columns", "virts_geometry_columns_auth", "virts_geometry_columns_field_infos", "virts_geometry_columns_statistics", + "data_licenses", "sql_statements_log", "SpatialIndex", + "gpkg_spatial_ref_sys", "gpkg_contents", "gpkg_geometry_columns", + "gpkg_tile_matrix", "gpkg_tile_matrix_set", "gpkg_extensions", + "gpkg_metadata", "gpkg_metadata_reference", "gpkg_data_columns", + "gpkg_data_column_constraints", "gpkg_ogr_contents" + }; + + private static readonly string[] ShadowTableSuffixes = new string[] + { + "data", "idx", "content", "docsize", "config", // FTS5 + "segments", "segdir", "stat", // FTS3/4 + "node", "rowid", "parent" // R-Tree + }; + + private static readonly char[] HexLookup = "0123456789ABCDEF".ToCharArray(); + + public SqliteSqlExportService(IDatabase db, string databaseName) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + } + + public List LoadTableCatalog() + { + var list = new List(); + + var virtualTables = GetVirtualTableNames(); + var spatialTableMap = GetSpatialTableDetails(); + + // 取得所有一般與虛擬資料表（排除 sqlite_% 系統表） + DataTable dt = null; + try + { + dt = db.SelectSQL("SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;", new Dictionary()); + } + catch + { + dt = new DataTable(); + } + + foreach (DataRow row in dt.Rows) + { + string tableName = row["name"]?.ToString(); + if (string.IsNullOrWhiteSpace(tableName)) continue; + + // 排除影子表 (FTS / RTree 內部實體表，還原時由虛擬表自動生成，若重複建立會報錯) + if (IsShadowTable(tableName, virtualTables)) continue; + + bool isVirtual = virtualTables.Contains(tableName); + bool isSpatialMetadata = KnownSpatialMetadataTables.Contains(tableName); + bool isSpatial = spatialTableMap.ContainsKey(tableName); + string spatialInfo = isSpatial ? spatialTableMap[tableName] : (isSpatialMetadata ? "Spatial Metadata" : (isVirtual ? "Virtual Table" : "")); + + long count = 0; + try + { + count = db.CountRows(databaseName, tableName); + } + catch + { + count = 0; + } + + // 預設選取：除巨量空間中繼資料表（如 spatial_ref_sys）外預設勾選 + bool defaultSelect = !(isSpatialMetadata && string.Equals(tableName, "spatial_ref_sys", StringComparison.OrdinalIgnoreCase)); + + list.Add(new SqliteTableExportItem( + tableName, + count, + defaultSelect, + isSpatial, + isSpatialMetadata, + isVirtual, + spatialInfo)); + } + + return list; + } + + public List LoadViews() + { + var views = new List(); + try + { + DataTable dt = db.SelectSQL("SELECT name FROM sqlite_master WHERE type='view' AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;", new Dictionary()); + foreach (DataRow row in dt.Rows) + { + string name = row[0]?.ToString(); + if (!string.IsNullOrWhiteSpace(name)) views.Add(name); + } + } + catch { } + return views; + } + + public List LoadTriggers() + { + var triggers = new List(); + try + { + DataTable dt = db.SelectSQL("SELECT name FROM sqlite_master WHERE type='trigger' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;", new Dictionary()); + foreach (DataRow row in dt.Rows) + { + string name = row[0]?.ToString(); + if (!string.IsNullOrWhiteSpace(name)) triggers.Add(name); + } + } + catch { } + return triggers; + } + + public Task ExportAsync( + SqliteExportWizardOptions options, + IProgress progress, + CancellationToken cancellation) + { + return Task.Run(() => ExportInternal(options, progress, cancellation), cancellation); + } + + private SqliteExportResult ExportInternal( + SqliteExportWizardOptions options, + IProgress progress, + CancellationToken cancellation) + { + var clock = Stopwatch.StartNew(); + if (string.IsNullOrWhiteSpace(options.TargetPath)) + throw new ArgumentException("Target path is required", nameof(options.TargetPath)); + + string targetPath = options.TargetPath; + string targetDir = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrWhiteSpace(targetDir) && !Directory.Exists(targetDir)) + Directory.CreateDirectory(targetDir); + + string stagingPath = targetPath + ".staging." + Guid.NewGuid().ToString("N") + ".tmp"; + long totalProcessedRows = 0; + long estimatedTotalRows = 0; + int tableCount = 0; + int viewCount = 0; + int triggerCount = 0; + int indexCount = 0; + var tableManifest = new List>(); + + var selectedTables = options.SelectedTables?.ToList() ?? new List(); + var selectedViews = options.IncludeViews ? (options.SelectedViews?.ToList() ?? new List()) : new List(); + var selectedTriggers = options.IncludeTriggers ? (options.SelectedTriggers?.ToList() ?? new List()) : new List(); + + // 空間中繼資料表排除過濾 + if (options.ExcludeSpatialMetadata) + { + selectedTables.RemoveAll(t => KnownSpatialMetadataTables.Contains(t)); + } + + // 預估總列數 + if (options.Mode != SqliteExportMode.SchemaOnly) + { + foreach (var table in selectedTables) + { + try { estimatedTotalRows += db.CountRows(databaseName, table); } catch { } + } + } + + try + { + cancellation.ThrowIfCancellationRequested(); + + using (var fileStream = new FileStream(stagingPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 65536)) + using (var writer = new StreamWriter(fileStream, new UTF8Encoding(false), 65536)) + { + writer.WriteLine("-- ========================================================"); + writer.WriteLine("-- mySQLPunk SQLite Logical Dump"); + writer.WriteLine($"-- Database: {databaseName}"); + writer.WriteLine($"-- Created: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + writer.WriteLine($"-- Mode: {options.Mode}"); + writer.WriteLine($"-- SingleTransaction: {options.SingleTransaction}"); + writer.WriteLine("-- GIS / SpatiaLite / GeoPackage Binary Fidelity: Enabled"); + writer.WriteLine("-- ========================================================"); + writer.WriteLine(); + + // PRAGMA 設定 + if (options.PragmaForeignKeysOff) + writer.WriteLine("PRAGMA foreign_keys = OFF;"); + if (options.PragmaSynchronousOff) + writer.WriteLine("PRAGMA synchronous = OFF;"); + writer.WriteLine(); + + // 交易起始 + if (options.SingleTransaction) + { + writer.WriteLine("BEGIN TRANSACTION;"); + writer.WriteLine(); + } + + // 1. DROP 語句 + if (options.IncludeDropStatements && options.Mode != SqliteExportMode.DataOnly) + { + foreach (var tr in selectedTriggers) + writer.WriteLine($"DROP TRIGGER IF EXISTS {QuoteIdentifier(tr)};"); + foreach (var v in selectedViews) + writer.WriteLine($"DROP VIEW IF EXISTS {QuoteIdentifier(v)};"); + foreach (var t in selectedTables) + writer.WriteLine($"DROP TABLE IF EXISTS {QuoteIdentifier(t)};"); + writer.WriteLine(); + } + + // 2. 結構 DDL (Tables) + if (options.Mode != SqliteExportMode.DataOnly) + { + ReportProgress(progress, SqliteExportStage.WritingSchema, "", 0, selectedTables.Count, 0, estimatedTotalRows, fileStream.Position, clock); + + foreach (var table in selectedTables) + { + cancellation.ThrowIfCancellationRequested(); + string ddl = GetTableDdl(table); + if (!string.IsNullOrWhiteSpace(ddl)) + { + writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + writer.WriteLine(); + } + + // 側車欄位註解 DDL + if (options.IncludeComments) + { + string commentDdl = BuildColumnCommentsDump(table); + if (!string.IsNullOrWhiteSpace(commentDdl)) + { + writer.WriteLine(commentDdl.TrimEnd()); + writer.WriteLine(); + } + } + } + } + + // 3. 資料列 (Data) + if (options.Mode != SqliteExportMode.SchemaOnly) + { + int batchSize = options.InsertBatchSize > 0 ? options.InsertBatchSize : 1000; + for (int i = 0; i < selectedTables.Count; i++) + { + cancellation.ThrowIfCancellationRequested(); + string table = selectedTables[i]; + long tableRows = 0; + long totalTableRows = 0; + try { totalTableRows = db.CountRows(databaseName, table); } catch { } + + ReportProgress(progress, SqliteExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + + while (tableRows < totalTableRows || totalTableRows == 0) + { + cancellation.ThrowIfCancellationRequested(); + DataTable page = db.SelectTablePage(databaseName, table, tableRows, batchSize); + if (page == null || page.Rows.Count == 0) break; + + string insertPrefix = "INSERT INTO " + QuoteIdentifier(table) + " ("; + var colNames = new string[page.Columns.Count]; + for (int c = 0; c < page.Columns.Count; c++) + { + colNames[c] = QuoteIdentifier(page.Columns[c].ColumnName); + } + string colsJoined = string.Join(", ", colNames); + + for (int r = 0; r < page.Rows.Count; r++) + { + writer.Write(insertPrefix); + writer.Write(colsJoined); + writer.Write(") VALUES ("); + + DataRow row = page.Rows[r]; + for (int c = 0; c < page.Columns.Count; c++) + { + if (c > 0) writer.Write(", "); + WriteSqliteLiteral(writer, row[c]); + } + writer.WriteLine(");"); + } + + tableRows += page.Rows.Count; + totalProcessedRows += page.Rows.Count; + ReportProgress(progress, SqliteExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + } + + if (tableRows > 0) writer.WriteLine(); + tableCount++; + tableManifest.Add(Tuple.Create(table, tableRows)); + } + } + else + { + tableCount = selectedTables.Count; + } + + // 4. 檢視 (Views) + if (options.Mode != SqliteExportMode.DataOnly && selectedViews.Count > 0) + { + ReportProgress(progress, SqliteExportStage.WritingSchema, "Views", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + foreach (var view in selectedViews) + { + cancellation.ThrowIfCancellationRequested(); + string ddl = GetViewDdl(view); + if (!string.IsNullOrWhiteSpace(ddl)) + { + writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + writer.WriteLine(); + } + viewCount++; + } + } + + // 5. 索引 (Indexes) + if (options.Mode != SqliteExportMode.DataOnly && options.IncludeIndexes) + { + ReportProgress(progress, SqliteExportStage.WritingIndexes, "Indexes", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + var indexes = GetIndexesForTables(selectedTables); + foreach (var idx in indexes) + { + cancellation.ThrowIfCancellationRequested(); + writer.WriteLine(idx.TrimEnd().TrimEnd(';') + ";"); + indexCount++; + } + if (indexes.Count > 0) writer.WriteLine(); + } + + // 6. 觸發器 (Triggers) + if (options.Mode != SqliteExportMode.DataOnly && selectedTriggers.Count > 0) + { + ReportProgress(progress, SqliteExportStage.WritingTriggers, "Triggers", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); + foreach (var triggerName in selectedTriggers) + { + cancellation.ThrowIfCancellationRequested(); + string ddl = GetTriggerDdl(triggerName); + if (!string.IsNullOrWhiteSpace(ddl)) + { + writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + writer.WriteLine(); + triggerCount++; + } + } + } + + // 交易結束 + if (options.SingleTransaction) + { + writer.WriteLine("COMMIT;"); + writer.WriteLine(); + } + + // 恢復 foreign_keys + if (options.PragmaForeignKeysOff) + { + writer.WriteLine("PRAGMA foreign_keys = ON;"); + } + + writer.Flush(); + + // 7. 寫入 Native Manifest 摘要 + writer.WriteLine(); + writer.WriteLine("-- ========================================================"); + writer.WriteLine("-- mySQLPunk Native Manifest"); + writer.WriteLine($"-- Tables: {tableCount}"); + writer.WriteLine($"-- TotalRows: {totalProcessedRows}"); + foreach (var pair in tableManifest) + { + writer.WriteLine($"-- Manifest-Table: {pair.Item1} | Rows: {pair.Item2}"); + } + writer.WriteLine("-- ========================================================"); + writer.Flush(); + } + + // 計算產出檔案的 SHA-256 + string sha256Hex; + long finalLength; + using (var fs = File.OpenRead(stagingPath)) + { + finalLength = fs.Length; + using (var sha = SHA256.Create()) + { + byte[] hash = sha.ComputeHash(fs); + sha256Hex = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); + } + } + + // 結尾附加 SHA256 到檔案中 + using (var appendWriter = new StreamWriter(stagingPath, true, new UTF8Encoding(false))) + { + appendWriter.WriteLine($"-- Manifest-SHA256: {sha256Hex}"); + } + + // 原子替換到目標路徑 + if (File.Exists(targetPath)) + { + File.Delete(targetPath); + } + File.Move(stagingPath, targetPath); + + clock.Stop(); + + ReportProgress(progress, SqliteExportStage.Completed, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, finalLength, clock); + + return new SqliteExportResult( + SqliteExportStage.Completed, + targetPath, + tableCount, + viewCount, + triggerCount, + indexCount, + totalProcessedRows, + finalLength, + sha256Hex, + clock.Elapsed); + } + catch (OperationCanceledException) + { + clock.Stop(); + TryDeleteFile(stagingPath); + ReportProgress(progress, SqliteExportStage.Cancelled, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, 0, clock); + return new SqliteExportResult( + SqliteExportStage.Cancelled, + targetPath, + tableCount, + viewCount, + triggerCount, + indexCount, + totalProcessedRows, + 0, + null, + clock.Elapsed, + "Export cancelled by user."); + } + catch (Exception ex) + { + clock.Stop(); + TryDeleteFile(stagingPath); + ReportProgress(progress, SqliteExportStage.Failed, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, 0, clock, ex.Message); + return new SqliteExportResult( + SqliteExportStage.Failed, + targetPath, + tableCount, + viewCount, + triggerCount, + indexCount, + totalProcessedRows, + 0, + null, + clock.Elapsed, + ex.Message); + } + } + + private static void WriteSqliteLiteral(StreamWriter writer, object value) + { + if (value == null || value == DBNull.Value) + { + writer.Write("NULL"); + return; + } + + if (value is byte[] bytes) + { + // SpatiaLite / GeoPackage GIS 幾何與一般 BLOB 二進位無損輸出 + writer.Write("X'"); + char[] hexBuffer = new char[Math.Min(bytes.Length * 2, 8192)]; + int bufIdx = 0; + for (int i = 0; i < bytes.Length; i++) + { + byte b = bytes[i]; + hexBuffer[bufIdx++] = HexLookup[b >> 4]; + hexBuffer[bufIdx++] = HexLookup[b & 0x0F]; + if (bufIdx >= hexBuffer.Length - 1) + { + writer.Write(hexBuffer, 0, bufIdx); + bufIdx = 0; + } + } + if (bufIdx > 0) + { + writer.Write(hexBuffer, 0, bufIdx); + } + writer.Write("'"); + return; + } + + if (value is string s) + { + writer.Write("'"); + writer.Write(s.Replace("'", "''")); + writer.Write("'"); + return; + } + + if (value is bool bVal) + { + writer.Write(bVal ? "1" : "0"); + return; + } + + if (value is DateTime dt) + { + writer.Write("'"); + writer.Write(dt.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture)); + writer.Write("'"); + return; + } + + if (value is float f) + { + writer.Write(f.ToString("R", CultureInfo.InvariantCulture)); + return; + } + + if (value is double d) + { + writer.Write(d.ToString("R", CultureInfo.InvariantCulture)); + return; + } + + if (value is decimal dec) + { + writer.Write(dec.ToString(CultureInfo.InvariantCulture)); + return; + } + + if (value is IFormattable formattable) + { + writer.Write(formattable.ToString(null, CultureInfo.InvariantCulture)); + return; + } + + writer.Write("'"); + writer.Write(value.ToString().Replace("'", "''")); + writer.Write("'"); + } + + public static string QuoteIdentifier(string name) + { + name = name ?? string.Empty; + return "\"" + name.Replace("\"", "\"\"") + "\""; + } + + private string GetTableDdl(string tableName) + { + try + { + string safe = tableName.Replace("'", "''"); + DataTable dt = db.SelectSQL($"SELECT sql FROM sqlite_master WHERE type='table' AND name='{safe}';", new Dictionary()); + if (dt != null && dt.Rows.Count > 0 && dt.Rows[0][0] != DBNull.Value) + return dt.Rows[0][0].ToString(); + } + catch { } + return ""; + } + + private string GetViewDdl(string viewName) + { + try + { + string safe = viewName.Replace("'", "''"); + DataTable dt = db.SelectSQL($"SELECT sql FROM sqlite_master WHERE type='view' AND name='{safe}';", new Dictionary()); + if (dt != null && dt.Rows.Count > 0 && dt.Rows[0][0] != DBNull.Value) + return dt.Rows[0][0].ToString(); + } + catch { } + return ""; + } + + private string GetTriggerDdl(string triggerName) + { + try + { + string safe = triggerName.Replace("'", "''"); + DataTable dt = db.SelectSQL($"SELECT sql FROM sqlite_master WHERE type='trigger' AND name='{safe}';", new Dictionary()); + if (dt != null && dt.Rows.Count > 0 && dt.Rows[0][0] != DBNull.Value) + return dt.Rows[0][0].ToString(); + } + catch { } + return ""; + } + + private List GetIndexesForTables(IEnumerable tableNames) + { + var result = new List(); + var set = new HashSet(tableNames, StringComparer.OrdinalIgnoreCase); + try + { + DataTable dt = db.SelectSQL("SELECT sql, tbl_name FROM sqlite_master WHERE type='index' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%';", new Dictionary()); + foreach (DataRow row in dt.Rows) + { + string tbl = row["tbl_name"]?.ToString(); + string sql = row["sql"]?.ToString(); + if (!string.IsNullOrWhiteSpace(tbl) && set.Contains(tbl) && !string.IsNullOrWhiteSpace(sql)) + { + result.Add(sql); + } + } + } + catch { } + return result; + } + + private string BuildColumnCommentsDump(string tableName) + { + try + { + DataTable columns = db.GetColumns(databaseName, tableName); + if (columns == null || columns.Rows.Count == 0) return ""; + + var inserts = new List(); + foreach (DataRow row in columns.Rows) + { + string colName = null; + foreach (string key in new[] { "name", "Name", "COLUMN_NAME", "column_name", "Field" }) + { + if (row.Table.Columns.Contains(key) && row[key] != DBNull.Value) + { + colName = row[key].ToString(); + break; + } + } + string comment = null; + foreach (string key in new[] { "Comment", "comment" }) + { + if (row.Table.Columns.Contains(key) && row[key] != DBNull.Value) + { + comment = row[key].ToString(); + break; + } + } + + if (string.IsNullOrWhiteSpace(colName) || string.IsNullOrWhiteSpace(comment)) continue; + + inserts.Add("INSERT OR REPLACE INTO " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + + " (table_name, column_name, comment) VALUES (" + + "'" + tableName.Replace("'", "''") + "', " + + "'" + colName.Replace("'", "''") + "', " + + "'" + comment.Replace("'", "''") + "');"); + } + + if (inserts.Count == 0) return ""; + + var sb = new StringBuilder(); + sb.AppendLine("-- SQLite column comments sidecar metadata"); + sb.AppendLine("CREATE TABLE IF NOT EXISTS " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + " (" + + "table_name TEXT NOT NULL, " + + "column_name TEXT NOT NULL, " + + "comment TEXT NOT NULL, " + + "PRIMARY KEY (table_name, column_name));"); + foreach (var ins in inserts) sb.AppendLine(ins); + return sb.ToString(); + } + catch + { + return ""; + } + } + + private HashSet GetVirtualTableNames() + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + try + { + DataTable dt = db.SelectSQL("SELECT name FROM sqlite_master WHERE type='table' AND sql LIKE 'CREATE VIRTUAL%';", new Dictionary()); + foreach (DataRow row in dt.Rows) + { + string n = row[0]?.ToString(); + if (!string.IsNullOrWhiteSpace(n)) names.Add(n); + } + } + catch { } + return names; + } + + private static bool IsShadowTable(string tableName, HashSet virtualNames) + { + if (virtualNames == null || virtualNames.Count == 0) return false; + int idx = tableName.LastIndexOf('_'); + if (idx <= 0) return false; + string prefix = tableName.Substring(0, idx); + if (!virtualNames.Contains(prefix)) return false; + string suffix = tableName.Substring(idx + 1); + foreach (string known in ShadowTableSuffixes) + { + if (string.Equals(suffix, known, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + + private Dictionary GetSpatialTableDetails() + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // 1. 探測 SpatiaLite geometry_columns + try + { + DataTable dt = db.SelectSQL("SELECT f_table_name, f_geometry_column, type, srid FROM geometry_columns;", new Dictionary()); + if (dt != null) + { + foreach (DataRow r in dt.Rows) + { + string tbl = r["f_table_name"]?.ToString(); + string col = r["f_geometry_column"]?.ToString(); + string type = r["type"]?.ToString(); + string srid = r["srid"]?.ToString(); + if (!string.IsNullOrWhiteSpace(tbl)) + { + map[tbl] = $"[SpatiaLite] {type} ({col}, SRID: {srid})"; + } + } + } + } + catch { } + + // 2. 探測 GeoPackage gpkg_geometry_columns + try + { + DataTable dt = db.SelectSQL("SELECT table_name, column_name, geometry_type_name, srid FROM gpkg_geometry_columns;", new Dictionary()); + if (dt != null) + { + foreach (DataRow r in dt.Rows) + { + string tbl = r["table_name"]?.ToString(); + string col = r["column_name"]?.ToString(); + string type = r["geometry_type_name"]?.ToString(); + string srid = r["srid"]?.ToString(); + if (!string.IsNullOrWhiteSpace(tbl)) + { + map[tbl] = $"[GeoPackage] {type} ({col}, SRID: {srid})"; + } + } + } + } + catch { } + + return map; + } + + private static void ReportProgress( + IProgress progress, + SqliteExportStage stage, + string currentObject, + int completedTables, + int totalTables, + long processedRows, + long estimatedTotalRows, + long writtenBytes, + Stopwatch clock, + string errorMessage = null) + { + if (progress == null) return; + + double elapsedSec = clock.Elapsed.TotalSeconds; + double rowsPerSec = elapsedSec > 0.05 ? processedRows / elapsedSec : 0; + TimeSpan? remaining = null; + if (rowsPerSec > 0 && estimatedTotalRows > processedRows) + { + double remainingSec = (estimatedTotalRows - processedRows) / rowsPerSec; + if (remainingSec < 86400) remaining = TimeSpan.FromSeconds(remainingSec); + } + + string msg; + switch (stage) + { + case SqliteExportStage.ReadingCatalog: + msg = "Reading database catalog..."; + break; + case SqliteExportStage.WritingSchema: + msg = $"Writing schema for {currentObject}..."; + break; + case SqliteExportStage.WritingData: + msg = $"Writing data rows for {currentObject} ({processedRows:N0} rows)..."; + break; + case SqliteExportStage.WritingIndexes: + msg = "Writing indexes..."; + break; + case SqliteExportStage.WritingTriggers: + msg = "Writing triggers..."; + break; + case SqliteExportStage.WritingManifest: + msg = "Generating native manifest..."; + break; + case SqliteExportStage.Completed: + msg = $"Export completed: {completedTables} tables, {processedRows:N0} rows."; + break; + case SqliteExportStage.Cancelled: + msg = "Export cancelled."; + break; + case SqliteExportStage.Failed: + msg = "Export failed: " + errorMessage; + break; + default: + msg = currentObject; + break; + } + + progress.Report(new SqliteExportProgress( + stage, + currentObject, + completedTables, + totalTables, + processedRows, + estimatedTotalRows, + writtenBytes, + rowsPerSec, + clock.Elapsed, + remaining, + msg)); + } + + private static void TryDeleteFile(string path) + { + try + { + if (File.Exists(path)) File.Delete(path); + } + catch { } + } + } +} diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportLog.cs b/mySQLPunk/lib/SqliteImport/SqliteImportLog.cs new file mode 100644 index 0000000..eeb31da --- /dev/null +++ b/mySQLPunk/lib/SqliteImport/SqliteImportLog.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace mySQLPunk.lib.SqliteImport +{ + public sealed class SqliteImportLog : IDisposable + { + private readonly Stream stream; + private readonly StreamWriter writer; + private readonly object gate = new object(); + private readonly List recentErrors = new List(); + private bool disposed; + + public string Path { get; } + public IReadOnlyList RecentErrors + { + get { lock (gate) return recentErrors.ToArray(); } + } + + public SqliteImportLog(Guid jobId, string directory = null) + { + directory = directory ?? System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "mySQLPunk", "ImportLogs"); + Directory.CreateDirectory(directory); + string stem = "import-sqlite-" + DateTime.Now.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture) + "-" + jobId.ToString("N").Substring(0, 8); + Path = System.IO.Path.Combine(directory, stem + ".log"); + + stream = new FileStream(Path, FileMode.Create, FileAccess.Write, FileShare.Read); + writer = new StreamWriter(stream, Encoding.UTF8); + WriteHeader(jobId); + } + + private void WriteHeader(Guid jobId) + { + lock (gate) + { + writer.WriteLine("================================================================================"); + writer.WriteLine("mySQLPunk SQLite Import Execution Log"); + writer.WriteLine($"Job ID : {jobId}"); + writer.WriteLine($"Started : {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + writer.WriteLine("================================================================================"); + writer.WriteLine(); + writer.Flush(); + } + } + + public void LogInfo(string message) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [INFO] {message}"); + } + } + + public void LogStatement(int lineNumber, string preview, bool success, string errorMessage = null, string action = null) + { + lock (gate) + { + if (disposed) return; + if (success) + { + string tag = string.IsNullOrWhiteSpace(action) ? "OK" : action; + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [{tag}] Line {lineNumber}: {preview}"); + } + else + { + string errText = $"[{DateTime.Now:HH:mm:ss.fff}] [ERROR] Line {lineNumber}: {errorMessage} | Action: {action} | SQL: {preview}"; + writer.WriteLine(errText); + recentErrors.Add($"Line {lineNumber}: {errorMessage}"); + if (recentErrors.Count > 100) recentErrors.RemoveAt(0); + } + } + } + + public void LogManifestVerification(string table, long expectedRows, long actualRows, bool match) + { + lock (gate) + { + if (disposed) return; + string icon = match ? "[PASS]" : "[WARN]"; + writer.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] {icon} Manifest Check: Table \"{table}\" -> Expected: {expectedRows}, Actual: {actualRows}"); + } + } + + public void Flush() + { + lock (gate) + { + if (disposed) return; + try { writer.Flush(); } catch { } + } + } + + public void WriteEnd(SqliteImportResult result) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine(); + writer.WriteLine("================================================================================"); + writer.WriteLine($"Import Execution Completed: Status={result.Status}"); + writer.WriteLine($"Elapsed : {result.Elapsed}"); + writer.WriteLine($"Statements : Executed={result.ExecutedStatements}, Failed={result.FailedStatements}, Skipped={result.SkippedStatements}"); + writer.WriteLine($"RowsAffected: {result.TotalRowsAffected:N0}"); + writer.WriteLine($"Manifest : Verified={result.ManifestVerified}"); + if (!string.IsNullOrWhiteSpace(result.ManifestReport)) + { + writer.WriteLine($"Manifest Details: {result.ManifestReport}"); + } + writer.WriteLine("================================================================================"); + writer.Flush(); + } + } + + public void Dispose() + { + lock (gate) + { + if (disposed) return; + disposed = true; + try { writer.Flush(); writer.Dispose(); } catch { } + try { stream.Dispose(); } catch { } + } + } + } +} diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs b/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs new file mode 100644 index 0000000..39137c2 --- /dev/null +++ b/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; + +namespace mySQLPunk.lib.SqliteImport +{ + public enum SqliteImportStatus + { + Ready, + Running, + Completed, + Cancelled, + Failed + } + + public enum SqliteExistingObjectStrategy + { + AbortIfExists, + SkipExisting, + DropAndRecreate, + OverwriteKeepExisting + } + + public sealed class SqliteImportRequest + { + public Guid JobId { get; } + public string SourcePath { get; } + public string DatabaseName { get; } + + public SqliteImportRequest(Guid jobId, string sourcePath, string databaseName) + { + JobId = jobId; + SourcePath = sourcePath; + DatabaseName = databaseName; + } + } + + public sealed class SqliteImportOptions + { + public bool ContinueOnError { get; set; } = false; + public SqliteExistingObjectStrategy ExistingObjectStrategy { get; set; } = SqliteExistingObjectStrategy.SkipExisting; + public bool AutoLoadSpatiaLite { get; set; } = true; + public bool WrapInTransaction { get; set; } = true; + public bool PragmaForeignKeysOff { get; set; } = true; + public bool PragmaSynchronousOff { get; set; } = true; + } + + public sealed class SqliteImportPreflight + { + public bool Passed { get; } + public long FileBytes { get; } + public int StatementCount { get; } + public bool HasManifest { get; } + public int ManifestTables { get; } + public long ManifestRows { get; } + public string ManifestSha256 { get; } + public bool DetectedSpatiaLite { get; } + public IReadOnlyList Issues { get; } + + public SqliteImportPreflight( + bool passed, + long fileBytes, + int statementCount, + bool hasManifest, + int manifestTables, + long manifestRows, + string manifestSha256, + bool detectedSpatiaLite, + IReadOnlyList issues) + { + Passed = passed; + FileBytes = fileBytes; + StatementCount = statementCount; + HasManifest = hasManifest; + ManifestTables = manifestTables; + ManifestRows = manifestRows; + ManifestSha256 = manifestSha256; + DetectedSpatiaLite = detectedSpatiaLite; + Issues = issues ?? Array.Empty(); + } + } + + public sealed class SqliteImportProgress + { + public long ProcessedBytes { get; } + public long TotalBytes { get; } + public int ExecutedStatements { get; } + public int FailedStatements { get; } + public int SkippedStatements { get; } + public long TotalRowsAffected { get; } + public double BytesPerSecond { get; } + public TimeSpan Elapsed { get; } + public TimeSpan? EstimatedRemaining { get; } + public string Message { get; } + public string CurrentStatementPreview { get; } + + public SqliteImportProgress( + long processedBytes, + long totalBytes, + int executedStatements, + int failedStatements, + int skippedStatements, + long totalRowsAffected, + double bytesPerSecond, + TimeSpan elapsed, + TimeSpan? estimatedRemaining, + string message, + string currentStatementPreview) + { + ProcessedBytes = processedBytes; + TotalBytes = totalBytes; + ExecutedStatements = executedStatements; + FailedStatements = failedStatements; + SkippedStatements = skippedStatements; + TotalRowsAffected = totalRowsAffected; + BytesPerSecond = bytesPerSecond; + Elapsed = elapsed; + EstimatedRemaining = estimatedRemaining; + Message = message; + CurrentStatementPreview = currentStatementPreview; + } + } + + public sealed class SqliteImportResult + { + public SqliteImportStatus Status { get; } + public int ExecutedStatements { get; } + public int FailedStatements { get; } + public int SkippedStatements { get; } + public long TotalRowsAffected { get; } + public string LogPath { get; } + public TimeSpan Elapsed { get; } + public IReadOnlyList Errors { get; } + public bool ManifestVerified { get; } + public string ManifestReport { get; } + + public SqliteImportResult( + SqliteImportStatus status, + int executedStatements, + int failedStatements, + int skippedStatements, + long totalRowsAffected, + string logPath, + TimeSpan elapsed, + IReadOnlyList errors, + bool manifestVerified = false, + string manifestReport = null) + { + Status = status; + ExecutedStatements = executedStatements; + FailedStatements = failedStatements; + SkippedStatements = skippedStatements; + TotalRowsAffected = totalRowsAffected; + LogPath = logPath; + Elapsed = elapsed; + Errors = errors ?? Array.Empty(); + ManifestVerified = manifestVerified; + ManifestReport = manifestReport; + } + } +} diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs b/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs new file mode 100644 index 0000000..b2cd91b --- /dev/null +++ b/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs @@ -0,0 +1,243 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; + +namespace mySQLPunk.lib.SqliteImport +{ + public static class SqliteImportPreflightScanner + { + private const int MaxBatchBytes = 32 * 1024 * 1024; // 32 MiB + private static readonly Regex ManifestTableRegex = new Regex(@"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestTablesRegex = new Regex(@"^--\s*Tables:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestRowsRegex = new Regex(@"^--\s*TotalRows:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestSha256Regex = new Regex(@"^--\s*Manifest-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly string[] SpatialKeywords = new string[] + { + "InitSpatialMetaData", "AddGeometryColumn", "CreateSpatialIndex", + "mod_spatialite", "spatial_ref_sys", "geometry_columns", + "gpkg_spatial_ref_sys", "gpkg_geometry_columns", "gpkg_contents", + "GeomFromText", "ST_GeomFromText", "AsText", "ST_AsText", + "PointFromText", "LineFromText", "PolyFromText" + }; + + public static SqliteImportPreflight Scan(string filePath, CancellationToken cancellation) + { + if (string.IsNullOrWhiteSpace(filePath)) + throw new ArgumentException("File path is required", nameof(filePath)); + if (!File.Exists(filePath)) + return new SqliteImportPreflight(false, 0, 0, false, 0, 0, null, false, new[] { "File not found: " + filePath }); + + long fileBytes = new FileInfo(filePath).Length; + var issues = new List(); + int statementCount = 0; + bool hasManifest = false; + int manifestTables = 0; + long manifestRows = 0; + string manifestSha = null; + bool detectedSpatiaLite = false; + + try + { + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) + using (var reader = new StreamReader(fs, Encoding.UTF8, true, 65536)) + { + bool inSingleQuote = false; + bool inDoubleQuote = false; + bool inBracket = false; + bool inBlockComment = false; + int currentBatchLength = 0; + bool hasStatementContent = false; + int lineIndex = 0; + string line; + + while ((line = reader.ReadLine()) != null) + { + cancellation.ThrowIfCancellationRequested(); + lineIndex++; + + // 檢查 embedded NUL (非法字元或編碼毀損) + if (line.IndexOf('\0') >= 0) + { + issues.Add($"Line {lineIndex}: Contains Embedded NUL character (possible malformed encoding or corrupted binary file)."); + break; + } + + string trimmed = line.Trim(); + + // 檢查 Manifest + if (trimmed.StartsWith("--")) + { + if (trimmed.Contains("mySQLPunk Native Manifest")) + { + hasManifest = true; + } + else if (hasManifest) + { + var mTables = ManifestTablesRegex.Match(trimmed); + if (mTables.Success && int.TryParse(mTables.Groups[1].Value, out int mt)) + { + manifestTables = mt; + continue; + } + var mRows = ManifestRowsRegex.Match(trimmed); + if (mRows.Success && long.TryParse(mRows.Groups[1].Value, out long mr)) + { + manifestRows = mr; + continue; + } + var mTable = ManifestTableRegex.Match(trimmed); + if (mTable.Success) + { + if (manifestTables == 0) manifestTables++; + if (long.TryParse(mTable.Groups[2].Value, out long r) && manifestRows == 0) manifestRows += r; + continue; + } + var mSha = ManifestSha256Regex.Match(trimmed); + if (mSha.Success) + { + manifestSha = mSha.Groups[1].Value; + } + } + + // 註解行若在引號外，不計入語句實質內容 + if (!inSingleQuote && !inDoubleQuote && !inBlockComment) + { + continue; + } + } + + // 探測 SpatiaLite / GeoPackage 關鍵字 + if (!detectedSpatiaLite) + { + for (int k = 0; k < SpatialKeywords.Length; k++) + { + if (line.IndexOf(SpatialKeywords[k], StringComparison.OrdinalIgnoreCase) >= 0) + { + detectedSpatiaLite = true; + break; + } + } + } + + // 逐字元狀態機解析，以精準識別分號 + int len = line.Length; + for (int i = 0; i < len; i++) + { + char c = line[i]; + + if (inBlockComment) + { + if (c == '*' && i + 1 < len && line[i + 1] == '/') + { + inBlockComment = false; + i++; + } + continue; + } + + if (!inSingleQuote && !inDoubleQuote && !inBracket) + { + if (c == '-' && i + 1 < len && line[i + 1] == '-') + { + // 遇到單行註解，本行後續忽略 + break; + } + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + inBlockComment = true; + i++; + continue; + } + } + + if (c == '\'' && !inDoubleQuote && !inBracket) + { + inSingleQuote = !inSingleQuote; + hasStatementContent = true; + currentBatchLength++; + continue; + } + + if (c == '"' && !inSingleQuote && !inBracket) + { + inDoubleQuote = !inDoubleQuote; + hasStatementContent = true; + currentBatchLength++; + continue; + } + + if (c == '[' && !inSingleQuote && !inDoubleQuote) + { + inBracket = true; + hasStatementContent = true; + currentBatchLength++; + continue; + } + + if (c == ']' && inBracket) + { + inBracket = false; + hasStatementContent = true; + currentBatchLength++; + continue; + } + + if (!char.IsWhiteSpace(c)) + { + hasStatementContent = true; + } + currentBatchLength++; + + // 檢查語句終止分號（必須在字串引號與註解之外） + if (c == ';' && !inSingleQuote && !inDoubleQuote && !inBracket) + { + if (hasStatementContent) + { + statementCount++; + hasStatementContent = false; + currentBatchLength = 0; + } + } + } + + // 單批次大小檢查 (32 MiB) + if (currentBatchLength > MaxBatchBytes) + { + issues.Add($"Line {lineIndex}: Single SQL statement exceeds safe limit of 32 MiB."); + break; + } + } + + if (hasStatementContent) + { + statementCount++; + } + } + } + catch (OperationCanceledException) + { + return new SqliteImportPreflight(false, fileBytes, statementCount, hasManifest, manifestTables, manifestRows, manifestSha, detectedSpatiaLite, new[] { "Scan cancelled by user." }); + } + catch (Exception ex) + { + issues.Add("Error during preflight scan: " + ex.Message); + } + + bool passed = issues.Count == 0; + return new SqliteImportPreflight( + passed, + fileBytes, + statementCount, + hasManifest, + manifestTables, + manifestRows, + manifestSha, + detectedSpatiaLite, + issues); + } + } +} diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs b/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs new file mode 100644 index 0000000..c44a879 --- /dev/null +++ b/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs @@ -0,0 +1,673 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.SqliteImport +{ + public sealed class SqliteImportRunner + { + private const string IdentifierPattern = @"""(?:""""|[^""])+""|`(?:``|[^`])+`|\[[^\]]+\]|'[^']+'|[A-Za-z0-9_$]+"; + + private static readonly Regex CreateObjectRegex = new Regex( + @"^\s*CREATE\s+(?:(?:TEMP|TEMPORARY|UNIQUE|VIRTUAL)\s+)?(?TABLE|VIEW|INDEX|TRIGGER)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?" + IdentifierPattern + @")(?:\s+USING\s+|\s+ON\s+(?" + IdentifierPattern + @"))?", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex DropObjectRegex = new Regex( + @"^\s*DROP\s+(?TABLE|VIEW|INDEX|TRIGGER)\s+(?:IF\s+EXISTS\s+)?(?" + IdentifierPattern + @")", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex InsertTargetRegex = new Regex( + @"^\s*INSERT(?:\s+OR\s+(?:REPLACE|IGNORE|ROLLBACK|ABORT|FAIL))?\s+INTO\s+(?" + IdentifierPattern + @")", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex BeginTxRegex = new Regex( + @"^\s*BEGIN(?:\s+(?:IMMEDIATE|DEFERRED|EXCLUSIVE))?(?:\s+TRANSACTION)?\s*;?$", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex CommitTxRegex = new Regex( + @"^\s*(?:COMMIT|END)(?:\s+TRANSACTION)?\s*;?$", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex RollbackTxRegex = new Regex( + @"^\s*ROLLBACK(?:\s+TRANSACTION)?\s*;?$", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex ManifestTableRegex = new Regex( + @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex ManifestSha256Regex = new Regex( + @"^--\s*Manifest-SHA256:\s*([a-fA-F0-9]{64})", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private readonly IDatabase db; + private readonly string databaseName; + + public SqliteImportRunner(IDatabase db, string databaseName) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + } + + public Task RunAsync( + SqliteImportRequest request, + SqliteImportOptions options, + IProgress progress, + CancellationToken cancellation) + { + return Task.Run(() => RunInternal(request, options, progress, cancellation), cancellation); + } + + private SqliteImportResult RunInternal( + SqliteImportRequest request, + SqliteImportOptions options, + IProgress progress, + CancellationToken cancellation) + { + var clock = Stopwatch.StartNew(); + var log = new SqliteImportLog(request.JobId); + long fileBytes = 0; + if (File.Exists(request.SourcePath)) fileBytes = new FileInfo(request.SourcePath).Length; + + log.LogInfo($"Target SQLite Database: {databaseName}"); + log.LogInfo($"Source SQL File: {request.SourcePath} ({fileBytes:N0} bytes)"); + log.LogInfo($"Options: ContinueOnError={options.ContinueOnError}, Strategy={options.ExistingObjectStrategy}, WrapInTransaction={options.WrapInTransaction}"); + + int executed = 0; + int failed = 0; + int skipped = 0; + long totalRowsAffected = 0; + var errors = new List(); + var manifestTables = new List>(); + bool readingManifest = false; + bool haltedOnError = false; + + var existingTables = new HashSet(StringComparer.OrdinalIgnoreCase); + var existingViews = new HashSet(StringComparer.OrdinalIgnoreCase); + var skippedTables = new HashSet(StringComparer.OrdinalIgnoreCase); + + bool runnerManagedTx = false; + + try + { + // 1. 若啟用 SpatiaLite 自動加載，嘗試載入模組 + if (options.AutoLoadSpatiaLite && db is my_sqlite sqliteDb) + { + try + { + sqliteDb.RetryLoadSpatiaLite(); + if (sqliteDb.SpatiaLiteEnabled) + { + log.LogInfo("SpatiaLite runtime dynamic extension (mod_spatialite) loaded successfully."); + } + else if (!string.IsNullOrWhiteSpace(sqliteDb.SpatiaLiteLoadError)) + { + log.LogInfo("SpatiaLite runtime extension note: " + sqliteDb.SpatiaLiteLoadError); + } + } + catch (Exception ex) + { + log.LogInfo("SpatiaLite runtime extension attempt: " + ex.Message); + } + } + + // 2. 設定 PRAGMA 環境變數以加快寫入 + if (options.PragmaForeignKeysOff) + { + try { db.ExecSQL("PRAGMA foreign_keys = OFF;"); } catch { } + } + if (options.PragmaSynchronousOff) + { + try { db.ExecSQL("PRAGMA synchronous = OFF;"); } catch { } + } + + // 3. 快取目標資料庫中既有物件 + try + { + var tList = db.GetTables(databaseName); + if (tList != null) existingTables.UnionWith(tList); + } + catch { } + + try + { + var vList = db.GetViews(databaseName); + if (vList != null) existingViews.UnionWith(vList); + } + catch { } + + // 4. 預先檢查腳本是否自帶 BEGIN TRANSACTION + bool scriptHasExplicitTx = CheckIfScriptContainsExplicitTx(request.SourcePath); + if (options.WrapInTransaction && !scriptHasExplicitTx) + { + var beginRes = db.ExecSQL("BEGIN IMMEDIATE;"); + if (IsExecutionSuccess(beginRes)) + { + runnerManagedTx = true; + log.LogInfo("Transaction started by runner: BEGIN IMMEDIATE;"); + } + } + + using (var fs = new FileStream(request.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) + using (var reader = new StreamReader(fs, Encoding.UTF8, true, 65536)) + { + var currentBatch = new StringBuilder(); + bool inSingleQuote = false; + bool inDoubleQuote = false; + bool inBracket = false; + bool inBlockComment = false; + int batchStartLine = 1; + int currentLine = 0; + string line; + long lastReportMs = 0; + + while ((line = reader.ReadLine()) != null) + { + cancellation.ThrowIfCancellationRequested(); + currentLine++; + + string trimmed = line.Trim(); + + // 檢查 Native Manifest + if (trimmed.StartsWith("--")) + { + if (trimmed.Contains("mySQLPunk Native Manifest") && !readingManifest) + { + readingManifest = true; + log.LogInfo("Detected Native Manifest in SQL script footer."); + } + + var match = ManifestTableRegex.Match(trimmed); + if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) + { + manifestTables.Add(Tuple.Create(match.Groups[1].Value, r)); + } + + if (!inSingleQuote && !inDoubleQuote && !inBlockComment) + { + continue; + } + } + + if (currentBatch.Length == 0) batchStartLine = currentLine; + + // 逐字元狀態機解析語句結束分號 + int len = line.Length; + int lastSegmentStart = 0; + + for (int i = 0; i < len; i++) + { + char c = line[i]; + + if (inBlockComment) + { + if (c == '*' && i + 1 < len && line[i + 1] == '/') + { + inBlockComment = false; + i++; + } + continue; + } + + if (!inSingleQuote && !inDoubleQuote && !inBracket) + { + if (c == '-' && i + 1 < len && line[i + 1] == '-') + { + // 遇到單行註解，本行此處後續為註解 + currentBatch.Append(line.Substring(lastSegmentStart, i - lastSegmentStart)); + lastSegmentStart = len; + break; + } + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + inBlockComment = true; + i++; + continue; + } + } + + if (c == '\'' && !inDoubleQuote && !inBracket) + { + inSingleQuote = !inSingleQuote; + continue; + } + + if (c == '"' && !inSingleQuote && !inBracket) + { + inDoubleQuote = !inDoubleQuote; + continue; + } + + if (c == '[' && !inSingleQuote && !inDoubleQuote) + { + inBracket = true; + continue; + } + + if (c == ']' && inBracket) + { + inBracket = false; + continue; + } + + // 偵測到語句結束分號 + if (c == ';' && !inSingleQuote && !inDoubleQuote && !inBracket) + { + currentBatch.Append(line.Substring(lastSegmentStart, i - lastSegmentStart + 1)); + lastSegmentStart = i + 1; + + string statementSql = currentBatch.ToString().Trim(); + currentBatch.Clear(); + + if (!string.IsNullOrWhiteSpace(statementSql)) + { + bool continueRunning = ExecuteStatement( + statementSql, + batchStartLine, + options, + log, + existingTables, + existingViews, + skippedTables, + ref executed, + ref failed, + ref skipped, + ref totalRowsAffected, + errors); + + if (!continueRunning) + { + haltedOnError = true; + break; + } + + // 每 150ms 節流回報進度至 UI + if (clock.ElapsedMilliseconds - lastReportMs > 150) + { + lastReportMs = clock.ElapsedMilliseconds; + ReportProgress(progress, fs.Position, fileBytes, executed, failed, skipped, totalRowsAffected, clock, statementSql); + } + } + + batchStartLine = currentLine; + } + } + + if (haltedOnError) break; + + if (lastSegmentStart < len) + { + currentBatch.AppendLine(line.Substring(lastSegmentStart)); + } + } + + // 處理檔尾若有未以分號結尾的最後一條語句 + if (!haltedOnError && currentBatch.Length > 0) + { + string remainingSql = currentBatch.ToString().Trim(); + if (!string.IsNullOrWhiteSpace(remainingSql) && !remainingSql.StartsWith("--")) + { + ExecuteStatement( + remainingSql, + batchStartLine, + options, + log, + existingTables, + existingViews, + skippedTables, + ref executed, + ref failed, + ref skipped, + ref totalRowsAffected, + errors); + } + } + } + + // 5. 交易提交或回滾 + if (runnerManagedTx) + { + if (haltedOnError && !options.ContinueOnError) + { + try { db.ExecSQL("ROLLBACK;"); } catch { } + log.LogInfo("Runner managed transaction rolled back due to error: ROLLBACK;"); + } + else + { + try { db.ExecSQL("COMMIT;"); } catch { } + log.LogInfo("Runner managed transaction committed successfully: COMMIT;"); + } + } + + // 恢復 foreign_keys + if (options.PragmaForeignKeysOff) + { + try { db.ExecSQL("PRAGMA foreign_keys = ON;"); } catch { } + } + + clock.Stop(); + + // 6. Native Manifest 自動比對 + bool manifestVerified = false; + string manifestReport = null; + if (manifestTables.Count > 0) + { + int matchCount = 0; + var reportBuilder = new StringBuilder(); + foreach (var pair in manifestTables) + { + string tName = pair.Item1; + long expected = pair.Item2; + long actual = 0; + try { actual = db.CountRows(databaseName, tName); } catch { } + bool matched = (actual == expected); + if (matched) matchCount++; + log.LogManifestVerification(tName, expected, actual, matched); + reportBuilder.Append($"{tName}: {actual}/{expected} {(matched ? "OK" : "MISMATCH")}; "); + } + manifestVerified = (matchCount == manifestTables.Count); + manifestReport = reportBuilder.ToString().Trim(); + log.LogInfo($"Manifest Verification: {matchCount}/{manifestTables.Count} tables matched row counts exactly. Verified={manifestVerified}"); + } + + var status = haltedOnError + ? SqliteImportStatus.Failed + : (failed > 0 && !options.ContinueOnError ? SqliteImportStatus.Failed : SqliteImportStatus.Completed); + + var finalResult = new SqliteImportResult( + status, + executed, + failed, + skipped, + totalRowsAffected, + log.Path, + clock.Elapsed, + errors, + manifestVerified, + manifestReport); + + log.WriteEnd(finalResult); + log.Dispose(); + + ReportProgress(progress, fileBytes, fileBytes, executed, failed, skipped, totalRowsAffected, clock, null, true); + + return finalResult; + } + catch (OperationCanceledException) + { + clock.Stop(); + if (runnerManagedTx) + { + try { db.ExecSQL("ROLLBACK;"); } catch { } + } + log.LogInfo("Import cancelled by user. Rolled back any runner-managed transaction."); + var cancelResult = new SqliteImportResult( + SqliteImportStatus.Cancelled, + executed, + failed, + skipped, + totalRowsAffected, + log.Path, + clock.Elapsed, + new[] { "Import cancelled by user." }); + log.WriteEnd(cancelResult); + log.Dispose(); + return cancelResult; + } + catch (Exception ex) + { + clock.Stop(); + if (runnerManagedTx) + { + try { db.ExecSQL("ROLLBACK;"); } catch { } + } + log.LogInfo("Import terminated with unhandled exception: " + ex.Message); + errors.Add(ex.Message); + var errResult = new SqliteImportResult( + SqliteImportStatus.Failed, + executed, + failed, + skipped, + totalRowsAffected, + log.Path, + clock.Elapsed, + errors); + log.WriteEnd(errResult); + log.Dispose(); + return errResult; + } + } + + private bool ExecuteStatement( + string sql, + int lineNumber, + SqliteImportOptions options, + SqliteImportLog log, + HashSet existingTables, + HashSet existingViews, + HashSet skippedTables, + ref int executed, + ref int failed, + ref int skipped, + ref long totalRowsAffected, + List errors) + { + string trimmed = sql.Trim(); + string preview = MakeStatementPreview(trimmed); + + // 1. 處理 CREATE 物件策略 + var createMatch = CreateObjectRegex.Match(trimmed); + if (createMatch.Success) + { + string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); + string objName = Unquote(createMatch.Groups["name"].Value); + + bool isExisting = false; + if (objType == "TABLE") isExisting = existingTables.Contains(objName); + else if (objType == "VIEW") isExisting = existingViews.Contains(objName); + + if (isExisting) + { + switch (options.ExistingObjectStrategy) + { + case SqliteExistingObjectStrategy.AbortIfExists: + string abortMsg = $"Object {objType} \"{objName}\" already exists in database (AbortIfExists policy)."; + log.LogStatement(lineNumber, preview, false, abortMsg, "ABORT"); + failed++; + errors.Add(abortMsg); + return false; // 中斷匯入 + + case SqliteExistingObjectStrategy.SkipExisting: + log.LogStatement(lineNumber, preview, true, null, "SKIP_EXISTING"); + skipped++; + if (objType == "TABLE") skippedTables.Add(objName); + return true; // 略過本句，繼續 + + case SqliteExistingObjectStrategy.DropAndRecreate: + string dropSql = $"DROP {objType} IF EXISTS {QuoteIdentifier(objName)};"; + db.ExecSQL(dropSql); + log.LogStatement(lineNumber, $"DROP {objType} \"{objName}\" for recreation", true, null, "DROP_EXISTING"); + if (objType == "TABLE") existingTables.Remove(objName); + else if (objType == "VIEW") existingViews.Remove(objName); + break; + + case SqliteExistingObjectStrategy.OverwriteKeepExisting: + log.LogStatement(lineNumber, preview, true, null, "KEEP_STRUCTURE"); + skipped++; + return true; + } + } + } + + // 2. 處理 INSERT 資料策略 (若該表於前面被標記為 SkipExisting，則一併略過 INSERT) + var insertMatch = InsertTargetRegex.Match(trimmed); + if (insertMatch.Success) + { + string targetTable = Unquote(insertMatch.Groups["name"].Value); + if (skippedTables.Contains(targetTable)) + { + skipped++; + log.LogStatement(lineNumber, preview, true, null, "SKIP_DATA_FOR_SKIPPED_TABLE"); + return true; + } + } + + // 3. 實際執行 SQL + var res = db.ExecSQL(trimmed); + bool success = IsExecutionSuccess(res); + + if (success) + { + executed++; + long affected = 0; + if (res != null && res.ContainsKey("rowsAffected") && long.TryParse(res["rowsAffected"], out long a)) + { + affected = a; + totalRowsAffected += a; + } + + // 若為 CREATE TABLE / VIEW 成功，更新快取 + if (createMatch.Success) + { + string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); + string objName = Unquote(createMatch.Groups["name"].Value); + if (objType == "TABLE") existingTables.Add(objName); + else if (objType == "VIEW") existingViews.Add(objName); + } + + log.LogStatement(lineNumber, preview, true, null, affected > 0 ? $"OK (+{affected} rows)" : "OK"); + return true; + } + else + { + failed++; + string reason = DatabaseExecutionResultService.GetFailureReason(res); + string errText = $"Line {lineNumber}: {reason}"; + errors.Add(errText); + log.LogStatement(lineNumber, preview, false, reason, "FAIL"); + + if (!options.ContinueOnError) + { + return false; // 停止匯入 + } + return true; // 繼續略過執行 + } + } + + private static bool IsExecutionSuccess(Dictionary result) + { + if (result == null) return false; + if (result.TryGetValue("status", out string status)) + { + return string.Equals(status, "OK", StringComparison.OrdinalIgnoreCase); + } + return true; + } + + private static string Unquote(string identifier) + { + if (string.IsNullOrWhiteSpace(identifier)) return string.Empty; + string s = identifier.Trim(); + if ((s.StartsWith("\"") && s.EndsWith("\"")) || + (s.StartsWith("`") && s.EndsWith("`")) || + (s.StartsWith("'") && s.EndsWith("'"))) + { + return s.Substring(1, s.Length - 2).Replace("\"\"", "\"").Replace("``", "`").Replace("''", "'"); + } + if (s.StartsWith("[") && s.EndsWith("]")) + { + return s.Substring(1, s.Length - 2); + } + return s; + } + + private static string QuoteIdentifier(string name) + { + name = name ?? string.Empty; + return "\"" + name.Replace("\"", "\"\"") + "\""; + } + + private static string MakeStatementPreview(string sql) + { + if (string.IsNullOrWhiteSpace(sql)) return string.Empty; + string singleLine = Regex.Replace(sql, @"\s+", " ").Trim(); + if (singleLine.Length > 120) return singleLine.Substring(0, 117) + "..."; + return singleLine; + } + + private static bool CheckIfScriptContainsExplicitTx(string filePath) + { + if (!File.Exists(filePath)) return false; + try + { + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096)) + using (var reader = new StreamReader(fs, Encoding.UTF8, true, 4096)) + { + string line; + int count = 0; + while ((line = reader.ReadLine()) != null && count < 200) + { + count++; + string trimmed = line.Trim(); + if (BeginTxRegex.IsMatch(trimmed)) return true; + } + } + } + catch { } + return false; + } + + private static void ReportProgress( + IProgress progress, + long currentBytes, + long totalBytes, + int executed, + int failed, + int skipped, + long totalRowsAffected, + Stopwatch clock, + string currentPreview, + bool isComplete = false) + { + if (progress == null) return; + + double elapsedSec = clock.Elapsed.TotalSeconds; + double bytesPerSec = elapsedSec > 0.05 ? currentBytes / elapsedSec : 0; + TimeSpan? remaining = null; + if (bytesPerSec > 0 && totalBytes > currentBytes) + { + double remainingSec = (totalBytes - currentBytes) / bytesPerSec; + if (remainingSec < 86400) remaining = TimeSpan.FromSeconds(remainingSec); + } + + string msg; + if (isComplete) + { + msg = $"Import execution finished: {executed} executed, {failed} failed, {skipped} skipped, {totalRowsAffected:N0} rows affected."; + } + else + { + msg = $"Executing: {executed} executed, {failed} failed, {skipped} skipped..."; + } + + progress.Report(new SqliteImportProgress( + currentBytes, + totalBytes, + executed, + failed, + skipped, + totalRowsAffected, + bytesPerSec, + clock.Elapsed, + remaining, + msg, + currentPreview)); + } + } +} diff --git a/mySQLPunk/mySQLPunk.csproj b/mySQLPunk/mySQLPunk.csproj index e3c0749..fdc275f 100644 --- a/mySQLPunk/mySQLPunk.csproj +++ b/mySQLPunk/mySQLPunk.csproj @@ -321,6 +321,10 @@ + + + + @@ -398,6 +402,12 @@ + + + + + + diff --git a/tests/Run-SmokeTests.ps1 b/tests/Run-SmokeTests.ps1 index acd85a4..7d2aba6 100644 --- a/tests/Run-SmokeTests.ps1 +++ b/tests/Run-SmokeTests.ps1 @@ -149,6 +149,8 @@ if (!(Test-Path -LiteralPath $mysqlConnector)) { (Join-Path $PSScriptRoot "SqlServerMigrationCatalogTests.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MySqlExport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MySqlImport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.SqliteExport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.SqliteImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.TransferTrial.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Live.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Parser.cs") $source $preferencesSource $schemaSyncSource $erModelSource (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Snapshot.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Schema.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.ViewReferences.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Properties.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Artifact.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Values.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Lob.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Routing.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.SnapshotIntegration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.CompatibilityIntegration.cs") if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE diff --git a/tests/SmokeTests.MssqlExport.Routing.cs b/tests/SmokeTests.MssqlExport.Routing.cs index 7c28c6b..899c486 100644 --- a/tests/SmokeTests.MssqlExport.Routing.cs +++ b/tests/SmokeTests.MssqlExport.Routing.cs @@ -121,7 +121,7 @@ private static void Routing_FourHandlers() private static void Routing_LegacyBranches() { - foreach (string provider in new[] { "sqlite", "postgresql" }) + foreach (string provider in new[] { "postgresql", "oracle" }) foreach (string handler in new[] { "DumpSelectedTableSql", "DumpCurrentSelectionSqlWithDialog", "DumpSelectedDatabaseSqlWithDialog", "BackupSelectedDatabaseWithDialog" }) using (var fixture = new RoutingFixture(provider, "Tables")) { diff --git a/tests/SmokeTests.MssqlImport.Wizard.cs b/tests/SmokeTests.MssqlImport.Wizard.cs index 73d4e2f..2da8ccb 100644 --- a/tests/SmokeTests.MssqlImport.Wizard.cs +++ b/tests/SmokeTests.MssqlImport.Wizard.cs @@ -384,7 +384,7 @@ public MssqlImportCleanup Cleanup() } private static void ImportWizard_Routing() { - foreach (string provider in new[] { "mssql", "SQLSERVER", "sqlite" }) + foreach (string provider in new[] { "mssql", "SQLSERVER", "postgresql" }) using (var fixture = new RoutingFixture(provider, "Tables")) { int opened = 0, legacy = 0; @@ -400,7 +400,7 @@ private static void ImportWizard_Routing() new Func(dialog => { legacy++; return DialogResult.Cancel; })); typeof(Form1).GetField("MssqlImportStatusSink", flags).SetValue(fixture.Form, new Action(text => fixture.Status.Add(text))); fixture.Invoke("ImportSqlWithDialog"); - bool native = provider != "sqlite"; + bool native = provider != "postgresql"; Assert(opened == (native ? 1 : 0) && legacy == (native ? 0 : 1), "Actual file handler routes only MSSQL before legacy file selection."); if (native) { diff --git a/tests/SmokeTests.SqliteExport.cs b/tests/SmokeTests.SqliteExport.cs new file mode 100644 index 0000000..7aaca84 --- /dev/null +++ b/tests/SmokeTests.SqliteExport.cs @@ -0,0 +1,251 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.SqliteExport; + +public static partial class SmokeTests +{ + private static void SqliteExport_OptionsAndModels_Defaults() + { + var options = new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData, @"C:\dump.sql"); + Assert(options.Mode == SqliteExportMode.SchemaAndData, "Default mode should be SchemaAndData"); + Assert(options.SingleTransaction, "SingleTransaction should be true by default"); + Assert(options.PragmaForeignKeysOff, "PragmaForeignKeysOff should be true by default"); + Assert(options.PragmaSynchronousOff, "PragmaSynchronousOff should be true by default"); + Assert(options.IncludeDropStatements, "IncludeDropStatements should be true by default"); + Assert(options.IncludeViews, "IncludeViews should be true by default"); + Assert(options.IncludeTriggers, "IncludeTriggers should be true by default"); + Assert(options.IncludeIndexes, "IncludeIndexes should be true by default"); + Assert(options.IncludeComments, "IncludeComments should be true by default"); + Assert(!options.ExcludeSpatialMetadata, "ExcludeSpatialMetadata should be false by default"); + Assert(options.InsertBatchSize == 1000, "Default batch size should be 1000"); + + var item = new SqliteTableExportItem("spatial_roads", 1500, true, true, false, false, "[GIS] LineString (SRID 3826)"); + Assert(item.Name == "spatial_roads", "TableName match"); + Assert(item.EstimatedRows == 1500, "EstimatedRows match"); + Assert(item.IsSelected, "IsSelected match"); + Assert(item.IsSpatial, "IsSpatial match"); + Assert(!item.IsSpatialMetadata, "IsSpatialMetadata match"); + AssertContains(item.SpatialInfo, "GIS", "SpatialInfo match"); + } + + private static void SqliteExport_ServiceAndManifest_StreamingAndHexBlob() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_export_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string targetFile = Path.Combine(tempDir, "sqlite_export_test.sql"); + + try + { + var mockDb = new MockSqliteExportDatabase(); + var service = new SqliteSqlExportService(mockDb, "gis_demo.sqlite"); + + // 1. 驗證 LoadTableCatalog 正確辨識空間圖層並過濾影子表 + var catalog = service.LoadTableCatalog(); + Assert(catalog.Count == 1, "Catalog should filter out shadow tables like idx_roads_geom_node, expected 1 table but got " + catalog.Count); + Assert(catalog[0].Name == "spatial_roads", "Export item table name match"); + Assert(catalog[0].IsSpatial, "spatial_roads should be recognized as spatial layer"); + AssertContains(catalog[0].SpatialInfo, "SpatiaLite", "Spatial info contains provider note"); + + var options = new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData, targetFile) + { + SingleTransaction = true, + PragmaForeignKeysOff = true, + PragmaSynchronousOff = true, + IncludeDropStatements = true, + IncludeIndexes = true, + IncludeViews = true, + IncludeTriggers = true + }; + options.SelectedTables.Add("spatial_roads"); + options.SelectedViews.Add("v_roads"); + options.SelectedTriggers.Add("trg_roads_audit"); + + SqliteExportProgress lastProgress = null; + var progress = new Progress(p => lastProgress = p); + + var task = service.ExportAsync(options, progress, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Success, "Export should succeed"); + Assert(File.Exists(targetFile), "Target file must exist after atomic move"); + Assert(result.TableCount == 1, "Exported table count should be 1"); + Assert(result.ViewCount == 1, "Exported view count should be 1"); + Assert(result.TriggerCount == 1, "Exported trigger count should be 1"); + Assert(result.IndexCount == 1, "Exported index count should be 1"); + Assert(result.TotalRows == 2, "Exported row count should be 2"); + + string content = File.ReadAllText(targetFile); + AssertContains(content, "-- mySQLPunk SQLite Logical Dump", "Header comment present"); + AssertContains(content, "PRAGMA foreign_keys = OFF;", "Pragma foreign_keys OFF present"); + AssertContains(content, "PRAGMA synchronous = OFF;", "Pragma synchronous OFF present"); + AssertContains(content, "BEGIN TRANSACTION;", "BEGIN TRANSACTION present"); + AssertContains(content, "DROP TABLE IF EXISTS \"spatial_roads\";", "DROP TABLE statement present"); + AssertContains(content, "CREATE TABLE \"spatial_roads\"", "CREATE TABLE DDL present"); + AssertNotContains(content, "CREATE TABLE \"idx_roads_geom_node\"", "Shadow table should NEVER be exported as CREATE TABLE"); + + // 驗證 BLOB 幾何二進位輸出為 X'...' 格式，無損精度 + AssertContains(content, "X'00010000007C0101000000'", "GIS binary geometry must be exported as X'...' hex literal"); + AssertContains(content, "COMMIT;", "COMMIT transaction present"); + AssertContains(content, "PRAGMA foreign_keys = ON;", "Restore foreign_keys ON present"); + + // 驗證 Native Manifest + AssertContains(content, "-- mySQLPunk Native Manifest", "Native Manifest header present"); + AssertContains(content, "-- Tables: 1", "Manifest total tables count present"); + AssertContains(content, "-- TotalRows: 2", "Manifest total rows count present"); + AssertContains(content, "-- Manifest-Table: spatial_roads | Rows: 2", "Manifest table row entry present"); + AssertContains(content, "-- Manifest-SHA256:", "Manifest SHA-256 present"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private sealed class MockSqliteExportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "sqlite"; + + public void SetConn(string connectionString) { } + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + var dt = new DataTable(); + if (sql.Contains("FROM sqlite_master WHERE type='table' AND sql LIKE 'CREATE VIRTUAL%'")) + { + dt.Columns.Add("name"); + dt.Rows.Add("idx_roads_geom"); + return dt; + } + if (sql.Contains("FROM geometry_columns")) + { + dt.Columns.Add("f_table_name"); + dt.Columns.Add("f_geometry_column"); + dt.Columns.Add("type"); + dt.Columns.Add("srid"); + dt.Rows.Add("spatial_roads", "geom", "POINT", "3826"); + return dt; + } + if (sql.Contains("FROM gpkg_geometry_columns")) + { + return dt; + } + if (sql.Contains("SELECT name, sql FROM sqlite_master WHERE type='table'")) + { + dt.Columns.Add("name"); + dt.Columns.Add("sql"); + dt.Rows.Add("spatial_roads", "CREATE TABLE \"spatial_roads\" (\"id\" INTEGER PRIMARY KEY, \"name\" TEXT, \"geom\" BLOB);"); + // 影子表 (R-Tree node) + dt.Rows.Add("idx_roads_geom_node", "CREATE TABLE \"idx_roads_geom_node\" (nodeno INTEGER PRIMARY KEY, data BLOB);"); + return dt; + } + if (sql.Contains("WHERE type='table' AND name='spatial_roads'")) + { + dt.Columns.Add("sql"); + dt.Rows.Add("CREATE TABLE \"spatial_roads\" (\"id\" INTEGER PRIMARY KEY, \"name\" TEXT, \"geom\" BLOB);"); + return dt; + } + if (sql.Contains("SELECT name FROM sqlite_master WHERE type='view'")) + { + dt.Columns.Add("name"); + dt.Rows.Add("v_roads"); + return dt; + } + if (sql.Contains("WHERE type='view' AND name='v_roads'")) + { + dt.Columns.Add("sql"); + dt.Rows.Add("CREATE VIEW \"v_roads\" AS SELECT \"id\", \"name\" FROM \"spatial_roads\";"); + return dt; + } + if (sql.Contains("SELECT name FROM sqlite_master WHERE type='trigger'")) + { + dt.Columns.Add("name"); + dt.Rows.Add("trg_roads_audit"); + return dt; + } + if (sql.Contains("WHERE type='trigger' AND name='trg_roads_audit'")) + { + dt.Columns.Add("sql"); + dt.Rows.Add("CREATE TRIGGER \"trg_roads_audit\" AFTER INSERT ON \"spatial_roads\" BEGIN SELECT 1; END;"); + return dt; + } + if (sql.Contains("WHERE type='index'")) + { + dt.Columns.Add("sql"); + dt.Columns.Add("tbl_name"); + dt.Rows.Add("CREATE INDEX \"idx_roads_name\" ON \"spatial_roads\" (\"name\");", "spatial_roads"); + return dt; + } + return dt; + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + return new Dictionary { { "status", "OK" } }; + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + + public List GetDatabases() => new List { "main" }; + public List GetTables(string databaseName) => new List { "spatial_roads", "idx_roads_geom_node" }; + public List GetViews(string databaseName) => new List { "v_roads" }; + + public DataTable GetColumns(string databaseName, string tableName) + { + var dt = new DataTable(); + dt.Columns.Add("COLUMN_NAME"); + dt.Columns.Add("DATA_TYPE"); + dt.Rows.Add("id", "INTEGER"); + dt.Rows.Add("name", "TEXT"); + dt.Rows.Add("geom", "BLOB"); + return dt; + } + + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => "CREATE TABLE \"spatial_roads\" (\"id\" INTEGER PRIMARY KEY, \"name\" TEXT, \"geom\" BLOB);"; + public bool TableExists(string databaseName, string tableName) => true; + public bool ViewExists(string databaseName, string viewName) => true; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 2; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) + { + var dt = new DataTable(); + dt.Columns.Add("id", typeof(int)); + dt.Columns.Add("name", typeof(string)); + dt.Columns.Add("geom", typeof(byte[])); + if (offset == 0) + { + byte[] sampleBlob = new byte[] { 0x00, 0x01, 0x00, 0x00, 0x00, 0x7C, 0x01, 0x01, 0x00, 0x00, 0x00 }; + dt.Rows.Add(1, "Main Street", sampleBlob); + dt.Rows.Add(2, "Broadway", sampleBlob); + } + return dt; + } + + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => "CREATE VIEW \"v_roads\" AS SELECT \"id\", \"name\" FROM \"spatial_roads\";"; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.SqliteImport.cs b/tests/SmokeTests.SqliteImport.cs new file mode 100644 index 0000000..e5d7c94 --- /dev/null +++ b/tests/SmokeTests.SqliteImport.cs @@ -0,0 +1,275 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.SqliteImport; + +public static partial class SmokeTests +{ + private static void SqliteImport_PreflightScanner_SpatiaLiteAndManifest() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_preflight_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "spatial_sample.sql"); + + try + { + string sqlContent = + "-- SQLite GIS Dump Header\r\n" + + "PRAGMA foreign_keys = OFF;\r\n" + + "BEGIN TRANSACTION;\r\n" + + "CREATE TABLE \"spatial_points\" (\"id\" INTEGER PRIMARY KEY, \"name\" TEXT, \"geom\" BLOB);\r\n" + + "SELECT InitSpatialMetaData(1);\r\n" + + "SELECT AddGeometryColumn('spatial_points', 'geom', 4326, 'POINT', 'XY');\r\n" + + "INSERT INTO \"spatial_points\" VALUES (1, 'Point with ; inside string', X'00010000007C0101000000');\r\n" + + "COMMIT;\r\n" + + "-- ========================================================\r\n" + + "-- mySQLPunk Native Manifest\r\n" + + "-- Tables: 1\r\n" + + "-- TotalRows: 1\r\n" + + "-- Manifest-Table: spatial_points | Rows: 1\r\n" + + "-- Manifest-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\r\n" + + "-- ========================================================\r\n"; + + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var preflight = SqliteImportPreflightScanner.Scan(sqlFile, CancellationToken.None); + Assert(preflight.Passed, "Preflight scan should pass"); + Assert(preflight.FileBytes > 0, "FileBytes > 0"); + Assert(preflight.DetectedSpatiaLite, "Preflight must detect SpatiaLite functions like InitSpatialMetaData / AddGeometryColumn"); + Assert(preflight.HasManifest, "Preflight must detect Native Manifest"); + Assert(preflight.ManifestTables == 1, "ManifestTables count match 1"); + Assert(preflight.ManifestRows == 1, "ManifestRows count match 1"); + AssertEquals("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", preflight.ManifestSha256, "Manifest SHA-256 match"); + + // 測試 Embedded NUL 偵測 + string badFile = Path.Combine(tempDir, "corrupted_nul.sql"); + File.WriteAllText(badFile, "CREATE TABLE \"bad\" (\0id int);", Encoding.UTF8); + var badPreflight = SqliteImportPreflightScanner.Scan(badFile, CancellationToken.None); + Assert(!badPreflight.Passed, "Preflight must fail on embedded NUL"); + AssertContains(badPreflight.Issues[0], "Embedded NUL", "NUL issue reported"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void SqliteImport_Runner_ExecutionAndManifestVerification() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_runner_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "run_sqlite_import.sql"); + + try + { + string sqlContent = + "PRAGMA foreign_keys = OFF;\r\n" + + "CREATE TABLE \"places\" (\"id\" INTEGER PRIMARY KEY, \"geom\" BLOB);\r\n" + + "INSERT INTO \"places\" VALUES (1, X'000100');\r\n" + + "INSERT INTO \"places\" VALUES (2, X'000200');\r\n" + + "-- ========================================================\r\n" + + "-- mySQLPunk Native Manifest\r\n" + + "-- Tables: 1\r\n" + + "-- TotalRows: 2\r\n" + + "-- Manifest-Table: places | Rows: 2\r\n" + + "-- ========================================================\r\n"; + + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var mockDb = new MockSqliteImportDatabase(); + var runner = new SqliteImportRunner(mockDb, "test_gis.sqlite"); + + var request = new SqliteImportRequest(Guid.NewGuid(), sqlFile, "test_gis.sqlite"); + var options = new SqliteImportOptions + { + ExistingObjectStrategy = SqliteExistingObjectStrategy.SkipExisting, + ContinueOnError = true, + WrapInTransaction = true + }; + + SqliteImportProgress lastProgress = null; + var progress = new Progress(p => lastProgress = p); + + var task = runner.RunAsync(request, options, progress, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Status == SqliteImportStatus.Completed, "Import status should be Completed"); + Assert(result.ExecutedStatements == 4, "Executed statements count should be 4 (PRAGMA, CREATE, 2x INSERT), got " + result.ExecutedStatements); + Assert(result.FailedStatements == 0, "Failed statements should be 0"); + Assert(result.TotalRowsAffected == 2, "TotalRowsAffected should be 2, got " + result.TotalRowsAffected); + Assert(result.ManifestVerified, "Native Manifest should verify table row counts match exactly (2 == 2)"); + Assert(File.Exists(result.LogPath), "Log file must exist at " + result.LogPath); + + string logContent = File.ReadAllText(result.LogPath); + AssertContains(logContent, "mySQLPunk SQLite Import Execution Log", "Log contains SQLite import header"); + AssertContains(logContent, "Manifest Check: Table \"places\" -> Expected: 2, Actual: 2", "Log contains manifest verification pass"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void SqliteImport_Runner_ExistingObjectStrategies() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_strat_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "conflict.sql"); + + try + { + string sqlContent = + "CREATE TABLE \"existing_tbl\" (\"id\" INTEGER);\r\n" + + "INSERT INTO \"existing_tbl\" VALUES (100);\r\n"; + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var request = new SqliteImportRequest(Guid.NewGuid(), sqlFile, "test.sqlite"); + + // 1. 測試 AbortIfExists：既有物件存在時立即終止 + var mockDb1 = new MockSqliteImportDatabase(); + mockDb1.ExistingTables.Add("existing_tbl"); + var runner1 = new SqliteImportRunner(mockDb1, "test.sqlite"); + var res1 = runner1.RunAsync(request, new SqliteImportOptions { ExistingObjectStrategy = SqliteExistingObjectStrategy.AbortIfExists, ContinueOnError = false }, null, CancellationToken.None).Result; + Assert(res1.Status == SqliteImportStatus.Failed, "AbortIfExists strategy must fail when table exists"); + Assert(res1.FailedStatements >= 1, "Failed statement recorded"); + + // 2. 測試 SkipExisting：既有物件存在時略過結構與資料 + var mockDb2 = new MockSqliteImportDatabase(); + mockDb2.ExistingTables.Add("existing_tbl"); + var runner2 = new SqliteImportRunner(mockDb2, "test.sqlite"); + var res2 = runner2.RunAsync(request, new SqliteImportOptions { ExistingObjectStrategy = SqliteExistingObjectStrategy.SkipExisting }, null, CancellationToken.None).Result; + Assert(res2.Status == SqliteImportStatus.Completed, "SkipExisting strategy should complete"); + Assert(res2.SkippedStatements == 2, "Both CREATE TABLE and INSERT should be skipped, got " + res2.SkippedStatements); + + // 3. 測試 DropAndRecreate：先 DROP 後 CREATE + var mockDb3 = new MockSqliteImportDatabase(); + mockDb3.ExistingTables.Add("existing_tbl"); + var runner3 = new SqliteImportRunner(mockDb3, "test.sqlite"); + var res3 = runner3.RunAsync(request, new SqliteImportOptions { ExistingObjectStrategy = SqliteExistingObjectStrategy.DropAndRecreate }, null, CancellationToken.None).Result; + Assert(res3.Status == SqliteImportStatus.Completed, "DropAndRecreate strategy should complete"); + Assert(res3.ExecutedStatements == 2, "Executed statements should be 2"); + Assert(mockDb3.ExecutedStatements.Exists(s => s.StartsWith("DROP TABLE IF EXISTS", StringComparison.OrdinalIgnoreCase)), "DROP TABLE must be executed before recreate"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private static void SqliteImport_Runner_TransactionRollbackOnFailure() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_tx_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string sqlFile = Path.Combine(tempDir, "fail_tx.sql"); + + try + { + string sqlContent = + "INSERT INTO \"tbl\" VALUES (1);\r\n" + + "FAILING_SQL_SYNTAX_ERROR;\r\n"; + File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + + var mockDb = new MockSqliteImportDatabase(); + mockDb.FailOnFailingSql = true; + + var runner = new SqliteImportRunner(mockDb, "test.sqlite"); + var request = new SqliteImportRequest(Guid.NewGuid(), sqlFile, "test.sqlite"); + var options = new SqliteImportOptions + { + WrapInTransaction = true, + ContinueOnError = false + }; + + var res = runner.RunAsync(request, options, null, CancellationToken.None).Result; + Assert(res.Status == SqliteImportStatus.Failed, "Import should fail on syntax error with ContinueOnError=false"); + Assert(mockDb.ExecutedStatements.Contains("ROLLBACK;"), "Transaction must be rolled back on fatal failure"); + } + finally + { + if (Directory.Exists(tempDir)) + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + } + + private sealed class MockSqliteImportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "sqlite"; + public List ExecutedStatements { get; } = new List(); + public HashSet ExistingTables { get; } = new HashSet(StringComparer.OrdinalIgnoreCase); + public bool FailOnFailingSql { get; set; } = false; + + public void SetConn(string connectionString) { } + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + return new DataTable(); + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + ExecutedStatements.Add(sql); + if (FailOnFailingSql && sql.Contains("FAILING_SQL_SYNTAX_ERROR")) + { + return new Dictionary + { + { "status", "NO" }, + { "reason", "near \"FAILING_SQL_SYNTAX_ERROR\": syntax error" } + }; + } + + var res = new Dictionary { { "status", "OK" } }; + if (sql.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase)) + { + res["rowsAffected"] = "1"; + } + return res; + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + + public List GetDatabases() => new List { "main" }; + public List GetTables(string databaseName) => new List(ExistingTables); + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => ""; + public bool TableExists(string databaseName, string tableName) => ExistingTables.Contains(tableName); + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 2; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index ec862e3..bb13975 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -40,6 +40,12 @@ public static int Main(string[] args) Run("MySQL import runner database redirection", MySqlImport_Runner_DatabaseRedirection, ref passed); Run("MySQL import runner existing object strategies", MySqlImport_Runner_ExistingObjectStrategies, ref passed); Run("MySQL import runner manifest mismatch detection", MySqlImport_Runner_ManifestMismatchDetection, ref passed); + Run("SQLite export options and models defaults", SqliteExport_OptionsAndModels_Defaults, ref passed); + Run("SQLite export service and manifest streaming and hex blob", SqliteExport_ServiceAndManifest_StreamingAndHexBlob, ref passed); + Run("SQLite import preflight scanner SpatiaLite and manifest", SqliteImport_PreflightScanner_SpatiaLiteAndManifest, ref passed); + Run("SQLite import runner execution and manifest verification", SqliteImport_Runner_ExecutionAndManifestVerification, ref passed); + Run("SQLite import runner existing object strategies", SqliteImport_Runner_ExistingObjectStrategies, ref passed); + Run("SQLite import runner transaction rollback on failure", SqliteImport_Runner_TransactionRollbackOnFailure, ref passed); Run("MSSQL import Transfer guards", MssqlImportTransferGuards, ref passed); Run("MSSQL import Transfer trial", MssqlImportTransferTrial, ref passed); Run("MSSQL import worker published cleanup receipt", MssqlImportPublishedReleaseRace, ref passed); From ee10c12549398bc7c92ec42ae963e58fec761208 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 15:26:55 +0800 Subject: [PATCH 07/12] feat(postgresql): add PostgreSql export/import wizard with PostGIS support and streaming state machine --- TODO.md | 14 +- mySQLPunk/Form1.PostgreSqlExport.cs | 65 ++ mySQLPunk/Form1.PostgreSqlImport.cs | 54 ++ mySQLPunk/Form1.cs | 30 + mySQLPunk/PostgreSqlExportWizardForm.cs | 702 ++++++++++++++++ mySQLPunk/PostgreSqlImportWizardForm.cs | 610 ++++++++++++++ .../PostgreSqlExportModels.cs | 114 +++ .../PostgreSqlSqlExportService.cs | 756 ++++++++++++++++++ .../PostgreSqlImport/PostgreSqlImportLog.cs | 132 +++ .../PostgreSqlImportModels.cs | 100 +++ .../PostgreSqlImportPreflightScanner.cs | 330 ++++++++ .../PostgreSqlImportRunner.cs | 573 +++++++++++++ mySQLPunk/mySQLPunk.csproj | 10 + tests/Run-SmokeTests.ps1 | 2 + tests/SmokeTests.MssqlExport.Routing.cs | 2 +- tests/SmokeTests.MssqlImport.Wizard.cs | 4 +- tests/SmokeTests.PostgreSqlExport.cs | 252 ++++++ tests/SmokeTests.PostgreSqlImport.cs | 278 +++++++ tests/SmokeTests.cs | 7 + 19 files changed, 4025 insertions(+), 10 deletions(-) create mode 100644 mySQLPunk/Form1.PostgreSqlExport.cs create mode 100644 mySQLPunk/Form1.PostgreSqlImport.cs create mode 100644 mySQLPunk/PostgreSqlExportWizardForm.cs create mode 100644 mySQLPunk/PostgreSqlImportWizardForm.cs create mode 100644 mySQLPunk/lib/PostgreSqlExport/PostgreSqlExportModels.cs create mode 100644 mySQLPunk/lib/PostgreSqlExport/PostgreSqlSqlExportService.cs create mode 100644 mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportLog.cs create mode 100644 mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportModels.cs create mode 100644 mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportPreflightScanner.cs create mode 100644 mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportRunner.cs create mode 100644 tests/SmokeTests.PostgreSqlExport.cs create mode 100644 tests/SmokeTests.PostgreSqlImport.cs diff --git a/TODO.md b/TODO.md index ae63b8c..45de392 100644 --- a/TODO.md +++ b/TODO.md @@ -34,7 +34,7 @@ | :--- | :--- | :--- | :--- | | **MySQL / MariaDB** | ✅ 已具備完整四步精靈 (`MySqlExportWizardForm`)，支援選表/View/Routine/Trigger、一致性快照、Native Manifest。 | ✅ 已具備完整四步精靈 (`MySqlImportWizardForm`)，支援串流預檢、4種衝突策略、Session鎖定、終端機滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格）。 | | **SQLite** | ✅ 已具備完整四步精靈 (`SqliteExportWizardForm`)，支援選表、SpatiaLite/GeoPackage 空間幾何無損 Hex 傾印、排除虛擬表影子表、原子替換與 Native Manifest。 | ✅ 已具備完整四步精靈 (`SqliteImportWizardForm`)，支援串流狀態機預檢、SpatiaLite 動態模組自動加載、4種衝突策略、滑動視窗終端機日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS 空間特性）。 | -| **PostgreSQL** | 走舊版 `DatabaseDumpService`，字串拼接；進階型別 (bytea, jsonb, array, uuid) 處理簡陋。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取後以分號切割。 | **高危 OOM 風險**；無法正確處理 Dollar-quoted (`$$...$$`) 預存程序或函數語法（分號會被誤切）；無 Repeatable Read 一致性快照；無進度。 | +| **PostgreSQL** | ✅ 已具備完整四步精靈 (`PostgreSqlExportWizardForm`)，支援選表/View、PostGIS 空間幾何無損 Hex EWKB 傾印、排除 spatial_ref_sys、REPEATABLE READ 快照與 Native Manifest。 | ✅ 已具備完整四步精靈 (`PostgreSqlImportWizardForm`)，支援狀態機 Lexer (精準解析 Dollar Quotes 與巢狀註解)、PostGIS 自動安裝開關、4種衝突策略、2,000行滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS PostGIS 空間特性）。 | | **Oracle** | 走舊版 `DatabaseDumpService`，字串拼接。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取。 | **高危 OOM 風險**；無法處理 PL/SQL 區塊 (`/` 結尾)；缺少 RAW/LOB 分塊支援。 | --- @@ -59,14 +59,14 @@ - [x] 匯出：改用真正的串流 StreamWriter 寫檔；支援 `BEGIN IMMEDIATE` 一致性交易；支援側車註解 (Sidecar Comment) 隨結構導出；空間幾何 BLOB 無損 Hex 傾印；排除 R-Tree 影子表；產生 Native Manifest。 - [x] 匯入：串流狀態機 Lexer 解析；支援 SpatiaLite 動態模組自動加載；交易包覆與 PRAGMA 性能優化；4種物件衝突策略；2,000 行滑動視窗終端機日誌；Manifest 逐表筆數核對。 -#### 🥉 第三優先：PostgreSQL +#### 🥉 第三優先：PostgreSQL（崩琦 GIS PostGIS 空間特長對齊） - **原因**： - 1. 企業與現代開發廣泛使用，重要性極高。 - 2. 語法複雜度較高：需要支援 Dollar-quoted (`$$` 或 `$tag$`) 函數/預存程序本體、Bytea Hex 轉義、Array 與 Jsonb 原生字面量。 - 3. 可導入 PostgreSQL 原生的 `REPEATABLE READ` 交易快照保證匯出一致性。 + 1. 企業與現代空間資訊系統（GIS）廣泛使用，為崩琦 GIS 核心亮點。 + 2. 語法複雜度較高：需要支援 Dollar-quoted (`$$` 或 `$tag$`) 函數/預存程序本體、Bytea Hex 轉義、Array、Jsonb 與幾何 (EWKB) 原生字面量。 + 3. 導入 PostgreSQL 原生的 `REPEATABLE READ` 交易快照保證匯出一致性，並無損傾印 PostGIS 幾何資料。 - **具體工作**： - - [ ] 匯出：Schema 命名空間分組、支援 `public` 及自訂 Schema 的 Table/View 選取；串流寫入與進階型別格式化。 - - [ ] 匯入：開發 PostgreSQL 專屬 Statement Lexer（精確識別 Dollar Quotes、SQL 註解與跳脫字串，不隨意在分號切斷）；支援錯誤略過與 Manifest 驗證。 + - [x] 匯出：支援四步精靈（選表/View ➔ 輸出設定/快照隔離/排除 spatial_ref_sys ➔ 物件清單 ➔ 串流執行 + Punky 動畫 + Manifest 生成）；PostGIS geometry/geography/raster 轉無損 EWKB 十六進位字面量；保留 GIST/SP-GIST 空間索引。 + - [x] 匯入：支援四步精靈（選檔 ➔ 串流狀態機預檢 Dollar Quotes 與巢狀註解 ➔ 4種物件衝突策略與 PostGIS 自動加載 ➔ 串流執行 + Punky 動畫 + 2,000行滑動視窗終端機日誌 + 獨立 Log + Manifest 逐表筆數核對）。 #### 🎖️ 第四優先：Oracle - **原因**： diff --git a/mySQLPunk/Form1.PostgreSqlExport.cs b/mySQLPunk/Form1.PostgreSqlExport.cs new file mode 100644 index 0000000..77fedb9 --- /dev/null +++ b/mySQLPunk/Form1.PostgreSqlExport.cs @@ -0,0 +1,65 @@ +using System; +using System.IO; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.PostgreSqlExport; +using PostgreSqlExportResult = mySQLPunk.lib.PostgreSqlExport.PostgreSqlExportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func PostgreSqlExportWizardRunner = null; + internal Action PostgreSqlExportStatusSink = null; + + private void SetPostgreSqlExportStatus(string message) + { + if (PostgreSqlExportStatusSink == null) UpdateMainStatus(message); + else PostgreSqlExportStatusSink(message); + } + + private void OpenPostgreSqlExportWizard(TreeDatabaseTarget target, PostgreSqlExportMode mode = PostgreSqlExportMode.SchemaAndData, string specificTable = null) + { + if (target == null) return; + try + { + string dbStem = string.IsNullOrWhiteSpace(target.DatabaseName) ? "pg_dump" : target.DatabaseName; + string defaultFileName = dbStem + (string.IsNullOrWhiteSpace(specificTable) ? "" : "_" + specificTable) + "_pg_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; + string targetPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultFileName); + var options = new PostgreSqlExportWizardOptions(mode, targetPath); + if (!string.IsNullOrWhiteSpace(specificTable)) + { + options.SelectedTables.Add(specificTable); + } + + PostgreSqlExportResult result; + if (PostgreSqlExportWizardRunner != null) + { + result = PostgreSqlExportWizardRunner(target.Database, target.DatabaseName, options); + } + else + { + using (var wizard = new PostgreSqlExportWizardForm(target.Database, target.DatabaseName, options)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Success + ? string.Format(ReportText.Get("PostgreSQL 傾印完成：{0} 個物件，共 {1:N0} 列 · {2}", "PostgreSQL dump completed: {0} objects, {1:N0} rows · {2}"), result.TableCount + result.ViewCount, result.TotalRows, result.TargetPath) + : (result.Stage == PostgreSqlExportStage.Cancelled + ? ReportText.Get("PostgreSQL 匯出已取消", "PostgreSQL export cancelled") + : ReportText.Get("PostgreSQL 匯出失敗：", "PostgreSQL export failed: ") + result.ErrorMessage); + SetPostgreSqlExportStatus(status); + } + } + catch (Exception ex) + { + SetPostgreSqlExportStatus(ReportText.Get("無法開啟 PostgreSQL 匯出精靈", "Cannot open PostgreSQL export wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.PostgreSqlImport.cs b/mySQLPunk/Form1.PostgreSqlImport.cs new file mode 100644 index 0000000..97acfc8 --- /dev/null +++ b/mySQLPunk/Form1.PostgreSqlImport.cs @@ -0,0 +1,54 @@ +using System; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.PostgreSqlImport; +using PostgreSqlImportResult = mySQLPunk.lib.PostgreSqlImport.PostgreSqlImportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func PostgreSqlImportWizardRunner = null; + internal Action PostgreSqlImportStatusSink = null; + + private void SetPostgreSqlImportStatus(string message) + { + if (PostgreSqlImportStatusSink == null) UpdateMainStatus(message); + else PostgreSqlImportStatusSink(message); + } + + private void OpenPostgreSqlImportWizard(TreeDatabaseTarget target) + { + if (target == null) return; + try + { + PostgreSqlImportResult result; + if (PostgreSqlImportWizardRunner != null) + { + result = PostgreSqlImportWizardRunner(target.Database, target.DatabaseName, ""); + } + else + { + using (var wizard = new PostgreSqlImportWizardForm(target.Database, target.DatabaseName)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Success + ? string.Format(ReportText.Get("PostgreSQL 匯入執行完成：{0} 句成功 · 影響 {1:N0} 列 · 日誌：{2}", "PostgreSQL import execution completed: {0} statements · {1:N0} rows · Log: {2}"), result.StatementsExecuted, result.TotalRowsAffected, result.LogFilePath) + : string.Format(ReportText.Get("PostgreSQL 匯入結束：{0} 失敗 · 日誌：{1}", "PostgreSQL import finished: {0} failed · Log: {1}"), result.StatementsFailed, result.LogFilePath); + SetPostgreSqlImportStatus(status); + RefreshDatabaseObjectNodes(target.DatabaseNode); + } + } + catch (Exception ex) + { + SetPostgreSqlImportStatus(ReportText.Get("無法開啟 PostgreSQL 匯入精靈", "Cannot open PostgreSQL import wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.cs b/mySQLPunk/Form1.cs index c86926d..55a7879 100644 --- a/mySQLPunk/Form1.cs +++ b/mySQLPunk/Form1.cs @@ -6685,6 +6685,11 @@ private void DumpSelectedTableSql(bool dataOnly) OpenSqliteExportWizard(target, dataOnly ? lib.SqliteExport.SqliteExportMode.DataOnly : lib.SqliteExport.SqliteExportMode.SchemaAndData, tableName); return; } + if (IsPostgreSqlTarget(target)) + { + OpenPostgreSqlExportWizard(target, dataOnly ? lib.PostgreSqlExport.PostgreSqlExportMode.DataOnly : lib.PostgreSqlExport.PostgreSqlExportMode.SchemaAndData, tableName); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -6951,6 +6956,12 @@ private void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) OpenSqliteExportWizard(target, tableDataOnly ? lib.SqliteExport.SqliteExportMode.DataOnly : lib.SqliteExport.SqliteExportMode.SchemaAndData, selection == null ? null : selection.ObjectName); return; } + if (IsPostgreSqlTarget(target) && (selection == null || selection.GroupName == "Tables" || selection.GroupName == "Views")) + { + bool tableDataOnly = selection != null && dataOnlyForTable; + OpenPostgreSqlExportWizard(target, tableDataOnly ? lib.PostgreSqlExport.PostgreSqlExportMode.DataOnly : lib.PostgreSqlExport.PostgreSqlExportMode.SchemaAndData, selection == null ? null : selection.ObjectName); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -7041,6 +7052,13 @@ private static bool IsMySqlTarget(TreeDatabaseTarget target) return target != null && string.Equals(target.ProviderName, "mysql", StringComparison.OrdinalIgnoreCase); } + private static bool IsPostgreSqlTarget(TreeDatabaseTarget target) + { + return target != null && + ((target.Database != null && string.Equals(target.Database.ProviderName, "postgresql", StringComparison.OrdinalIgnoreCase)) || + string.Equals(target.ProviderName, "postgresql", StringComparison.OrdinalIgnoreCase)); + } + private async Task ExportMySqlDatabaseWithProgress(TreeDatabaseTarget target, string targetPath, MySqlExportOptions options) { @@ -7302,6 +7320,12 @@ private void DumpSelectedDatabaseSqlWithDialog() return; } + if (IsPostgreSqlTarget(target)) + { + OpenPostgreSqlExportWizard(target, lib.PostgreSqlExport.PostgreSqlExportMode.SchemaAndData); + return; + } + using (SaveFileDialog dialog = new SaveFileDialog()) { dialog.Filter = Localization.T("Common.SqlFilesFilter"); @@ -7371,6 +7395,12 @@ private void ImportSqlWithDialog() return; } + if (IsPostgreSqlTarget(target)) + { + OpenPostgreSqlImportWizard(target); + return; + } + using (OpenFileDialog dialog = new OpenFileDialog()) { dialog.Title = Localization.T("ImportSql.Title"); diff --git a/mySQLPunk/PostgreSqlExportWizardForm.cs b/mySQLPunk/PostgreSqlExportWizardForm.cs new file mode 100644 index 0000000..7515ee3 --- /dev/null +++ b/mySQLPunk/PostgreSqlExportWizardForm.cs @@ -0,0 +1,702 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.PostgreSqlExport; +using PostgreSqlExportResult = mySQLPunk.lib.PostgreSqlExport.PostgreSqlExportResult; + +namespace mySQLPunk +{ + internal sealed class PostgreSqlExportWizardForm : Form + { + private readonly IDatabase db; + private readonly string databaseName; + private readonly PostgreSqlSqlExportService service; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); + private readonly DataGridView tableGrid = new DataGridView(); + private readonly RadioButton schemaData = new RadioButton(), schemaOnly = new RadioButton(), dataOnly = new RadioButton(); + private readonly CheckBox repeatableRead = new CheckBox(), createPostgisExt = new CheckBox(), excludeSpatialMetadata = new CheckBox(); + private readonly CheckBox includeDrop = new CheckBox(), truncateBefore = new CheckBox(); + private readonly CheckBox includeViews = new CheckBox(), includeIndexes = new CheckBox(), includeComments = new CheckBox(), includeManifest = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(), selectionSummary = new Label(), confirmSummary = new Label(); + private readonly Label currentObjectLabel = new Label(), percentLabel = new Label(), writtenLabel = new Label(); + private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); + private readonly Button openFileButton = new Button(), openFolderButton = new Button(); + private readonly ProgressBar overallBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox logList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private List catalog = new List(); + private CancellationTokenSource cancellation; + private PostgreSqlExportProgress lastProgress; + private bool busy, closing, exporting; + private int step; + + internal PostgreSqlExportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal PostgreSqlExportWizardForm(IDatabase db, string databaseName, PostgreSqlExportWizardOptions initialOptions = null) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + this.service = new PostgreSqlSqlExportService(db, databaseName); + + Name = "postgreSqlExportWizard"; + Text = T("匯出資料庫 · PostgreSQL (支援 PostGIS 空間幾何無損備份)", "Export database · PostgreSQL (with PostGIS spatial GIS support)"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1120, 740); + MinimumSize = new Size(680, 500); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("PostgreSQL 資料庫邏輯傾印精靈", "PostgreSQL database logical dump wizard") + " · " + databaseName, Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(); + BuildPage1(initialOptions); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var footer = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.RightToLeft, AutoSize = true, Margin = new Padding(0, 8, 0, 0) }; + nextButton.AutoSize = true; + backButton.AutoSize = true; + cancelButton.AutoSize = true; + nextButton.Text = T("下一步 >", "Next >"); + backButton.Text = T("< 上一步", "< Back"); + cancelButton.Text = T("取消", "Cancel"); + nextButton.Click += (s, e) => Navigate(step + 1); + backButton.Click += (s, e) => Navigate(step - 1); + cancelButton.Click += (s, e) => HandleCancel(); + footer.Controls.Add(nextButton); + footer.Controls.Add(backButton); + footer.Controls.Add(cancelButton); + root.Controls.Add(footer, 0, 3); + + timer.Tick += (s, e) => UpdateLiveTicker(); + + Shown += async (s, e) => + { + await LoadCatalogAsync(); + ApplyInitialSelection(initialOptions); + UpdateStep(); + }; + FormClosing += (s, e) => + { + if (busy && !closing) + { + closing = true; + cancellation?.Cancel(); + } + }; + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, AutoSize = true, WrapContents = false }; + + private void BuildPage0() + { + var p = pages[0]; + p.Padding = new Padding(8); + var layout = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 1, RowCount = 4 }; + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + p.Controls.Add(layout); + + var bar = new FlowLayoutPanel { Dock = DockStyle.Fill, AutoSize = true, Margin = new Padding(0, 0, 0, 8) }; + search.Width = 260; + search.Margin = new Padding(0, 2, 8, 0); + search.TextChanged += (s, e) => FilterGrid(); + var all = new Button { AutoSize = true, Text = T("全選", "Select all"), Margin = new Padding(0, 0, 6, 0) }; + var none = new Button { AutoSize = true, Text = T("全不選", "Clear all"), Margin = new Padding(0, 0, 6, 0) }; + var spatialOnly = new Button { AutoSize = true, Text = T("僅選 PostGIS 空間表 🌐", "Select PostGIS spatial tables 🌐"), Margin = new Padding(0, 0, 6, 0) }; + all.Click += (s, e) => SetAllSelection(true); + none.Click += (s, e) => SetAllSelection(false); + spatialOnly.Click += (s, e) => SetSpatialSelectionOnly(); + + bar.Controls.Add(new Label { Text = T("搜尋物件:", "Filter objects:"), AutoSize = true, Margin = new Padding(0, 6, 4, 0) }); + bar.Controls.Add(search); + bar.Controls.Add(all); + bar.Controls.Add(none); + bar.Controls.Add(spatialOnly); + layout.Controls.Add(bar, 0, 0); + + var tip = new Label + { + Text = T("💡 支援 PostGIS 幾何/地理欄位 (Point, MultiPolygon, 3D/4D)，將以無損十六進位 EWKB 匯出。預設排除龐大空間中繼表 (spatial_ref_sys) 防範還原主鍵衝突。", + "💡 Supports PostGIS geometry/geography (Point, MultiPolygon, 3D/4D) in lossless hex EWKB. Bulky spatial metadata (spatial_ref_sys) is excluded by default."), + AutoSize = true, + ForeColor = Color.DarkSlateBlue, + Margin = new Padding(0, 0, 0, 8) + }; + layout.Controls.Add(tip, 0, 1); + + tableGrid.Dock = DockStyle.Fill; + tableGrid.AllowUserToAddRows = false; + tableGrid.AllowUserToDeleteRows = false; + tableGrid.RowHeadersVisible = false; + tableGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect; + tableGrid.AutoGenerateColumns = false; + tableGrid.Columns.Add(new DataGridViewCheckBoxColumn { Name = "Export", HeaderText = T("匯出", "Export"), Width = 60 }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "Schema", HeaderText = T("結構結構 (Schema)", "Schema"), Width = 110, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "Name", HeaderText = T("物件名稱", "Name"), Width = 220, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "Kind", HeaderText = T("類型", "Type"), Width = 80, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "Spatial", HeaderText = T("🌐 空間特色 (PostGIS)", "🌐 Spatial (PostGIS)"), Width = 280, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "Rows", HeaderText = T("估計列數", "Rows"), Width = 110, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "Bytes", HeaderText = T("大小", "Size"), Width = 110, ReadOnly = true }); + tableGrid.CellValueChanged += (s, e) => UpdateSelectionCounts(); + tableGrid.CurrentCellDirtyStateChanged += (s, e) => { if (tableGrid.IsCurrentCellDirty) tableGrid.CommitEdit(DataGridViewDataErrorContexts.Commit); }; + layout.Controls.Add(tableGrid, 0, 2); + + selectionSummary.AutoSize = true; + selectionSummary.Margin = new Padding(0, 8, 0, 0); + layout.Controls.Add(selectionSummary, 0, 3); + } + + private void BuildPage1(PostgreSqlExportWizardOptions opt) + { + var p = pages[1]; + p.Padding = new Padding(8); + var layout = Stack(); + p.Controls.Add(layout); + + layout.Controls.Add(new Label { Text = T("匯出範圍與模式：", "Export scope and mode:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); + schemaData.Text = T("結構與資料 (完整邏輯 SQL 傾印備份)", "Schema and Data (Full logical SQL dump backup)"); + schemaData.Checked = true; + schemaData.AutoSize = true; + schemaOnly.Text = T("僅結構 (僅產出 DDL、索引、檢視與註解)", "Schema Only (DDL, indexes, views, and comments)"); + schemaOnly.AutoSize = true; + dataOnly.Text = T("僅資料 (僅產出 INSERT 語句)", "Data Only (INSERT statements)"); + dataOnly.AutoSize = true; + layout.Controls.Add(schemaData); + layout.Controls.Add(schemaOnly); + layout.Controls.Add(dataOnly); + + layout.Controls.Add(new Label { Text = T("輸出 SQL 檔案位置：", "Output SQL file destination:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 16, 0, 4) }); + var pathRow = new FlowLayoutPanel { AutoSize = true, Margin = new Padding(0, 0, 0, 12) }; + targetPathBox.Width = 620; + browseButton.Text = T("瀏覽...", "Browse..."); + browseButton.AutoSize = true; + browseButton.Click += (s, e) => ChooseTargetFile(); + pathRow.Controls.Add(targetPathBox); + pathRow.Controls.Add(browseButton); + layout.Controls.Add(pathRow); + + layout.Controls.Add(new Label { Text = T("PostgreSQL / PostGIS 進階選項：", "PostgreSQL / PostGIS advanced options:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 8, 0, 4) }); + repeatableRead.Text = T("開啟 REPEATABLE READ 一致性交易快照 (保證匯出期間點對點資料一致)", "Enable REPEATABLE READ transaction snapshot (guarantees point-in-time consistency)"); + repeatableRead.Checked = true; + repeatableRead.AutoSize = true; + + createPostgisExt.Text = T("於腳本開頭加入 CREATE EXTENSION IF NOT EXISTS postgis; (空間環境自動初始化)", "Include CREATE EXTENSION IF NOT EXISTS postgis; at script header"); + createPostgisExt.Checked = true; + createPostgisExt.AutoSize = true; + + excludeSpatialMetadata.Text = T("排除龐大空間中繼表 (spatial_ref_sys，防範還原時主鍵衝突)", "Exclude bulky spatial metadata tables (spatial_ref_sys, avoids PK collision on restore)"); + excludeSpatialMetadata.Checked = true; + excludeSpatialMetadata.AutoSize = true; + excludeSpatialMetadata.CheckedChanged += (s, e) => + { + foreach (var item in catalog) + { + if (item.IsSpatialMetadata) item.Selected = !excludeSpatialMetadata.Checked; + } + FilterGrid(); + }; + + includeDrop.Text = T("建立前先加入 DROP TABLE IF EXISTS ... CASCADE;", "Include DROP TABLE IF EXISTS ... CASCADE; before creation"); + includeDrop.AutoSize = true; + + truncateBefore.Text = T("插入前先加入 TRUNCATE TABLE ... CASCADE; (清空目標表)", "Include TRUNCATE TABLE ... CASCADE; before inserting data"); + truncateBefore.AutoSize = true; + + includeViews.Text = T("包含檢視表 (CREATE OR REPLACE VIEW)", "Include views (CREATE OR REPLACE VIEW)"); + includeViews.Checked = true; + includeViews.AutoSize = true; + + includeIndexes.Text = T("包含索引 (包含 PostGIS GIST / SP-GIST 空間幾何索引)", "Include indexes (including PostGIS GIST / SP-GIST spatial indexes)"); + includeIndexes.Checked = true; + includeIndexes.AutoSize = true; + + includeComments.Text = T("包含結構註解 (COMMENT ON TABLE / COLUMN)", "Include sidecar comments (COMMENT ON TABLE / COLUMN)"); + includeComments.Checked = true; + includeComments.AutoSize = true; + + includeManifest.Text = T("附加 Native Manifest 清單與 SHA-256 核對碼 (作為還原檢驗依據)", "Append Native Manifest summary & SHA-256 checksum"); + includeManifest.Checked = true; + includeManifest.AutoSize = true; + + layout.Controls.Add(repeatableRead); + layout.Controls.Add(createPostgisExt); + layout.Controls.Add(excludeSpatialMetadata); + layout.Controls.Add(includeDrop); + layout.Controls.Add(truncateBefore); + layout.Controls.Add(includeViews); + layout.Controls.Add(includeIndexes); + layout.Controls.Add(includeComments); + layout.Controls.Add(includeManifest); + + if (opt != null) + { + if (opt.Mode == PostgreSqlExportMode.SchemaOnly) schemaOnly.Checked = true; + else if (opt.Mode == PostgreSqlExportMode.DataOnly) dataOnly.Checked = true; + if (!string.IsNullOrWhiteSpace(opt.TargetPath)) targetPathBox.Text = opt.TargetPath; + } + + if (string.IsNullOrWhiteSpace(targetPathBox.Text)) + { + string desktop = Environment.GetFolderPath(Environment.SpecialFolder.Desktop); + targetPathBox.Text = Path.Combine(desktop, databaseName + "_pg_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"); + } + } + + private void BuildPage2() + { + var p = pages[2]; + p.Padding = new Padding(12); + var layout = Stack(); + p.Controls.Add(layout); + + layout.Controls.Add(new Label { Text = T("請確認以下匯出設定：", "Please confirm export settings:"), Font = UiKit.Title, AutoSize = true, Margin = new Padding(0, 4, 0, 16) }); + confirmSummary.AutoSize = true; + confirmSummary.Font = UiKit.Body; + layout.Controls.Add(confirmSummary); + } + + private void BuildPage3() + { + var p = pages[3]; + p.Padding = new Padding(8); + var layout = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 2, RowCount = 5 }; + layout.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 65)); + layout.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 35)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + p.Controls.Add(layout); + + var leftTop = Stack(); + currentObjectLabel.Text = T("準備開始匯出...", "Preparing export..."); + currentObjectLabel.Font = UiKit.Subtitle; + currentObjectLabel.AutoSize = true; + leftTop.Controls.Add(currentObjectLabel); + + overallBar.Dock = DockStyle.Top; + overallBar.Height = 24; + leftTop.Controls.Add(overallBar); + + var statRow = new FlowLayoutPanel { AutoSize = true, Margin = new Padding(0, 4, 0, 4) }; + percentLabel.Text = "0%"; + writtenLabel.Text = "0 KB"; + elapsedLabel.Text = "00:00:00"; + speedLabel.Text = "0 KB/s"; + statRow.Controls.Add(percentLabel); + statRow.Controls.Add(new Label { Text = " · ", AutoSize = true }); + statRow.Controls.Add(writtenLabel); + statRow.Controls.Add(new Label { Text = " · ", AutoSize = true }); + statRow.Controls.Add(elapsedLabel); + statRow.Controls.Add(new Label { Text = " · ", AutoSize = true }); + statRow.Controls.Add(speedLabel); + leftTop.Controls.Add(statRow); + layout.Controls.Add(leftTop, 0, 0); + + var rightBox = Stack(); + punky.Size = new Size(160, 160); + rightBox.Controls.Add(punky); + + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + soundButton.Click += (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }; + rightBox.Controls.Add(soundButton); + layout.Controls.Add(rightBox, 1, 0); + + logList.Dock = DockStyle.Fill; + logList.Font = new Font("Consolas", 9f); + layout.Controls.Add(logList, 0, 2); + layout.SetColumnSpan(logList, 2); + + resultText.AutoSize = true; + resultText.Font = UiKit.Subtitle; + resultText.ForeColor = Color.DarkGreen; + resultText.Margin = new Padding(0, 8, 0, 8); + layout.Controls.Add(resultText, 0, 3); + layout.SetColumnSpan(resultText, 2); + + var finishActions = new FlowLayoutPanel { AutoSize = true }; + openFileButton.Text = T("開啟輸出檔案", "Open export file"); + openFileButton.AutoSize = true; + openFileButton.Visible = false; + openFileButton.Click += (s, e) => + { + if (File.Exists(targetPathBox.Text)) Process.Start("explorer.exe", "/select,\"" + targetPathBox.Text + "\""); + }; + + openFolderButton.Text = T("開啟所在資料夾", "Open containing folder"); + openFolderButton.AutoSize = true; + openFolderButton.Visible = false; + openFolderButton.Click += (s, e) => + { + string dir = Path.GetDirectoryName(targetPathBox.Text); + if (Directory.Exists(dir)) Process.Start("explorer.exe", dir); + }; + + finishActions.Controls.Add(openFileButton); + finishActions.Controls.Add(openFolderButton); + layout.Controls.Add(finishActions, 0, 4); + layout.SetColumnSpan(finishActions, 2); + } + + private async Task LoadCatalogAsync() + { + try + { + statusLabel.Text = T("正在探測 PostgreSQL 物件結構與 PostGIS 空間中繼資料...", "Scanning PostgreSQL objects and PostGIS metadata..."); + catalog = await service.GetTableCatalogAsync().ConfigureAwait(true); + FilterGrid(); + UpdateSelectionCounts(); + statusLabel.Text = T("物件載入完成。", "Objects loaded."); + } + catch (Exception ex) + { + statusLabel.Text = T("載入失敗：", "Failed to load: ") + ex.Message; + } + } + + private void FilterGrid() + { + string q = search.Text.Trim(); + tableGrid.Rows.Clear(); + + foreach (var item in catalog) + { + if (!string.IsNullOrWhiteSpace(q)) + { + bool match = item.Name.IndexOf(q, StringComparison.OrdinalIgnoreCase) >= 0 || + item.Schema.IndexOf(q, StringComparison.OrdinalIgnoreCase) >= 0 || + item.SpatialFeatureDescription.IndexOf(q, StringComparison.OrdinalIgnoreCase) >= 0; + if (!match) continue; + } + + int idx = tableGrid.Rows.Add( + item.Selected, + item.Schema, + item.Name, + item.IsView ? "View" : "Table", + item.IsSpatial ? "🌐 " + item.SpatialFeatureDescription : "-", + string.Format("{0:N0}", item.RowCount), + FormatBytes(item.EstimatedBytes) + ); + tableGrid.Rows[idx].Tag = item; + } + UpdateSelectionCounts(); + } + + private void SetAllSelection(bool selected) + { + foreach (var item in catalog) + { + item.Selected = selected; + } + FilterGrid(); + } + + private void SetSpatialSelectionOnly() + { + foreach (var item in catalog) + { + item.Selected = item.IsSpatial && !item.IsSpatialMetadata; + } + FilterGrid(); + } + + private void ApplyInitialSelection(PostgreSqlExportWizardOptions initialOptions) + { + if (initialOptions == null || initialOptions.SelectedTables.Count == 0) return; + var set = new HashSet(initialOptions.SelectedTables, StringComparer.OrdinalIgnoreCase); + foreach (var item in catalog) + { + item.Selected = set.Contains(item.FullQualifiedName) || set.Contains(item.DisplayName) || set.Contains(item.Name); + } + FilterGrid(); + } + + private void UpdateSelectionCounts() + { + foreach (DataGridViewRow row in tableGrid.Rows) + { + var item = row.Tag as PostgreSqlTableExportItem; + if (item != null && row.Cells["Export"].Value is bool b) + { + item.Selected = b; + } + } + + int selectedCount = catalog.Count(i => i.Selected); + int spatialCount = catalog.Count(i => i.Selected && i.IsSpatial); + long totalRows = catalog.Where(i => i.Selected).Sum(i => i.RowCount); + selectionSummary.Text = string.Format(T("已選取 {0} / {1} 個物件 (含 {2} 張 PostGIS 空間幾何表，預估共 {3:N0} 列資料)", + "Selected {0} / {1} objects ({2} PostGIS spatial tables, ~{3:N0} total rows)"), + selectedCount, catalog.Count, spatialCount, totalRows); + } + + private void Navigate(int nextStep) + { + if (nextStep < 0 || nextStep > 3) return; + + if (step == 0 && nextStep == 1) + { + UpdateSelectionCounts(); + if (!catalog.Any(i => i.Selected)) + { + MessageBox.Show(this, T("請至少選取一個欲匯出的物件！", "Please select at least one object to export!"), Text, MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + } + else if (step == 1 && nextStep == 2) + { + if (string.IsNullOrWhiteSpace(targetPathBox.Text)) + { + MessageBox.Show(this, T("請指定輸出檔案路徑！", "Please specify output target file path!"), Text, MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + UpdateConfirmation(); + } + else if (step == 2 && nextStep == 3) + { + StartExport(); + } + + step = nextStep; + UpdateStep(); + } + + private void UpdateConfirmation() + { + int selectedCount = catalog.Count(i => i.Selected); + int spatialCount = catalog.Count(i => i.Selected && i.IsSpatial); + long totalRows = catalog.Where(i => i.Selected).Sum(i => i.RowCount); + long totalBytes = catalog.Where(i => i.Selected).Sum(i => i.EstimatedBytes); + + string modeStr = schemaData.Checked ? T("結構與資料 (完整備份)", "Schema and Data (Full backup)") : + schemaOnly.Checked ? T("僅結構 (DDL)", "Schema Only (DDL)") : + T("僅資料 (INSERT)", "Data Only (INSERT)"); + + var sb = new StringBuilder(); + sb.AppendLine(string.Format("• {0}: {1}", T("目標資料庫", "Target Database"), databaseName)); + sb.AppendLine(string.Format("• {0}: {1}", T("匯出模式", "Export Mode"), modeStr)); + sb.AppendLine(string.Format("• {0}: {1}", T("輸出路徑", "Destination Path"), targetPathBox.Text)); + sb.AppendLine(string.Format("• {0}: {1} {2}", T("選取物件", "Selected Objects"), selectedCount, T("個", "items"))); + sb.AppendLine(string.Format("• {0}: {1} {2}", T("PostGIS 空間幾何表", "PostGIS Spatial Tables"), spatialCount, T("個 (零失真 EWKB Hex 傾印)", "items (lossless EWKB Hex)"))); + sb.AppendLine(string.Format("• {0}: ~{1:N0} {2}", T("預估資料列數", "Estimated Rows"), totalRows, T("列", "rows"))); + sb.AppendLine(string.Format("• {0}: ~{1}", T("預估物件體積", "Estimated Size"), FormatBytes(totalBytes))); + sb.AppendLine(string.Format("• {0}: {1}", T("一致性快照", "Consistency Snapshot"), repeatableRead.Checked ? "REPEATABLE READ" : T("無", "None"))); + sb.AppendLine(string.Format("• {0}: {1}", T("PostGIS 擴充模組開關", "CREATE EXTENSION postgis"), createPostgisExt.Checked ? T("是 (自動初始化)", "Yes (auto initialize)") : T("否", "No"))); + sb.AppendLine(string.Format("• {0}: {1}", T("排除空間中繼表", "Exclude Spatial Metadata"), excludeSpatialMetadata.Checked ? T("是 (spatial_ref_sys 已排除)", "Yes (spatial_ref_sys excluded)") : T("否", "No"))); + + confirmSummary.Text = sb.ToString(); + } + + private void UpdateStep() + { + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + string[] stepTitles = new[] + { + T("步驟 1/4：選取欲匯出的 PostgreSQL 物件與空間幾何表", "Step 1/4: Select PostgreSQL objects and spatial tables"), + T("步驟 2/4：輸出路徑與 PostGIS 空間備份進階設定", "Step 2/4: Output path and PostGIS spatial backup settings"), + T("步驟 3/4：確認匯出設定與物件清單摘要", "Step 3/4: Confirm export settings and object summary"), + T("步驟 4/4：執行串流傾印匯出進度", "Step 4/4: Streaming export execution progress") + }; + + stepsLabel.Text = stepTitles[step]; + backButton.Enabled = (step > 0 && step < 3 && !busy); + nextButton.Enabled = (!busy || step == 3); + nextButton.Text = (step == 2) ? T("開始匯出 🚀", "Start Export 🚀") : (step == 3 ? T("完成", "Finished") : T("下一步 >", "Next >")); + cancelButton.Enabled = !busy || exporting; + + if (step == 3 && !exporting) + { + nextButton.Enabled = true; + cancelButton.Enabled = false; + } + } + + private async void StartExport() + { + exporting = true; + busy = true; + backButton.Enabled = false; + nextButton.Enabled = false; + cancellation = new CancellationTokenSource(); + clock.Restart(); + timer.Start(); + + punky.BeginExport(); + + var opt = new PostgreSqlExportWizardOptions + { + Mode = schemaData.Checked ? PostgreSqlExportMode.SchemaAndData : (schemaOnly.Checked ? PostgreSqlExportMode.SchemaOnly : PostgreSqlExportMode.DataOnly), + TargetPath = targetPathBox.Text.Trim(), + UseTransactionSnapshot = repeatableRead.Checked, + IncludeCreateExtensionPostgis = createPostgisExt.Checked, + ExcludeSpatialMetadataTables = excludeSpatialMetadata.Checked, + DropTableIfExists = includeDrop.Checked, + TruncateBeforeInsert = truncateBefore.Checked, + IncludeViews = includeViews.Checked, + IncludeIndexes = includeIndexes.Checked, + IncludeComments = includeComments.Checked, + IncludeManifest = includeManifest.Checked + }; + + foreach (var item in catalog) + { + if (item.Selected) opt.SelectedTables.Add(item.FullQualifiedName); + } + + var progressReporter = new Progress(p => + { + lastProgress = p; + currentObjectLabel.Text = p.Message; + percentLabel.Text = string.Format("{0:0.0}%", p.PercentComplete); + writtenLabel.Text = FormatBytes(p.BytesExported); + overallBar.Value = Math.Min(100, (int)p.PercentComplete); + if (!string.IsNullOrWhiteSpace(p.Message)) + { + logList.Items.Add(string.Format("[{0:HH:mm:ss}] {1}", DateTime.Now, p.Message)); + logList.TopIndex = logList.Items.Count - 1; + } + }); + + try + { + Result = await service.ExportAsync(opt, progressReporter, cancellation.Token).ConfigureAwait(true); + } + catch (Exception ex) + { + Result = new PostgreSqlExportResult { Success = false, ErrorMessage = ex.Message }; + } + finally + { + exporting = false; + busy = false; + clock.Stop(); + timer.Stop(); + punky.EndExport(); + + if (Result != null && Result.Success) + { + resultText.Text = string.Format(T("✅ 傾印成功！共匯出 {0} 個物件，{1:N0} 列資料，輸出至 {2} (耗時 {3:hh\\:mm\\:ss\\.fff})", + "✅ Dump succeeded! Exported {0} objects, {1:N0} rows to {2} (took {3:hh\\:mm\\:ss\\.fff})"), + Result.TableCount + Result.ViewCount, Result.TotalRows, Result.TargetPath, Result.Elapsed); + resultText.ForeColor = Color.DarkGreen; + openFileButton.Visible = true; + openFolderButton.Visible = true; + } + else + { + resultText.Text = T("❌ 匯出未完成：", "❌ Export failed: ") + (Result?.ErrorMessage ?? "Unknown error"); + resultText.ForeColor = Color.Crimson; + } + + UpdateStep(); + } + } + + private void UpdateLiveTicker() + { + if (!clock.IsRunning) return; + var el = clock.Elapsed; + elapsedLabel.Text = string.Format("{0:00}:{1:00}:{2:00}", (int)el.TotalHours, el.Minutes, el.Seconds); + + if (lastProgress != null && el.TotalSeconds > 0) + { + double bytesPerSec = lastProgress.BytesExported / el.TotalSeconds; + speedLabel.Text = FormatBytes((long)bytesPerSec) + "/s"; + } + } + + private void ChooseTargetFile() + { + using (var dialog = new SaveFileDialog()) + { + dialog.Filter = T("SQL 腳本檔案 (*.sql)|*.sql|所有檔案 (*.*)|*.*", "SQL Script Files (*.sql)|*.sql|All Files (*.*)|*.*"); + dialog.FileName = Path.GetFileName(targetPathBox.Text); + if (dialog.ShowDialog(this) == DialogResult.OK) + { + targetPathBox.Text = dialog.FileName; + } + } + } + + private void HandleCancel() + { + if (busy && exporting) + { + if (MessageBox.Show(this, T("匯出正在執行中，確定要中途取消嗎？", "Export is running, are you sure to cancel?"), Text, MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes) + { + cancellation?.Cancel(); + } + } + else + { + Close(); + } + } + + private static string FormatBytes(long bytes) + { + if (bytes < 1024) return bytes + " B"; + if (bytes < 1024 * 1024) return (bytes / 1024.0).ToString("0.0") + " KB"; + if (bytes < 1024 * 1024 * 1024) return (bytes / (1024.0 * 1024.0)).ToString("0.0") + " MB"; + return (bytes / (1024.0 * 1024.0 * 1024.0)).ToString("0.0") + " GB"; + } + } +} diff --git a/mySQLPunk/PostgreSqlImportWizardForm.cs b/mySQLPunk/PostgreSqlImportWizardForm.cs new file mode 100644 index 0000000..72bf237 --- /dev/null +++ b/mySQLPunk/PostgreSqlImportWizardForm.cs @@ -0,0 +1,610 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.PostgreSqlImport; +using PostgreSqlImportResult = mySQLPunk.lib.PostgreSqlImport.PostgreSqlImportResult; + +namespace mySQLPunk +{ + internal sealed class PostgreSqlImportWizardForm : Form + { + private const int MaxLogLines = 2000; + private readonly IDatabase db; + private readonly string databaseName; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox filePathBox = new TextBox(); + private readonly Button browseFileButton = new Button(); + private readonly Label preflightSummary = new Label(), preflightDetails = new Label(); + private readonly ListBox issuesList = new ListBox(); + private readonly RadioButton strategyKeep = new RadioButton(), strategyReplace = new RadioButton(); + private readonly RadioButton strategyDrop = new RadioButton(), strategyTruncate = new RadioButton(); + private readonly CheckBox autoInstallPostgis = new CheckBox(), stopOnError = new CheckBox(), useTransaction = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(); + private readonly Label currentStatementLabel = new Label(), coverageLabel = new Label(), speedLabel = new Label(); + private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), soundButton = new Button(); + private readonly Button openLogButton = new Button(), openFolderButton = new Button(), copyPathButton = new Button(), clearLogButton = new Button(); + private readonly CheckBox autoScrollCheck = new CheckBox(); + private readonly ProgressBar execBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox logList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private PostgreSqlImportPreflightResult preflight; + private CancellationTokenSource cancellation; + private bool busy, executing, closing; + private int step; + private long lastBytes; + + internal PostgreSqlImportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal PostgreSqlImportWizardForm(IDatabase db, string databaseName, string initialSourcePath = "") + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); + + Name = "postgreSqlImportWizard"; + Text = T("匯入 SQL 檔案 · PostgreSQL (支援 PostGIS 空間幾何還原)", "Import SQL file · PostgreSQL (with PostGIS spatial GIS support)"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1120, 750); + MinimumSize = new Size(680, 500); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("PostgreSQL 原生 SQL 匯入精靈", "PostgreSQL native SQL import wizard") + " · " + databaseName, Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(initialSourcePath); + BuildPage1(); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var footer = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.RightToLeft, AutoSize = true, Margin = new Padding(0, 8, 0, 0) }; + nextButton.AutoSize = true; + backButton.AutoSize = true; + cancelButton.AutoSize = true; + nextButton.Text = T("下一步 >", "Next >"); + backButton.Text = T("< 上一步", "< Back"); + cancelButton.Text = T("取消", "Cancel"); + nextButton.Click += (s, e) => Navigate(step + 1); + backButton.Click += (s, e) => Navigate(step - 1); + cancelButton.Click += (s, e) => HandleCancel(); + footer.Controls.Add(nextButton); + footer.Controls.Add(backButton); + footer.Controls.Add(cancelButton); + root.Controls.Add(footer, 0, 3); + + timer.Tick += (s, e) => UpdateLiveTicker(); + + Shown += (s, e) => UpdateStep(); + FormClosing += (s, e) => + { + if (busy && !closing) + { + closing = true; + cancellation?.Cancel(); + } + }; + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, AutoSize = true, WrapContents = false }; + + private void BuildPage0(string initialPath) + { + var p = pages[0]; + p.Padding = new Padding(12); + var layout = Stack(); + p.Controls.Add(layout); + + layout.Controls.Add(new Label { Text = T("選取欲匯入的 PostgreSQL SQL 檔案：", "Select PostgreSQL SQL file to import:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 4, 0, 6) }); + + var row = new FlowLayoutPanel { AutoSize = true, Margin = new Padding(0, 0, 0, 16) }; + filePathBox.Width = 620; + filePathBox.Text = initialPath ?? string.Empty; + browseFileButton.Text = T("瀏覽...", "Browse..."); + browseFileButton.AutoSize = true; + browseFileButton.Click += (s, e) => ChooseSqlFile(); + row.Controls.Add(filePathBox); + row.Controls.Add(browseFileButton); + layout.Controls.Add(row); + + var info = new Label + { + Text = string.Format(T("目標資料庫：{0}\r\n💡 精靈將以狀態機串流掃描腳本編碼、語句數、Dollar-Quotes 區塊與 PostGIS 空間特徵。", + "Target database: {0}\r\n💡 Wizard will stream-scan encoding, statement count, dollar quotes, and PostGIS spatial features."), databaseName), + AutoSize = true, + Font = UiKit.Body, + ForeColor = Color.DarkSlateBlue + }; + layout.Controls.Add(info); + } + + private void BuildPage1() + { + var p = pages[1]; + p.Padding = new Padding(12); + var layout = Stack(); + p.Controls.Add(layout); + + layout.Controls.Add(new Label { Text = T("SQL 腳本預檢報告 (Preflight Report)：", "SQL script preflight report:"), Font = UiKit.Title, AutoSize = true, Margin = new Padding(0, 4, 0, 8) }); + + preflightSummary.Font = UiKit.Subtitle; + preflightSummary.AutoSize = true; + preflightSummary.Margin = new Padding(0, 0, 0, 8); + layout.Controls.Add(preflightSummary); + + preflightDetails.Font = UiKit.Body; + preflightDetails.AutoSize = true; + preflightDetails.Margin = new Padding(0, 0, 0, 12); + layout.Controls.Add(preflightDetails); + + layout.Controls.Add(new Label { Text = T("預檢警示或問題清單：", "Preflight warnings and issues:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); + issuesList.Height = 140; + issuesList.Width = 800; + layout.Controls.Add(issuesList); + } + + private void BuildPage2() + { + var p = pages[2]; + p.Padding = new Padding(12); + var layout = Stack(); + p.Controls.Add(layout); + + layout.Controls.Add(new Label { Text = T("既有物件衝突處理策略：", "Existing objects conflict strategy:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 4, 0, 6) }); + strategyKeep.Text = T("保留現有物件 (CREATE TABLE IF NOT EXISTS，遇衝突略過)", "Keep existing objects (CREATE TABLE IF NOT EXISTS, ignore collisions)"); + strategyKeep.Checked = true; + strategyKeep.AutoSize = true; + + strategyReplace.Text = T("全部替換 (覆寫/替換既有物件)", "Replace all (overwrite existing objects)"); + strategyReplace.AutoSize = true; + + strategyDrop.Text = T("刪表重建 (DROP TABLE IF EXISTS ... CASCADE 後重新建表)", "Drop and recreate (DROP TABLE IF EXISTS ... CASCADE then recreate)"); + strategyDrop.AutoSize = true; + + strategyTruncate.Text = T("清空重匯 (TRUNCATE TABLE ... CASCADE 後寫入資料)", "Truncate and insert (TRUNCATE TABLE ... CASCADE then insert)"); + strategyTruncate.AutoSize = true; + + layout.Controls.Add(strategyKeep); + layout.Controls.Add(strategyReplace); + layout.Controls.Add(strategyDrop); + layout.Controls.Add(strategyTruncate); + + layout.Controls.Add(new Label { Text = T("執行保護與 PostGIS 選項：", "Execution protection and PostGIS options:"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 16, 0, 6) }); + + autoInstallPostgis.Text = T("自動載入 PostGIS 擴充套件 (CREATE EXTENSION IF NOT EXISTS postgis;)", "Auto install PostGIS extension (CREATE EXTENSION IF NOT EXISTS postgis;)"); + autoInstallPostgis.Checked = true; + autoInstallPostgis.AutoSize = true; + + stopOnError.Text = T("發生錯誤時立即停止 (Stop on error)", "Stop immediately on error"); + stopOnError.Checked = true; + stopOnError.AutoSize = true; + + useTransaction.Text = T("以單一交易包覆全腳本 (BEGIN ... COMMIT，遇錯自動 ROLLBACK)", "Wrap entire execution in single transaction (Rollback on failure)"); + useTransaction.Checked = true; + useTransaction.AutoSize = true; + + layout.Controls.Add(autoInstallPostgis); + layout.Controls.Add(stopOnError); + layout.Controls.Add(useTransaction); + } + + private void BuildPage3() + { + var p = pages[3]; + p.Padding = new Padding(8); + var layout = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 2, RowCount = 5 }; + layout.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 65)); + layout.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 35)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + layout.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + p.Controls.Add(layout); + + var leftTop = Stack(); + currentStatementLabel.Text = T("準備開始匯入...", "Preparing import..."); + currentStatementLabel.Font = UiKit.Subtitle; + currentStatementLabel.AutoSize = true; + leftTop.Controls.Add(currentStatementLabel); + + execBar.Dock = DockStyle.Top; + execBar.Height = 24; + leftTop.Controls.Add(execBar); + + var statRow = new FlowLayoutPanel { AutoSize = true, Margin = new Padding(0, 4, 0, 4) }; + coverageLabel.Text = "0%"; + elapsedLabel.Text = "00:00:00"; + speedLabel.Text = "0 KB/s"; + statRow.Controls.Add(coverageLabel); + statRow.Controls.Add(new Label { Text = " · ", AutoSize = true }); + statRow.Controls.Add(elapsedLabel); + statRow.Controls.Add(new Label { Text = " · ", AutoSize = true }); + statRow.Controls.Add(speedLabel); + leftTop.Controls.Add(statRow); + layout.Controls.Add(leftTop, 0, 0); + + var rightBox = Stack(); + punky.Size = new Size(160, 160); + rightBox.Controls.Add(punky); + + mascotHint.Text = T("崩琦正在為你匯入 PostgreSQL GIS 資料！", "Punky is importing PostgreSQL GIS data for you!"); + mascotHint.AutoSize = true; + mascotHint.Font = UiKit.Body; + rightBox.Controls.Add(mascotHint); + + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + soundButton.Click += (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }; + rightBox.Controls.Add(soundButton); + layout.Controls.Add(rightBox, 1, 0); + + // 終端機風格 2,000 行滑動視窗即時日誌 + var logPanel = new Panel { Dock = DockStyle.Fill }; + var logHeader = new FlowLayoutPanel { Dock = DockStyle.Top, Height = 30, AutoSize = false }; + logHeader.Controls.Add(new Label { Text = T("即時執行日誌 (保留最新 2,000 行)：", "Live Execution Log (rolling 2,000 lines):"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 4, 12, 0) }); + autoScrollCheck.Text = T("自動滾動", "Auto-scroll"); + autoScrollCheck.Checked = true; + autoScrollCheck.AutoSize = true; + logHeader.Controls.Add(autoScrollCheck); + clearLogButton.Text = T("清空日誌", "Clear"); + clearLogButton.AutoSize = true; + clearLogButton.Click += (s, e) => logList.Items.Clear(); + logHeader.Controls.Add(clearLogButton); + logPanel.Controls.Add(logHeader); + + logList.Dock = DockStyle.Fill; + logList.Font = new Font("Consolas", 9f); + logList.BackColor = Color.FromArgb(20, 20, 20); + logList.ForeColor = Color.FromArgb(200, 240, 200); + logPanel.Controls.Add(logList); + logList.BringToFront(); + + layout.Controls.Add(logPanel, 0, 2); + layout.SetColumnSpan(logPanel, 2); + + resultText.AutoSize = true; + resultText.Font = UiKit.Subtitle; + resultText.ForeColor = Color.DarkGreen; + resultText.Margin = new Padding(0, 8, 0, 8); + layout.Controls.Add(resultText, 0, 3); + layout.SetColumnSpan(resultText, 2); + + var actRow = new FlowLayoutPanel { AutoSize = true }; + openLogButton.Text = T("開啟詳細 Log 檔案", "Open Log File"); + openLogButton.AutoSize = true; + openLogButton.Visible = false; + openLogButton.Click += (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogFilePath) && File.Exists(Result.LogFilePath)) + { + Process.Start(new ProcessStartInfo("notepad.exe", Result.LogFilePath) { UseShellExecute = true }); + } + }; + + openFolderButton.Text = T("開啟 Log 資料夾", "Open Log Folder"); + openFolderButton.AutoSize = true; + openFolderButton.Visible = false; + openFolderButton.Click += (s, e) => + { + if (!string.IsNullOrWhiteSpace(Result?.LogFilePath) && File.Exists(Result.LogFilePath)) + { + Process.Start("explorer.exe", string.Format("/select,\"{0}\"", Result.LogFilePath)); + } + }; + + actRow.Controls.Add(openLogButton); + actRow.Controls.Add(openFolderButton); + layout.Controls.Add(actRow, 0, 4); + layout.SetColumnSpan(actRow, 2); + } + + private void ChooseSqlFile() + { + using (var dialog = new OpenFileDialog()) + { + dialog.Filter = T("SQL 腳本檔案 (*.sql)|*.sql|所有檔案 (*.*)|*.*", "SQL Script Files (*.sql)|*.sql|All Files (*.*)|*.*"); + if (dialog.ShowDialog(this) == DialogResult.OK) + { + filePathBox.Text = dialog.FileName; + } + } + } + + private async void Navigate(int nextStep) + { + if (nextStep < 0 || nextStep > 3) return; + + if (step == 0 && nextStep == 1) + { + string path = filePathBox.Text.Trim(); + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + MessageBox.Show(this, T("請指定有效的 SQL 腳本檔案！", "Please specify a valid SQL script file!"), Text, MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + + await RunPreflightAsync(path); + if (preflight != null && !preflight.CanProceed) + { + MessageBox.Show(this, T("SQL 腳本未通過安全預檢，無法繼續匯入！", "SQL script failed preflight checks!"), Text, MessageBoxButtons.OK, MessageBoxIcon.Error); + return; + } + } + else if (step == 2 && nextStep == 3) + { + StartImport(); + } + + step = nextStep; + UpdateStep(); + } + + private async Task RunPreflightAsync(string path) + { + busy = true; + statusLabel.Text = T("正在對 SQL 檔案執行狀態機串流預檢...", "Scanning SQL script with state-machine preflight..."); + try + { + var scanner = new PostgreSqlImportPreflightScanner(); + preflight = await scanner.ScanAsync(path).ConfigureAwait(true); + + preflightSummary.Text = preflight.CanProceed + ? T("✅ 預檢通過！腳本結構正常。", "✅ Preflight passed! Script syntax is normal.") + : T("❌ 預檢發現重大問題！", "❌ Preflight found critical issues!"); + preflightSummary.ForeColor = preflight.CanProceed ? Color.DarkGreen : Color.Crimson; + + var sb = new StringBuilder(); + sb.AppendLine(string.Format("• {0}: {1}", T("檔案編碼", "Encoding"), preflight.DetectedEncoding.WebName.ToUpperInvariant())); + sb.AppendLine(string.Format("• {0}: {1:N0} KB", T("檔案體積", "File Size"), preflight.FileSize / 1024)); + sb.AppendLine(string.Format("• {0}: {1:N0} {2}", T("估計語句總數", "Estimated Statements"), preflight.StatementCount, T("條", "statements"))); + sb.AppendLine(string.Format("• {0}: {1}", T("PostGIS 空間特徵", "PostGIS Spatial"), preflight.HasPostgisFeatures ? "🌐 " + T("偵測到 PostGIS 空間特徵 (含 geometry / geography / raster 關鍵字)", "PostGIS spatial features detected") : T("無", "None"))); + sb.AppendLine(string.Format("• {0}: {1}", T("Dollar Quotes 區塊", "Dollar Quotes Blocks"), preflight.HasDollarQuotes ? string.Format(T("偵測到 {0} 個 $$...$$ 函數區塊 (將精確解析)", "{0} blocks detected"), preflight.DollarQuoteBlocksCount) : T("無", "None"))); + sb.AppendLine(string.Format("• {0}: {1}", T("Native Manifest 驗證標記", "Native Manifest Signature"), preflight.ContainsManifest ? T("已偵測到結尾清單與 SHA-256 核對碼", "Detected footer summary & SHA-256") : T("未附帶 Manifest 清單", "No Manifest signature"))); + + preflightDetails.Text = sb.ToString(); + + issuesList.Items.Clear(); + if (preflight.Issues.Count == 0) + { + issuesList.Items.Add(T("（無任何警告或錯誤）", "(No warnings or errors)")); + } + else + { + foreach (var issue in preflight.Issues) issuesList.Items.Add(issue); + } + + statusLabel.Text = T("預檢完成。", "Preflight complete."); + } + catch (Exception ex) + { + preflightSummary.Text = T("❌ 預檢過程發生例外：", "❌ Preflight exception: ") + ex.Message; + preflightSummary.ForeColor = Color.Crimson; + } + finally + { + busy = false; + } + } + + private void UpdateStep() + { + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + string[] stepTitles = new[] + { + T("步驟 1/4：選取欲匯入的 PostgreSQL SQL 檔案與目標庫", "Step 1/4: Select PostgreSQL SQL file and target database"), + T("步驟 2/4：SQL 腳本安全預檢與 PostGIS 空間特徵報告", "Step 2/4: SQL script preflight and PostGIS spatial report"), + T("步驟 3/4：物件衝突處理策略與交易保護設定", "Step 3/4: Conflict strategy and transaction settings"), + T("步驟 4/4：執行串流匯入與即時終端機日誌", "Step 4/4: Streaming execution and live terminal log") + }; + + stepsLabel.Text = stepTitles[step]; + backButton.Enabled = (step > 0 && step < 3 && !busy); + nextButton.Enabled = (!busy || step == 3); + nextButton.Text = (step == 2) ? T("開始匯入 🚀", "Start Import 🚀") : (step == 3 ? T("完成", "Finished") : T("下一步 >", "Next >")); + cancelButton.Enabled = !busy || executing; + + if (step == 3 && !executing) + { + nextButton.Enabled = true; + cancelButton.Enabled = false; + } + } + + private async void StartImport() + { + executing = true; + busy = true; + backButton.Enabled = false; + nextButton.Enabled = false; + cancellation = new CancellationTokenSource(); + clock.Restart(); + timer.Start(); + + punky.BeginExport(); + + var strategy = strategyKeep.Checked ? PostgreSqlConflictStrategy.KeepExisting : + strategyReplace.Checked ? PostgreSqlConflictStrategy.ReplaceAll : + strategyDrop.Checked ? PostgreSqlConflictStrategy.DropAndRecreate : + PostgreSqlConflictStrategy.TruncateAndInsert; + + var opt = new PostgreSqlImportOptions + { + ConflictStrategy = strategy, + AutoInstallPostgis = autoInstallPostgis.Checked, + StopOnError = stopOnError.Checked, + UseTransaction = useTransaction.Checked + }; + + var req = new PostgreSqlImportRequest(filePathBox.Text.Trim(), databaseName, opt); + var runner = new PostgreSqlImportRunner(db, req); + + var progressReporter = new Progress(p => + { + coverageLabel.Text = string.Format("{0:0.0}% ({1:N0} / {2:N0})", p.PercentComplete, p.CurrentStatement, p.TotalStatements); + execBar.Value = Math.Min(100, (int)p.PercentComplete); + currentStatementLabel.Text = p.Message; + lastBytes = p.BytesRead; + }); + + Action terminalSink = line => + { + if (InvokeRequired) + { + BeginInvoke(new Action(() => AppendTerminalLine(line))); + } + else + { + AppendTerminalLine(line); + } + }; + + try + { + Result = await runner.RunAsync(progressReporter, terminalSink, cancellation.Token).ConfigureAwait(true); + } + catch (Exception ex) + { + Result = new PostgreSqlImportResult { Success = false, ErrorMessage = ex.Message }; + } + finally + { + executing = false; + busy = false; + clock.Stop(); + timer.Stop(); + punky.EndExport(); + + if (Result != null && Result.Success) + { + resultText.Text = string.Format(T("✅ 匯入成功！共執行 {0:N0} 條語句，寫入/影響 {1:N0} 列資料 (耗時 {2:hh\\:mm\\:ss\\.fff})", + "✅ Import succeeded! Executed {0:N0} statements, affected {1:N0} rows (took {2:hh\\:mm\\:ss\\.fff})"), + Result.StatementsExecuted, Result.TotalRowsAffected, Result.Elapsed); + resultText.ForeColor = Color.DarkGreen; + openLogButton.Visible = true; + openFolderButton.Visible = true; + } + else + { + resultText.Text = T("❌ 匯入未完成：", "❌ Import failed: ") + (Result?.ErrorMessage ?? "Unknown error"); + resultText.ForeColor = Color.Crimson; + } + + UpdateStep(); + } + } + + private void AppendTerminalLine(string line) + { + if (logList.Items.Count >= MaxLogLines) + { + logList.BeginUpdate(); + try + { + int excess = logList.Items.Count - MaxLogLines + 1; + for (int i = 0; i < excess && logList.Items.Count > 0; i++) + { + logList.Items.RemoveAt(0); + } + logList.Items.Add(line); + } + finally + { + logList.EndUpdate(); + } + } + else + { + logList.Items.Add(line); + } + + if (autoScrollCheck.Checked && logList.Items.Count > 0) + { + logList.TopIndex = logList.Items.Count - 1; + } + } + + private void UpdateLiveTicker() + { + if (!clock.IsRunning) return; + var el = clock.Elapsed; + elapsedLabel.Text = string.Format("{0:00}:{1:00}:{2:00}", (int)el.TotalHours, el.Minutes, el.Seconds); + + if (lastBytes > 0 && el.TotalSeconds > 0) + { + double bytesPerSec = lastBytes / el.TotalSeconds; + speedLabel.Text = FormatBytes((long)bytesPerSec) + "/s"; + } + } + + private void HandleCancel() + { + if (busy && executing) + { + if (MessageBox.Show(this, T("匯入正在執行中，確定要中止嗎？", "Import is running, are you sure to cancel?"), Text, MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes) + { + cancellation?.Cancel(); + } + } + else + { + Close(); + } + } + + private static string FormatBytes(long bytes) + { + if (bytes < 1024) return bytes + " B"; + if (bytes < 1024 * 1024) return (bytes / 1024.0).ToString("0.0") + " KB"; + if (bytes < 1024 * 1024 * 1024) return (bytes / (1024.0 * 1024.0)).ToString("0.0") + " MB"; + return (bytes / (1024.0 * 1024.0 * 1024.0)).ToString("0.0") + " GB"; + } + } +} diff --git a/mySQLPunk/lib/PostgreSqlExport/PostgreSqlExportModels.cs b/mySQLPunk/lib/PostgreSqlExport/PostgreSqlExportModels.cs new file mode 100644 index 0000000..579f389 --- /dev/null +++ b/mySQLPunk/lib/PostgreSqlExport/PostgreSqlExportModels.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; + +namespace mySQLPunk.lib.PostgreSqlExport +{ + public enum PostgreSqlExportMode + { + SchemaAndData, + DataOnly, + SchemaOnly + } + + public enum PostgreSqlExportStage + { + NotStarted, + Preflight, + ExportingSchema, + ExportingData, + Finalizing, + Completed, + Cancelled, + Failed + } + + public sealed class PostgreSqlTableExportItem + { + public string Schema { get; set; } = "public"; + public string Name { get; set; } = string.Empty; + public string FullQualifiedName => "\"" + (string.IsNullOrWhiteSpace(Schema) ? "public" : Schema) + "\".\"" + Name + "\""; + public string DisplayName => string.Equals(Schema, "public", StringComparison.OrdinalIgnoreCase) ? Name : Schema + "." + Name; + public long RowCount { get; set; } + public long EstimatedBytes { get; set; } + public bool IsView { get; set; } + public bool IsSpatial { get; set; } + public bool IsSpatialMetadata { get; set; } + public List SpatialColumnNames { get; } = new List(); + public string SpatialFeatureDescription { get; set; } = string.Empty; + public bool Selected { get; set; } = true; + } + + public sealed class PostgreSqlExportWizardOptions + { + public PostgreSqlExportMode Mode { get; set; } = PostgreSqlExportMode.SchemaAndData; + public string TargetPath { get; set; } = string.Empty; + public List SelectedTables { get; } = new List(); + public bool UseTransactionSnapshot { get; set; } = true; // REPEATABLE READ + public bool ExcludeSpatialMetadataTables { get; set; } = true; // 排除龐大空間中繼表 (spatial_ref_sys) + public bool IncludeCreateExtensionPostgis { get; set; } = true; // CREATE EXTENSION IF NOT EXISTS postgis + public bool DropTableIfExists { get; set; } = false; + public bool TruncateBeforeInsert { get; set; } = false; + public bool UseMultiRowInsert { get; set; } = true; + public int BatchSize { get; set; } = 1000; + public bool IncludeViews { get; set; } = true; + public bool IncludeIndexes { get; set; } = true; + public bool IncludeTriggers { get; set; } = true; + public bool IncludeComments { get; set; } = true; + public bool IncludeSequences { get; set; } = true; + public bool IncludeManifest { get; set; } = true; + + public PostgreSqlExportWizardOptions() { } + + public PostgreSqlExportWizardOptions(PostgreSqlExportMode mode, string targetPath) + { + Mode = mode; + TargetPath = targetPath ?? string.Empty; + } + } + + public sealed class PostgreSqlExportProgress + { + public PostgreSqlExportStage Stage { get; set; } = PostgreSqlExportStage.NotStarted; + public string CurrentTable { get; set; } = string.Empty; + public string CurrentSchema { get; set; } = string.Empty; + public int TablesCompleted { get; set; } + public int TotalTables { get; set; } + public long RowsExported { get; set; } + public long TotalRows { get; set; } + public long BytesExported { get; set; } + public string Message { get; set; } = string.Empty; + + public double PercentComplete + { + get + { + if (TotalRows > 0) + { + double p = (double)RowsExported / TotalRows * 100.0; + return p > 100.0 ? 100.0 : p; + } + if (TotalTables > 0) + { + double p = (double)TablesCompleted / TotalTables * 100.0; + return p > 100.0 ? 100.0 : p; + } + return 0.0; + } + } + } + + public sealed class PostgreSqlExportResult + { + public bool Success { get; set; } + public PostgreSqlExportStage Stage { get; set; } + public int TableCount { get; set; } + public int ViewCount { get; set; } + public long TotalRows { get; set; } + public long BytesWritten { get; set; } + public string TargetPath { get; set; } = string.Empty; + public TimeSpan Elapsed { get; set; } + public string ErrorMessage { get; set; } = string.Empty; + public Dictionary ManifestTableRows { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public string ManifestSha256 { get; set; } = string.Empty; + } +} diff --git a/mySQLPunk/lib/PostgreSqlExport/PostgreSqlSqlExportService.cs b/mySQLPunk/lib/PostgreSqlExport/PostgreSqlSqlExportService.cs new file mode 100644 index 0000000..a5dce41 --- /dev/null +++ b/mySQLPunk/lib/PostgreSqlExport/PostgreSqlSqlExportService.cs @@ -0,0 +1,756 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.PostgreSqlExport +{ + public sealed class PostgreSqlSqlExportService + { + private const int BufferSize = 65536; // 64KB + private readonly IDatabase db; + private readonly string databaseName; + + private static readonly HashSet SpatialMetadataTableNames = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "spatial_ref_sys", "spatial_ref_sys_aux", "geometry_columns", "geography_columns", + "raster_columns", "raster_overviews", "layer", "topology" + }; + + public PostgreSqlSqlExportService(IDatabase db, string databaseName) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = databaseName ?? string.Empty; + } + + public async Task> GetTableCatalogAsync(CancellationToken cancellationToken = default(CancellationToken)) + { + return await Task.Run(() => GetTableCatalog(cancellationToken), cancellationToken).ConfigureAwait(false); + } + + public List GetTableCatalog(CancellationToken cancellationToken = default(CancellationToken)) + { + var list = new List(); + + // 1. 取得所有一般資料表與檢視 + string catalogSql = @" + SELECT + n.nspname AS ""Schema"", + c.relname AS ""Name"", + c.relkind AS ""Kind"", + pg_total_relation_size(c.oid) AS ""EstimatedBytes"", + COALESCE(s.n_live_tup, 0) AS ""RowCount"" + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_stat_user_tables s ON s.relid = c.oid + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + AND c.relkind IN ('r', 'p', 'v', 'm') + ORDER BY n.nspname, c.relname;"; + + DataTable dt = db.SelectSQL(catalogSql); + var itemMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (DataRow row in dt.Rows) + { + cancellationToken.ThrowIfCancellationRequested(); + string schema = row["Schema"]?.ToString() ?? "public"; + string name = row["Name"]?.ToString() ?? string.Empty; + string kind = row["Kind"]?.ToString() ?? "r"; + long bytes = 0; + long rows = 0; + if (row["EstimatedBytes"] != DBNull.Value) long.TryParse(row["EstimatedBytes"].ToString(), out bytes); + if (row["RowCount"] != DBNull.Value) long.TryParse(row["RowCount"].ToString(), out rows); + + bool isView = (kind == "v" || kind == "m"); + bool isMeta = SpatialMetadataTableNames.Contains(name); + + var item = new PostgreSqlTableExportItem + { + Schema = schema, + Name = name, + EstimatedBytes = bytes, + RowCount = rows, + IsView = isView, + IsSpatialMetadata = isMeta, + Selected = !isMeta // 龐大空間中繼表預設不勾選 + }; + + list.Add(item); + itemMap[schema + "." + name] = item; + } + + // 2. 探測 PostGIS 空間幾何欄位 (geometry, geography, raster) + try + { + string spatialColsSql = @" + SELECT + c.table_schema, + c.table_name, + c.column_name, + c.udt_name + FROM information_schema.columns c + WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + AND (c.udt_name IN ('geometry', 'geography', 'raster') OR c.data_type = 'USER-DEFINED');"; + + DataTable spatialDt = db.SelectSQL(spatialColsSql); + foreach (DataRow row in spatialDt.Rows) + { + string schema = row["table_schema"]?.ToString() ?? ""; + string name = row["table_name"]?.ToString() ?? ""; + string col = row["column_name"]?.ToString() ?? ""; + string udt = row["udt_name"]?.ToString() ?? "geometry"; + + string key = schema + "." + name; + if (itemMap.TryGetValue(key, out var item)) + { + item.IsSpatial = true; + if (!item.SpatialColumnNames.Contains(col)) + { + item.SpatialColumnNames.Add(col); + } + item.SpatialFeatureDescription = "PostGIS " + udt.ToUpperInvariant(); + } + } + + // 3. 嘗試讀取 geometry_columns 取得精準 SRID 與空間類型 + try + { + string geomViewSql = @" + SELECT f_table_schema, f_table_name, f_geometry_column, coord_dimension, srid, type + FROM geometry_columns;"; + DataTable geomViewDt = db.SelectSQL(geomViewSql); + foreach (DataRow row in geomViewDt.Rows) + { + string schema = row["f_table_schema"]?.ToString() ?? ""; + string name = row["f_table_name"]?.ToString() ?? ""; + string col = row["f_geometry_column"]?.ToString() ?? ""; + string dim = row["coord_dimension"]?.ToString() ?? "2"; + string srid = row["srid"]?.ToString() ?? "0"; + string type = row["type"]?.ToString() ?? "GEOMETRY"; + + string key = schema + "." + name; + if (itemMap.TryGetValue(key, out var item)) + { + item.IsSpatial = true; + item.SpatialFeatureDescription = string.Format("PostGIS: {0} ({1}, SRID {2}, {3}D)", col, type, srid, dim); + } + } + } + catch + { + // geometry_columns view 可能不存在或未裝 PostGIS，安全略過 + } + } + catch + { + // information_schema 查空間欄位若失敗不中斷基本結構清單 + } + + return list; + } + + public async Task ExportAsync( + PostgreSqlExportWizardOptions options, + IProgress progress = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrWhiteSpace(options.TargetPath)) throw new ArgumentException("Target path cannot be empty.", nameof(options.TargetPath)); + + var result = new PostgreSqlExportResult + { + TargetPath = options.TargetPath, + Stage = PostgreSqlExportStage.Preflight + }; + + var clock = Stopwatch.StartNew(); + string stagingFile = options.TargetPath + ".tmp_" + Guid.NewGuid().ToString("N"); + string dir = Path.GetDirectoryName(options.TargetPath); + if (!string.IsNullOrWhiteSpace(dir) && !Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + FileStream fs = null; + StreamWriter writer = null; + IncrementalHash sha256 = null; + + try + { + progress?.Report(new PostgreSqlExportProgress + { + Stage = PostgreSqlExportStage.Preflight, + Message = "正在探測資料庫結構與 PostGIS 空間中繼資料..." + }); + + var catalog = await GetTableCatalogAsync(cancellationToken).ConfigureAwait(false); + var selectedItems = new List(); + var selectedSet = new HashSet(options.SelectedTables, StringComparer.OrdinalIgnoreCase); + + bool hasSpatialObjects = false; + foreach (var item in catalog) + { + bool match = selectedSet.Count == 0 || + selectedSet.Contains(item.FullQualifiedName) || + selectedSet.Contains(item.DisplayName) || + selectedSet.Contains(item.Name); + + if (match) + { + selectedItems.Add(item); + if (item.IsSpatial) hasSpatialObjects = true; + } + } + + fs = new FileStream(stagingFile, FileMode.Create, FileAccess.Write, FileShare.None, BufferSize); + writer = new StreamWriter(fs, new UTF8Encoding(false), BufferSize); + sha256 = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + void WriteLine(string text) + { + cancellationToken.ThrowIfCancellationRequested(); + writer.WriteLine(text); + byte[] bytes = Encoding.UTF8.GetBytes(text + "\n"); + sha256.AppendData(bytes); + result.BytesWritten += bytes.Length; + } + + // 腳本檔頭 + WriteLine("-- ===================================================================="); + WriteLine("-- mySQLPunk PostgreSQL Logical SQL Dump"); + WriteLine(string.Format("-- Database: {0}", databaseName)); + WriteLine(string.Format("-- Generated At: {0:yyyy-MM-dd HH:mm:ss} UTC", DateTime.UtcNow)); + WriteLine("-- Provider: PostgreSQL / PostGIS"); + WriteLine("-- ===================================================================="); + WriteLine(""); + WriteLine("SET statement_timeout = 0;"); + WriteLine("SET lock_timeout = 0;"); + WriteLine("SET client_encoding = 'UTF8';"); + WriteLine("SET standard_conforming_strings = on;"); + WriteLine("SET check_function_bodies = false;"); + WriteLine("SET client_min_messages = warning;"); + WriteLine("SET row_security = off;"); + WriteLine(""); + + // PostGIS 擴充模組開頭宣告 + if (options.IncludeCreateExtensionPostgis && hasSpatialObjects) + { + WriteLine("-- 🌐 PostGIS 空間延伸模組自動載入"); + WriteLine("CREATE EXTENSION IF NOT EXISTS postgis;"); + WriteLine(""); + } + + // 交易快照 (REPEATABLE READ) + if (options.UseTransactionSnapshot) + { + WriteLine("-- 開啟 REPEATABLE READ 一致性交易快照"); + WriteLine("BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); + WriteLine(""); + } + + int totalTables = selectedItems.Count; + int tableIndex = 0; + long totalRowsExported = 0; + + // 1. 結構 (DDL) 匯出 + if (options.Mode == PostgreSqlExportMode.SchemaAndData || options.Mode == PostgreSqlExportMode.SchemaOnly) + { + result.Stage = PostgreSqlExportStage.ExportingSchema; + foreach (var item in selectedItems) + { + cancellationToken.ThrowIfCancellationRequested(); + tableIndex++; + + progress?.Report(new PostgreSqlExportProgress + { + Stage = PostgreSqlExportStage.ExportingSchema, + CurrentSchema = item.Schema, + CurrentTable = item.Name, + TablesCompleted = tableIndex - 1, + TotalTables = totalTables, + BytesExported = result.BytesWritten, + Message = string.Format("正在產生結構 DDL: {0} ({1}/{2})", item.DisplayName, tableIndex, totalTables) + }); + + WriteLine(string.Format("-- --------------------------------------------------------------------")); + WriteLine(string.Format("-- Table Structure for: {0}", item.FullQualifiedName)); + if (item.IsSpatial) + { + WriteLine(string.Format("-- Spatial Feature: {0}", item.SpatialFeatureDescription)); + } + WriteLine(string.Format("-- --------------------------------------------------------------------")); + + if (options.DropTableIfExists) + { + WriteLine(string.Format("DROP TABLE IF EXISTS {0} CASCADE;", item.FullQualifiedName)); + } + + string ddl = GetTableDdl(item.Schema, item.Name, options); + if (!string.IsNullOrWhiteSpace(ddl)) + { + WriteLine(ddl.Trim()); + WriteLine(""); + } + + if (item.IsView) result.ViewCount++; + else result.TableCount++; + } + } + + // 2. 資料 (Data) 匯出 + if (options.Mode == PostgreSqlExportMode.SchemaAndData || options.Mode == PostgreSqlExportMode.DataOnly) + { + result.Stage = PostgreSqlExportStage.ExportingData; + tableIndex = 0; + + foreach (var item in selectedItems) + { + cancellationToken.ThrowIfCancellationRequested(); + tableIndex++; + if (item.IsView) continue; // 檢視不傾印資料 + + progress?.Report(new PostgreSqlExportProgress + { + Stage = PostgreSqlExportStage.ExportingData, + CurrentSchema = item.Schema, + CurrentTable = item.Name, + TablesCompleted = tableIndex - 1, + TotalTables = totalTables, + RowsExported = totalRowsExported, + BytesExported = result.BytesWritten, + Message = string.Format("正在匯出資料: {0} ({1}/{2})", item.DisplayName, tableIndex, totalTables) + }); + + WriteLine(string.Format("-- --------------------------------------------------------------------")); + WriteLine(string.Format("-- Data for Table: {0}", item.FullQualifiedName)); + WriteLine(string.Format("-- --------------------------------------------------------------------")); + + if (options.TruncateBeforeInsert) + { + WriteLine(string.Format("TRUNCATE TABLE {0} CASCADE;", item.FullQualifiedName)); + } + + long tableRows = ExportTableData(item, writer, sha256, WriteLine, options, cancellationToken); + totalRowsExported += tableRows; + result.ManifestTableRows[item.DisplayName] = tableRows; + WriteLine(""); + } + } + + // 3. 關閉交易 + if (options.UseTransactionSnapshot) + { + WriteLine("COMMIT;"); + WriteLine(""); + } + + // 4. Native Manifest 結尾清單與 SHA-256 + if (options.IncludeManifest) + { + WriteLine("-- ===================================================================="); + WriteLine("-- mySQLPunk Native Manifest"); + WriteLine("-- ===================================================================="); + WriteLine(string.Format("-- Manifest-Version: 1.0")); + WriteLine(string.Format("-- Manifest-Database: {0}", databaseName)); + WriteLine(string.Format("-- Manifest-Provider: postgresql")); + WriteLine(string.Format("-- Manifest-Total-Tables: {0}", result.ManifestTableRows.Count)); + + foreach (var kvp in result.ManifestTableRows) + { + WriteLine(string.Format("-- Manifest-Table: {0} | Rows: {1}", kvp.Key, kvp.Value)); + } + + byte[] hashBytes = sha256.GetHashAndReset(); + string sha256Hex = BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant(); + result.ManifestSha256 = sha256Hex; + WriteLine(string.Format("-- Manifest-Checksum-SHA256: {0}", sha256Hex)); + WriteLine("-- ===================================================================="); + } + + await writer.FlushAsync().ConfigureAwait(false); + writer.Close(); + writer = null; + fs.Close(); + fs = null; + + // 原子替換 + if (File.Exists(options.TargetPath)) + { + File.Delete(options.TargetPath); + } + File.Move(stagingFile, options.TargetPath); + + result.Success = true; + result.Stage = PostgreSqlExportStage.Completed; + result.TotalRows = totalRowsExported; + clock.Stop(); + result.Elapsed = clock.Elapsed; + + progress?.Report(new PostgreSqlExportProgress + { + Stage = PostgreSqlExportStage.Completed, + TablesCompleted = totalTables, + TotalTables = totalTables, + RowsExported = totalRowsExported, + TotalRows = totalRowsExported, + BytesExported = result.BytesWritten, + Message = string.Format("PostgreSQL 傾印完成：{0} 張物件，共 {1:N0} 列資料。", totalTables, totalRowsExported) + }); + + return result; + } + catch (OperationCanceledException) + { + result.Success = false; + result.Stage = PostgreSqlExportStage.Cancelled; + result.ErrorMessage = "使用者取消了匯出操作。"; + return result; + } + catch (Exception ex) + { + result.Success = false; + result.Stage = PostgreSqlExportStage.Failed; + result.ErrorMessage = ex.Message; + return result; + } + finally + { + if (writer != null) { try { writer.Dispose(); } catch { } } + if (fs != null) { try { fs.Dispose(); } catch { } } + if (File.Exists(stagingFile)) + { + try { File.Delete(stagingFile); } catch { } + } + } + } + + private string GetTableDdl(string schema, string tableName, PostgreSqlExportWizardOptions options) + { + var p = new Dictionary { { "schema", schema }, { "tableName", tableName } }; + + // 查詢精確欄位定義 (包含 pg_catalog.format_type 原生型別、預設值、Generated Columns 與註解) + string colSql = @" + SELECT + c.column_name, + c.data_type, + c.is_nullable, + c.column_default, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS ""ProviderType"", + c.is_identity AS ""IsIdentity"", + c.identity_generation AS ""IdentityGeneration"", + c.is_generated AS ""IsGenerated"", + COALESCE(pg_get_expr(ad.adbin, ad.adrelid), '') AS ""GenerationExpression"", + COALESCE(col_description(cls.oid, a.attnum), '') AS ""Comment"" + FROM information_schema.columns c + LEFT JOIN pg_namespace n ON n.nspname = c.table_schema + LEFT JOIN pg_class cls ON cls.relnamespace = n.oid AND cls.relname = c.table_name + LEFT JOIN pg_attribute a ON a.attrelid = cls.oid AND a.attname = c.column_name AND a.attnum > 0 AND NOT a.attisdropped + LEFT JOIN pg_attrdef ad ON ad.adrelid = cls.oid AND ad.adnum = a.attnum + WHERE c.table_schema = :schema AND c.table_name = :tableName + ORDER BY c.ordinal_position;"; + + DataTable columns; + try + { + columns = db.SelectSQL(colSql, p); + } + catch + { + // PG 11 以下退回簡化欄位查詢 + string fallbackSql = @" + SELECT + c.column_name, + c.data_type, + c.is_nullable, + c.column_default, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS ""ProviderType"", + 'NO' AS ""IsIdentity"", + '' AS ""IdentityGeneration"", + 'NEVER' AS ""IsGenerated"", + '' AS ""GenerationExpression"", + COALESCE(col_description(cls.oid, a.attnum), '') AS ""Comment"" + FROM information_schema.columns c + LEFT JOIN pg_namespace n ON n.nspname = c.table_schema + LEFT JOIN pg_class cls ON cls.relnamespace = n.oid AND cls.relname = c.table_name + LEFT JOIN pg_attribute a ON a.attrelid = cls.oid AND a.attname = c.column_name AND a.attnum > 0 AND NOT a.attisdropped + WHERE c.table_schema = :schema AND c.table_name = :tableName + ORDER BY c.ordinal_position;"; + columns = db.SelectSQL(fallbackSql, p); + } + + if (columns.Rows.Count == 0) return string.Empty; + + var defs = new List(); + var columnComments = new StringBuilder(); + + foreach (DataRow row in columns.Rows) + { + string name = row["column_name"]?.ToString() ?? ""; + string providerType = row["ProviderType"]?.ToString(); + if (string.IsNullOrWhiteSpace(providerType)) + { + providerType = row["data_type"]?.ToString() ?? "text"; + } + + string nullable = string.Equals(row["is_nullable"]?.ToString(), "YES", StringComparison.OrdinalIgnoreCase) ? "NULL" : "NOT NULL"; + string def = " \"" + name.Replace("\"", "\"\"") + "\" " + providerType; + + string isIdentity = row["IsIdentity"]?.ToString(); + string isGenerated = row["IsGenerated"]?.ToString(); + + if (string.Equals(isGenerated, "ALWAYS", StringComparison.OrdinalIgnoreCase)) + { + string expr = row["GenerationExpression"]?.ToString(); + def += " GENERATED ALWAYS AS (" + expr + ") STORED"; + } + else if (string.Equals(isIdentity, "YES", StringComparison.OrdinalIgnoreCase)) + { + string gen = row["IdentityGeneration"]?.ToString(); + def += string.Equals(gen, "ALWAYS", StringComparison.OrdinalIgnoreCase) + ? " GENERATED ALWAYS AS IDENTITY" + : " GENERATED BY DEFAULT AS IDENTITY"; + } + else + { + string defVal = row["column_default"]?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(defVal)) + { + def += " DEFAULT " + defVal; + } + def += " " + nullable; + } + + defs.Add(def); + + if (options.IncludeComments) + { + string comment = row["Comment"]?.ToString(); + if (!string.IsNullOrWhiteSpace(comment)) + { + columnComments.Append("\r\nCOMMENT ON COLUMN \"" + schema + "\".\"" + tableName + "\".\"" + name.Replace("\"", "\"\"") + + "\" IS '" + comment.Replace("'", "''") + "';"); + } + } + } + + // 主鍵 + try + { + string pkSql = @" + SELECT a.attname + FROM pg_index ix + JOIN pg_attribute a ON a.attrelid = ix.indrelid AND a.attnum = ANY(ix.indkey) + JOIN pg_class t ON t.oid = ix.indrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = :schema AND t.relname = :tableName AND ix.indisprimary + ORDER BY array_position(ix.indkey, a.attnum);"; + DataTable pkDt = db.SelectSQL(pkSql, p); + if (pkDt.Rows.Count > 0) + { + var pkCols = new List(); + foreach (DataRow r in pkDt.Rows) + { + pkCols.Add("\"" + r["attname"]?.ToString().Replace("\"", "\"\"") + "\""); + } + defs.Add(" PRIMARY KEY (" + string.Join(", ", pkCols) + ")"); + } + } + catch { } + + var sb = new StringBuilder(); + sb.Append("CREATE TABLE \"" + schema + "\".\"" + tableName + "\" (\r\n"); + sb.Append(string.Join(",\r\n", defs)); + sb.Append("\r\n);"); + + // 表註解 + if (options.IncludeComments) + { + try + { + string tableCommentSql = @" + SELECT obj_description(c.oid, 'pg_class') AS comment + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = :schema AND c.relname = :tableName;"; + DataTable tcDt = db.SelectSQL(tableCommentSql, p); + if (tcDt.Rows.Count > 0 && tcDt.Rows[0]["comment"] != DBNull.Value) + { + string cText = tcDt.Rows[0]["comment"]?.ToString(); + if (!string.IsNullOrWhiteSpace(cText)) + { + sb.Append("\r\nCOMMENT ON TABLE \"" + schema + "\".\"" + tableName + "\" IS '" + cText.Replace("'", "''") + "';"); + } + } + } + catch { } + + sb.Append(columnComments.ToString()); + } + + // 索引 (包含 PostGIS GIST / SP-GIST 空間索引) + if (options.IncludeIndexes) + { + try + { + string indexSql = @" + SELECT i.indexdef + FROM pg_indexes i + WHERE i.schemaname = :schema AND i.tablename = :tableName + AND NOT EXISTS ( + SELECT 1 FROM pg_constraint con + JOIN pg_class ic ON ic.oid = con.conindid + WHERE con.contype = 'p' AND ic.relname = i.indexname + ) + ORDER BY i.indexname;"; + DataTable idxDt = db.SelectSQL(indexSql, p); + foreach (DataRow r in idxDt.Rows) + { + string idxDef = r["indexdef"]?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(idxDef)) + { + sb.Append("\r\n" + idxDef + ";"); + } + } + } + catch { } + } + + return sb.ToString(); + } + + private long ExportTableData( + PostgreSqlTableExportItem item, + StreamWriter writer, + IncrementalHash sha256, + Action writeLine, + PostgreSqlExportWizardOptions options, + CancellationToken cancellationToken) + { + long rowCount = 0; + string selectSql = string.Format("SELECT * FROM {0};", item.FullQualifiedName); + + DataTable dt = db.SelectSQL(selectSql); + if (dt == null || dt.Rows.Count == 0) return 0; + + int colCount = dt.Columns.Count; + var colNames = new List(); + var isSpatialCol = new bool[colCount]; + var isByteaCol = new bool[colCount]; + + for (int i = 0; i < colCount; i++) + { + string colName = dt.Columns[i].ColumnName; + colNames.Add("\"" + colName.Replace("\"", "\"\"") + "\""); + + if (item.SpatialColumnNames.Contains(colName)) + { + isSpatialCol[i] = true; + } + } + + string insertHeader = string.Format("INSERT INTO {0} ({1}) VALUES", item.FullQualifiedName, string.Join(", ", colNames)); + var batchRows = new List(); + + foreach (DataRow row in dt.Rows) + { + cancellationToken.ThrowIfCancellationRequested(); + rowCount++; + + var values = new List(); + for (int i = 0; i < colCount; i++) + { + object val = row[i]; + values.Add(FormatPostgreSqlValue(val, isSpatialCol[i])); + } + + batchRows.Add("(" + string.Join(", ", values) + ")"); + + if (batchRows.Count >= options.BatchSize) + { + writeLine(insertHeader); + writeLine(string.Join(",\r\n", batchRows) + ";"); + batchRows.Clear(); + } + } + + if (batchRows.Count > 0) + { + writeLine(insertHeader); + writeLine(string.Join(",\r\n", batchRows) + ";"); + batchRows.Clear(); + } + + return rowCount; + } + + private static string FormatPostgreSqlValue(object val, bool isSpatial) + { + if (val == null || val == DBNull.Value) return "NULL"; + + if (val is bool b) + { + return b ? "TRUE" : "FALSE"; + } + + if (val is byte[] bytes) + { + string hex = BitConverter.ToString(bytes).Replace("-", ""); + if (isSpatial) + { + // PostGIS 空間幾何無損 Hex 傾印 (EWKB) + return "'\\x" + hex + "'::geometry"; + } + return "'\\x" + hex + "'::bytea"; + } + + if (val is sbyte || val is byte || val is short || val is ushort || + val is int || val is uint || val is long || val is ulong) + { + return Convert.ToString(val, CultureInfo.InvariantCulture); + } + + if (val is float || val is double || val is decimal) + { + return Convert.ToString(val, CultureInfo.InvariantCulture); + } + + if (val is DateTime dt) + { + return "'" + dt.ToString("yyyy-MM-dd HH:mm:ss.ffffff", CultureInfo.InvariantCulture) + "'"; + } + + if (val is Guid g) + { + return "'" + g.ToString() + "'::uuid"; + } + + string str = val.ToString(); + + // 若欄位為空間幾何且值以文字傳入 (如 WKT 或 Hex 字串) + if (isSpatial) + { + string trimmed = str.Trim(); + if (trimmed.StartsWith("01", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("\\x", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("SRID=", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("POINT", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("LINESTRING", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("POLYGON", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("MULTIPOINT", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("MULTILINESTRING", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("MULTIPOLYGON", StringComparison.OrdinalIgnoreCase) || + trimmed.StartsWith("GEOMETRYCOLLECTION", StringComparison.OrdinalIgnoreCase)) + { + return "'" + str.Replace("'", "''") + "'::geometry"; + } + } + + return "'" + str.Replace("'", "''") + "'"; + } + } +} diff --git a/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportLog.cs b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportLog.cs new file mode 100644 index 0000000..7dc7992 --- /dev/null +++ b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportLog.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace mySQLPunk.lib.PostgreSqlImport +{ + public sealed class PostgreSqlImportLog : IDisposable + { + private readonly Stream stream; + private readonly StreamWriter writer; + private readonly object gate = new object(); + private readonly List recentErrors = new List(); + private bool disposed; + + public string Path { get; } + public IReadOnlyList RecentErrors + { + get { lock (gate) return recentErrors.ToArray(); } + } + + public PostgreSqlImportLog(Guid jobId, string directory = null) + { + directory = directory ?? System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "mySQLPunk", "ImportLogs"); + Directory.CreateDirectory(directory); + string stem = "import-postgresql-" + DateTime.Now.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture) + "-" + jobId.ToString("N").Substring(0, 8); + Path = System.IO.Path.Combine(directory, stem + ".log"); + + stream = new FileStream(Path, FileMode.Create, FileAccess.Write, FileShare.Read); + writer = new StreamWriter(stream, Encoding.UTF8); + WriteHeader(jobId); + } + + private void WriteHeader(Guid jobId) + { + lock (gate) + { + writer.WriteLine("================================================================================"); + writer.WriteLine("mySQLPunk PostgreSQL / PostGIS Import Execution Log"); + writer.WriteLine(string.Format("Job ID : {0}", jobId)); + writer.WriteLine(string.Format("Started : {0:yyyy-MM-dd HH:mm:ss}", DateTime.Now)); + writer.WriteLine("================================================================================"); + writer.WriteLine(); + writer.Flush(); + } + } + + public void LogInfo(string message) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine(string.Format("[{0:HH:mm:ss.fff}] [INFO] {1}", DateTime.Now, message)); + } + } + + public void LogStatement(int lineNumber, string preview, bool success, string errorMessage = null, string action = null) + { + lock (gate) + { + if (disposed) return; + string status = success ? "OK" : "ERROR"; + string extra = string.IsNullOrWhiteSpace(action) ? "" : " (" + action + ")"; + writer.WriteLine(string.Format("[{0:HH:mm:ss.fff}] [{1}] Line {2}{3}: {4}", DateTime.Now, status, lineNumber, extra, preview)); + if (!success && !string.IsNullOrWhiteSpace(errorMessage)) + { + writer.WriteLine(string.Format(" Error: {0}", errorMessage)); + if (recentErrors.Count < 20) + { + recentErrors.Add(string.Format("Line {0}: {1}", lineNumber, errorMessage)); + } + } + } + } + + public void LogManifest(Dictionary expected, Dictionary actual, bool match) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine(); + writer.WriteLine("--- Manifest Verification ---"); + writer.WriteLine(string.Format("Result: {0}", match ? "PASSED" : "MISMATCH")); + foreach (var kvp in expected) + { + long actualRows = actual.ContainsKey(kvp.Key) ? actual[kvp.Key] : -1; + writer.WriteLine(string.Format(" Table {0}: Expected {1}, Actual {2}", kvp.Key, kvp.Value, actualRows)); + } + writer.WriteLine("-----------------------------"); + writer.WriteLine(); + } + } + + public void LogSummary(int executed, int failed, long rowsAffected, TimeSpan elapsed, bool success) + { + lock (gate) + { + if (disposed) return; + writer.WriteLine(); + writer.WriteLine("================================================================================"); + writer.WriteLine(string.Format("Execution Finished: {0}", success ? "SUCCESS" : "FAILED")); + writer.WriteLine(string.Format("Statements Executed: {0}", executed)); + writer.WriteLine(string.Format("Statements Failed : {0}", failed)); + writer.WriteLine(string.Format("Rows Affected : {0:N0}", rowsAffected)); + writer.WriteLine(string.Format("Duration : {0:hh\\:mm\\:ss\\.fff}", elapsed)); + writer.WriteLine("================================================================================"); + writer.Flush(); + } + } + + public void Flush() + { + lock (gate) + { + if (disposed) return; + writer.Flush(); + } + } + + public void Dispose() + { + lock (gate) + { + if (disposed) return; + disposed = true; + try { writer.Flush(); writer.Dispose(); } catch { } + try { stream.Dispose(); } catch { } + } + } + } +} diff --git a/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportModels.cs b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportModels.cs new file mode 100644 index 0000000..77f5459 --- /dev/null +++ b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportModels.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace mySQLPunk.lib.PostgreSqlImport +{ + public enum PostgreSqlConflictStrategy + { + KeepExisting, // 保留現存物件 (遇衝突略過) + ReplaceAll, // 全部替換 (CREATE OR REPLACE / 覆寫) + DropAndRecreate, // 刪表重建 (DROP TABLE IF EXISTS ... CASCADE) + TruncateAndInsert // 清空重匯 (TRUNCATE TABLE ... CASCADE) + } + + public sealed class PostgreSqlImportOptions + { + public PostgreSqlConflictStrategy ConflictStrategy { get; set; } = PostgreSqlConflictStrategy.KeepExisting; + public bool StopOnError { get; set; } = true; + public bool AutoInstallPostgis { get; set; } = true; // 偵測到 PostGIS 空間特徵時自動安裝擴充套件 + public bool UseTransaction { get; set; } = true; // 交易保護，遇錯 ROLLBACK + public string TargetSchema { get; set; } = string.Empty; + } + + public sealed class PostgreSqlImportRequest + { + public string FilePath { get; set; } = string.Empty; + public string TargetDatabase { get; set; } = string.Empty; + public PostgreSqlImportOptions Options { get; set; } = new PostgreSqlImportOptions(); + + public PostgreSqlImportRequest() { } + + public PostgreSqlImportRequest(string filePath, string targetDatabase, PostgreSqlImportOptions options = null) + { + FilePath = filePath ?? string.Empty; + TargetDatabase = targetDatabase ?? string.Empty; + Options = options ?? new PostgreSqlImportOptions(); + } + } + + public sealed class PostgreSqlImportPreflightResult + { + public bool CanProceed { get; set; } = true; + public Encoding DetectedEncoding { get; set; } = Encoding.UTF8; + public int StatementCount { get; set; } + public bool HasPostgisFeatures { get; set; } + public List DetectedSpatialKeywords { get; } = new List(); + public bool HasDollarQuotes { get; set; } + public int DollarQuoteBlocksCount { get; set; } + public bool ContainsManifest { get; set; } + public Dictionary ManifestDeclaredTables { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public string ManifestSha256 { get; set; } = string.Empty; + public List Issues { get; } = new List(); + public long FileSize { get; set; } + } + + public sealed class PostgreSqlImportProgress + { + public string Stage { get; set; } = string.Empty; + public int CurrentStatement { get; set; } + public int TotalStatements { get; set; } + public long LinesRead { get; set; } + public long TotalLines { get; set; } + public long BytesRead { get; set; } + public long TotalBytes { get; set; } + public long RowsAffected { get; set; } + public string CurrentObject { get; set; } = string.Empty; + public string Message { get; set; } = string.Empty; + + public double PercentComplete + { + get + { + if (TotalBytes > 0 && BytesRead > 0) + { + double p = (double)BytesRead / TotalBytes * 100.0; + return p > 100.0 ? 100.0 : p; + } + if (TotalStatements > 0 && CurrentStatement > 0) + { + double p = (double)CurrentStatement / TotalStatements * 100.0; + return p > 100.0 ? 100.0 : p; + } + return 0.0; + } + } + } + + public sealed class PostgreSqlImportResult + { + public bool Success { get; set; } + public int StatementsExecuted { get; set; } + public int StatementsFailed { get; set; } + public long TotalRowsAffected { get; set; } + public Dictionary RestoredTables { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public bool ManifestMatched { get; set; } + public TimeSpan Elapsed { get; set; } + public string LogFilePath { get; set; } = string.Empty; + public string ErrorMessage { get; set; } = string.Empty; + } +} diff --git a/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportPreflightScanner.cs b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportPreflightScanner.cs new file mode 100644 index 0000000..812685d --- /dev/null +++ b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportPreflightScanner.cs @@ -0,0 +1,330 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.PostgreSqlImport +{ + public sealed class PostgreSqlImportPreflightScanner + { + private const int MaxBatchSizeBytes = 32 * 1024 * 1024; // 32 MiB + private const int BufferSize = 65536; // 64KB + + private static readonly string[] PostgisKeywords = new[] + { + "geometry", "geography", "raster", "postgis", "spatial_ref_sys", + "st_geomfromtext", "st_geomfromewkb", "st_astext", "st_asewkt", + "st_point", "st_polygon", "st_buffer", "st_intersects", "st_dwithin", + "addgeometrycolumn", "create extension postgis" + }; + + public async Task ScanAsync(string filePath, CancellationToken cancellationToken = default(CancellationToken)) + { + return await Task.Run(() => Scan(filePath, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + public PostgreSqlImportPreflightResult Scan(string filePath, CancellationToken cancellationToken = default(CancellationToken)) + { + if (string.IsNullOrWhiteSpace(filePath)) throw new ArgumentException("File path cannot be empty.", nameof(filePath)); + if (!File.Exists(filePath)) throw new FileNotFoundException("SQL script not found.", filePath); + + var result = new PostgreSqlImportPreflightResult(); + var fileInfo = new FileInfo(filePath); + result.FileSize = fileInfo.Length; + + Encoding encoding = DetectEncoding(filePath); + result.DetectedEncoding = encoding; + + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize)) + using (var reader = new StreamReader(fs, encoding, true, BufferSize)) + { + int statementCount = 0; + long currentStatementBytes = 0; + bool hasStatementContent = false; + bool inSingleQuote = false; + bool inDoubleQuote = false; + bool inLineComment = false; + int blockCommentDepth = 0; + string dollarTag = null; // 非 null 代表正在此特定 $tag$ 內 + + var currentStmtSb = new StringBuilder(); + var spatialFound = new HashSet(StringComparer.OrdinalIgnoreCase); + + string line; + int lineNumber = 0; + + while ((line = reader.ReadLine()) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + lineNumber++; + + // 檢查 Native Manifest 標記 + string trimmed = line.Trim(); + if (trimmed.StartsWith("--", StringComparison.Ordinal)) + { + if (trimmed.Contains("mySQLPunk Native Manifest")) + { + result.ContainsManifest = true; + } + + var matchTable = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (matchTable.Success && long.TryParse(matchTable.Groups[2].Value, out long rows)) + { + result.ManifestDeclaredTables[matchTable.Groups[1].Value] = rows; + } + + var matchSha = Regex.Match(trimmed, @"^--\s*Manifest-Checksum-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase); + if (matchSha.Success) + { + result.ManifestSha256 = matchSha.Groups[1].Value.ToLowerInvariant(); + } + } + + // 探測 PostGIS 關鍵字 + string lowerLine = line.ToLowerInvariant(); + foreach (var kw in PostgisKeywords) + { + if (lowerLine.Contains(kw)) + { + result.HasPostgisFeatures = true; + if (spatialFound.Add(kw)) + { + result.DetectedSpatialKeywords.Add(kw); + } + } + } + + int len = line.Length; + for (int i = 0; i < len; i++) + { + char c = line[i]; + + // NUL byte 檢查 + if (c == '\0') + { + result.CanProceed = false; + result.Issues.Add(string.Format("第 {0} 行包含二進位 NUL 空字元，檔案可能已損毀或非純文字 SQL 腳本。", lineNumber)); + return result; + } + + // 1. 若在單行註解中 + if (inLineComment) + { + // 單行註解由換行符號終結 + continue; + } + + // 2. 若在塊註解中 (支援 PG 巢狀 /* /* */ */) + if (blockCommentDepth > 0) + { + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + blockCommentDepth++; + i++; + } + else if (c == '*' && i + 1 < len && line[i + 1] == '/') + { + blockCommentDepth--; + i++; + } + continue; + } + + // 3. 若在單引號字串中 + if (inSingleQuote) + { + hasStatementContent = true; + currentStatementBytes += (c > 127) ? 2 : 1; + if (c == '\'') + { + if (i + 1 < len && line[i + 1] == '\'') + { + i++; // 跳過跳脫的 '' + currentStatementBytes++; + } + else + { + inSingleQuote = false; + } + } + continue; + } + + // 4. 若在雙引號識別字中 + if (inDoubleQuote) + { + hasStatementContent = true; + currentStatementBytes += (c > 127) ? 2 : 1; + if (c == '"') + { + if (i + 1 < len && line[i + 1] == '"') + { + i++; // 跳過跳脫的 "" + currentStatementBytes++; + } + else + { + inDoubleQuote = false; + } + } + continue; + } + + // 5. 若在 Dollar Quotes 中 ($tag$...$tag$) + if (dollarTag != null) + { + hasStatementContent = true; + currentStatementBytes += (c > 127) ? 2 : 1; + if (c == '$') + { + int tagLen = dollarTag.Length; + if (i + tagLen <= len && string.Compare(line, i, dollarTag, 0, tagLen, StringComparison.Ordinal) == 0) + { + dollarTag = null; // 結束 Dollar Quote + i += tagLen - 1; + currentStatementBytes += tagLen - 1; + } + } + continue; + } + + // 6. 一般狀態：檢查開始符號 + if (c == '-' && i + 1 < len && line[i + 1] == '-') + { + inLineComment = true; + i++; + continue; + } + + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + blockCommentDepth = 1; + i++; + continue; + } + + if (c == '\'') + { + inSingleQuote = true; + hasStatementContent = true; + currentStatementBytes += 1; + continue; + } + + if (c == '"') + { + inDoubleQuote = true; + hasStatementContent = true; + currentStatementBytes += 1; + continue; + } + + // 探測開始 Dollar Quote + if (c == '$') + { + string potentialTag = TryMatchDollarTag(line, i); + if (potentialTag != null) + { + dollarTag = potentialTag; + result.HasDollarQuotes = true; + result.DollarQuoteBlocksCount++; + hasStatementContent = true; + currentStatementBytes += potentialTag.Length; + i += potentialTag.Length - 1; + continue; + } + } + + // 分號終結語句 + if (c == ';') + { + if (hasStatementContent) + { + statementCount++; + hasStatementContent = false; + currentStatementBytes = 0; + } + continue; + } + + if (!char.IsWhiteSpace(c)) + { + hasStatementContent = true; + } + + if (hasStatementContent) + { + currentStatementBytes += (c > 127) ? 2 : 1; + if (currentStatementBytes > MaxBatchSizeBytes) + { + result.CanProceed = false; + result.Issues.Add(string.Format("第 {0} 行單一語句大小已超過 32 MiB 安全上限。", lineNumber)); + return result; + } + } + } + + inLineComment = false; // 換行結束單行註解 + } + + if (hasStatementContent) + { + statementCount++; + } + + result.StatementCount = statementCount; + + if (statementCount == 0) + { + result.Issues.Add("SQL 腳本中未發現任何有效的執行語句。"); + } + } + + return result; + } + + private static string TryMatchDollarTag(string line, int startIndex) + { + if (startIndex >= line.Length || line[startIndex] != '$') return null; + + int end = -1; + for (int i = startIndex + 1; i < line.Length; i++) + { + char c = line[i]; + if (c == '$') + { + end = i; + break; + } + // Tag 內部必須是合法的識別字元 (字母、數字、底線) + if (!char.IsLetterOrDigit(c) && c != '_') + { + return null; + } + } + + if (end > startIndex) + { + return line.Substring(startIndex, end - startIndex + 1); + } + + return null; + } + + private static Encoding DetectEncoding(string filePath) + { + byte[] bom = new byte[4]; + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + int read = fs.Read(bom, 0, 4); + if (read >= 3 && bom[0] == 0xef && bom[1] == 0xbb && bom[2] == 0xbf) return Encoding.UTF8; + if (read >= 2 && bom[0] == 0xff && bom[1] == 0xfe) return Encoding.Unicode; + if (read >= 2 && bom[0] == 0xfe && bom[1] == 0xff) return Encoding.BigEndianUnicode; + } + return new UTF8Encoding(false); + } + } +} diff --git a/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportRunner.cs b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportRunner.cs new file mode 100644 index 0000000..5a43462 --- /dev/null +++ b/mySQLPunk/lib/PostgreSqlImport/PostgreSqlImportRunner.cs @@ -0,0 +1,573 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.PostgreSqlImport +{ + public sealed class PostgreSqlImportRunner + { + private const int BufferSize = 65536; // 64KB + private readonly IDatabase db; + private readonly PostgreSqlImportRequest request; + + public PostgreSqlImportRunner(IDatabase db, PostgreSqlImportRequest request) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.request = request ?? throw new ArgumentNullException(nameof(request)); + } + + public async Task RunAsync( + IProgress progress = null, + Action terminalSink = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + return await Task.Run(() => Run(progress, terminalSink, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + public PostgreSqlImportResult Run( + IProgress progress = null, + Action terminalSink = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + var result = new PostgreSqlImportResult(); + var clock = Stopwatch.StartNew(); + Guid jobId = Guid.NewGuid(); + + var log = new PostgreSqlImportLog(jobId); + result.LogFilePath = log.Path; + + void LogAndTerminal(string message) + { + log.LogInfo(message); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] {1}", DateTime.Now, message)); + } + + LogAndTerminal(string.Format("開始執行 PostgreSQL 匯入作業 (Job ID: {0})", jobId)); + LogAndTerminal(string.Format("目標檔案: {0}", request.FilePath)); + LogAndTerminal(string.Format("目標資料庫: {0}", request.TargetDatabase)); + LogAndTerminal(string.Format("衝突處理策略: {0}", request.Options.ConflictStrategy)); + + var scanner = new PostgreSqlImportPreflightScanner(); + var preflight = scanner.Scan(request.FilePath, cancellationToken); + int totalStatements = preflight.StatementCount; + long totalBytes = preflight.FileSize; + + if (preflight.HasPostgisFeatures) + { + LogAndTerminal(string.Format("🌐 偵測到 PostGIS 空間特徵 (關鍵字: {0})", string.Join(", ", preflight.DetectedSpatialKeywords))); + if (request.Options.AutoInstallPostgis) + { + try + { + LogAndTerminal("正在確保目標資料庫已載入 PostGIS 擴充套件 (CREATE EXTENSION IF NOT EXISTS postgis)..."); + db.ExecSQL("CREATE EXTENSION IF NOT EXISTS postgis;"); + LogAndTerminal("PostGIS 擴充套件已就緒。"); + } + catch (Exception ex) + { + LogAndTerminal("PostGIS 擴充套件檢查注意: " + ex.Message); + } + } + } + + if (preflight.HasDollarQuotes) + { + LogAndTerminal(string.Format("偵測到 {0} 個 Dollar-Quoted 區塊 ($$...$$)，將使用狀態機精準解析。", preflight.DollarQuoteBlocksCount)); + } + + // 效能優化旗標 + try + { + db.ExecSQL("SET synchronous_commit = off;"); + } + catch { } + + // 判斷是否需要外部 Transaction + bool outerTransaction = request.Options.UseTransaction; + bool transactionStarted = false; + + if (outerTransaction) + { + try + { + db.ExecSQL("BEGIN;"); + transactionStarted = true; + LogAndTerminal("已開啟單一交易包覆 (BEGIN)。"); + } + catch (Exception ex) + { + LogAndTerminal("開啟交易時發生警告: " + ex.Message); + } + } + + int currentStmtIndex = 0; + int failedStatements = 0; + long totalRowsAffected = 0; + long bytesRead = 0; + var truncatedTables = new HashSet(StringComparer.OrdinalIgnoreCase); + + var expectedManifestTables = new Dictionary(preflight.ManifestDeclaredTables, StringComparer.OrdinalIgnoreCase); + bool readingManifest = false; + + Action checkManifestLine = comment => + { + if (comment.Contains("mySQLPunk Native Manifest") && !readingManifest) + { + readingManifest = true; + log.LogInfo("Detected Native Manifest in SQL script footer."); + } + + var match = Regex.Match(comment, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) + { + expectedManifestTables[match.Groups[1].Value] = r; + } + }; + + try + { + using (var fs = new FileStream(request.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize)) + using (var reader = new StreamReader(fs, preflight.DetectedEncoding, true, BufferSize)) + { + string statement; + int lineNum = 0; + + foreach (var stmtInfo in EnumerateStatements(reader, cancellationToken, checkManifestLine)) + { + statement = stmtInfo.Sql.Trim(); + lineNum = stmtInfo.LineNumber; + bytesRead += stmtInfo.ByteLength; + + if (string.IsNullOrWhiteSpace(statement)) continue; + + currentStmtIndex++; + + // 套用衝突處理策略 + string effectiveSql = ApplyConflictStrategy(statement, request.Options.ConflictStrategy, truncatedTables, db, LogAndTerminal); + + // 判斷當前語句是否為交易控制 + string upperSql = effectiveSql.ToUpperInvariant().Trim(); + if (upperSql.StartsWith("BEGIN") || upperSql.StartsWith("START TRANSACTION")) + { + if (transactionStarted) + { + // 避免巢狀 BEGIN + continue; + } + transactionStarted = true; + } + else if (upperSql.StartsWith("COMMIT")) + { + transactionStarted = false; + } + else if (upperSql.StartsWith("ROLLBACK")) + { + transactionStarted = false; + } + + string preview = effectiveSql.Length > 80 ? effectiveSql.Substring(0, 80).Replace("\r", " ").Replace("\n", " ") + "..." : effectiveSql.Replace("\r", " ").Replace("\n", " "); + + try + { + var execRes = db.ExecSQL(effectiveSql); + long affected = 0; + if (execRes != null && execRes.ContainsKey("rowsAffected")) + { + long.TryParse(execRes["rowsAffected"], out affected); + } + + if (affected > 0) totalRowsAffected += affected; + + log.LogStatement(lineNum, preview, true); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [OK] Line {1}: {2} (Rows: {3})", DateTime.Now, lineNum, preview, affected)); + } + catch (Exception ex) + { + failedStatements++; + log.LogStatement(lineNum, preview, false, ex.Message); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [ERROR] Line {1}: {2} -> {3}", DateTime.Now, lineNum, preview, ex.Message)); + + if (request.Options.StopOnError) + { + if (transactionStarted) + { + try { db.ExecSQL("ROLLBACK;"); LogAndTerminal("遇錯停止，已執行 ROLLBACK。"); } catch { } + } + throw new InvalidOperationException(string.Format("第 {0} 行執行失敗：{1}", lineNum, ex.Message), ex); + } + } + + if (currentStmtIndex % 50 == 0 || currentStmtIndex == totalStatements) + { + progress?.Report(new PostgreSqlImportProgress + { + Stage = "Executing", + CurrentStatement = currentStmtIndex, + TotalStatements = totalStatements, + BytesRead = bytesRead, + TotalBytes = totalBytes, + RowsAffected = totalRowsAffected, + Message = string.Format("正在執行語句: {0}/{1} (已影響 {2:N0} 列)", currentStmtIndex, totalStatements, totalRowsAffected) + }); + } + } + } + + // 正常結束提交 + if (transactionStarted) + { + try + { + db.ExecSQL("COMMIT;"); + LogAndTerminal("交易已成功提交 (COMMIT)。"); + } + catch (Exception ex) + { + LogAndTerminal("提交交易時發生警告: " + ex.Message); + } + } + + result.Success = (failedStatements == 0); + } + catch (Exception ex) + { + result.Success = false; + result.ErrorMessage = ex.Message; + LogAndTerminal("匯入過程中斷: " + ex.Message); + } + + clock.Stop(); + result.Elapsed = clock.Elapsed; + result.StatementsExecuted = currentStmtIndex; + result.StatementsFailed = failedStatements; + result.TotalRowsAffected = totalRowsAffected; + + // 核對 Native Manifest + if (expectedManifestTables.Count > 0) + { + LogAndTerminal(string.Format("正在核對腳本結尾 Manifest 清單 (共 {0} 張資料表)...", expectedManifestTables.Count)); + bool allMatch = true; + + foreach (var kvp in expectedManifestTables) + { + string tableName = kvp.Key; + long expectedRows = kvp.Value; + long actualRows = -1; + + try + { + string countSql = tableName.Contains(".") + ? string.Format("SELECT COUNT(*) FROM \"{0}\".\"{1}\";", tableName.Split('.')[0], tableName.Split('.')[1]) + : string.Format("SELECT COUNT(*) FROM \"{0}\";", tableName); + + DataTable dt = db.SelectSQL(countSql); + if (dt != null && dt.Rows.Count > 0) + { + actualRows = Convert.ToInt64(dt.Rows[0][0]); + } + } + catch + { + actualRows = -1; + } + + result.RestoredTables[tableName] = actualRows; + if (actualRows != expectedRows) + { + allMatch = false; + LogAndTerminal(string.Format("⚠️ 資料表 [{0}] 筆數不符：Manifest 宣告 {1} 列，還原後實際 {2} 列。", tableName, expectedRows, actualRows)); + } + else + { + LogAndTerminal(string.Format("✅ 資料表 [{0}] 筆數相符：{1} 列。", tableName, actualRows)); + } + } + + result.ManifestMatched = allMatch; + log.LogManifest(expectedManifestTables, result.RestoredTables, allMatch); + } + + log.LogSummary(currentStmtIndex, failedStatements, totalRowsAffected, result.Elapsed, result.Success); + log.Flush(); + log.Dispose(); + + LogAndTerminal(string.Format("PostgreSQL 匯入作業完成，耗時 {0:hh\\:mm\\:ss\\.fff}，詳細日誌已寫入: {1}", result.Elapsed, result.LogFilePath)); + return result; + } + + private static string ApplyConflictStrategy( + string sql, + PostgreSqlConflictStrategy strategy, + HashSet truncatedTables, + IDatabase db, + Action logAction) + { + if (strategy == PostgreSqlConflictStrategy.KeepExisting) + { + // CREATE TABLE -> CREATE TABLE IF NOT EXISTS + var match = Regex.Match(sql, @"^CREATE\s+TABLE\s+(?!IF\s+NOT\s+EXISTS)([""\w\.]+)", RegexOptions.IgnoreCase); + if (match.Success) + { + return Regex.Replace(sql, @"^CREATE\s+TABLE\s+", "CREATE TABLE IF NOT EXISTS ", RegexOptions.IgnoreCase); + } + } + else if (strategy == PostgreSqlConflictStrategy.DropAndRecreate || strategy == PostgreSqlConflictStrategy.ReplaceAll) + { + // 在 CREATE TABLE 前執行 DROP TABLE IF EXISTS ... CASCADE + var match = Regex.Match(sql, @"^CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([""\w\.]+)", RegexOptions.IgnoreCase); + if (match.Success) + { + string target = match.Groups[1].Value; + string dropSql = string.Format("DROP TABLE IF EXISTS {0} CASCADE;", target); + try + { + db.ExecSQL(dropSql); + logAction(string.Format("[衝突處理] 已預先清除資料表: {0}", target)); + } + catch { } + } + } + else if (strategy == PostgreSqlConflictStrategy.TruncateAndInsert) + { + // 當第一次遇到該表的 INSERT 時，先執行 TRUNCATE + var match = Regex.Match(sql, @"^INSERT\s+INTO\s+([""\w\.]+)", RegexOptions.IgnoreCase); + if (match.Success) + { + string target = match.Groups[1].Value; + if (truncatedTables.Add(target)) + { + string truncSql = string.Format("TRUNCATE TABLE {0} CASCADE;", target); + try + { + db.ExecSQL(truncSql); + logAction(string.Format("[衝突處理] 已清空資料表內容: {0}", target)); + } + catch { } + } + } + } + + return sql; + } + + private struct StatementInfo + { + public string Sql; + public int LineNumber; + public int ByteLength; + } + + private static IEnumerable EnumerateStatements(StreamReader reader, CancellationToken cancellationToken, Action onCommentLine = null) + { + var sb = new StringBuilder(); + int startLine = 1; + int currentLine = 1; + int byteLen = 0; + + bool inSingleQuote = false; + bool inDoubleQuote = false; + bool inLineComment = false; + int blockCommentDepth = 0; + string dollarTag = null; + + string line; + while ((line = reader.ReadLine()) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + byteLen += line.Length + 1; + + string trimmed = line.Trim(); + + // 獨立行註解不作為執行語句，僅觸發註解回呼 (如 Manifest 解析) + if (sb.Length == 0 && trimmed.StartsWith("--", StringComparison.Ordinal)) + { + onCommentLine?.Invoke(trimmed); + currentLine++; + startLine = currentLine; + continue; + } + + if (sb.Length == 0) startLine = currentLine; + + int len = line.Length; + for (int i = 0; i < len; i++) + { + char c = line[i]; + + if (inLineComment) continue; + + if (blockCommentDepth > 0) + { + if (c == '/' && i + 1 < len && line[i + 1] == '*') { blockCommentDepth++; i++; } + else if (c == '*' && i + 1 < len && line[i + 1] == '/') { blockCommentDepth--; i++; } + sb.Append(c); + continue; + } + + if (inSingleQuote) + { + sb.Append(c); + if (c == '\'') + { + if (i + 1 < len && line[i + 1] == '\'') + { + sb.Append(line[i + 1]); + i++; + } + else + { + inSingleQuote = false; + } + } + continue; + } + + if (inDoubleQuote) + { + sb.Append(c); + if (c == '"') + { + if (i + 1 < len && line[i + 1] == '"') + { + sb.Append(line[i + 1]); + i++; + } + else + { + inDoubleQuote = false; + } + } + continue; + } + + if (dollarTag != null) + { + sb.Append(c); + if (c == '$') + { + int tagLen = dollarTag.Length; + if (i + tagLen <= len && string.Compare(line, i, dollarTag, 0, tagLen, StringComparison.Ordinal) == 0) + { + // 附加剩餘 tag 字元 + for (int k = 1; k < tagLen; k++) sb.Append(line[i + k]); + dollarTag = null; + i += tagLen - 1; + } + } + continue; + } + + // 一般符號檢查 + if (c == '-' && i + 1 < len && line[i + 1] == '-') + { + inLineComment = true; + i++; + continue; + } + + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + blockCommentDepth = 1; + sb.Append("/*"); + i++; + continue; + } + + if (c == '\'') + { + inSingleQuote = true; + sb.Append(c); + continue; + } + + if (c == '"') + { + inDoubleQuote = true; + sb.Append(c); + continue; + } + + if (c == '$') + { + string potentialTag = TryMatchDollarTag(line, i); + if (potentialTag != null) + { + dollarTag = potentialTag; + sb.Append(potentialTag); + i += potentialTag.Length - 1; + continue; + } + } + + if (c == ';') + { + sb.Append(';'); + yield return new StatementInfo + { + Sql = sb.ToString(), + LineNumber = startLine, + ByteLength = byteLen + }; + sb.Clear(); + byteLen = 0; + startLine = currentLine; + continue; + } + + sb.Append(c); + } + + inLineComment = false; + if (sb.Length > 0) sb.Append("\r\n"); + currentLine++; + } + + if (sb.Length > 0) + { + string rem = sb.ToString().Trim(); + if (!string.IsNullOrWhiteSpace(rem) && !rem.StartsWith("--", StringComparison.Ordinal)) + { + yield return new StatementInfo + { + Sql = rem, + LineNumber = startLine, + ByteLength = byteLen + }; + } + } + } + + private static string TryMatchDollarTag(string line, int startIndex) + { + if (startIndex >= line.Length || line[startIndex] != '$') return null; + + int end = -1; + for (int i = startIndex + 1; i < line.Length; i++) + { + char c = line[i]; + if (c == '$') + { + end = i; + break; + } + if (!char.IsLetterOrDigit(c) && c != '_') + { + return null; + } + } + + if (end > startIndex) + { + return line.Substring(startIndex, end - startIndex + 1); + } + + return null; + } + } +} diff --git a/mySQLPunk/mySQLPunk.csproj b/mySQLPunk/mySQLPunk.csproj index fdc275f..204a035 100644 --- a/mySQLPunk/mySQLPunk.csproj +++ b/mySQLPunk/mySQLPunk.csproj @@ -325,6 +325,10 @@ + + + + @@ -408,6 +412,12 @@ + + + + + + diff --git a/tests/Run-SmokeTests.ps1 b/tests/Run-SmokeTests.ps1 index 7d2aba6..7bca91b 100644 --- a/tests/Run-SmokeTests.ps1 +++ b/tests/Run-SmokeTests.ps1 @@ -151,6 +151,8 @@ if (!(Test-Path -LiteralPath $mysqlConnector)) { (Join-Path $PSScriptRoot "SmokeTests.MySqlImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.SqliteExport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.SqliteImport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.PostgreSqlExport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.PostgreSqlImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.TransferTrial.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Live.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Parser.cs") $source $preferencesSource $schemaSyncSource $erModelSource (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Snapshot.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Schema.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.ViewReferences.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Properties.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Artifact.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Values.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Lob.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Routing.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.SnapshotIntegration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.CompatibilityIntegration.cs") if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE diff --git a/tests/SmokeTests.MssqlExport.Routing.cs b/tests/SmokeTests.MssqlExport.Routing.cs index 899c486..4f8171b 100644 --- a/tests/SmokeTests.MssqlExport.Routing.cs +++ b/tests/SmokeTests.MssqlExport.Routing.cs @@ -121,7 +121,7 @@ private static void Routing_FourHandlers() private static void Routing_LegacyBranches() { - foreach (string provider in new[] { "postgresql", "oracle" }) + foreach (string provider in new[] { "oracle" }) foreach (string handler in new[] { "DumpSelectedTableSql", "DumpCurrentSelectionSqlWithDialog", "DumpSelectedDatabaseSqlWithDialog", "BackupSelectedDatabaseWithDialog" }) using (var fixture = new RoutingFixture(provider, "Tables")) { diff --git a/tests/SmokeTests.MssqlImport.Wizard.cs b/tests/SmokeTests.MssqlImport.Wizard.cs index 2da8ccb..ac74bc9 100644 --- a/tests/SmokeTests.MssqlImport.Wizard.cs +++ b/tests/SmokeTests.MssqlImport.Wizard.cs @@ -384,7 +384,7 @@ public MssqlImportCleanup Cleanup() } private static void ImportWizard_Routing() { - foreach (string provider in new[] { "mssql", "SQLSERVER", "postgresql" }) + foreach (string provider in new[] { "mssql", "SQLSERVER", "oracle" }) using (var fixture = new RoutingFixture(provider, "Tables")) { int opened = 0, legacy = 0; @@ -400,7 +400,7 @@ private static void ImportWizard_Routing() new Func(dialog => { legacy++; return DialogResult.Cancel; })); typeof(Form1).GetField("MssqlImportStatusSink", flags).SetValue(fixture.Form, new Action(text => fixture.Status.Add(text))); fixture.Invoke("ImportSqlWithDialog"); - bool native = provider != "postgresql"; + bool native = provider != "oracle"; Assert(opened == (native ? 1 : 0) && legacy == (native ? 0 : 1), "Actual file handler routes only MSSQL before legacy file selection."); if (native) { diff --git a/tests/SmokeTests.PostgreSqlExport.cs b/tests/SmokeTests.PostgreSqlExport.cs new file mode 100644 index 0000000..74b008e --- /dev/null +++ b/tests/SmokeTests.PostgreSqlExport.cs @@ -0,0 +1,252 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.PostgreSqlExport; + +public static partial class SmokeTests +{ + private static void PostgreSqlExport_OptionsAndModels_Defaults() + { + var options = new PostgreSqlExportWizardOptions(PostgreSqlExportMode.SchemaAndData, @"C:\pg_dump.sql"); + Assert(options.Mode == PostgreSqlExportMode.SchemaAndData, "Default mode should be SchemaAndData"); + Assert(options.UseTransactionSnapshot, "UseTransactionSnapshot should be true by default (REPEATABLE READ)"); + Assert(options.IncludeCreateExtensionPostgis, "IncludeCreateExtensionPostgis should be true by default"); + Assert(options.ExcludeSpatialMetadataTables, "ExcludeSpatialMetadataTables should be true by default"); + Assert(options.IncludeIndexes, "IncludeIndexes should be true by default"); + Assert(options.IncludeComments, "IncludeComments should be true by default"); + Assert(options.IncludeViews, "IncludeViews should be true by default"); + Assert(options.IncludeManifest, "IncludeManifest should be true by default"); + Assert(options.BatchSize == 1000, "Default BatchSize should be 1000"); + + var item = new PostgreSqlTableExportItem + { + Schema = "public", + Name = "parcels", + IsSpatial = true, + RowCount = 2500, + SpatialFeatureDescription = "PostGIS Geometry (POLYGON, SRID 3826)" + }; + Assert(item.FullQualifiedName == "\"public\".\"parcels\"", "FullQualifiedName should match"); + Assert(item.DisplayName == "parcels", "DisplayName for public schema should omit public prefix"); + + var customSchemaItem = new PostgreSqlTableExportItem + { + Schema = "geo", + Name = "cadastre" + }; + Assert(customSchemaItem.FullQualifiedName == "\"geo\".\"cadastre\"", "FullQualifiedName for custom schema"); + Assert(customSchemaItem.DisplayName == "geo.cadastre", "DisplayName for custom schema should include schema prefix"); + } + + private static void PostgreSqlExport_ServiceAndManifest_StreamingAndHexEwkb() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_pg_export_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string targetFile = Path.Combine(tempDir, "pg_export_test.sql"); + + try + { + var mockDb = new MockPostgreSqlExportDatabase(); + var service = new PostgreSqlSqlExportService(mockDb, "gis_postgres"); + + // 1. 驗證 Catalog 正確辨識 PostGIS 空間表與空間中繼表 + var catalog = service.GetTableCatalog(); + Assert(catalog.Count == 2, "Expected 2 objects in catalog, got " + catalog.Count); + + var parcels = catalog.Find(i => i.Name == "parcels"); + Assert(parcels != null, "parcels table should exist in catalog"); + Assert(parcels.IsSpatial, "parcels should be recognized as PostGIS spatial layer"); + Assert(parcels.Selected, "parcels should be selected by default"); + + var srs = catalog.Find(i => i.Name == "spatial_ref_sys"); + Assert(srs != null, "spatial_ref_sys table should exist in catalog"); + Assert(srs.IsSpatialMetadata, "spatial_ref_sys should be recognized as spatial metadata"); + Assert(!srs.Selected, "spatial_ref_sys should be unchecked by default to prevent PK conflict on restore"); + + var options = new PostgreSqlExportWizardOptions(PostgreSqlExportMode.SchemaAndData, targetFile) + { + UseTransactionSnapshot = true, + IncludeCreateExtensionPostgis = true, + IncludeIndexes = true, + IncludeComments = true, + IncludeManifest = true + }; + options.SelectedTables.Add("parcels"); + + var task = service.ExportAsync(options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Success, "Export should succeed"); + Assert(File.Exists(targetFile), "Target file must exist after atomic move"); + Assert(result.TableCount == 1, "Exported table count should be 1"); + Assert(result.TotalRows == 2, "Total exported rows should be 2"); + + string sql = File.ReadAllText(targetFile); + + // 2. 驗證 PostGIS 空間檔頭宣告與 REPEATABLE READ 交易 + AssertContains(sql, "CREATE EXTENSION IF NOT EXISTS postgis;", "Header must initialize PostGIS extension"); + AssertContains(sql, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;", "Snapshot transaction must be REPEATABLE READ"); + + // 3. 驗證 PostGIS 幾何資料轉換為零失真 \\x HEX EWKB ::geometry + AssertContains(sql, "::geometry", "Geometry values must be formatted as \\x...::geometry"); + AssertContains(sql, "INSERT INTO \"public\".\"parcels\"", "Insert header should target qualified table name"); + + // 4. 驗證 GIST 空間索引與結構註解 + AssertContains(sql, "CREATE INDEX idx_parcels_geom ON public.parcels USING gist (geom);", "PostGIS GIST spatial index preserved"); + AssertContains(sql, "COMMENT ON TABLE \"public\".\"parcels\"", "Table comment preserved"); + + // 5. 驗證 Native Manifest + AssertContains(sql, "-- mySQLPunk Native Manifest", "Manifest header present"); + AssertContains(sql, "-- Manifest-Table: parcels | Rows: 2", "Manifest table row count accurate"); + AssertContains(sql, "-- Manifest-Checksum-SHA256:", "Manifest SHA-256 checksum present"); + Assert(!string.IsNullOrWhiteSpace(result.ManifestSha256), "Manifest SHA256 string populated in result"); + } + finally + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + + private sealed class MockPostgreSqlExportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "postgresql"; + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + var dt = new DataTable(); + + if (sql.Contains("obj_description(c.oid, 'pg_class')")) + { + dt.Columns.Add("comment", typeof(string)); + dt.Rows.Add("全國地籍空間圖層"); + return dt; + } + + if (sql.Contains("FROM pg_class c") && sql.Contains("relkind")) + { + dt.Columns.Add("Schema", typeof(string)); + dt.Columns.Add("Name", typeof(string)); + dt.Columns.Add("Kind", typeof(string)); + dt.Columns.Add("EstimatedBytes", typeof(long)); + dt.Columns.Add("RowCount", typeof(long)); + + dt.Rows.Add("public", "parcels", "r", 65536L, 2L); + dt.Rows.Add("public", "spatial_ref_sys", "r", 8500000L, 8500L); + return dt; + } + + if (sql.Contains("FROM information_schema.columns c") && sql.Contains("udt_name")) + { + dt.Columns.Add("table_schema", typeof(string)); + dt.Columns.Add("table_name", typeof(string)); + dt.Columns.Add("column_name", typeof(string)); + dt.Columns.Add("udt_name", typeof(string)); + dt.Columns.Add("data_type", typeof(string)); + + dt.Rows.Add("public", "parcels", "geom", "geometry", "USER-DEFINED"); + return dt; + } + + if (sql.Contains("FROM geometry_columns")) + { + dt.Columns.Add("f_table_schema", typeof(string)); + dt.Columns.Add("f_table_name", typeof(string)); + dt.Columns.Add("f_geometry_column", typeof(string)); + dt.Columns.Add("coord_dimension", typeof(int)); + dt.Columns.Add("srid", typeof(int)); + dt.Columns.Add("type", typeof(string)); + + dt.Rows.Add("public", "parcels", "geom", 2, 3826, "MULTIPOLYGON"); + return dt; + } + + if (sql.Contains("WHERE c.table_schema = :schema AND c.table_name = :tableName")) + { + dt.Columns.Add("column_name", typeof(string)); + dt.Columns.Add("data_type", typeof(string)); + dt.Columns.Add("is_nullable", typeof(string)); + dt.Columns.Add("column_default", typeof(string)); + dt.Columns.Add("ProviderType", typeof(string)); + dt.Columns.Add("IsIdentity", typeof(string)); + dt.Columns.Add("IdentityGeneration", typeof(string)); + dt.Columns.Add("IsGenerated", typeof(string)); + dt.Columns.Add("GenerationExpression", typeof(string)); + dt.Columns.Add("Comment", typeof(string)); + + dt.Rows.Add("id", "integer", "NO", "nextval('parcels_id_seq'::regclass)", "integer", "NO", "", "NEVER", "", "地籍宗地編號"); + dt.Rows.Add("owner_name", "character varying", "YES", null, "character varying(100)", "NO", "", "NEVER", "", "地主姓名"); + dt.Rows.Add("geom", "USER-DEFINED", "YES", null, "geometry(MultiPolygon,3826)", "NO", "", "NEVER", "", "宗地邊界多邊形"); + return dt; + } + + if (sql.Contains("SELECT a.attname") && sql.Contains("ix.indisprimary")) + { + dt.Columns.Add("attname", typeof(string)); + dt.Rows.Add("id"); + return dt; + } + + + if (sql.Contains("SELECT i.indexdef") && sql.Contains("FROM pg_indexes i")) + { + dt.Columns.Add("indexdef", typeof(string)); + dt.Rows.Add("CREATE INDEX idx_parcels_geom ON public.parcels USING gist (geom)"); + return dt; + } + + if (sql.Contains("SELECT * FROM \"public\".\"parcels\"")) + { + dt.Columns.Add("id", typeof(int)); + dt.Columns.Add("owner_name", typeof(string)); + dt.Columns.Add("geom", typeof(byte[])); + + byte[] ewkbSample = new byte[] { 0x01, 0x06, 0x00, 0x00, 0x20, 0xF2, 0x0E, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; + dt.Rows.Add(1, "王大明", ewkbSample); + dt.Rows.Add(2, "林小花", ewkbSample); + return dt; + } + + return dt; + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + return new Dictionary { { "status", "OK" }, { "rowsAffected", "1" } }; + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + public List GetDatabases() => new List { "gis_postgres" }; + public List GetTables(string databaseName) => new List { "parcels", "spatial_ref_sys" }; + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => ""; + public bool TableExists(string databaseName, string tableName) => true; + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 2; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void SetConn(string connection) { } + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.PostgreSqlImport.cs b/tests/SmokeTests.PostgreSqlImport.cs new file mode 100644 index 0000000..305b2e2 --- /dev/null +++ b/tests/SmokeTests.PostgreSqlImport.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.PostgreSqlImport; + +public static partial class SmokeTests +{ + private static void PostgreSqlImport_PreflightScanner_PostgisAndDollarQuotes() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_pg_preflight_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +-- mySQLPunk PostgreSQL Logical Dump +CREATE EXTENSION IF NOT EXISTS postgis; + +CREATE TABLE ""public"".""gis_points"" ( + ""id"" serial PRIMARY KEY, + ""geom"" geometry(Point, 4326), + ""updated_at"" timestamp +); + +-- PL/pgSQL Dollar-quoted function with internal semicolons +CREATE OR REPLACE FUNCTION update_timestamp() RETURNS trigger AS $body$ +BEGIN + NEW.updated_at := CURRENT_TIMESTAMP; + RETURN NEW; +END; +$body$ LANGUAGE plpgsql; + +INSERT INTO ""public"".""gis_points"" (""geom"") VALUES +('\x0101000020E6100000000000000000F03F000000000000F03F'::geometry), +('\x0101000020E610000000000000000000400000000000000040'::geometry); + +-- ==================================================================== +-- mySQLPunk Native Manifest +-- Manifest-Table: gis_points | Rows: 2 +-- Manifest-Checksum-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +-- ==================================================================== +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + var scanner = new PostgreSqlImportPreflightScanner(); + var result = scanner.Scan(tempFile); + + Assert(result.CanProceed, "Valid PostgreSQL script should pass preflight"); + Assert(result.HasPostgisFeatures, "Should detect PostGIS keywords (geometry, postgis)"); + Assert(result.HasDollarQuotes, "Should detect Dollar-quoted PL/pgSQL function block ($body$)"); + Assert(result.DollarQuoteBlocksCount == 1, "Expected 1 dollar-quoted block, got " + result.DollarQuoteBlocksCount); + + // 驗證語句計數：CREATE EXTENSION (1) + CREATE TABLE (2) + CREATE FUNCTION (3) + INSERT (4) = 4 條語句 (函數本體內的 2 個分號不可被誤切！) + Assert(result.StatementCount == 4, "StatementCount should be 4, semicolons inside $body$ must NOT be split. Got: " + result.StatementCount); + + Assert(result.ContainsManifest, "Should detect Native Manifest footer"); + Assert(result.ManifestDeclaredTables.ContainsKey("gis_points"), "Manifest should contain gis_points"); + Assert(result.ManifestDeclaredTables["gis_points"] == 2, "Declared rows should be 2"); + Assert(result.ManifestSha256 == "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "SHA256 should match"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void PostgreSqlImport_PreflightScanner_EmbeddedNul() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_pg_nul_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + byte[] corrupt = new byte[] { (byte)'S', (byte)'E', (byte)'L', (byte)'E', (byte)'C', (byte)'T', 0x00, (byte)';' }; + File.WriteAllBytes(tempFile, corrupt); + + var scanner = new PostgreSqlImportPreflightScanner(); + var result = scanner.Scan(tempFile); + + Assert(!result.CanProceed, "Embedded NUL byte must fail preflight"); + Assert(result.Issues.Count > 0, "Issues list should report corrupted NUL character"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void PostgreSqlImport_Runner_ExecutionAndManifestVerification() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_pg_run_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE TABLE ""parcels"" (""id"" int, ""geom"" geometry); +INSERT INTO ""parcels"" VALUES (1, '\x0101'::geometry), (2, '\x0102'::geometry); +-- mySQLPunk Native Manifest +-- Manifest-Table: parcels | Rows: 2 +-- Manifest-Checksum-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + var mockDb = new MockPostgreSqlImportDatabase(); + var options = new PostgreSqlImportOptions + { + AutoInstallPostgis = true, + UseTransaction = true, + StopOnError = true + }; + var request = new PostgreSqlImportRequest(tempFile, "gis_db", options); + var runner = new PostgreSqlImportRunner(mockDb, request); + + var result = runner.Run(); + + Assert(result.Success, "Runner execution should succeed"); + Assert(result.StatementsExecuted == 3, "Expected 3 statements executed, got " + result.StatementsExecuted); + Assert(result.TotalRowsAffected == 2, "Expected 2 rows affected, got " + result.TotalRowsAffected); + Assert(result.ManifestMatched, "Manifest verification should pass (declared 2 == actual 2)"); + Assert(File.Exists(result.LogFilePath), "Independent log file must exist: " + result.LogFilePath); + + // 驗證交易包覆有正確 COMMIT + Assert(mockDb.ExecutedQueries.Contains("BEGIN;"), "Transaction BEGIN must be executed"); + Assert(mockDb.ExecutedQueries.Contains("COMMIT;"), "Transaction COMMIT must be executed"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void PostgreSqlImport_Runner_ExistingObjectStrategies() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_pg_strat_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +CREATE TABLE ""roads"" (""id"" int); +INSERT INTO ""roads"" VALUES (1); +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + // 1. KeepExisting 策略：CREATE TABLE 應轉換為 CREATE TABLE IF NOT EXISTS + var mockDb1 = new MockPostgreSqlImportDatabase(); + var opt1 = new PostgreSqlImportOptions { ConflictStrategy = PostgreSqlConflictStrategy.KeepExisting }; + var runner1 = new PostgreSqlImportRunner(mockDb1, new PostgreSqlImportRequest(tempFile, "gis_db", opt1)); + runner1.Run(); + + bool hasIfNotExists = mockDb1.ExecutedQueries.Exists(q => q.Contains("CREATE TABLE IF NOT EXISTS \"roads\"")); + Assert(hasIfNotExists, "KeepExisting should convert to CREATE TABLE IF NOT EXISTS"); + + // 2. DropAndRecreate 策略：應在建表前預先執行 DROP TABLE IF EXISTS ... CASCADE + var mockDb2 = new MockPostgreSqlImportDatabase(); + var opt2 = new PostgreSqlImportOptions { ConflictStrategy = PostgreSqlConflictStrategy.DropAndRecreate }; + var runner2 = new PostgreSqlImportRunner(mockDb2, new PostgreSqlImportRequest(tempFile, "gis_db", opt2)); + runner2.Run(); + + bool hasDrop = mockDb2.ExecutedQueries.Exists(q => q.Contains("DROP TABLE IF EXISTS \"roads\" CASCADE;")); + Assert(hasDrop, "DropAndRecreate should execute DROP TABLE IF EXISTS CASCADE"); + + // 3. TruncateAndInsert 策略：應在第一次 INSERT 前執行 TRUNCATE TABLE ... CASCADE + var mockDb3 = new MockPostgreSqlImportDatabase(); + var opt3 = new PostgreSqlImportOptions { ConflictStrategy = PostgreSqlConflictStrategy.TruncateAndInsert }; + var runner3 = new PostgreSqlImportRunner(mockDb3, new PostgreSqlImportRequest(tempFile, "gis_db", opt3)); + runner3.Run(); + + bool hasTruncate = mockDb3.ExecutedQueries.Exists(q => q.Contains("TRUNCATE TABLE \"roads\" CASCADE;")); + Assert(hasTruncate, "TruncateAndInsert should execute TRUNCATE TABLE CASCADE"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void PostgreSqlImport_Runner_TransactionRollbackOnFailure() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_pg_fail_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +CREATE TABLE ""valid"" (""id"" int); +INVALID SQL SYNTAX HERE; +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + var mockDb = new MockPostgreSqlImportDatabase(); + mockDb.FailOnSql = "INVALID SQL"; + + var opt = new PostgreSqlImportOptions + { + UseTransaction = true, + StopOnError = true + }; + var request = new PostgreSqlImportRequest(tempFile, "gis_db", opt); + var runner = new PostgreSqlImportRunner(mockDb, request); + + var result = runner.Run(); + + Assert(!result.Success, "Runner should report failure"); + Assert(mockDb.ExecutedQueries.Contains("ROLLBACK;"), "Transaction ROLLBACK must be executed upon statement failure"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private sealed class MockPostgreSqlImportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "postgresql"; + public List ExecutedQueries { get; } = new List(); + public string FailOnSql { get; set; } = null; + + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + ExecutedQueries.Add(sql); + var dt = new DataTable(); + + if (sql.Contains("SELECT COUNT(*) FROM \"parcels\"")) + { + dt.Columns.Add("count", typeof(long)); + dt.Rows.Add(2L); + return dt; + } + + return dt; + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + ExecutedQueries.Add(sql); + + if (!string.IsNullOrWhiteSpace(FailOnSql) && sql.Contains(FailOnSql)) + { + throw new InvalidOperationException("Simulated syntax error: " + sql); + } + + long rows = sql.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase) ? 2 : 0; + return new Dictionary + { + { "status", "OK" }, + { "rowsAffected", rows.ToString() } + }; + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + public List GetDatabases() => new List { "gis_db" }; + public List GetTables(string databaseName) => new List(); + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => ""; + public bool TableExists(string databaseName, string tableName) => false; + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 0; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void SetConn(string connection) { } + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index bb13975..b1e50a3 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -46,6 +46,13 @@ public static int Main(string[] args) Run("SQLite import runner execution and manifest verification", SqliteImport_Runner_ExecutionAndManifestVerification, ref passed); Run("SQLite import runner existing object strategies", SqliteImport_Runner_ExistingObjectStrategies, ref passed); Run("SQLite import runner transaction rollback on failure", SqliteImport_Runner_TransactionRollbackOnFailure, ref passed); + Run("PostgreSQL export options and models defaults", PostgreSqlExport_OptionsAndModels_Defaults, ref passed); + Run("PostgreSQL export service and manifest streaming and hex EWKB", PostgreSqlExport_ServiceAndManifest_StreamingAndHexEwkb, ref passed); + Run("PostgreSQL import preflight scanner PostGIS and dollar quotes", PostgreSqlImport_PreflightScanner_PostgisAndDollarQuotes, ref passed); + Run("PostgreSQL import preflight scanner embedded NUL", PostgreSqlImport_PreflightScanner_EmbeddedNul, ref passed); + Run("PostgreSQL import runner execution and manifest verification", PostgreSqlImport_Runner_ExecutionAndManifestVerification, ref passed); + Run("PostgreSQL import runner existing object strategies", PostgreSqlImport_Runner_ExistingObjectStrategies, ref passed); + Run("PostgreSQL import runner transaction rollback on failure", PostgreSqlImport_Runner_TransactionRollbackOnFailure, ref passed); Run("MSSQL import Transfer guards", MssqlImportTransferGuards, ref passed); Run("MSSQL import Transfer trial", MssqlImportTransferTrial, ref passed); Run("MSSQL import worker published cleanup receipt", MssqlImportPublishedReleaseRace, ref passed); From d41825cead810bc67f263753bb5f986a7f69c65c Mon Sep 17 00:00:00 2001 From: John Date: Thu, 15 Oct 2026 16:06:52 +0800 Subject: [PATCH 08/12] feat(oracle): add 4-step SQL export and import wizards with Oracle Spatial and PL/SQL support - Implement OracleSqlExportService with 64KB StreamWriter and .writing staging atomic replacement to prevent OOM - Losslessly dump MDSYS.SDO_GEOMETRY via SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('...')) and register USER_SDO_GEOM_METADATA - Automatically filter out internal R-Tree spatial shadow tables (MDRT_*, MDXT_*, MDRS_*) - Add OracleImportPreflightScanner with PL/SQL slash (/) block lexer and NUL byte corruption detection - Implement OracleImportRunner supporting 4 conflict strategies (KeepExisting, ReplaceAll, DropAndRecreate, TruncateAndInsert) and FIFO 2,000-line terminal log - Add 4-step wizard UI (OracleExportWizardForm, OracleImportWizardForm) integrated with Punky mascot animation and sound - Register full regression suite in SmokeTests (268/268 passed, auto-release policy passed) - Update TODO.md marking Oracle alignment as completed --- TODO.md | 11 +- mySQLPunk/Form1.OracleExport.cs | 63 ++ mySQLPunk/Form1.OracleImport.cs | 54 ++ mySQLPunk/Form1.cs | 30 + mySQLPunk/OracleExportWizardForm.cs | 669 +++++++++++++++++ mySQLPunk/OracleImportWizardForm.cs | 564 +++++++++++++++ .../lib/OracleExport/OracleExportModels.cs | 111 +++ .../OracleExport/OracleSqlExportService.cs | 675 ++++++++++++++++++ mySQLPunk/lib/OracleImport/OracleImportLog.cs | 105 +++ .../lib/OracleImport/OracleImportModels.cs | 111 +++ .../OracleImportPreflightScanner.cs | 303 ++++++++ .../lib/OracleImport/OracleImportRunner.cs | 522 ++++++++++++++ mySQLPunk/mySQLPunk.csproj | 10 + tests/Run-SmokeTests.ps1 | 2 + tests/SmokeTests.MssqlExport.Routing.cs | 2 +- tests/SmokeTests.MssqlImport.Wizard.cs | 4 +- tests/SmokeTests.OracleExport.cs | 211 ++++++ tests/SmokeTests.OracleImport.cs | 279 ++++++++ tests/SmokeTests.cs | 7 + 19 files changed, 3725 insertions(+), 8 deletions(-) create mode 100644 mySQLPunk/Form1.OracleExport.cs create mode 100644 mySQLPunk/Form1.OracleImport.cs create mode 100644 mySQLPunk/OracleExportWizardForm.cs create mode 100644 mySQLPunk/OracleImportWizardForm.cs create mode 100644 mySQLPunk/lib/OracleExport/OracleExportModels.cs create mode 100644 mySQLPunk/lib/OracleExport/OracleSqlExportService.cs create mode 100644 mySQLPunk/lib/OracleImport/OracleImportLog.cs create mode 100644 mySQLPunk/lib/OracleImport/OracleImportModels.cs create mode 100644 mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs create mode 100644 mySQLPunk/lib/OracleImport/OracleImportRunner.cs create mode 100644 tests/SmokeTests.OracleExport.cs create mode 100644 tests/SmokeTests.OracleImport.cs diff --git a/TODO.md b/TODO.md index 45de392..a736e06 100644 --- a/TODO.md +++ b/TODO.md @@ -35,7 +35,7 @@ | **MySQL / MariaDB** | ✅ 已具備完整四步精靈 (`MySqlExportWizardForm`)，支援選表/View/Routine/Trigger、一致性快照、Native Manifest。 | ✅ 已具備完整四步精靈 (`MySqlImportWizardForm`)，支援串流預檢、4種衝突策略、Session鎖定、終端機滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格）。 | | **SQLite** | ✅ 已具備完整四步精靈 (`SqliteExportWizardForm`)，支援選表、SpatiaLite/GeoPackage 空間幾何無損 Hex 傾印、排除虛擬表影子表、原子替換與 Native Manifest。 | ✅ 已具備完整四步精靈 (`SqliteImportWizardForm`)，支援串流狀態機預檢、SpatiaLite 動態模組自動加載、4種衝突策略、滑動視窗終端機日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS 空間特性）。 | | **PostgreSQL** | ✅ 已具備完整四步精靈 (`PostgreSqlExportWizardForm`)，支援選表/View、PostGIS 空間幾何無損 Hex EWKB 傾印、排除 spatial_ref_sys、REPEATABLE READ 快照與 Native Manifest。 | ✅ 已具備完整四步精靈 (`PostgreSqlImportWizardForm`)，支援狀態機 Lexer (精準解析 Dollar Quotes 與巢狀註解)、PostGIS 自動安裝開關、4種衝突策略、2,000行滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS PostGIS 空間特性）。 | -| **Oracle** | 走舊版 `DatabaseDumpService`，字串拼接。 | 走舊版 `ImportSqlScript`，一次性記憶體讀取。 | **高危 OOM 風險**；無法處理 PL/SQL 區塊 (`/` 結尾)；缺少 RAW/LOB 分塊支援。 | +| **Oracle** | ✅ 已具備完整四步精靈 (`OracleExportWizardForm`)，支援選表/View、Oracle Spatial (`SDO_GEOMETRY`) 無損 WKB 傾印、排除 R-Tree 內部影子表 (`MDRT_*`)、`USER_SDO_GEOM_METADATA` 註冊、`SET TRANSACTION READ ONLY;` 快照與 Native Manifest。 | ✅ 已具備完整四步精靈 (`OracleImportWizardForm`)，支援串流狀態機預檢 (精準解析 PL/SQL 區塊與獨立斜線 `/`)、4種衝突策略、2,000行滑動視窗終端機日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS Oracle Spatial 與 PL/SQL 特長）。 | --- @@ -68,12 +68,13 @@ - [x] 匯出：支援四步精靈（選表/View ➔ 輸出設定/快照隔離/排除 spatial_ref_sys ➔ 物件清單 ➔ 串流執行 + Punky 動畫 + Manifest 生成）；PostGIS geometry/geography/raster 轉無損 EWKB 十六進位字面量；保留 GIST/SP-GIST 空間索引。 - [x] 匯入：支援四步精靈（選檔 ➔ 串流狀態機預檢 Dollar Quotes 與巢狀註解 ➔ 4種物件衝突策略與 PostGIS 自動加載 ➔ 串流執行 + Punky 動畫 + 2,000行滑動視窗終端機日誌 + 獨立 Log + Manifest 逐表筆數核對）。 -#### 🎖️ 第四優先：Oracle +#### 🎖️ 第四優先：Oracle（崩琦 GIS Oracle Spatial 空間特長對齊） - **原因**： - 1. 企業級封閉環境，使用情境相對特定。 - 2. 需處理 PL/SQL 特殊分隔語意（`/` 獨立行代表編譯區塊結束）。 + 1. 企業級大型資料庫與政府空間資訊系統（如國土測繪、地籍圖資、水利圖資）廣泛採用 Oracle Spatial (`MDSYS.SDO_GEOMETRY`)，為崩琦 GIS 核心場景。 + 2. 需處理 PL/SQL 特殊分隔語意（內部多個邏輯分號 `;`，必須以獨立行 `/` 作為區塊終止符）。 - **具體工作**： - - [ ] 匯出/匯入改為串流架構，支援 PL/SQL 區塊解析與 LOB 欄位分段。 + - [x] 匯出：支援四步精靈（選表/View ➔ 輸出設定/唯讀交易快照/排除空間影子表 ➔ 物件清單 ➔ 串流執行 + Punky 動畫 + Manifest 生成）；探測 `SDO_GEOMETRY` 空間幾何欄位並轉為無損 `SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('...'))`；產出 `USER_SDO_GEOM_METADATA` 空間中繼字典註冊；保留 `INDEXTYPE IS MDSYS.SPATIAL_INDEX` 空間索引；預設排除 `MDRT_*`、`MDXT_*`、`MDRS_*` 內部影子表。 + - [x] 匯入：支援四步精靈（選檔 ➔ 串流狀態機預檢 PL/SQL 獨立斜線 `/` 與空間幾何關鍵字 ➔ 4種物件衝突策略 ➔ 串流執行 + Punky 動畫 + 2,000行滑動視窗終端機日誌 + 獨立 Log + Manifest 逐表筆數核對）。 --- diff --git a/mySQLPunk/Form1.OracleExport.cs b/mySQLPunk/Form1.OracleExport.cs new file mode 100644 index 0000000..4e37b8d --- /dev/null +++ b/mySQLPunk/Form1.OracleExport.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.OracleExport; +using OracleExportResult = mySQLPunk.lib.OracleExport.OracleExportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func OracleExportWizardRunner = null; + internal Action OracleExportStatusSink = null; + + private void SetOracleExportStatus(string message) + { + if (OracleExportStatusSink == null) UpdateMainStatus(message); + else OracleExportStatusSink(message); + } + + private void OpenOracleExportWizard(TreeDatabaseTarget target, OracleExportMode mode = OracleExportMode.SchemaAndData, string specificTable = null) + { + if (target == null) return; + try + { + string dbStem = string.IsNullOrWhiteSpace(target.DatabaseName) ? "oracle_dump" : target.DatabaseName; + string defaultFileName = dbStem + (string.IsNullOrWhiteSpace(specificTable) ? "" : "_" + specificTable) + "_oracle_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; + string targetPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultFileName); + var options = new OracleExportWizardOptions(mode, targetPath); + if (!string.IsNullOrWhiteSpace(specificTable)) + { + options.SelectedTables.Add(specificTable); + } + + OracleExportResult result; + if (OracleExportWizardRunner != null) + { + result = OracleExportWizardRunner(target.Database, target.DatabaseName, options); + } + else + { + using (var wizard = new OracleExportWizardForm(target.Database, target.DatabaseName, options)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Success + ? string.Format(ReportText.Get("Oracle 傾印完成：{0} 個物件，共 {1:N0} 列 · {2}", "Oracle dump completed: {0} objects, {1:N0} rows · {2}"), result.TablesExported + result.ViewsExported, result.TotalRowsExported, result.FilePath) + : ReportText.Get("Oracle 匯出失敗：", "Oracle export failed: ") + result.ErrorMessage; + SetOracleExportStatus(status); + } + } + catch (Exception ex) + { + SetOracleExportStatus(ReportText.Get("無法開啟 Oracle 匯出精靈", "Cannot open Oracle export wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.OracleImport.cs b/mySQLPunk/Form1.OracleImport.cs new file mode 100644 index 0000000..0975571 --- /dev/null +++ b/mySQLPunk/Form1.OracleImport.cs @@ -0,0 +1,54 @@ +using System; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.OracleImport; +using OracleImportResult = mySQLPunk.lib.OracleImport.OracleImportResult; + +namespace mySQLPunk +{ + public partial class Form1 + { + internal Func OracleImportWizardRunner = null; + internal Action OracleImportStatusSink = null; + + private void SetOracleImportStatus(string message) + { + if (OracleImportStatusSink == null) UpdateMainStatus(message); + else OracleImportStatusSink(message); + } + + private void OpenOracleImportWizard(TreeDatabaseTarget target) + { + if (target == null) return; + try + { + OracleImportResult result; + if (OracleImportWizardRunner != null) + { + result = OracleImportWizardRunner(target.Database, target.DatabaseName, ""); + } + else + { + using (var wizard = new OracleImportWizardForm(target.Database, target.DatabaseName)) + { + wizard.ShowDialog(this); + result = wizard.Result; + } + } + + if (result != null) + { + string status = result.Success + ? string.Format(ReportText.Get("Oracle 匯入執行完成：{0} 句成功 · 影響 {1:N0} 列 · 日誌：{2}", "Oracle import execution completed: {0} statements · {1:N0} rows · Log: {2}"), result.StatementsExecuted, result.TotalRowsAffected, result.LogFilePath) + : string.Format(ReportText.Get("Oracle 匯入結束：{0} 失敗 · 日誌：{1}", "Oracle import finished: {0} failed · Log: {1}"), result.StatementsFailed, result.LogFilePath); + SetOracleImportStatus(status); + RefreshDatabaseObjectNodes(target.DatabaseNode); + } + } + catch (Exception ex) + { + SetOracleImportStatus(ReportText.Get("無法開啟 Oracle 匯入精靈", "Cannot open Oracle import wizard") + ": " + ex.Message); + } + } + } +} diff --git a/mySQLPunk/Form1.cs b/mySQLPunk/Form1.cs index 55a7879..5deee2c 100644 --- a/mySQLPunk/Form1.cs +++ b/mySQLPunk/Form1.cs @@ -6690,6 +6690,11 @@ private void DumpSelectedTableSql(bool dataOnly) OpenPostgreSqlExportWizard(target, dataOnly ? lib.PostgreSqlExport.PostgreSqlExportMode.DataOnly : lib.PostgreSqlExport.PostgreSqlExportMode.SchemaAndData, tableName); return; } + if (IsOracleTarget(target)) + { + OpenOracleExportWizard(target, dataOnly ? lib.OracleExport.OracleExportMode.DataOnly : lib.OracleExport.OracleExportMode.SchemaAndData, tableName); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -6962,6 +6967,12 @@ private void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) OpenPostgreSqlExportWizard(target, tableDataOnly ? lib.PostgreSqlExport.PostgreSqlExportMode.DataOnly : lib.PostgreSqlExport.PostgreSqlExportMode.SchemaAndData, selection == null ? null : selection.ObjectName); return; } + if (IsOracleTarget(target) && (selection == null || selection.GroupName == "Tables" || selection.GroupName == "Views")) + { + bool tableDataOnly = selection != null && dataOnlyForTable; + OpenOracleExportWizard(target, tableDataOnly ? lib.OracleExport.OracleExportMode.DataOnly : lib.OracleExport.OracleExportMode.SchemaAndData, selection == null ? null : selection.ObjectName); + return; + } using (SaveFileDialog dialog = new SaveFileDialog()) { @@ -7059,6 +7070,13 @@ private static bool IsPostgreSqlTarget(TreeDatabaseTarget target) string.Equals(target.ProviderName, "postgresql", StringComparison.OrdinalIgnoreCase)); } + private static bool IsOracleTarget(TreeDatabaseTarget target) + { + return target != null && + ((target.Database != null && string.Equals(target.Database.ProviderName, "oracle", StringComparison.OrdinalIgnoreCase)) || + string.Equals(target.ProviderName, "oracle", StringComparison.OrdinalIgnoreCase)); + } + private async Task ExportMySqlDatabaseWithProgress(TreeDatabaseTarget target, string targetPath, MySqlExportOptions options) { @@ -7326,6 +7344,12 @@ private void DumpSelectedDatabaseSqlWithDialog() return; } + if (IsOracleTarget(target)) + { + OpenOracleExportWizard(target, lib.OracleExport.OracleExportMode.SchemaAndData); + return; + } + using (SaveFileDialog dialog = new SaveFileDialog()) { dialog.Filter = Localization.T("Common.SqlFilesFilter"); @@ -7401,6 +7425,12 @@ private void ImportSqlWithDialog() return; } + if (IsOracleTarget(target)) + { + OpenOracleImportWizard(target); + return; + } + using (OpenFileDialog dialog = new OpenFileDialog()) { dialog.Title = Localization.T("ImportSql.Title"); diff --git a/mySQLPunk/OracleExportWizardForm.cs b/mySQLPunk/OracleExportWizardForm.cs new file mode 100644 index 0000000..dba8609 --- /dev/null +++ b/mySQLPunk/OracleExportWizardForm.cs @@ -0,0 +1,669 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.OracleExport; +using OracleExportResult = mySQLPunk.lib.OracleExport.OracleExportResult; + +namespace mySQLPunk +{ + internal sealed class OracleExportWizardForm : Form + { + private readonly IDatabase db; + private readonly string databaseName; // Owner / Schema + private readonly OracleSqlExportService service; + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); + private readonly DataGridView tableGrid = new DataGridView(); + private readonly RadioButton schemaData = new RadioButton(), schemaOnly = new RadioButton(), dataOnly = new RadioButton(); + private readonly CheckBox readOnlySnapshot = new CheckBox(), excludeShadowTables = new CheckBox(), includeSpatialMetadata = new CheckBox(), includeSpatialIndexes = new CheckBox(); + private readonly CheckBox includeDrop = new CheckBox(), truncateBefore = new CheckBox(); + private readonly CheckBox includeViews = new CheckBox(), includeIndexes = new CheckBox(), includeComments = new CheckBox(), includeManifest = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(), selectionSummary = new Label(), confirmSummary = new Label(); + private readonly Label currentObjectLabel = new Label(), percentLabel = new Label(), writtenLabel = new Label(); + private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); + private readonly Button openFileButton = new Button(), openFolderButton = new Button(); + private readonly ProgressBar overallBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox logList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private List catalog = new List(); + private CancellationTokenSource cancellation; + private OracleExportProgress lastProgress; + private bool busy, closing, exporting; + private int step; + + internal OracleExportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal OracleExportWizardForm(IDatabase db, string databaseName, OracleExportWizardOptions initialOptions = null) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : "USER"; + this.service = new OracleSqlExportService(db, databaseName); + + Name = "oracleExportWizard"; + Text = T("匯出資料庫 · Oracle (支援 SDO_GEOMETRY 空間幾何無損備份)", "Export database · Oracle (with SDO_GEOMETRY spatial support)"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1120, 740); + MinimumSize = new Size(680, 500); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("Oracle 資料庫邏輯傾印精靈", "Oracle database logical dump wizard") + " · " + databaseName, Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(); + BuildPage1(initialOptions); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var nav = new FlowLayoutPanel { Dock = DockStyle.Fill, AutoSize = true, FlowDirection = FlowDirection.RightToLeft, Padding = new Padding(0, 10, 0, 0) }; + nextButton.AutoSize = true; + nextButton.Click += async (s, e) => await OnNextAsync(); + backButton.AutoSize = true; + backButton.Text = T("◀ 上一步", "◀ Back"); + backButton.Click += (s, e) => MoveStep(-1); + cancelButton.AutoSize = true; + cancelButton.Text = T("取消", "Cancel"); + cancelButton.Click += (s, e) => Close(); + nav.Controls.Add(nextButton); + nav.Controls.Add(backButton); + nav.Controls.Add(cancelButton); + root.Controls.Add(nav, 0, 3); + + timer.Tick += (s, e) => UpdateLiveTicker(); + FormClosing += (s, e) => + { + if (exporting && !closing) + { + if (MessageBox.Show(T("正在匯出 Oracle 資料庫中，確定要中止嗎？", "Export is currently running. Cancel it?"), T("確認", "Confirm"), MessageBoxButtons.YesNo, MessageBoxIcon.Question) != DialogResult.Yes) + { + e.Cancel = true; + return; + } + closing = true; + cancellation?.Cancel(); + } + timer.Stop(); + clock.Stop(); + }; + + Shown += async (s, e) => await LoadCatalogAsync(); + SetStep(0); + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel { Dock = DockStyle.Top, AutoSize = true, FlowDirection = FlowDirection.TopDown, WrapContents = false }; + private static FlowLayoutPanel Row() => new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, WrapContents = true, Margin = new Padding(0, 0, 0, 8) }; + + private void BuildPage0() + { + var p = pages[0]; + p.Padding = new Padding(0, 4, 0, 0); + + var filterRow = Row(); + search.Width = 260; + search.TextChanged += (s, e) => ApplyFilter(); + var allBtn = new Button { Text = T("全選", "Select all"), AutoSize = true }; + allBtn.Click += (s, e) => SetAllSelection(true); + var clearBtn = new Button { Text = T("全不選", "Clear all"), AutoSize = true }; + clearBtn.Click += (s, e) => SetAllSelection(false); + + filterRow.Controls.Add(new Label { Text = T("搜尋：", "Search:"), AutoSize = true, Margin = new Padding(0, 5, 4, 0) }); + filterRow.Controls.Add(search); + filterRow.Controls.Add(allBtn); + filterRow.Controls.Add(clearBtn); + + tableGrid.Dock = DockStyle.Fill; + tableGrid.AllowUserToAddRows = false; + tableGrid.AllowUserToDeleteRows = false; + tableGrid.RowHeadersVisible = false; + tableGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect; + tableGrid.MultiSelect = false; + tableGrid.AutoGenerateColumns = false; + tableGrid.BackgroundColor = SystemColors.Window; + + tableGrid.Columns.Add(new DataGridViewCheckBoxColumn { Name = "sel", HeaderText = T("勾選", "Export"), Width = 60, DataPropertyName = "Selected" }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "owner", HeaderText = T("綱要 (Owner)", "Owner"), Width = 110, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "name", HeaderText = T("物件名稱", "Object name"), Width = 230, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "type", HeaderText = T("類型", "Type"), Width = 70, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "spatial", HeaderText = T("空間特性 (Oracle Spatial)", "Spatial Features"), Width = 260, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "rows", HeaderText = T("列數預估", "Rows (est.)"), Width = 110, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "bytes", HeaderText = T("大小 (Bytes)", "Bytes"), Width = 110, ReadOnly = true }); + + tableGrid.CellContentClick += (s, e) => + { + if (e.RowIndex >= 0 && e.ColumnIndex == 0) + { + tableGrid.CommitEdit(DataGridViewDataErrorContexts.Commit); + UpdateSelectionSummary(); + } + }; + + var main = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 1, RowCount = 3 }; + main.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + main.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + main.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + main.Controls.Add(filterRow, 0, 0); + main.Controls.Add(tableGrid, 0, 1); + selectionSummary.AutoSize = true; + selectionSummary.Margin = new Padding(0, 6, 0, 0); + main.Controls.Add(selectionSummary, 0, 2); + + p.Controls.Add(main); + } + + private void BuildPage1(OracleExportWizardOptions opt) + { + var p = pages[1]; + var host = Stack(); + + var pathRow = Row(); + targetPathBox.Width = 560; + browseButton.Text = T("瀏覽...", "Browse..."); + browseButton.AutoSize = true; + browseButton.Click += (s, e) => + { + using (var sfd = new SaveFileDialog()) + { + sfd.Filter = "SQL Files (*.sql)|*.sql|All Files (*.*)|*.*"; + sfd.FileName = string.Format("oracle_{0}_dump_{1:yyyyMMdd_HHmmss}.sql", databaseName, DateTime.Now); + if (sfd.ShowDialog(this) == DialogResult.OK) + { + targetPathBox.Text = sfd.FileName; + } + } + }; + pathRow.Controls.Add(new Label { Text = T("匯出檔案路徑：", "Target file path:"), AutoSize = true, Margin = new Padding(0, 5, 4, 0) }); + pathRow.Controls.Add(targetPathBox); + pathRow.Controls.Add(browseButton); + host.Controls.Add(pathRow); + + var modeBox = new GroupBox { Text = T("匯出模式", "Export mode"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 4, 0, 8) }; + var modeFlow = Stack(); + schemaData.Text = T("結構與資料 (Schema and Data)", "Schema and Data"); + schemaData.Checked = true; + schemaData.AutoSize = true; + dataOnly.Text = T("僅資料 (Data only)", "Data only"); + dataOnly.AutoSize = true; + schemaOnly.Text = T("僅結構 (Schema only)", "Schema only"); + schemaOnly.AutoSize = true; + modeFlow.Controls.Add(schemaData); + modeFlow.Controls.Add(dataOnly); + modeFlow.Controls.Add(schemaOnly); + modeBox.Controls.Add(modeFlow); + host.Controls.Add(modeBox); + + var spatialBox = new GroupBox { Text = T("Oracle Spatial 崩琦 GIS 特徵與交易防護", "Oracle Spatial GIS & Transaction Settings"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 4, 0, 8) }; + var spatialFlow = Stack(); + readOnlySnapshot.Text = T("使用交易唯讀快照隔離 (SET TRANSACTION READ ONLY) 防範資料與外鍵不一致", "Use transaction READ ONLY snapshot isolation to guarantee foreign key consistency"); + readOnlySnapshot.Checked = true; + readOnlySnapshot.AutoSize = true; + + excludeShadowTables.Text = T("排除 Oracle Spatial 內部 R-Tree 空間索引影子表 (MDRT_*, MDXT_*, MDRS_*)", "Exclude Oracle Spatial internal R-Tree index shadow tables (MDRT_*, MDXT_*)"); + excludeShadowTables.Checked = true; + excludeShadowTables.AutoSize = true; + + includeSpatialMetadata.Text = T("產生空間中繼字典註冊語句 (INSERT INTO USER_SDO_GEOM_METADATA)", "Generate spatial metadata registration (INSERT INTO USER_SDO_GEOM_METADATA)"); + includeSpatialMetadata.Checked = true; + includeSpatialMetadata.AutoSize = true; + + includeSpatialIndexes.Text = T("導出 Oracle Spatial 空間網格幾何索引 (INDEXTYPE IS MDSYS.SPATIAL_INDEX)", "Export Oracle Spatial domain indexes (INDEXTYPE IS MDSYS.SPATIAL_INDEX)"); + includeSpatialIndexes.Checked = true; + includeSpatialIndexes.AutoSize = true; + + spatialFlow.Controls.Add(readOnlySnapshot); + spatialFlow.Controls.Add(excludeShadowTables); + spatialFlow.Controls.Add(includeSpatialMetadata); + spatialFlow.Controls.Add(includeSpatialIndexes); + spatialBox.Controls.Add(spatialFlow); + host.Controls.Add(spatialBox); + + var ddlBox = new GroupBox { Text = T("進階選項與腳本 Manifest", "Advanced options & Script Manifest"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 4, 0, 8) }; + var ddlFlow = Stack(); + includeDrop.Text = T("在建表前加入 DROP TABLE CASCADE CONSTRAINTS", "Add DROP TABLE CASCADE CONSTRAINTS before CREATE"); + includeDrop.AutoSize = true; + truncateBefore.Text = T("在插入前加入 TRUNCATE TABLE", "Add TRUNCATE TABLE before INSERT"); + truncateBefore.AutoSize = true; + includeViews.Text = T("包含檢視 (Views)", "Include views"); + includeViews.Checked = true; + includeViews.AutoSize = true; + includeIndexes.Text = T("包含常規索引 (Indexes)", "Include regular indexes"); + includeIndexes.Checked = true; + includeIndexes.AutoSize = true; + includeComments.Text = T("包含資料表與欄位註解 (Comments)", "Include comments"); + includeComments.Checked = true; + includeComments.AutoSize = true; + includeManifest.Text = T("在檔尾附帶 Native Manifest 清單與 SHA-256 核對碼", "Append Native Manifest summary & SHA-256 footer"); + includeManifest.Checked = true; + includeManifest.AutoSize = true; + + ddlFlow.Controls.Add(includeDrop); + ddlFlow.Controls.Add(truncateBefore); + ddlFlow.Controls.Add(includeViews); + ddlFlow.Controls.Add(includeIndexes); + ddlFlow.Controls.Add(includeComments); + ddlFlow.Controls.Add(includeManifest); + ddlBox.Controls.Add(ddlFlow); + host.Controls.Add(ddlBox); + + if (opt != null) + { + if (opt.Mode == OracleExportMode.DataOnly) dataOnly.Checked = true; + else if (opt.Mode == OracleExportMode.SchemaOnly) schemaOnly.Checked = true; + if (!string.IsNullOrWhiteSpace(opt.TargetPath)) targetPathBox.Text = opt.TargetPath; + } + if (string.IsNullOrWhiteSpace(targetPathBox.Text)) + { + targetPathBox.Text = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), + string.Format("oracle_{0}_dump_{1:yyyyMMdd_HHmmss}.sql", databaseName, DateTime.Now)); + } + + p.Controls.Add(host); + } + + private void BuildPage2() + { + var p = pages[2]; + var host = Stack(); + host.Controls.Add(new Label { Text = T("請確認即將執行的 Oracle 邏輯傾印清單與設定：", "Confirm export settings and objects:"), AutoSize = true, Font = UiKit.Subtitle, Margin = new Padding(0, 0, 0, 12) }); + confirmSummary.AutoSize = true; + confirmSummary.Font = UiKit.Body; + confirmSummary.Margin = new Padding(12, 0, 0, 0); + host.Controls.Add(confirmSummary); + p.Controls.Add(host); + } + + private void BuildPage3() + { + var p = pages[3]; + var root = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 2, RowCount = 3 }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 320)); + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + + var mascotPane = Stack(); + punky.Size = new Size(240, 240); + mascotPane.Controls.Add(punky); + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + soundButton.Margin = new Padding(12, 8, 0, 0); + soundButton.Click += (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }; + mascotPane.Controls.Add(soundButton); + mascotHint.AutoSize = true; + mascotHint.Margin = new Padding(12, 8, 0, 0); + mascotHint.Text = T("Punky 看板娘正在為您嚴格把關空間座標與傾印串流...", "Punky is streaming Oracle data with spatial integrity..."); + mascotPane.Controls.Add(mascotHint); + root.Controls.Add(mascotPane, 0, 0); + root.SetRowSpan(mascotPane, 3); + + var statStack = Stack(); + currentObjectLabel.AutoSize = true; + currentObjectLabel.Font = UiKit.Subtitle; + statStack.Controls.Add(currentObjectLabel); + + overallBar.Height = 24; + overallBar.Dock = DockStyle.Top; + overallBar.Margin = new Padding(0, 8, 0, 8); + statStack.Controls.Add(overallBar); + + var grid = new TableLayoutPanel { AutoSize = true, ColumnCount = 4, Margin = new Padding(0, 4, 0, 8) }; + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + + percentLabel.AutoSize = true; grid.Controls.Add(percentLabel, 0, 0); + writtenLabel.AutoSize = true; grid.Controls.Add(writtenLabel, 1, 0); + speedLabel.AutoSize = true; grid.Controls.Add(speedLabel, 2, 0); + elapsedLabel.AutoSize = true; grid.Controls.Add(elapsedLabel, 3, 0); + statStack.Controls.Add(grid); + + root.Controls.Add(statStack, 1, 0); + + var buttonRow = Row(); + openFileButton.Text = T("開啟匯出檔", "Open file"); + openFileButton.AutoSize = true; + openFileButton.Visible = false; + openFileButton.Click += (s, e) => + { + if (File.Exists(targetPathBox.Text)) Process.Start(new ProcessStartInfo(targetPathBox.Text) { UseShellExecute = true }); + }; + openFolderButton.Text = T("開啟目錄", "Open folder"); + openFolderButton.AutoSize = true; + openFolderButton.Visible = false; + openFolderButton.Click += (s, e) => + { + string dir = Path.GetDirectoryName(targetPathBox.Text); + if (Directory.Exists(dir)) Process.Start(new ProcessStartInfo(dir) { UseShellExecute = true }); + }; + resultText.AutoSize = true; + resultText.Margin = new Padding(0, 4, 12, 0); + buttonRow.Controls.Add(resultText); + buttonRow.Controls.Add(openFileButton); + buttonRow.Controls.Add(openFolderButton); + root.Controls.Add(buttonRow, 1, 1); + + logList.Dock = DockStyle.Fill; + logList.Font = new Font("Consolas", 9f); + root.Controls.Add(logList, 1, 2); + + p.Controls.Add(root); + } + + private async Task LoadCatalogAsync() + { + if (busy) return; + busy = true; + statusLabel.Text = T("正在探測 Oracle 資料庫物件與空間幾何欄位...", "Discovering Oracle tables and spatial features..."); + try + { + catalog = await service.GetTableCatalogAsync().ConfigureAwait(true); + ApplyFilter(); + statusLabel.Text = string.Format(T("探測完成：共找到 {0} 個資料表與檢視。", "Catalog loaded: {0} tables/views found."), catalog.Count); + } + catch (Exception ex) + { + statusLabel.Text = T("探測失敗: ", "Discovery failed: ") + ex.Message; + } + finally + { + busy = false; + } + } + + private void ApplyFilter() + { + string q = search.Text.Trim(); + tableGrid.Rows.Clear(); + + foreach (var item in catalog) + { + if (!string.IsNullOrWhiteSpace(q)) + { + if (item.Name.IndexOf(q, StringComparison.OrdinalIgnoreCase) < 0 && + item.Owner.IndexOf(q, StringComparison.OrdinalIgnoreCase) < 0 && + item.SpatialFeatureDescription.IndexOf(q, StringComparison.OrdinalIgnoreCase) < 0) + continue; + } + + int rowIdx = tableGrid.Rows.Add( + item.Selected, + item.Owner, + item.Name, + item.IsView ? "VIEW" : "TABLE", + item.IsSpatial ? "⚡ " + item.SpatialFeatureDescription : (item.IsSpatialMetadata ? "⚙️ Spatial Shadow" : "-"), + item.IsView ? "-" : item.RowCount.ToString("N0"), + item.IsView ? "-" : item.EstimatedBytes.ToString("N0") + ); + + tableGrid.Rows[rowIdx].Tag = item; + if (item.IsSpatial) + { + tableGrid.Rows[rowIdx].DefaultCellStyle.BackColor = Color.FromArgb(240, 248, 255); + } + } + + UpdateSelectionSummary(); + } + + private void SetAllSelection(bool select) + { + foreach (DataGridViewRow row in tableGrid.Rows) + { + if (row.Tag is OracleTableExportItem item) + { + if (item.IsSpatialMetadata && select && excludeShadowTables.Checked) + { + // 影子表維持排除 + continue; + } + item.Selected = select; + row.Cells[0].Value = select; + } + } + UpdateSelectionSummary(); + } + + private void UpdateSelectionSummary() + { + int tables = 0, views = 0, spatial = 0; + long rows = 0; + + foreach (var item in catalog) + { + if (item.Selected) + { + if (item.IsView) views++; + else { tables++; rows += item.RowCount; } + if (item.IsSpatial) spatial++; + } + } + + selectionSummary.Text = string.Format( + T("已選取：{0} 張資料表 ({1:N0} 列)，{2} 個檢視。空間幾何物件：{3} 個。", + "Selected: {0} tables ({1:N0} rows), {2} views. Spatial objects: {3}."), + tables, rows, views, spatial); + } + + private void SetStep(int s) + { + step = s; + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + backButton.Enabled = (step > 0 && step < 3); + cancelButton.Enabled = (step < 3); + + if (step == 0) + { + stepsLabel.Text = T("步驟 1/4：選取要匯出的 Oracle 資料表與檢視", "Step 1/4: Select Oracle tables and views to export"); + nextButton.Text = T("下一步 (設定選項) ▶", "Next (Options) ▶"); + } + else if (step == 1) + { + stepsLabel.Text = T("步驟 2/4：指定匯出檔案與 Oracle Spatial / 快照選項", "Step 2/4: Set export options and Oracle Spatial settings"); + nextButton.Text = T("下一步 (確認清單) ▶", "Next (Review) ▶"); + } + else if (step == 2) + { + stepsLabel.Text = T("步驟 3/4：確認匯出設定與即將傾印物件清單", "Step 3/4: Review export settings and objects"); + nextButton.Text = T("開始匯出 🚀", "Start Export 🚀"); + UpdateConfirmSummary(); + } + else if (step == 3) + { + stepsLabel.Text = T("步驟 4/4：正在串流匯出 Oracle 資料庫...", "Step 4/4: Exporting Oracle database streaming..."); + nextButton.Text = T("完成", "Done"); + nextButton.Enabled = false; + backButton.Enabled = false; + } + } + + private void UpdateConfirmSummary() + { + var sb = new StringBuilder(); + sb.AppendLine("• " + T("目標輸出路徑: ", "Target path: ") + targetPathBox.Text); + sb.AppendLine("• " + T("匯出模式: ", "Export mode: ") + (schemaData.Checked ? "結構與資料 (Schema and Data)" : (dataOnly.Checked ? "僅資料 (Data only)" : "僅結構 (Schema only)"))); + sb.AppendLine("• " + T("交易快照隔離: ", "Snapshot: ") + (readOnlySnapshot.Checked ? "SET TRANSACTION READ ONLY (一致性快照)" : "無 (不開啟快照)")); + sb.AppendLine("• " + T("空間中繼註冊: ", "Spatial metadata: ") + (includeSpatialMetadata.Checked ? "是 (USER_SDO_GEOM_METADATA)" : "否")); + sb.AppendLine("• " + T("空間網格索引: ", "Spatial index: ") + (includeSpatialIndexes.Checked ? "是 (MDSYS.SPATIAL_INDEX)" : "否")); + sb.AppendLine("• " + T("Native Manifest: ", "Native Manifest: ") + (includeManifest.Checked ? "包含 SHA-256 與逐表列數" : "不包含")); + sb.AppendLine(); + sb.AppendLine(selectionSummary.Text); + + confirmSummary.Text = sb.ToString(); + } + + private void MoveStep(int delta) + { + int n = step + delta; + if (n >= 0 && n < 4) SetStep(n); + } + + private async Task OnNextAsync() + { + if (step == 0) + { + int count = catalog.Count(i => i.Selected); + if (count == 0) + { + MessageBox.Show(T("請至少選取一個資料表或檢視！", "Please select at least one table or view!"), T("提示", "Notice"), MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + MoveStep(1); + } + else if (step == 1) + { + if (string.IsNullOrWhiteSpace(targetPathBox.Text)) + { + MessageBox.Show(T("請指定目標檔案路徑！", "Please specify target file path!"), T("提示", "Notice"), MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + MoveStep(1); + } + else if (step == 2) + { + MoveStep(1); + await StartExportAsync(); + } + else if (step == 3) + { + Close(); + } + } + + private async Task StartExportAsync() + { + exporting = true; + clock.Restart(); + timer.Start(); + punky.BeginExport(); + + var opt = new OracleExportWizardOptions + { + Mode = schemaData.Checked ? OracleExportMode.SchemaAndData : (dataOnly.Checked ? OracleExportMode.DataOnly : OracleExportMode.SchemaOnly), + TargetPath = targetPathBox.Text.Trim(), + UseTransactionSnapshot = readOnlySnapshot.Checked, + ExcludeSpatialIndexInternalTables = excludeShadowTables.Checked, + IncludeSpatialMetadata = includeSpatialMetadata.Checked, + IncludeSpatialIndexes = includeSpatialIndexes.Checked, + DropTableIfExists = includeDrop.Checked, + TruncateBeforeInsert = truncateBefore.Checked, + IncludeViews = includeViews.Checked, + IncludeIndexes = includeIndexes.Checked, + IncludeComments = includeComments.Checked, + IncludeManifest = includeManifest.Checked + }; + + foreach (var it in catalog) + { + if (it.Selected) opt.SelectedTables.Add(it.Name); + } + + cancellation = new CancellationTokenSource(); + var prog = new Progress(p => + { + lastProgress = p; + overallBar.Value = (int)p.PercentComplete; + percentLabel.Text = string.Format("{0:0.0} %", p.PercentComplete); + currentObjectLabel.Text = p.Message; + writtenLabel.Text = string.Format(T("已寫入: {0:N0} KB", "Written: {0:N0} KB"), p.BytesExported / 1024); + + if (!string.IsNullOrWhiteSpace(p.Message)) + { + logList.Items.Add(string.Format("[{0:HH:mm:ss}] {1}", DateTime.Now, p.Message)); + logList.TopIndex = logList.Items.Count - 1; + } + }); + + try + { + Result = await Task.Run(() => service.ExportAsync(opt, prog, cancellation.Token)).ConfigureAwait(true); + timer.Stop(); + clock.Stop(); + punky.EndExport(); + + resultText.Text = string.Format(T("✅ 匯出成功！耗時 {0:hh\\:mm\\:ss}，檔案大小: {1:N0} KB", "✅ Export succeeded! Elapsed {0:hh\\:mm\\:ss}, Size: {1:N0} KB"), Result.Elapsed, Result.FileSizeBytes / 1024); + resultText.ForeColor = Color.DarkGreen; + openFileButton.Visible = true; + openFolderButton.Visible = true; + nextButton.Enabled = true; + cancelButton.Enabled = true; + cancelButton.Text = T("關閉", "Close"); + } + catch (Exception ex) + { + timer.Stop(); + clock.Stop(); + punky.EndExport(); + + resultText.Text = T("❌ 匯出失敗: ", "❌ Export failed: ") + ex.Message; + resultText.ForeColor = Color.Red; + nextButton.Enabled = true; + cancelButton.Enabled = true; + } + finally + { + exporting = false; + } + } + + private void UpdateLiveTicker() + { + elapsedLabel.Text = string.Format(T("已耗時: {0:hh\\:mm\\:ss}", "Elapsed: {0:hh\\:mm\\:ss}"), clock.Elapsed); + if (lastProgress != null && clock.Elapsed.TotalSeconds > 0) + { + double kbSec = (lastProgress.BytesExported / 1024.0) / clock.Elapsed.TotalSeconds; + speedLabel.Text = string.Format("{0:N0} KB/s", kbSec); + } + } + } +} diff --git a/mySQLPunk/OracleImportWizardForm.cs b/mySQLPunk/OracleImportWizardForm.cs new file mode 100644 index 0000000..4b78941 --- /dev/null +++ b/mySQLPunk/OracleImportWizardForm.cs @@ -0,0 +1,564 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using mySQLPunk.lib; +using mySQLPunk.lib.OracleImport; +using OracleImportResult = mySQLPunk.lib.OracleImport.OracleImportResult; + +namespace mySQLPunk +{ + internal sealed class OracleImportWizardForm : Form + { + private const int MaxLogLines = 2000; + private readonly IDatabase db; + private readonly string databaseName; // Owner / Schema + private readonly Panel[] pages = new Panel[4]; + private readonly TextBox filePathBox = new TextBox(); + private readonly Button browseFileButton = new Button(); + private readonly Label preflightSummary = new Label(), preflightDetails = new Label(); + private readonly ListBox issuesList = new ListBox(); + private readonly RadioButton strategyKeep = new RadioButton(), strategyReplace = new RadioButton(); + private readonly RadioButton strategyDrop = new RadioButton(), strategyTruncate = new RadioButton(); + private readonly CheckBox autoRegisterSpatial = new CheckBox(), stopOnError = new CheckBox(), useTransaction = new CheckBox(); + private readonly Label stepsLabel = new Label(), statusLabel = new Label(); + private readonly Label currentStatementLabel = new Label(), coverageLabel = new Label(), speedLabel = new Label(); + private readonly Label elapsedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); + private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), soundButton = new Button(); + private readonly Button openLogButton = new Button(), openFolderButton = new Button(), copyPathButton = new Button(), clearLogButton = new Button(); + private readonly CheckBox autoScrollCheck = new CheckBox(); + private readonly ProgressBar execBar = new ProgressBar(); + private readonly PunkyReportLoadingControl punky = new PunkyReportLoadingControl(); + private readonly ListBox logList = new ListBox(); + private readonly System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer { Interval = 250 }; + private readonly Stopwatch clock = new Stopwatch(); + + private OracleImportPreflight preflight; + private CancellationTokenSource cancellation; + private bool executing, closing; + private int step; + private long lastBytes; + + internal OracleImportResult Result { get; private set; } + + private static string T(string zh, string en) => ReportText.Get(zh, en); + + internal OracleImportWizardForm(IDatabase db, string databaseName, string initialSourcePath = "") + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : "USER"; + + Name = "oracleImportWizard"; + Text = T("匯入 SQL 檔案 · Oracle (支援 SDO_GEOMETRY 空間幾何還原)", "Import SQL file · Oracle (with SDO_GEOMETRY spatial support)"); + AutoScaleMode = AutoScaleMode.Dpi; + Font = UiKit.Body; + ClientSize = new Size(1120, 750); + MinimumSize = new Size(680, 500); + StartPosition = FormStartPosition.CenterParent; + + var root = new TableLayoutPanel + { + Dock = DockStyle.Fill, + Padding = new Padding(24), + ColumnCount = 1, + RowCount = 4 + }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + Controls.Add(root); + + var header = Stack(); + header.Controls.Add(new Label { AutoSize = true, Text = T("Oracle 原生 SQL 匯入精靈", "Oracle native SQL import wizard") + " · " + databaseName, Font = UiKit.Title, Margin = new Padding(0, 4, 0, 8) }); + stepsLabel.AutoSize = true; + stepsLabel.Margin = new Padding(0, 4, 0, 12); + header.Controls.Add(stepsLabel); + root.Controls.Add(header, 0, 0); + + var body = new Panel { Dock = DockStyle.Fill }; + root.Controls.Add(body, 0, 1); + for (int i = 0; i < 4; i++) + { + pages[i] = new Panel { Dock = DockStyle.Fill, AutoScroll = true }; + body.Controls.Add(pages[i]); + } + + BuildPage0(initialSourcePath); + BuildPage1(); + BuildPage2(); + BuildPage3(); + + statusLabel.AutoSize = true; + statusLabel.Margin = new Padding(0, 8, 0, 6); + root.Controls.Add(statusLabel, 0, 2); + + var nav = new FlowLayoutPanel { Dock = DockStyle.Fill, AutoSize = true, FlowDirection = FlowDirection.RightToLeft, Padding = new Padding(0, 10, 0, 0) }; + nextButton.AutoSize = true; + nextButton.Click += async (s, e) => await OnNextAsync(); + backButton.AutoSize = true; + backButton.Text = T("◀ 上一步", "◀ Back"); + backButton.Click += (s, e) => MoveStep(-1); + cancelButton.AutoSize = true; + cancelButton.Text = T("取消", "Cancel"); + cancelButton.Click += (s, e) => Close(); + nav.Controls.Add(nextButton); + nav.Controls.Add(backButton); + nav.Controls.Add(cancelButton); + root.Controls.Add(nav, 0, 3); + + timer.Tick += (s, e) => UpdateLiveTicker(); + FormClosing += (s, e) => + { + if (executing && !closing) + { + if (MessageBox.Show(T("正在匯入 Oracle SQL 腳本中，確定要中途停止嗎？", "Import is currently executing. Cancel it?"), T("確認", "Confirm"), MessageBoxButtons.YesNo, MessageBoxIcon.Question) != DialogResult.Yes) + { + e.Cancel = true; + return; + } + closing = true; + cancellation?.Cancel(); + } + timer.Stop(); + clock.Stop(); + }; + + SetStep(0); + } + + private static FlowLayoutPanel Stack() => new FlowLayoutPanel { Dock = DockStyle.Top, AutoSize = true, FlowDirection = FlowDirection.TopDown, WrapContents = false }; + private static FlowLayoutPanel Row() => new FlowLayoutPanel { AutoSize = true, FlowDirection = FlowDirection.LeftToRight, WrapContents = true, Margin = new Padding(0, 0, 0, 8) }; + + private void BuildPage0(string initialPath) + { + var p = pages[0]; + var host = Stack(); + + var fileRow = Row(); + filePathBox.Width = 560; + filePathBox.Text = initialPath ?? string.Empty; + browseFileButton.Text = T("瀏覽...", "Browse..."); + browseFileButton.AutoSize = true; + browseFileButton.Click += (s, e) => + { + using (var ofd = new OpenFileDialog()) + { + ofd.Filter = "SQL Files (*.sql)|*.sql|All Files (*.*)|*.*"; + if (ofd.ShowDialog(this) == DialogResult.OK) + { + filePathBox.Text = ofd.FileName; + } + } + }; + + fileRow.Controls.Add(new Label { Text = T("選取 SQL 腳本檔案：", "Select SQL script file:"), AutoSize = true, Margin = new Padding(0, 5, 4, 0) }); + fileRow.Controls.Add(filePathBox); + fileRow.Controls.Add(browseFileButton); + host.Controls.Add(fileRow); + + var infoBox = new GroupBox { Text = T("匯入環境說明", "Import target info"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 8, 0, 0) }; + var infoFlow = Stack(); + infoFlow.Controls.Add(new Label { Text = "• " + T("目標資料庫綱要 (Owner)：", "Target Schema/Owner: ") + databaseName, AutoSize = true, Margin = new Padding(0, 2, 0, 4) }); + infoFlow.Controls.Add(new Label { Text = "• " + T("執行核心：串流狀態機 Lexer (支援 PL/SQL 區塊斜線 / 分隔符與 32 MiB 上限保護)", "Engine: Streaming State Machine Lexer with PL/SQL block delimiter / & 32 MiB safety check"), AutoSize = true, Margin = new Padding(0, 2, 0, 4) }); + infoFlow.Controls.Add(new Label { Text = "• " + T("空間支援：MDSYS.SDO_GEOMETRY、WKB 二進位與 USER_SDO_GEOM_METADATA 註冊", "Spatial Support: MDSYS.SDO_GEOMETRY, WKB & USER_SDO_GEOM_METADATA"), AutoSize = true, Margin = new Padding(0, 2, 0, 4) }); + infoBox.Controls.Add(infoFlow); + host.Controls.Add(infoBox); + + p.Controls.Add(host); + } + + private void BuildPage1() + { + var p = pages[1]; + var host = Stack(); + + host.Controls.Add(new Label { Text = T("預檢掃描結果 (Preflight Inspection)", "Preflight Inspection Result"), Font = UiKit.Subtitle, AutoSize = true, Margin = new Padding(0, 0, 0, 8) }); + preflightSummary.AutoSize = true; + preflightSummary.Font = UiKit.Body; + host.Controls.Add(preflightSummary); + + preflightDetails.AutoSize = true; + preflightDetails.Font = new Font("Consolas", 9.5f); + preflightDetails.Margin = new Padding(0, 8, 0, 8); + host.Controls.Add(preflightDetails); + + host.Controls.Add(new Label { Text = T("注意事項與潛在風險：", "Issues and Notices:"), AutoSize = true, Margin = new Padding(0, 4, 0, 4) }); + issuesList.Height = 160; + issuesList.Width = 820; + issuesList.Font = new Font("Consolas", 9f); + host.Controls.Add(issuesList); + + p.Controls.Add(host); + } + + private void BuildPage2() + { + var p = pages[2]; + var host = Stack(); + + var stratBox = new GroupBox { Text = T("既有物件衝突處理策略", "Object conflict resolution strategy"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 0, 0, 8) }; + var stratFlow = Stack(); + strategyKeep.Text = T("保留既有物件 (Keep Existing) - 若物件已存在則忽略錯誤繼續", "Keep Existing - Ignore ORA-00955 if table/object already exists"); + strategyKeep.Checked = true; + strategyKeep.AutoSize = true; + strategyReplace.Text = T("全部取代 (Replace All) - 自動執行 DROP TABLE CASCADE CONSTRAINTS 後重建", "Replace All - Automatically execute DROP TABLE CASCADE CONSTRAINTS"); + strategyReplace.AutoSize = true; + strategyDrop.Text = T("先刪除後重建 (Drop and Recreate) - 在建表前刪除同名資料表", "Drop and Recreate - Drop table before creating"); + strategyDrop.AutoSize = true; + strategyTruncate.Text = T("清空既有內容 (Truncate and Insert) - 清空資料表後再匯入新資料", "Truncate and Insert - Truncate table before inserting rows"); + strategyTruncate.AutoSize = true; + + stratFlow.Controls.Add(strategyKeep); + stratFlow.Controls.Add(strategyReplace); + stratFlow.Controls.Add(strategyDrop); + stratFlow.Controls.Add(strategyTruncate); + stratBox.Controls.Add(stratFlow); + host.Controls.Add(stratBox); + + var optBox = new GroupBox { Text = T("執行保護與 Oracle Spatial 選項", "Execution protection & Spatial settings"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 0, 0, 8) }; + var optFlow = Stack(); + + autoRegisterSpatial.Text = T("自動補全 USER_SDO_GEOM_METADATA 空間中繼字典 (若腳本內未包含)", "Auto-register USER_SDO_GEOM_METADATA if missing"); + autoRegisterSpatial.Checked = true; + autoRegisterSpatial.AutoSize = true; + + useTransaction.Text = T("使用交易包覆 (COMMIT / ROLLBACK)", "Wrap execution in transaction (COMMIT / ROLLBACK)"); + useTransaction.Checked = true; + useTransaction.AutoSize = true; + + stopOnError.Text = T("發生錯誤時立即停止並還原 (Stop on Error)", "Stop immediately and rollback on error"); + stopOnError.Checked = true; + stopOnError.AutoSize = true; + + optFlow.Controls.Add(autoRegisterSpatial); + optFlow.Controls.Add(useTransaction); + optFlow.Controls.Add(stopOnError); + optBox.Controls.Add(optFlow); + host.Controls.Add(optBox); + + p.Controls.Add(host); + } + + private void BuildPage3() + { + var p = pages[3]; + var root = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 2, RowCount = 3 }; + root.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 320)); + root.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + root.RowStyles.Add(new RowStyle(SizeType.Percent, 100)); + + var mascotPane = Stack(); + punky.Size = new Size(240, 240); + mascotPane.Controls.Add(punky); + + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + soundButton.Enabled = punky.AudioAvailable; + soundButton.AutoSize = true; + soundButton.Margin = new Padding(12, 8, 0, 0); + soundButton.Click += (s, e) => + { + punky.Muted = !punky.Muted; + soundButton.Text = punky.Muted ? T("開啟聲音", "Unmute") : T("靜音", "Mute"); + }; + mascotPane.Controls.Add(soundButton); + + mascotHint.AutoSize = true; + mascotHint.Margin = new Padding(12, 8, 0, 0); + mascotHint.Text = T("Punky 看板娘正在監視 Oracle 匯入狀態機與終端機日誌...", "Punky is streaming Oracle SQL into the database..."); + mascotPane.Controls.Add(mascotHint); + root.Controls.Add(mascotPane, 0, 0); + root.SetRowSpan(mascotPane, 3); + + var statStack = Stack(); + currentStatementLabel.AutoSize = true; + currentStatementLabel.Font = UiKit.Subtitle; + statStack.Controls.Add(currentStatementLabel); + + execBar.Height = 24; + execBar.Dock = DockStyle.Top; + execBar.Margin = new Padding(0, 8, 0, 8); + statStack.Controls.Add(execBar); + + var grid = new TableLayoutPanel { AutoSize = true, ColumnCount = 4, Margin = new Padding(0, 4, 0, 8) }; + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 25)); + + coverageLabel.AutoSize = true; grid.Controls.Add(coverageLabel, 0, 0); + speedLabel.AutoSize = true; grid.Controls.Add(speedLabel, 1, 0); + elapsedLabel.AutoSize = true; grid.Controls.Add(elapsedLabel, 2, 0); + etaLabel.AutoSize = true; grid.Controls.Add(etaLabel, 3, 0); + statStack.Controls.Add(grid); + + root.Controls.Add(statStack, 1, 0); + + var buttonRow = Row(); + openLogButton.Text = T("開啟詳細 Log", "Open Log"); + openLogButton.AutoSize = true; + openLogButton.Visible = false; + openLogButton.Click += (s, e) => + { + if (Result != null && File.Exists(Result.LogFilePath)) Process.Start(new ProcessStartInfo(Result.LogFilePath) { UseShellExecute = true }); + }; + openFolderButton.Text = T("開啟 Log 目錄", "Open Log Folder"); + openFolderButton.AutoSize = true; + openFolderButton.Visible = false; + openFolderButton.Click += (s, e) => + { + if (Result != null && !string.IsNullOrWhiteSpace(Result.LogFilePath)) + { + string dir = Path.GetDirectoryName(Result.LogFilePath); + if (Directory.Exists(dir)) Process.Start(new ProcessStartInfo(dir) { UseShellExecute = true }); + } + }; + copyPathButton.Text = T("複製 Log 路徑", "Copy Log Path"); + copyPathButton.AutoSize = true; + copyPathButton.Visible = false; + copyPathButton.Click += (s, e) => + { + if (Result != null && !string.IsNullOrWhiteSpace(Result.LogFilePath)) + { + Clipboard.SetText(Result.LogFilePath); + MessageBox.Show(T("已複製 Log 完整路徑至剪貼簿！", "Log path copied!"), T("提示", "Notice"), MessageBoxButtons.OK, MessageBoxIcon.Information); + } + }; + clearLogButton.Text = T("清除終端畫面", "Clear View"); + clearLogButton.AutoSize = true; + clearLogButton.Click += (s, e) => logList.Items.Clear(); + + autoScrollCheck.Text = T("自動捲動 (最多維持 2,000 行)", "Auto-scroll (Max 2,000 lines)"); + autoScrollCheck.Checked = true; + autoScrollCheck.AutoSize = true; + + resultText.AutoSize = true; + resultText.Margin = new Padding(0, 4, 12, 0); + + buttonRow.Controls.Add(resultText); + buttonRow.Controls.Add(openLogButton); + buttonRow.Controls.Add(openFolderButton); + buttonRow.Controls.Add(copyPathButton); + buttonRow.Controls.Add(clearLogButton); + buttonRow.Controls.Add(autoScrollCheck); + root.Controls.Add(buttonRow, 1, 1); + + logList.Dock = DockStyle.Fill; + logList.Font = new Font("Consolas", 9f); + root.Controls.Add(logList, 1, 2); + + p.Controls.Add(root); + } + + private void SetStep(int s) + { + step = s; + for (int i = 0; i < 4; i++) pages[i].Visible = (i == step); + + backButton.Enabled = (step > 0 && step < 3); + cancelButton.Enabled = (step < 3); + + if (step == 0) + { + stepsLabel.Text = T("步驟 1/4：選取要匯入的 Oracle SQL 腳本檔案", "Step 1/4: Select Oracle SQL script file"); + nextButton.Text = T("下一步 (預檢掃描) ▶", "Next (Preflight) ▶"); + } + else if (step == 1) + { + stepsLabel.Text = T("步驟 2/4：檢視預檢分析與 Oracle Spatial 特性", "Step 2/4: Review preflight inspection and spatial features"); + nextButton.Text = T("下一步 (選擇策略) ▶", "Next (Strategy) ▶"); + } + else if (step == 2) + { + stepsLabel.Text = T("步驟 3/4：設定物件衝突策略與交易防護", "Step 3/4: Configure conflict strategy & transaction protection"); + nextButton.Text = T("開始匯入 🚀", "Start Import 🚀"); + } + else if (step == 3) + { + stepsLabel.Text = T("步驟 4/4：正在串流執行 Oracle SQL 腳本...", "Step 4/4: Executing Oracle SQL script streaming..."); + nextButton.Text = T("完成", "Done"); + nextButton.Enabled = false; + backButton.Enabled = false; + } + } + + private void MoveStep(int delta) + { + int n = step + delta; + if (n >= 0 && n < 4) SetStep(n); + } + + private async Task OnNextAsync() + { + if (step == 0) + { + string path = filePathBox.Text.Trim(); + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + MessageBox.Show(T("請指定存在的 SQL 腳本檔案！", "Please specify a valid SQL script file!"), T("提示", "Notice"), MessageBoxButtons.OK, MessageBoxIcon.Warning); + return; + } + + await RunPreflightAsync(path); + if (preflight.CanProceed) + { + MoveStep(1); + } + } + else if (step == 1) + { + MoveStep(1); + } + else if (step == 2) + { + MoveStep(1); + await StartImportAsync(); + } + else if (step == 3) + { + Close(); + } + } + + private async Task RunPreflightAsync(string path) + { + statusLabel.Text = T("正在對 SQL 檔案進行狀態機預檢分析...", "Scanning SQL file with preflight lexer..."); + var scanner = new OracleImportPreflightScanner(); + preflight = await Task.Run(() => scanner.Scan(path)).ConfigureAwait(true); + + if (!preflight.CanProceed) + { + MessageBox.Show(string.Join("\n", preflight.Issues.ToArray()), T("預檢未通過", "Preflight Failed"), MessageBoxButtons.OK, MessageBoxIcon.Error); + statusLabel.Text = T("預檢未通過，請排除錯誤後再試。", "Preflight failed."); + return; + } + + preflightSummary.Text = string.Format( + T("✅ 預檢分析通過！檔案大小: {0:N0} 位元組，編碼: {1}，預估執行語句: {2:N0} 條。", + "✅ Preflight passed! Size: {0:N0} bytes, Encoding: {1}, Estimated statements: {2:N0}."), + preflight.FileSizeBytes, preflight.DetectedEncoding.WebName, preflight.StatementCount); + + var sb = new StringBuilder(); + sb.AppendLine("• " + T("PL/SQL 區塊偵測: ", "PL/SQL Blocks: ") + (preflight.HasPlSqlBlocks ? string.Format("包含 {0} 個預存程序／觸發程序區塊 (以獨立行 / 結尾)", preflight.PlSqlBlocksCount) : "無")); + sb.AppendLine("• " + T("Oracle Spatial 特徵: ", "Spatial Features: ") + (preflight.HasSpatialFeatures ? "偵測到 SDO_GEOMETRY 空間幾何 (" + string.Join(", ", preflight.DetectedSpatialKeywords.ToArray()) + ")" : "無空間欄位")); + sb.AppendLine("• " + T("Native Manifest 清單: ", "Native Manifest: ") + (preflight.ContainsManifest ? string.Format("包含 (共宣告 {0} 張資料表，附帶 SHA-256 核對)", preflight.ManifestDeclaredTables.Count) : "未包含")); + preflightDetails.Text = sb.ToString(); + + issuesList.Items.Clear(); + if (preflight.Issues.Count > 0) + { + foreach (var issue in preflight.Issues) issuesList.Items.Add(issue); + } + else + { + issuesList.Items.Add(T("無任何阻擋性異常，腳本結構完整且符合 Oracle 語法標準。", "No issues found. Script is ready for execution.")); + } + + statusLabel.Text = T("預檢完成，請確認設定。", "Preflight complete."); + } + + private async Task StartImportAsync() + { + executing = true; + clock.Restart(); + timer.Start(); + punky.BeginExport(); + + var opt = new OracleImportOptions + { + ConflictStrategy = strategyKeep.Checked ? OracleConflictStrategy.KeepExisting : + (strategyReplace.Checked ? OracleConflictStrategy.ReplaceAll : + (strategyDrop.Checked ? OracleConflictStrategy.DropAndRecreate : OracleConflictStrategy.TruncateAndInsert)), + AutoRegisterSpatialMetadata = autoRegisterSpatial.Checked, + UseTransaction = useTransaction.Checked, + StopOnError = stopOnError.Checked + }; + + var req = new OracleImportRequest(filePathBox.Text.Trim(), databaseName, opt); + cancellation = new CancellationTokenSource(); + + Action appendTerminal = null; + appendTerminal = line => + { + if (InvokeRequired) + { + try { Invoke(new Action(appendTerminal), line); } catch { } + return; + } + + if (logList.Items.Count >= MaxLogLines) + { + logList.Items.RemoveAt(0); // FIFO 滑動視窗 + } + logList.Items.Add(line); + if (autoScrollCheck.Checked) + { + logList.TopIndex = logList.Items.Count - 1; + } + }; + + var runner = new OracleImportRunner(db, req, appendTerminal); + + var prog = new Progress(p => + { + execBar.Value = (int)p.PercentComplete; + coverageLabel.Text = string.Format("{0:0.0} %", p.PercentComplete); + currentStatementLabel.Text = p.Message; + lastBytes = p.BytesRead; + }); + + try + { + Result = await Task.Run(() => runner.RunAsync(prog, cancellation.Token)).ConfigureAwait(true); + timer.Stop(); + clock.Stop(); + punky.EndExport(); + + resultText.Text = Result.Success + ? string.Format(T("✅ 匯入成功！耗時 {0:hh\\:mm\\:ss}，執行 {1:N0} 條語句", "✅ Import succeeded! Elapsed {0:hh\\:mm\\:ss}, Executed {1:N0} statements"), Result.Elapsed, Result.StatementsExecuted) + : string.Format(T("❌ 匯入中斷: {0}", "❌ Import failed: {0}"), Result.ErrorMessage); + + resultText.ForeColor = Result.Success ? Color.DarkGreen : Color.Red; + openLogButton.Visible = true; + openFolderButton.Visible = true; + copyPathButton.Visible = true; + nextButton.Enabled = true; + cancelButton.Enabled = true; + cancelButton.Text = T("關閉", "Close"); + } + catch (Exception ex) + { + timer.Stop(); + clock.Stop(); + punky.EndExport(); + + resultText.Text = T("❌ 執行失敗: ", "❌ Execution failed: ") + ex.Message; + resultText.ForeColor = Color.Red; + nextButton.Enabled = true; + cancelButton.Enabled = true; + } + finally + { + executing = false; + } + } + + private void UpdateLiveTicker() + { + elapsedLabel.Text = string.Format(T("已耗時: {0:hh\\:mm\\:ss}", "Elapsed: {0:hh\\:mm\\:ss}"), clock.Elapsed); + if (clock.Elapsed.TotalSeconds > 0 && lastBytes > 0) + { + double kbSec = (lastBytes / 1024.0) / clock.Elapsed.TotalSeconds; + speedLabel.Text = string.Format("{0:N0} KB/s", kbSec); + } + } + } +} diff --git a/mySQLPunk/lib/OracleExport/OracleExportModels.cs b/mySQLPunk/lib/OracleExport/OracleExportModels.cs new file mode 100644 index 0000000..2596aee --- /dev/null +++ b/mySQLPunk/lib/OracleExport/OracleExportModels.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; + +namespace mySQLPunk.lib.OracleExport +{ + public enum OracleExportMode + { + SchemaAndData, + DataOnly, + SchemaOnly + } + + public enum OracleExportStage + { + NotStarted, + Preflight, + ExportingSchema, + ExportingData, + Finalizing, + Completed, + Cancelled, + Failed + } + + public sealed class OracleTableExportItem + { + public string Owner { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string FullQualifiedName => "\"" + (string.IsNullOrWhiteSpace(Owner) ? "SYSTEM" : Owner) + "\".\"" + Name + "\""; + public string DisplayName => string.IsNullOrWhiteSpace(Owner) ? Name : Owner + "." + Name; + public long RowCount { get; set; } + public long EstimatedBytes { get; set; } + public bool IsView { get; set; } + public bool IsSpatial { get; set; } + public bool IsSpatialMetadata { get; set; } + public List SpatialColumnNames { get; } = new List(); + public string SpatialFeatureDescription { get; set; } = string.Empty; + public bool Selected { get; set; } = true; + } + + public sealed class OracleExportWizardOptions + { + public OracleExportMode Mode { get; set; } = OracleExportMode.SchemaAndData; + public string TargetPath { get; set; } = string.Empty; + public List SelectedTables { get; } = new List(); + public bool UseTransactionSnapshot { get; set; } = true; // SET TRANSACTION READ ONLY; + public bool ExcludeSpatialIndexInternalTables { get; set; } = true; // 排除 Oracle Spatial 內部 R-Tree 影子表 (MDRT_*, MDXT_*, MDRS_*) + public bool IncludeSpatialMetadata { get; set; } = true; // USER_SDO_GEOM_METADATA 註冊語句 + public bool IncludeSpatialIndexes { get; set; } = true; // INDEXTYPE IS MDSYS.SPATIAL_INDEX + public bool DropTableIfExists { get; set; } = false; + public bool TruncateBeforeInsert { get; set; } = false; + public int BatchSize { get; set; } = 500; + public bool IncludeViews { get; set; } = true; + public bool IncludeIndexes { get; set; } = true; + public bool IncludeComments { get; set; } = true; + public bool IncludeManifest { get; set; } = true; + + public OracleExportWizardOptions() { } + + public OracleExportWizardOptions(OracleExportMode mode, string targetPath) + { + Mode = mode; + TargetPath = targetPath ?? string.Empty; + } + } + + public sealed class OracleExportProgress + { + public OracleExportStage Stage { get; set; } = OracleExportStage.NotStarted; + public string CurrentTable { get; set; } = string.Empty; + public string CurrentOwner { get; set; } = string.Empty; + public int TablesCompleted { get; set; } + public int TotalTables { get; set; } + public long RowsExported { get; set; } + public long TotalRows { get; set; } + public long BytesExported { get; set; } + public string Message { get; set; } = string.Empty; + + public double PercentComplete + { + get + { + if (TotalRows > 0) + { + double p = (double)RowsExported / TotalRows * 100.0; + return p > 100.0 ? 100.0 : p; + } + if (TotalTables > 0) + { + double p = (double)TablesCompleted / TotalTables * 100.0; + return p > 100.0 ? 100.0 : p; + } + return 0.0; + } + } + } + + public sealed class OracleExportResult + { + public bool Success { get; set; } + public string FilePath { get; set; } = string.Empty; + public int TablesExported { get; set; } + public int ViewsExported { get; set; } + public long TotalRowsExported { get; set; } + public long FileSizeBytes { get; set; } + public TimeSpan Elapsed { get; set; } + public string ErrorMessage { get; set; } = string.Empty; + public string Sha256Checksum { get; set; } = string.Empty; + public Dictionary ManifestTables { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } +} diff --git a/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs b/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs new file mode 100644 index 0000000..259e161 --- /dev/null +++ b/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs @@ -0,0 +1,675 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.OracleExport +{ + public sealed class OracleSqlExportService + { + private const int BufferSize = 65536; // 64KB + private readonly IDatabase db; + private readonly string databaseName; // In Oracle, this represents the Schema / Owner + + private static readonly HashSet SpatialIndexPrefixes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "MDRT_", "MDXT_", "MDNT_", "MDRS_" + }; + + public OracleSqlExportService(IDatabase db, string databaseName) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.databaseName = databaseName ?? string.Empty; + } + + public async Task> GetTableCatalogAsync(CancellationToken cancellationToken = default(CancellationToken)) + { + return await Task.Run(() => GetTableCatalog(cancellationToken), cancellationToken).ConfigureAwait(false); + } + + public List GetTableCatalog(CancellationToken cancellationToken = default(CancellationToken)) + { + var list = new List(); + string owner = string.IsNullOrWhiteSpace(databaseName) ? "USER" : databaseName.Trim().ToUpperInvariant(); + + // 1. 取得所有一般資料表 + string tablesSql = @" + SELECT + t.OWNER, + t.TABLE_NAME, + COALESCE(t.NUM_ROWS, 0) AS ROW_COUNT, + COALESCE(t.BLOCKS * 8192, 0) AS ESTIMATED_BYTES + FROM ALL_TABLES t + WHERE (:owner = 'USER' OR t.OWNER = :owner) + AND t.TABLE_NAME NOT LIKE 'BIN$%' + ORDER BY t.OWNER, t.TABLE_NAME"; + + var p = new Dictionary { { "owner", owner } }; + DataTable dtTables; + try + { + dtTables = db.SelectSQL(tablesSql, p); + } + catch + { + // Fallback 若無 bind 支援 + dtTables = db.SelectSQL("SELECT OWNER, TABLE_NAME, COALESCE(NUM_ROWS, 0) AS ROW_COUNT, 0 AS ESTIMATED_BYTES FROM ALL_TABLES WHERE TABLE_NAME NOT LIKE 'BIN$%' ORDER BY OWNER, TABLE_NAME"); + } + + var itemMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (DataRow row in dtTables.Rows) + { + cancellationToken.ThrowIfCancellationRequested(); + string tblOwner = row["OWNER"]?.ToString() ?? ""; + string name = row["TABLE_NAME"]?.ToString() ?? ""; + long rows = 0; + long bytes = 0; + if (row["ROW_COUNT"] != DBNull.Value) long.TryParse(row["ROW_COUNT"].ToString(), out rows); + if (row["ESTIMATED_BYTES"] != DBNull.Value) long.TryParse(row["ESTIMATED_BYTES"].ToString(), out bytes); + + bool isInternalSpatial = false; + foreach (var prefix in SpatialIndexPrefixes) + { + if (name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + isInternalSpatial = true; + break; + } + } + + var item = new OracleTableExportItem + { + Owner = tblOwner, + Name = name, + RowCount = rows, + EstimatedBytes = bytes, + IsView = false, + IsSpatialMetadata = isInternalSpatial, + Selected = !isInternalSpatial // 內部 R-Tree 影子表預設排除 + }; + + list.Add(item); + itemMap[tblOwner + "." + name] = item; + } + + // 2. 取得檢視 (Views) + try + { + string viewsSql = @" + SELECT OWNER, VIEW_NAME + FROM ALL_VIEWS + WHERE (:owner = 'USER' OR OWNER = :owner) + ORDER BY OWNER, VIEW_NAME"; + + DataTable dtViews = db.SelectSQL(viewsSql, p); + foreach (DataRow row in dtViews.Rows) + { + cancellationToken.ThrowIfCancellationRequested(); + string vOwner = row["OWNER"]?.ToString() ?? ""; + string vName = row["VIEW_NAME"]?.ToString() ?? ""; + + var item = new OracleTableExportItem + { + Owner = vOwner, + Name = vName, + RowCount = 0, + EstimatedBytes = 0, + IsView = true, + IsSpatialMetadata = false, + Selected = true + }; + + list.Add(item); + itemMap[vOwner + "." + vName] = item; + } + } + catch { } + + // 3. 探測 Oracle Spatial (SDO_GEOMETRY) 空間欄位 + try + { + string spatialSql = @" + SELECT OWNER, TABLE_NAME, COLUMN_NAME, DATA_TYPE + FROM ALL_TAB_COLUMNS + WHERE (:owner = 'USER' OR OWNER = :owner) + AND (DATA_TYPE = 'SDO_GEOMETRY' OR (DATA_TYPE_OWNER = 'MDSYS' AND DATA_TYPE = 'SDO_GEOMETRY'))"; + + DataTable dtSpatial = db.SelectSQL(spatialSql, p); + foreach (DataRow row in dtSpatial.Rows) + { + string sOwner = row["OWNER"]?.ToString() ?? ""; + string sTable = row["TABLE_NAME"]?.ToString() ?? ""; + string sCol = row["COLUMN_NAME"]?.ToString() ?? ""; + + string key = sOwner + "." + sTable; + if (itemMap.TryGetValue(key, out var item)) + { + item.IsSpatial = true; + if (!item.SpatialColumnNames.Contains(sCol)) + { + item.SpatialColumnNames.Add(sCol); + } + item.SpatialFeatureDescription = "Oracle Spatial SDO_GEOMETRY"; + } + } + } + catch { } + + return list; + } + + public async Task ExportAsync( + OracleExportWizardOptions options, + IProgress progress = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrWhiteSpace(options.TargetPath)) throw new ArgumentException("TargetPath is required.", nameof(options)); + + var result = new OracleExportResult { FilePath = options.TargetPath }; + var stopwatch = Stopwatch.StartNew(); + + string dir = Path.GetDirectoryName(options.TargetPath); + if (!string.IsNullOrWhiteSpace(dir) && !Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + string stagingPath = options.TargetPath + ".writing"; + if (File.Exists(stagingPath)) + { + try { File.Delete(stagingPath); } catch { } + } + + try + { + // 1. 取得完整目標清單 + progress?.Report(new OracleExportProgress + { + Stage = OracleExportStage.Preflight, + Message = "正在探測 Oracle 綱要與空間物件結構..." + }); + + var catalog = await GetTableCatalogAsync(cancellationToken).ConfigureAwait(false); + var selectedSet = new HashSet(options.SelectedTables, StringComparer.OrdinalIgnoreCase); + + var itemsToExport = catalog.FindAll(i => + (options.SelectedTables.Count == 0 && i.Selected) || + selectedSet.Contains(i.Name) || + selectedSet.Contains(i.DisplayName) || + selectedSet.Contains(i.FullQualifiedName)); + + // 排除影子表 + if (options.ExcludeSpatialIndexInternalTables) + { + itemsToExport.RemoveAll(i => i.IsSpatialMetadata); + } + + int totalItems = itemsToExport.Count; + long totalRows = 0; + foreach (var it in itemsToExport) + { + if (!it.IsView) totalRows += it.RowCount; + } + + int itemsProcessed = 0; + long rowsExportedTotal = 0; + + using (var fs = new FileStream(stagingPath, FileMode.Create, FileAccess.Write, FileShare.None, BufferSize)) + using (var sw = new StreamWriter(fs, new UTF8Encoding(false), BufferSize)) + { + // 寫入 Header 註解 + sw.WriteLine("-- ===================================================================="); + sw.WriteLine("-- mySQLPunk Oracle Logical Dump"); + sw.WriteLine("-- Host Database: " + (string.IsNullOrWhiteSpace(databaseName) ? "Current Schema" : databaseName)); + sw.WriteLine("-- Created: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + sw.WriteLine("-- Architecture: Streaming StreamWriter (64KB Buffer, Zero Memory Bloat)"); + sw.WriteLine("-- Oracle Spatial: MDSYS.SDO_GEOMETRY with WKB / Hex literal support"); + sw.WriteLine("-- ===================================================================="); + sw.WriteLine(); + + // 一致性快照保護 (SET TRANSACTION READ ONLY) + if (options.UseTransactionSnapshot) + { + sw.WriteLine("-- 啟用交易一致性唯讀快照 (防範外鍵與空間關聯導出時不一致)"); + sw.WriteLine("SET TRANSACTION READ ONLY;"); + sw.WriteLine(); + } + + // 2. 結構 DDL 階段 (CREATE TABLE, COMMENTS, SPATIAL METADATA) + if (options.Mode != OracleExportMode.DataOnly) + { + progress?.Report(new OracleExportProgress + { + Stage = OracleExportStage.ExportingSchema, + TotalTables = totalItems, + TotalRows = totalRows, + Message = "正在產生 Oracle 資料表與空間中繼結構 DDL..." + }); + + foreach (var item in itemsToExport) + { + cancellationToken.ThrowIfCancellationRequested(); + if (item.IsView) continue; // View 於最後匯出 + + sw.WriteLine("-- --------------------------------------------------"); + sw.WriteLine(string.Format("-- Table structure for: {0}", item.FullQualifiedName)); + if (item.IsSpatial) + { + sw.WriteLine(string.Format("-- Spatial Feature: {0} ({1})", item.SpatialFeatureDescription, string.Join(", ", item.SpatialColumnNames.ToArray()))); + } + sw.WriteLine("-- --------------------------------------------------"); + + if (options.DropTableIfExists) + { + sw.WriteLine("BEGIN"); + sw.WriteLine(string.Format(" EXECUTE IMMEDIATE 'DROP TABLE {0} CASCADE CONSTRAINTS';", item.FullQualifiedName)); + sw.WriteLine("EXCEPTION WHEN OTHERS THEN NULL;"); + sw.WriteLine("END;"); + sw.WriteLine("/"); + } + + string ddl = db.GetTableCreateStatement(item.Owner, item.Name); + if (!string.IsNullOrWhiteSpace(ddl)) + { + sw.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + } + + // 空間中繼表註冊 (USER_SDO_GEOM_METADATA) + if (options.IncludeSpatialMetadata && item.IsSpatial && item.SpatialColumnNames.Count > 0) + { + foreach (var geomCol in item.SpatialColumnNames) + { + sw.WriteLine(string.Format( + "DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME = '{0}' AND COLUMN_NAME = '{1}';", + item.Name.ToUpperInvariant(), geomCol.ToUpperInvariant())); + sw.WriteLine(string.Format( + "INSERT INTO USER_SDO_GEOM_METADATA (TABLE_NAME, COLUMN_NAME, DIMINFO, SRID) " + + "VALUES ('{0}', '{1}', MDSYS.SDO_DIM_ARRAY(MDSYS.SDO_DIM_ELEMENT('X', -180, 180, 0.005), MDSYS.SDO_DIM_ELEMENT('Y', -90, 90, 0.005)), 4326);", + item.Name.ToUpperInvariant(), geomCol.ToUpperInvariant())); + } + } + + sw.WriteLine(); + } + } + + // 3. 資料傾印階段 (INSERT STATEMENTS) + if (options.Mode != OracleExportMode.SchemaOnly) + { + foreach (var item in itemsToExport) + { + cancellationToken.ThrowIfCancellationRequested(); + if (item.IsView) continue; + + progress?.Report(new OracleExportProgress + { + Stage = OracleExportStage.ExportingData, + CurrentTable = item.Name, + CurrentOwner = item.Owner, + TablesCompleted = itemsProcessed, + TotalTables = totalItems, + RowsExported = rowsExportedTotal, + TotalRows = totalRows, + BytesExported = fs.Position, + Message = string.Format("正在串流傾印資料表: {0}...", item.DisplayName) + }); + + sw.WriteLine("-- --------------------------------------------------"); + sw.WriteLine(string.Format("-- Dumping data for: {0}", item.FullQualifiedName)); + sw.WriteLine("-- --------------------------------------------------"); + + if (options.TruncateBeforeInsert) + { + sw.WriteLine(string.Format("TRUNCATE TABLE {0};", item.FullQualifiedName)); + } + + long tableRowsExported = ExportTableDataStreaming(db, item, sw, options, cancellationToken); + rowsExportedTotal += tableRowsExported; + result.ManifestTables[item.Name] = tableRowsExported; + result.TablesExported++; + + sw.WriteLine("COMMIT;"); + sw.WriteLine(); + + itemsProcessed++; + } + } + + // 4. 檢視與空間索引階段 + if (options.Mode != OracleExportMode.DataOnly) + { + // 建立檢視 (Views) + if (options.IncludeViews) + { + foreach (var item in itemsToExport) + { + if (!item.IsView) continue; + cancellationToken.ThrowIfCancellationRequested(); + + sw.WriteLine("-- --------------------------------------------------"); + sw.WriteLine(string.Format("-- View structure for: {0}", item.FullQualifiedName)); + sw.WriteLine("-- --------------------------------------------------"); + + string viewDdl = db.GetViewCreateStatement(item.Owner, item.Name); + if (!string.IsNullOrWhiteSpace(viewDdl)) + { + sw.WriteLine(viewDdl.TrimEnd().TrimEnd(';') + ";"); + sw.WriteLine(); + } + result.ViewsExported++; + } + } + + // 建立空間索引 (MDSYS.SPATIAL_INDEX) + if (options.IncludeSpatialIndexes) + { + foreach (var item in itemsToExport) + { + if (item.IsView || !item.IsSpatial) continue; + + foreach (var col in item.SpatialColumnNames) + { + string idxName = string.Format("SIDX_{0}_{1}", item.Name, col); + if (idxName.Length > 30) idxName = idxName.Substring(0, 30); + + sw.WriteLine(string.Format("-- Oracle Spatial Domain Index for {0}.{1}", item.Name, col)); + sw.WriteLine(string.Format( + "CREATE INDEX \"{0}\".\"{1}\" ON {2} (\"{3}\") INDEXTYPE IS MDSYS.SPATIAL_INDEX;", + item.Owner, idxName, item.FullQualifiedName, col)); + sw.WriteLine(); + } + } + } + } + + // 5. 結尾交易完成與 Native Manifest + sw.WriteLine("COMMIT;"); + sw.WriteLine(); + + if (options.IncludeManifest) + { + sw.WriteLine("-- ===================================================================="); + sw.WriteLine("-- mySQLPunk Native Manifest"); + sw.WriteLine(string.Format("-- Manifest-Provider: oracle")); + sw.WriteLine(string.Format("-- Manifest-Database: {0}", databaseName)); + sw.WriteLine(string.Format("-- Manifest-Generated: {0}", DateTime.UtcNow.ToString("o"))); + foreach (var kvp in result.ManifestTables) + { + sw.WriteLine(string.Format("-- Manifest-Table: {0} | Rows: {1}", kvp.Key, kvp.Value)); + } + sw.WriteLine("-- ===================================================================="); + } + + sw.Flush(); + } + + // 6. 計算 SHA-256 Checksum 並原子替換檔案 + result.FileSizeBytes = new FileInfo(stagingPath).Length; + result.Sha256Checksum = ComputeSha256(stagingPath); + + // 在檔案末尾附加 Checksum + if (options.IncludeManifest) + { + using (var sw = new StreamWriter(stagingPath, true, Encoding.UTF8)) + { + sw.WriteLine(string.Format("-- Manifest-Checksum-SHA256: {0}", result.Sha256Checksum)); + sw.WriteLine("-- ===================================================================="); + } + result.FileSizeBytes = new FileInfo(stagingPath).Length; + } + + if (File.Exists(options.TargetPath)) + { + File.Delete(options.TargetPath); + } + File.Move(stagingPath, options.TargetPath); + + stopwatch.Stop(); + result.Success = true; + result.Elapsed = stopwatch.Elapsed; + result.TotalRowsExported = rowsExportedTotal; + + progress?.Report(new OracleExportProgress + { + Stage = OracleExportStage.Completed, + TablesCompleted = totalItems, + TotalTables = totalItems, + RowsExported = rowsExportedTotal, + TotalRows = totalRows, + BytesExported = result.FileSizeBytes, + Message = string.Format("Oracle 匯出完成！共 {0} 張資料表，{1:N0} 列，耗時 {2:hh\\:mm\\:ss}。", result.TablesExported, rowsExportedTotal, result.Elapsed) + }); + } + catch (Exception ex) + { + stopwatch.Stop(); + result.Success = false; + result.ErrorMessage = ex.Message; + result.Elapsed = stopwatch.Elapsed; + + if (File.Exists(stagingPath)) + { + try { File.Delete(stagingPath); } catch { } + } + + progress?.Report(new OracleExportProgress + { + Stage = OracleExportStage.Failed, + Message = "Oracle 匯出失敗: " + ex.Message + }); + throw; + } + + return result; + } + + private static long ExportTableDataStreaming( + IDatabase db, + OracleTableExportItem item, + StreamWriter sw, + OracleExportWizardOptions options, + CancellationToken cancellationToken) + { + DataTable colsMeta; + try + { + colsMeta = db.GetColumns(item.Owner, item.Name); + } + catch + { + colsMeta = null; + } + + // 空間欄位集合 + var spatialSet = new HashSet(item.SpatialColumnNames, StringComparer.OrdinalIgnoreCase); + + // 建構查詢 SELECT 語句 + // 若包含 SDO_GEOMETRY 欄位，轉換為 WKB 以避免 ODP.NET UDT 映射錯誤 + string selectSql; + if (colsMeta != null && colsMeta.Rows.Count > 0) + { + var selectParts = new List(); + foreach (DataRow r in colsMeta.Rows) + { + string colName = r["COLUMN_NAME"]?.ToString() ?? ""; + if (spatialSet.Contains(colName)) + { + // 轉換為 WKB 二進位 (BLOB) + selectParts.Add(string.Format("CASE WHEN \"{0}\" IS NOT NULL THEN SDO_UTIL.TO_WKBGEOMETRY(\"{0}\") ELSE NULL END AS \"{0}\"", colName)); + } + else + { + selectParts.Add(string.Format("\"{0}\"", colName)); + } + } + selectSql = string.Format("SELECT {0} FROM {1}", string.Join(", ", selectParts.ToArray()), item.FullQualifiedName); + } + else + { + selectSql = string.Format("SELECT * FROM {0}", item.FullQualifiedName); + } + + long rowCount = 0; + int batchCount = 0; + + DataTable dt; + try + { + dt = db.SelectSQL(selectSql); + } + catch + { + // Fallback 直接 SELECT * + dt = db.SelectSQL(string.Format("SELECT * FROM {0}", item.FullQualifiedName)); + } + + if (dt == null || dt.Rows.Count == 0) return 0; + + int colCount = dt.Columns.Count; + var colNames = new string[colCount]; + var isSpatialCol = new bool[colCount]; + + for (int i = 0; i < colCount; i++) + { + colNames[i] = "\"" + dt.Columns[i].ColumnName + "\""; + isSpatialCol[i] = spatialSet.Contains(dt.Columns[i].ColumnName); + } + + string insertPrefix = string.Format("INSERT INTO {0} ({1}) VALUES (", item.FullQualifiedName, string.Join(", ", colNames)); + + foreach (DataRow row in dt.Rows) + { + cancellationToken.ThrowIfCancellationRequested(); + rowCount++; + batchCount++; + + sw.Write(insertPrefix); + for (int c = 0; c < colCount; c++) + { + if (c > 0) sw.Write(", "); + object val = row[c]; + bool spatial = isSpatialCol[c]; + + sw.Write(FormatOracleLiteral(val, spatial)); + } + sw.WriteLine(");"); + + if (batchCount >= options.BatchSize) + { + sw.WriteLine("COMMIT;"); + batchCount = 0; + } + } + + return rowCount; + } + + public static string FormatOracleLiteral(object val, bool isSpatial) + { + if (val == null || val == DBNull.Value) return "NULL"; + + // SDO_GEOMETRY 空間幾何無損還原 + if (isSpatial) + { + if (val is byte[] geomBytes) + { + var hexSb = new StringBuilder(geomBytes.Length * 2); + for (int i = 0; i < geomBytes.Length; i++) + { + hexSb.Append(geomBytes[i].ToString("X2")); + } + return string.Format("SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('{0}'))", hexSb.ToString()); + } + + string strVal = val.ToString().Trim(); + if (strVal.StartsWith("SDO_GEOMETRY", StringComparison.OrdinalIgnoreCase) || + strVal.StartsWith("MDSYS.SDO_GEOMETRY", StringComparison.OrdinalIgnoreCase)) + { + return strVal; + } + if (strVal.StartsWith("POINT", StringComparison.OrdinalIgnoreCase) || + strVal.StartsWith("POLYGON", StringComparison.OrdinalIgnoreCase) || + strVal.StartsWith("LINESTRING", StringComparison.OrdinalIgnoreCase) || + strVal.StartsWith("MULTIPOINT", StringComparison.OrdinalIgnoreCase) || + strVal.StartsWith("MULTILINESTRING", StringComparison.OrdinalIgnoreCase) || + strVal.StartsWith("MULTIPOLYGON", StringComparison.OrdinalIgnoreCase)) + { + return string.Format("SDO_GEOMETRY('{0}', 4326)", strVal.Replace("'", "''")); + } + } + + // 二進位 (RAW / BLOB) + if (val is byte[] bytes) + { + var hex = new StringBuilder(bytes.Length * 2); + for (int i = 0; i < bytes.Length; i++) + { + hex.Append(bytes[i].ToString("X2")); + } + return string.Format("HEXTORAW('{0}')", hex.ToString()); + } + + if (val is bool b) + { + return b ? "1" : "0"; + } + + if (val is sbyte || val is byte || val is short || val is ushort || + val is int || val is uint || val is long || val is ulong || + val is float || val is double || val is decimal) + { + return Convert.ToString(val, CultureInfo.InvariantCulture); + } + + if (val is DateTime dt) + { + return string.Format( + "TO_TIMESTAMP('{0}', 'YYYY-MM-DD HH24:MI:SS.FF7')", + dt.ToString("yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.InvariantCulture)); + } + + string s = val.ToString(); + // CLOB 超長字串支援 + if (s.Length > 4000) + { + // 超過 4000 字元切段為 TO_CLOB(...) || TO_CLOB(...) + var sb = new StringBuilder(); + int idx = 0; + while (idx < s.Length) + { + int chunkLen = Math.Min(2000, s.Length - idx); + string chunk = s.Substring(idx, chunkLen); + if (sb.Length > 0) sb.Append(" || "); + sb.Append("TO_CLOB('").Append(chunk.Replace("'", "''")).Append("')"); + idx += chunkLen; + } + return sb.ToString(); + } + + return "'" + s.Replace("'", "''") + "'"; + } + + private static string ComputeSha256(string filePath) + { + using (var sha = SHA256.Create()) + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize)) + { + byte[] hash = sha.ComputeHash(fs); + var sb = new StringBuilder(hash.Length * 2); + for (int i = 0; i < hash.Length; i++) + { + sb.Append(hash[i].ToString("x2")); + } + return sb.ToString(); + } + } + } +} diff --git a/mySQLPunk/lib/OracleImport/OracleImportLog.cs b/mySQLPunk/lib/OracleImport/OracleImportLog.cs new file mode 100644 index 0000000..04bd1c6 --- /dev/null +++ b/mySQLPunk/lib/OracleImport/OracleImportLog.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace mySQLPunk.lib.OracleImport +{ + public sealed class OracleImportLog : IDisposable + { + private readonly string logPath; + private readonly StreamWriter writer; + + public string LogPath => logPath; + + public OracleImportLog(string targetDatabase) + { + string appData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + string logDir = Path.Combine(appData, "mySQLPunk", "ImportLogs"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + string dbSanitized = string.IsNullOrWhiteSpace(targetDatabase) ? "default" : targetDatabase.Replace(':', '_').Replace('/', '_').Replace('\\', '_'); + string fileName = string.Format("import-oracle-{0}-{1:yyyyMMdd_HHmmss}.log", dbSanitized, DateTime.Now); + logPath = Path.Combine(logDir, fileName); + + writer = new StreamWriter(new FileStream(logPath, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8); + writer.WriteLine("===================================================================="); + writer.WriteLine(" mySQLPunk Oracle Import Execution Log"); + writer.WriteLine(string.Format(" Target Schema/Owner : {0}", targetDatabase)); + writer.WriteLine(string.Format(" Start Time : {0:yyyy-MM-dd HH:mm:ss}", DateTime.Now)); + writer.WriteLine(" Architecture : Streaming State Machine Lexer with PL/SQL Block Delimiters"); + writer.WriteLine(" Spatial Support : MDSYS.SDO_GEOMETRY, WKB & USER_SDO_GEOM_METADATA"); + writer.WriteLine("===================================================================="); + writer.WriteLine(); + writer.Flush(); + } + + public void LogInfo(string message) + { + if (writer == null) return; + writer.WriteLine(string.Format("[{0:HH:mm:ss.fff}] [INFO] {1}", DateTime.Now, message)); + } + + public void LogStatement(int line, string sqlPreview, bool success, string error = null) + { + if (writer == null) return; + string status = success ? "OK" : "ERROR"; + writer.WriteLine(string.Format("[{0:HH:mm:ss.fff}] [{1}] Line {2}: {3}", DateTime.Now, status, line, sqlPreview)); + if (!success && !string.IsNullOrWhiteSpace(error)) + { + writer.WriteLine(string.Format(" └─ Detail: {0}", error)); + } + } + + public void LogManifest(Dictionary expected, Dictionary actual, bool allMatched) + { + if (writer == null) return; + writer.WriteLine(); + writer.WriteLine("--------------------------------------------------------------------"); + writer.WriteLine(" Native Manifest Verification Summary"); + writer.WriteLine(string.Format(" Overall Result: {0}", allMatched ? "PASS (All tables match declared rows)" : "FAIL (Discrepancy detected)")); + writer.WriteLine("--------------------------------------------------------------------"); + foreach (var kvp in expected) + { + long act = actual.ContainsKey(kvp.Key) ? actual[kvp.Key] : -1; + string status = (act == kvp.Value) ? "[MATCH]" : "[MISMATCH]"; + writer.WriteLine(string.Format(" {0,-10} Table [{1}] -> Expected: {2:N0}, Restored: {3:N0}", status, kvp.Key, kvp.Value, act)); + } + writer.WriteLine("--------------------------------------------------------------------"); + writer.WriteLine(); + } + + public void LogSummary(int executed, int failed, long affectedRows, TimeSpan elapsed, bool success) + { + if (writer == null) return; + writer.WriteLine(); + writer.WriteLine("===================================================================="); + writer.WriteLine(" Oracle Import Completed Summary"); + writer.WriteLine(string.Format(" Status : {0}", success ? "SUCCESS" : "FAILED / COMPLETED WITH ERRORS")); + writer.WriteLine(string.Format(" Statements Executed: {0:N0}", executed)); + writer.WriteLine(string.Format(" Statements Failed : {0:N0}", failed)); + writer.WriteLine(string.Format(" Total Rows Affected: {0:N0}", affectedRows)); + writer.WriteLine(string.Format(" Elapsed Time : {0:hh\\:mm\\:ss\\.fff}", elapsed)); + writer.WriteLine("===================================================================="); + writer.Flush(); + } + + public void Flush() + { + try { writer?.Flush(); } catch { } + } + + public void Dispose() + { + try + { + writer?.Flush(); + writer?.Dispose(); + } + catch { } + } + } +} diff --git a/mySQLPunk/lib/OracleImport/OracleImportModels.cs b/mySQLPunk/lib/OracleImport/OracleImportModels.cs new file mode 100644 index 0000000..05c1448 --- /dev/null +++ b/mySQLPunk/lib/OracleImport/OracleImportModels.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace mySQLPunk.lib.OracleImport +{ + public enum OracleConflictStrategy + { + KeepExisting, + ReplaceAll, + DropAndRecreate, + TruncateAndInsert + } + + public sealed class OracleImportOptions + { + public OracleConflictStrategy ConflictStrategy { get; set; } = OracleConflictStrategy.KeepExisting; + public bool UseTransaction { get; set; } = true; + public bool StopOnError { get; set; } = true; + public bool AutoRegisterSpatialMetadata { get; set; } = true; + public int CommitBatchSize { get; set; } = 1000; + } + + public sealed class OracleImportRequest + { + public string FilePath { get; set; } + public string DatabaseName { get; set; } // Owner / Schema + public OracleImportOptions Options { get; set; } + + public OracleImportRequest() + { + Options = new OracleImportOptions(); + } + + public OracleImportRequest(string filePath, string databaseName, OracleImportOptions options = null) + { + FilePath = filePath; + DatabaseName = databaseName; + Options = options ?? new OracleImportOptions(); + } + } + + public sealed class OracleImportPreflight + { + public bool CanProceed { get; set; } = true; + public long FileSizeBytes { get; set; } + public Encoding DetectedEncoding { get; set; } = Encoding.UTF8; + public int StatementCount { get; set; } + public bool ContainsManifest { get; set; } + public Dictionary ManifestDeclaredTables { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public string ManifestSha256 { get; set; } = string.Empty; + public bool HasSpatialFeatures { get; set; } + public List DetectedSpatialKeywords { get; } = new List(); + public bool HasPlSqlBlocks { get; set; } + public int PlSqlBlocksCount { get; set; } + public List Issues { get; } = new List(); + + public OracleImportPreflight() { } + + public OracleImportPreflight(bool canProceed, string issue) + { + CanProceed = canProceed; + if (!string.IsNullOrWhiteSpace(issue)) + { + Issues.Add(issue); + } + } + } + + public sealed class OracleImportProgress + { + public string Stage { get; set; } = "Initializing"; + public int CurrentStatement { get; set; } + public int TotalStatements { get; set; } + public long BytesRead { get; set; } + public long TotalBytes { get; set; } + public long RowsAffected { get; set; } + public string Message { get; set; } = string.Empty; + + public double PercentComplete + { + get + { + if (TotalBytes > 0) + { + double p = (double)BytesRead / TotalBytes * 100.0; + return p > 100.0 ? 100.0 : p; + } + if (TotalStatements > 0) + { + double p = (double)CurrentStatement / TotalStatements * 100.0; + return p > 100.0 ? 100.0 : p; + } + return 0.0; + } + } + } + + public sealed class OracleImportResult + { + public bool Success { get; set; } + public int StatementsExecuted { get; set; } + public int StatementsFailed { get; set; } + public long TotalRowsAffected { get; set; } + public TimeSpan Elapsed { get; set; } + public string ErrorMessage { get; set; } = string.Empty; + public string LogFilePath { get; set; } = string.Empty; + public bool ManifestMatched { get; set; } = true; + public Dictionary RestoredTables { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } +} diff --git a/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs b/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs new file mode 100644 index 0000000..5ef8399 --- /dev/null +++ b/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs @@ -0,0 +1,303 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; + +namespace mySQLPunk.lib.OracleImport +{ + public sealed class OracleImportPreflightScanner + { + private const int MaxBatchSizeBytes = 32 * 1024 * 1024; // 32 MiB + + private static readonly string[] SpatialKeywords = new[] + { + "sdo_geometry", "mdsys.sdo_geometry", "sdo_util", "user_sdo_geom_metadata", + "spatial_index", "sdo_dim_array", "sdo_dim_element", "sdo_point_type" + }; + + private static readonly Regex PlSqlStartRegex = new Regex( + @"^\s*(DECLARE\b|BEGIN\b|CREATE\s+(OR\s+REPLACE\s+)?(PROCEDURE|FUNCTION|PACKAGE|TRIGGER|TYPE)\b)", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public OracleImportPreflight Scan(string filePath, CancellationToken cancellationToken = default(CancellationToken)) + { + if (string.IsNullOrWhiteSpace(filePath)) + return new OracleImportPreflight(false, "未指定匯入檔案路徑。"); + + if (!File.Exists(filePath)) + return new OracleImportPreflight(false, string.Format("找不到指定的 SQL 檔案: {0}", filePath)); + + var fi = new FileInfo(filePath); + var result = new OracleImportPreflight + { + FileSizeBytes = fi.Length, + CanProceed = true + }; + + Encoding detectedEncoding = DetectEncoding(filePath); + result.DetectedEncoding = detectedEncoding; + + try + { + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) + using (var reader = new StreamReader(fs, detectedEncoding, true, 65536)) + { + int statementCount = 0; + long currentStatementBytes = 0; + bool hasStatementContent = false; + bool inSingleQuote = false; + bool inDoubleQuote = false; + bool inLineComment = false; + int blockCommentDepth = 0; + bool inPlSqlBlock = false; + + var spatialFound = new HashSet(StringComparer.OrdinalIgnoreCase); + + string line; + int lineNumber = 0; + + while ((line = reader.ReadLine()) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + lineNumber++; + + // NUL byte 損毀快速檢查 + if (line.IndexOf('\0') >= 0) + { + result.CanProceed = false; + result.Issues.Add(string.Format("第 {0} 行包含二進位 NUL 空字元，檔案可能已損毀或非純文字 SQL 腳本。", lineNumber)); + return result; + } + + // 檢查 Native Manifest 標記 + string trimmed = line.Trim(); + if (trimmed.StartsWith("--", StringComparison.Ordinal)) + { + if (trimmed.Contains("mySQLPunk Native Manifest")) + { + result.ContainsManifest = true; + } + + var matchTable = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (matchTable.Success && long.TryParse(matchTable.Groups[2].Value, out long rows)) + { + result.ManifestDeclaredTables[matchTable.Groups[1].Value] = rows; + } + + var matchSha = Regex.Match(trimmed, @"^--\s*Manifest-Checksum-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase); + if (matchSha.Success) + { + result.ManifestSha256 = matchSha.Groups[1].Value.ToLowerInvariant(); + } + } + + // 探測 Oracle Spatial 空間關鍵字 + string lowerLine = line.ToLowerInvariant(); + foreach (var kw in SpatialKeywords) + { + if (lowerLine.Contains(kw)) + { + result.HasSpatialFeatures = true; + if (spatialFound.Add(kw)) + { + result.DetectedSpatialKeywords.Add(kw); + } + } + } + + // 檢查是否開展 PL/SQL 區塊 (DECLARE / BEGIN / CREATE PROCEDURE 等) + if (!inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) + { + if (PlSqlStartRegex.IsMatch(line)) + { + inPlSqlBlock = true; + result.HasPlSqlBlocks = true; + result.PlSqlBlocksCount++; + } + } + + // 檢查 PL/SQL 區塊結束符：獨立行斜線 / + if (inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) + { + if (trimmed == "/") + { + inPlSqlBlock = false; + if (hasStatementContent) + { + statementCount++; + hasStatementContent = false; + currentStatementBytes = 0; + } + continue; + } + } + + int len = line.Length; + for (int i = 0; i < len; i++) + { + char c = line[i]; + + // 1. 單行註解 + if (inLineComment) + { + continue; + } + + // 2. 塊註解 /* ... */ + if (blockCommentDepth > 0) + { + if (c == '*' && i + 1 < len && line[i + 1] == '/') + { + blockCommentDepth--; + i++; + } + continue; + } + + // 3. 單引號字串 '...' + if (inSingleQuote) + { + hasStatementContent = true; + currentStatementBytes += (c > 127) ? 2 : 1; + if (c == '\'') + { + if (i + 1 < len && line[i + 1] == '\'') + { + i++; // 跳過 '' + currentStatementBytes++; + } + else + { + inSingleQuote = false; + } + } + continue; + } + + // 4. 雙引號識別字 "..." + if (inDoubleQuote) + { + hasStatementContent = true; + currentStatementBytes += (c > 127) ? 2 : 1; + if (c == '"') + { + if (i + 1 < len && line[i + 1] == '"') + { + i++; // 跳過 "" + currentStatementBytes++; + } + else + { + inDoubleQuote = false; + } + } + continue; + } + + // 5. 符號開始判定 + if (c == '-' && i + 1 < len && line[i + 1] == '-') + { + inLineComment = true; + i++; + continue; + } + + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + blockCommentDepth = 1; + i++; + continue; + } + + if (c == '\'') + { + inSingleQuote = true; + hasStatementContent = true; + currentStatementBytes += 1; + continue; + } + + if (c == '"') + { + inDoubleQuote = true; + hasStatementContent = true; + currentStatementBytes += 1; + continue; + } + + // 6. 語句結束判定 + if (c == ';' && !inPlSqlBlock) + { + if (hasStatementContent) + { + statementCount++; + hasStatementContent = false; + currentStatementBytes = 0; + } + continue; + } + + if (!char.IsWhiteSpace(c)) + { + hasStatementContent = true; + } + + if (hasStatementContent) + { + currentStatementBytes += (c > 127) ? 2 : 1; + if (currentStatementBytes > MaxBatchSizeBytes) + { + result.CanProceed = false; + result.Issues.Add(string.Format("第 {0} 行單一語句大小已超過 32 MiB 安全上限。", lineNumber)); + return result; + } + } + } + + inLineComment = false; // 換行結束單行註解 + } + + if (hasStatementContent) + { + statementCount++; + } + + result.StatementCount = statementCount; + + if (statementCount == 0) + { + result.Issues.Add("SQL 腳本中未發現任何有效的執行語句。"); + } + } + } + catch (Exception ex) + { + result.CanProceed = false; + result.Issues.Add("預檢掃描檔案時發生例外錯誤: " + ex.Message); + } + + return result; + } + + private static Encoding DetectEncoding(string filePath) + { + byte[] bom = new byte[4]; + using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + int read = fs.Read(bom, 0, 4); + if (read >= 3 && bom[0] == 0xEF && bom[1] == 0xBB && bom[2] == 0xBF) + return Encoding.UTF8; + if (read >= 2 && bom[0] == 0xFF && bom[1] == 0xFE) + return Encoding.Unicode; // UTF-16LE + if (read >= 2 && bom[0] == 0xFE && bom[1] == 0xFF) + return Encoding.BigEndianUnicode; // UTF-16BE + if (read >= 4 && bom[0] == 0 && bom[1] == 0 && bom[2] == 0xFE && bom[3] == 0xFF) + return Encoding.UTF32; + } + + return new UTF8Encoding(false); + } + } +} diff --git a/mySQLPunk/lib/OracleImport/OracleImportRunner.cs b/mySQLPunk/lib/OracleImport/OracleImportRunner.cs new file mode 100644 index 0000000..55c0ca8 --- /dev/null +++ b/mySQLPunk/lib/OracleImport/OracleImportRunner.cs @@ -0,0 +1,522 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace mySQLPunk.lib.OracleImport +{ + public sealed class OracleImportRunner + { + private const int BufferSize = 65536; + private readonly IDatabase db; + private readonly OracleImportRequest request; + private readonly Action terminalSink; + + private static readonly Regex PlSqlStartRegex = new Regex( + @"^\s*(DECLARE\b|BEGIN\b|CREATE\s+(OR\s+REPLACE\s+)?(PROCEDURE|FUNCTION|PACKAGE|TRIGGER|TYPE)\b)", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public OracleImportRunner(IDatabase db, OracleImportRequest request, Action terminalSink = null) + { + this.db = db ?? throw new ArgumentNullException(nameof(db)); + this.request = request ?? throw new ArgumentNullException(nameof(request)); + this.terminalSink = terminalSink; + } + + public OracleImportResult Run(IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) + { + return RunInternal(progress, cancellationToken); + } + + public async Task RunAsync(IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) + { + return await Task.Run(() => RunInternal(progress, cancellationToken), cancellationToken).ConfigureAwait(false); + } + + private OracleImportResult RunInternal(IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) + { + var clock = Stopwatch.StartNew(); + var log = new OracleImportLog(request.DatabaseName); + var result = new OracleImportResult { LogFilePath = log.LogPath }; + + Action LogAndTerminal = msg => + { + log.LogInfo(msg); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] {1}", DateTime.Now, msg)); + }; + + LogAndTerminal(string.Format("正在初始化 Oracle 匯入任務: {0}", Path.GetFileName(request.FilePath))); + LogAndTerminal(string.Format("目標綱要 (Owner): {0}", string.IsNullOrWhiteSpace(request.DatabaseName) ? "Current Schema" : request.DatabaseName)); + + // 1. 預檢掃描 + var scanner = new OracleImportPreflightScanner(); + var preflight = scanner.Scan(request.FilePath, cancellationToken); + if (!preflight.CanProceed) + { + clock.Stop(); + result.Success = false; + result.ErrorMessage = string.Join("; ", preflight.Issues.ToArray()); + LogAndTerminal("❌ 預檢未通過：" + result.ErrorMessage); + log.LogSummary(0, 0, 0, clock.Elapsed, false); + log.Flush(); + log.Dispose(); + return result; + } + + LogAndTerminal(string.Format("預檢通過：共探測到約 {0:N0} 條語句 (檔案大小: {1:N0} 位元組)。", preflight.StatementCount, preflight.FileSizeBytes)); + if (preflight.HasSpatialFeatures) + { + LogAndTerminal(string.Format("偵測到 Oracle Spatial (SDO_GEOMETRY) 空間特性關鍵字: {0}", string.Join(", ", preflight.DetectedSpatialKeywords.ToArray()))); + } + if (preflight.HasPlSqlBlocks) + { + LogAndTerminal(string.Format("偵測到 PL/SQL 預存區塊或觸發程序 (約 {0} 個區塊)。", preflight.PlSqlBlocksCount)); + } + + long totalBytes = preflight.FileSizeBytes; + int totalStatements = preflight.StatementCount; + + int currentStmtIndex = 0; + int failedStatements = 0; + long totalRowsAffected = 0; + long bytesRead = 0; + var truncatedTables = new HashSet(StringComparer.OrdinalIgnoreCase); + + var expectedManifestTables = new Dictionary(preflight.ManifestDeclaredTables, StringComparer.OrdinalIgnoreCase); + bool readingManifest = false; + + Action checkManifestLine = comment => + { + if (comment.Contains("mySQLPunk Native Manifest") && !readingManifest) + { + readingManifest = true; + log.LogInfo("Detected Native Manifest in SQL script footer."); + } + + var match = Regex.Match(comment, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) + { + expectedManifestTables[match.Groups[1].Value] = r; + } + }; + + try + { + using (var fs = new FileStream(request.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize)) + using (var reader = new StreamReader(fs, preflight.DetectedEncoding, true, BufferSize)) + { + string statement; + int lineNum = 0; + + foreach (var stmtInfo in EnumerateStatements(reader, cancellationToken, checkManifestLine)) + { + statement = stmtInfo.Sql.Trim(); + lineNum = stmtInfo.LineNumber; + bytesRead += stmtInfo.ByteLength; + + if (string.IsNullOrWhiteSpace(statement)) continue; + + currentStmtIndex++; + + // 套用衝突處理策略 + string effectiveSql = ApplyConflictStrategy(statement, request.Options.ConflictStrategy, truncatedTables, db, LogAndTerminal); + + // 去除獨立行結尾斜線（若有） + if (effectiveSql.EndsWith("/") && effectiveSql.Length > 1) + { + effectiveSql = effectiveSql.Substring(0, effectiveSql.Length - 1).TrimEnd(); + } + + string preview = effectiveSql.Length > 80 ? effectiveSql.Substring(0, 80).Replace("\r", " ").Replace("\n", " ") + "..." : effectiveSql.Replace("\r", " ").Replace("\n", " "); + + try + { + var execRes = db.ExecSQL(effectiveSql); + long affected = 0; + if (execRes != null && execRes.ContainsKey("rowsAffected")) + { + long.TryParse(execRes["rowsAffected"], out affected); + } + + if (affected > 0) totalRowsAffected += affected; + + log.LogStatement(lineNum, preview, true); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [OK] Line {1}: {2} (Rows: {3})", DateTime.Now, lineNum, preview, affected)); + } + catch (Exception ex) + { + // 若衝突策略為 KeepExisting 且為物件已存在錯誤 (ORA-00955 / ORA-01430 / ORA-02260 等) + if (request.Options.ConflictStrategy == OracleConflictStrategy.KeepExisting && IsObjectExistsError(ex.Message)) + { + log.LogStatement(lineNum, preview, true, "物件已存在，依 KeepExisting 策略略過。"); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [SKIP] Line {1}: {2} (物件已存在)", DateTime.Now, lineNum, preview)); + } + else + { + failedStatements++; + log.LogStatement(lineNum, preview, false, ex.Message); + terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [ERROR] Line {1}: {2} -> {3}", DateTime.Now, lineNum, preview, ex.Message)); + + if (request.Options.StopOnError) + { + try { db.ExecSQL("ROLLBACK;"); LogAndTerminal("遇錯停止，已執行 ROLLBACK。"); } catch { } + throw new InvalidOperationException(string.Format("第 {0} 行執行失敗：{1}", lineNum, ex.Message), ex); + } + } + } + + if (currentStmtIndex % 50 == 0 || currentStmtIndex == totalStatements) + { + progress?.Report(new OracleImportProgress + { + Stage = "Executing", + CurrentStatement = currentStmtIndex, + TotalStatements = totalStatements, + BytesRead = bytesRead, + TotalBytes = totalBytes, + RowsAffected = totalRowsAffected, + Message = string.Format("正在執行語句: {0}/{1} (已影響 {2:N0} 列)", currentStmtIndex, totalStatements, totalRowsAffected) + }); + } + } + } + + // 結尾提交交易 + try + { + db.ExecSQL("COMMIT;"); + LogAndTerminal("交易已成功提交 (COMMIT)。"); + } + catch (Exception ex) + { + LogAndTerminal("提交交易時發生警告: " + ex.Message); + } + + result.Success = (failedStatements == 0); + } + catch (Exception ex) + { + result.Success = false; + result.ErrorMessage = ex.Message; + LogAndTerminal("匯入過程中斷: " + ex.Message); + } + + clock.Stop(); + result.Elapsed = clock.Elapsed; + result.StatementsExecuted = currentStmtIndex; + result.StatementsFailed = failedStatements; + result.TotalRowsAffected = totalRowsAffected; + + // 7. 核對 Native Manifest + if (expectedManifestTables.Count > 0) + { + LogAndTerminal(string.Format("正在核對腳本結尾 Manifest 清單 (共 {0} 張資料表)...", expectedManifestTables.Count)); + bool allMatch = true; + + foreach (var kvp in expectedManifestTables) + { + string tableName = kvp.Key; + long expectedRows = kvp.Value; + long actualRows = -1; + + try + { + string countSql = tableName.Contains(".") + ? string.Format("SELECT COUNT(*) FROM \"{0}\".\"{1}\"", tableName.Split('.')[0], tableName.Split('.')[1]) + : string.Format("SELECT COUNT(*) FROM \"{0}\"", tableName); + + DataTable dt = db.SelectSQL(countSql); + if (dt != null && dt.Rows.Count > 0) + { + actualRows = Convert.ToInt64(dt.Rows[0][0]); + } + } + catch + { + actualRows = -1; + } + + result.RestoredTables[tableName] = actualRows; + if (actualRows != expectedRows) + { + allMatch = false; + LogAndTerminal(string.Format("⚠️ 資料表 [{0}] 筆數不符：Manifest 宣告 {1} 列，還原後實際 {2} 列。", tableName, expectedRows, actualRows)); + } + else + { + LogAndTerminal(string.Format("✅ 資料表 [{0}] 筆數相符：{1} 列。", tableName, actualRows)); + } + } + + result.ManifestMatched = allMatch; + log.LogManifest(expectedManifestTables, result.RestoredTables, allMatch); + } + + LogAndTerminal(string.Format("Oracle 匯入作業完成，耗時 {0:hh\\:mm\\:ss\\.fff}，詳細日誌已寫入: {1}", result.Elapsed, result.LogFilePath)); + log.LogSummary(currentStmtIndex, failedStatements, totalRowsAffected, result.Elapsed, result.Success); + log.Flush(); + log.Dispose(); + + return result; + } + + private static bool IsObjectExistsError(string message) + { + if (string.IsNullOrWhiteSpace(message)) return false; + // ORA-00955: name is already used by an existing object + // ORA-02260: table can have only one primary key + // ORA-01430: column being added already exists in table + return message.IndexOf("ORA-00955", StringComparison.OrdinalIgnoreCase) >= 0 || + message.IndexOf("ORA-02260", StringComparison.OrdinalIgnoreCase) >= 0 || + message.IndexOf("ORA-01430", StringComparison.OrdinalIgnoreCase) >= 0; + } + + private static string ApplyConflictStrategy( + string sql, + OracleConflictStrategy strategy, + HashSet truncatedTables, + IDatabase db, + Action logAction) + { + if (strategy == OracleConflictStrategy.DropAndRecreate || strategy == OracleConflictStrategy.ReplaceAll) + { + // 在 CREATE TABLE 前執行 DROP TABLE ... CASCADE CONSTRAINTS + var match = Regex.Match(sql, @"^CREATE\s+TABLE\s+([""\w\.]+)", RegexOptions.IgnoreCase); + if (match.Success) + { + string target = match.Groups[1].Value; + string dropSql = string.Format("BEGIN EXECUTE IMMEDIATE 'DROP TABLE {0} CASCADE CONSTRAINTS'; EXCEPTION WHEN OTHERS THEN NULL; END;", target); + try + { + db.ExecSQL(dropSql); + logAction(string.Format("[衝突處理] 已預先清除資料表: {0}", target)); + } + catch { } + } + } + else if (strategy == OracleConflictStrategy.TruncateAndInsert) + { + // 當第一次遇到該表的 INSERT 時，先執行 TRUNCATE + var match = Regex.Match(sql, @"^INSERT\s+INTO\s+([""\w\.]+)", RegexOptions.IgnoreCase); + if (match.Success) + { + string target = match.Groups[1].Value; + if (truncatedTables.Add(target)) + { + string truncSql = string.Format("TRUNCATE TABLE {0};", target); + try + { + db.ExecSQL(truncSql); + logAction(string.Format("[衝突處理] 已清空資料表內容: {0}", target)); + } + catch { } + } + } + } + + return sql; + } + + private struct StatementInfo + { + public string Sql; + public int LineNumber; + public int ByteLength; + } + + private static IEnumerable EnumerateStatements(StreamReader reader, CancellationToken cancellationToken, Action onCommentLine = null) + { + var sb = new StringBuilder(); + int startLine = 1; + int currentLine = 1; + int byteLen = 0; + + bool inSingleQuote = false; + bool inDoubleQuote = false; + bool inLineComment = false; + int blockCommentDepth = 0; + bool inPlSqlBlock = false; + + string line; + while ((line = reader.ReadLine()) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + byteLen += line.Length + 1; + + string trimmed = line.Trim(); + + // 獨立行註解不作為執行語句，僅觸發註解回呼 (如 Manifest 解析) + if (sb.Length == 0 && trimmed.StartsWith("--", StringComparison.Ordinal)) + { + onCommentLine?.Invoke(trimmed); + currentLine++; + startLine = currentLine; + continue; + } + + if (sb.Length == 0) startLine = currentLine; + + // 檢查是否開展 PL/SQL 區塊 + if (!inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) + { + if (PlSqlStartRegex.IsMatch(line)) + { + inPlSqlBlock = true; + } + } + + // 檢查 PL/SQL 結束斜線 / + if (inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) + { + if (trimmed == "/") + { + inPlSqlBlock = false; + string plsql = sb.ToString().Trim(); + sb.Clear(); + byteLen = 0; + startLine = currentLine + 1; + + if (!string.IsNullOrWhiteSpace(plsql)) + { + yield return new StatementInfo + { + Sql = plsql, + LineNumber = startLine, + ByteLength = byteLen + }; + } + currentLine++; + continue; + } + } + + int len = line.Length; + for (int i = 0; i < len; i++) + { + char c = line[i]; + + if (inLineComment) continue; + + if (blockCommentDepth > 0) + { + if (c == '*' && i + 1 < len && line[i + 1] == '/') { blockCommentDepth--; i++; } + sb.Append(c); + continue; + } + + if (inSingleQuote) + { + sb.Append(c); + if (c == '\'') + { + if (i + 1 < len && line[i + 1] == '\'') + { + sb.Append(line[i + 1]); + i++; + } + else + { + inSingleQuote = false; + } + } + continue; + } + + if (inDoubleQuote) + { + sb.Append(c); + if (c == '"') + { + if (i + 1 < len && line[i + 1] == '"') + { + sb.Append(line[i + 1]); + i++; + } + else + { + inDoubleQuote = false; + } + } + continue; + } + + // 符號檢查 + if (c == '-' && i + 1 < len && line[i + 1] == '-') + { + inLineComment = true; + i++; + continue; + } + + if (c == '/' && i + 1 < len && line[i + 1] == '*') + { + blockCommentDepth = 1; + sb.Append("/*"); + i++; + continue; + } + + if (c == '\'') + { + inSingleQuote = true; + sb.Append(c); + continue; + } + + if (c == '"') + { + inDoubleQuote = true; + sb.Append(c); + continue; + } + + // 非 PL/SQL 區塊下的分號終止符 + if (c == ';' && !inPlSqlBlock) + { + sb.Append(';'); + string sql = sb.ToString().Trim(); + sb.Clear(); + byteLen = 0; + startLine = currentLine; + + if (!string.IsNullOrWhiteSpace(sql)) + { + yield return new StatementInfo + { + Sql = sql, + LineNumber = startLine, + ByteLength = byteLen + }; + } + continue; + } + + sb.Append(c); + } + + inLineComment = false; + if (sb.Length > 0) sb.Append("\r\n"); + currentLine++; + } + + if (sb.Length > 0) + { + string rem = sb.ToString().Trim(); + if (!string.IsNullOrWhiteSpace(rem) && !rem.StartsWith("--", StringComparison.Ordinal)) + { + yield return new StatementInfo + { + Sql = rem, + LineNumber = startLine, + ByteLength = byteLen + }; + } + } + } + } +} diff --git a/mySQLPunk/mySQLPunk.csproj b/mySQLPunk/mySQLPunk.csproj index 204a035..9ce21b0 100644 --- a/mySQLPunk/mySQLPunk.csproj +++ b/mySQLPunk/mySQLPunk.csproj @@ -329,6 +329,10 @@ + + + + @@ -418,6 +422,12 @@ + + + + + + diff --git a/tests/Run-SmokeTests.ps1 b/tests/Run-SmokeTests.ps1 index 7bca91b..63bf168 100644 --- a/tests/Run-SmokeTests.ps1 +++ b/tests/Run-SmokeTests.ps1 @@ -153,6 +153,8 @@ if (!(Test-Path -LiteralPath $mysqlConnector)) { (Join-Path $PSScriptRoot "SmokeTests.SqliteImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.PostgreSqlExport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.PostgreSqlImport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.OracleExport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.OracleImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.TransferTrial.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Live.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlImport.Parser.cs") $source $preferencesSource $schemaSyncSource $erModelSource (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Snapshot.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Schema.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.ViewReferences.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Properties.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Artifact.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Values.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Lob.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Worker.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Wizard.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Routing.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.Integration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.SnapshotIntegration.cs") (Join-Path $PSScriptRoot "SmokeTests.MssqlExport.CompatibilityIntegration.cs") if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE diff --git a/tests/SmokeTests.MssqlExport.Routing.cs b/tests/SmokeTests.MssqlExport.Routing.cs index 4f8171b..3fd94e4 100644 --- a/tests/SmokeTests.MssqlExport.Routing.cs +++ b/tests/SmokeTests.MssqlExport.Routing.cs @@ -121,7 +121,7 @@ private static void Routing_FourHandlers() private static void Routing_LegacyBranches() { - foreach (string provider in new[] { "oracle" }) + foreach (string provider in new[] { "sybase" }) foreach (string handler in new[] { "DumpSelectedTableSql", "DumpCurrentSelectionSqlWithDialog", "DumpSelectedDatabaseSqlWithDialog", "BackupSelectedDatabaseWithDialog" }) using (var fixture = new RoutingFixture(provider, "Tables")) { diff --git a/tests/SmokeTests.MssqlImport.Wizard.cs b/tests/SmokeTests.MssqlImport.Wizard.cs index ac74bc9..79b2ede 100644 --- a/tests/SmokeTests.MssqlImport.Wizard.cs +++ b/tests/SmokeTests.MssqlImport.Wizard.cs @@ -384,7 +384,7 @@ public MssqlImportCleanup Cleanup() } private static void ImportWizard_Routing() { - foreach (string provider in new[] { "mssql", "SQLSERVER", "oracle" }) + foreach (string provider in new[] { "mssql", "SQLSERVER", "sybase" }) using (var fixture = new RoutingFixture(provider, "Tables")) { int opened = 0, legacy = 0; @@ -400,7 +400,7 @@ private static void ImportWizard_Routing() new Func(dialog => { legacy++; return DialogResult.Cancel; })); typeof(Form1).GetField("MssqlImportStatusSink", flags).SetValue(fixture.Form, new Action(text => fixture.Status.Add(text))); fixture.Invoke("ImportSqlWithDialog"); - bool native = provider != "oracle"; + bool native = provider != "sybase"; Assert(opened == (native ? 1 : 0) && legacy == (native ? 0 : 1), "Actual file handler routes only MSSQL before legacy file selection."); if (native) { diff --git a/tests/SmokeTests.OracleExport.cs b/tests/SmokeTests.OracleExport.cs new file mode 100644 index 0000000..78c78a6 --- /dev/null +++ b/tests/SmokeTests.OracleExport.cs @@ -0,0 +1,211 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.OracleExport; + +public static partial class SmokeTests +{ + private static void OracleExport_OptionsAndModels_Defaults() + { + var options = new OracleExportWizardOptions(OracleExportMode.SchemaAndData, @"C:\oracle_dump.sql"); + Assert(options.Mode == OracleExportMode.SchemaAndData, "Default mode should be SchemaAndData"); + Assert(options.UseTransactionSnapshot, "UseTransactionSnapshot should be true by default (SET TRANSACTION READ ONLY)"); + Assert(options.ExcludeSpatialIndexInternalTables, "ExcludeSpatialIndexInternalTables should be true by default"); + Assert(options.IncludeSpatialMetadata, "IncludeSpatialMetadata should be true by default (USER_SDO_GEOM_METADATA)"); + Assert(options.IncludeSpatialIndexes, "IncludeSpatialIndexes should be true by default (MDSYS.SPATIAL_INDEX)"); + Assert(options.IncludeIndexes, "IncludeIndexes should be true by default"); + Assert(options.IncludeComments, "IncludeComments should be true by default"); + Assert(options.IncludeViews, "IncludeViews should be true by default"); + Assert(options.IncludeManifest, "IncludeManifest should be true by default"); + Assert(options.BatchSize == 500, "Default BatchSize should be 500"); + + var item = new OracleTableExportItem + { + Owner = "SCOTT", + Name = "PARCELS", + IsSpatial = true, + RowCount = 1200, + SpatialFeatureDescription = "Oracle Spatial SDO_GEOMETRY" + }; + Assert(item.FullQualifiedName == "\"SCOTT\".\"PARCELS\"", "FullQualifiedName should match"); + Assert(item.DisplayName == "SCOTT.PARCELS", "DisplayName should match"); + } + + private static void OracleExport_ServiceAndManifest_StreamingAndSdoGeometry() + { + string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_ora_export_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + string targetFile = Path.Combine(tempDir, "oracle_export_test.sql"); + + try + { + var mockDb = new MockOracleExportDatabase(); + var service = new OracleSqlExportService(mockDb, "SCOTT"); + + // 1. 驗證 Catalog 正確辨識 Oracle Spatial (SDO_GEOMETRY) 空間表與 R-Tree 內部影子表 + var catalog = service.GetTableCatalog(); + Assert(catalog.Count == 2, "Expected 2 objects in catalog, got " + catalog.Count); + + var parcels = catalog.Find(i => i.Name == "PARCELS"); + Assert(parcels != null, "PARCELS table should exist in catalog"); + Assert(parcels.IsSpatial, "PARCELS should be recognized as Oracle Spatial layer"); + Assert(parcels.Selected, "PARCELS should be selected by default"); + + var shadow = catalog.Find(i => i.Name == "MDRT_12345$"); + Assert(shadow != null, "MDRT_12345$ table should exist in catalog"); + Assert(shadow.IsSpatialMetadata, "MDRT_* should be recognized as internal spatial shadow table"); + Assert(!shadow.Selected, "MDRT_* should be unchecked by default"); + + var options = new OracleExportWizardOptions(OracleExportMode.SchemaAndData, targetFile) + { + UseTransactionSnapshot = true, + IncludeSpatialMetadata = true, + IncludeSpatialIndexes = true, + IncludeIndexes = true, + IncludeComments = true, + IncludeManifest = true + }; + options.SelectedTables.Add("PARCELS"); + + var task = service.ExportAsync(options, null, CancellationToken.None); + task.Wait(); + var result = task.Result; + + Assert(result.Success, "Export should succeed"); + Assert(File.Exists(targetFile), "Target file must exist after atomic move"); + Assert(result.TablesExported == 1, "Exported table count should be 1"); + Assert(result.TotalRowsExported == 2, "Total exported rows should be 2"); + + string sql = File.ReadAllText(targetFile); + + // 2. 驗證 Oracle SET TRANSACTION READ ONLY 交易快照 + AssertContains(sql, "SET TRANSACTION READ ONLY;", "Header must initialize READ ONLY snapshot"); + + // 3. 驗證 USER_SDO_GEOM_METADATA 空間中繼註冊 + AssertContains(sql, "INSERT INTO USER_SDO_GEOM_METADATA", "Must generate USER_SDO_GEOM_METADATA insert"); + AssertContains(sql, "MDSYS.SDO_DIM_ARRAY", "Must generate dimension array"); + + // 4. 驗證 SDO_GEOMETRY 幾何資料轉換為無損 SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('...')) + AssertContains(sql, "SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW(", "Geometry values must be formatted as WKB HEXTORAW"); + AssertContains(sql, "INSERT INTO \"SCOTT\".\"PARCELS\"", "Insert header should target qualified table name"); + + // 5. 驗證 Oracle Spatial 空間索引 + AssertContains(sql, "INDEXTYPE IS MDSYS.SPATIAL_INDEX", "Must generate MDSYS.SPATIAL_INDEX domain index"); + + // 6. 驗證 Native Manifest + AssertContains(sql, "-- mySQLPunk Native Manifest", "Manifest header present"); + AssertContains(sql, "-- Manifest-Table: PARCELS | Rows: 2", "Manifest table row count accurate"); + AssertContains(sql, "-- Manifest-Checksum-SHA256:", "Manifest SHA-256 checksum present"); + Assert(!string.IsNullOrWhiteSpace(result.Sha256Checksum), "Manifest SHA256 string populated in result"); + } + finally + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + + private sealed class MockOracleExportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "oracle"; + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + var dt = new DataTable(); + + if (sql.Contains("FROM ALL_TABLES")) + { + dt.Columns.Add("OWNER", typeof(string)); + dt.Columns.Add("TABLE_NAME", typeof(string)); + dt.Columns.Add("ROW_COUNT", typeof(long)); + dt.Columns.Add("ESTIMATED_BYTES", typeof(long)); + + dt.Rows.Add("SCOTT", "PARCELS", 2L, 65536L); + dt.Rows.Add("SCOTT", "MDRT_12345$", 50L, 16384L); + return dt; + } + + if (sql.Contains("FROM ALL_VIEWS")) + { + dt.Columns.Add("OWNER", typeof(string)); + dt.Columns.Add("VIEW_NAME", typeof(string)); + return dt; + } + + if (sql.Contains("FROM ALL_TAB_COLUMNS") && sql.Contains("SDO_GEOMETRY")) + { + dt.Columns.Add("OWNER", typeof(string)); + dt.Columns.Add("TABLE_NAME", typeof(string)); + dt.Columns.Add("COLUMN_NAME", typeof(string)); + dt.Columns.Add("DATA_TYPE", typeof(string)); + + dt.Rows.Add("SCOTT", "PARCELS", "SHAPE", "SDO_GEOMETRY"); + return dt; + } + + if (sql.Contains("FROM \"SCOTT\".\"PARCELS\"")) + { + dt.Columns.Add("ID", typeof(int)); + dt.Columns.Add("NAME", typeof(string)); + dt.Columns.Add("SHAPE", typeof(byte[])); + + byte[] wkbSample = new byte[] { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F }; + dt.Rows.Add(1, "宗地一", wkbSample); + dt.Rows.Add(2, "宗地二", wkbSample); + return dt; + } + + return dt; + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + return new Dictionary { { "status", "OK" }, { "rowsAffected", "1" } }; + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + public List GetDatabases() => new List { "SCOTT" }; + public List GetTables(string databaseName) => new List { "PARCELS", "MDRT_12345$" }; + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) + { + var dt = new DataTable(); + dt.Columns.Add("COLUMN_NAME", typeof(string)); + dt.Columns.Add("DATA_TYPE", typeof(string)); + dt.Rows.Add("ID", "NUMBER"); + dt.Rows.Add("NAME", "VARCHAR2"); + dt.Rows.Add("SHAPE", "SDO_GEOMETRY"); + return dt; + } + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) + { + return "CREATE TABLE \"SCOTT\".\"PARCELS\" (\"ID\" NUMBER, \"NAME\" VARCHAR2(50), \"SHAPE\" MDSYS.SDO_GEOMETRY);"; + } + public bool TableExists(string databaseName, string tableName) => true; + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 2; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void SetConn(string connection) { } + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.OracleImport.cs b/tests/SmokeTests.OracleImport.cs new file mode 100644 index 0000000..f91e8c3 --- /dev/null +++ b/tests/SmokeTests.OracleImport.cs @@ -0,0 +1,279 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.OracleImport; + +public static partial class SmokeTests +{ + private static void OracleImport_PreflightScanner_SdoGeometryAndPlSql() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_ora_preflight_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +-- mySQLPunk Oracle Logical Dump +CREATE TABLE ""SCOTT"".""PARCELS"" ( + ""ID"" NUMBER PRIMARY KEY, + ""NAME"" VARCHAR2(100), + ""SHAPE"" MDSYS.SDO_GEOMETRY +); + +-- PL/SQL Trigger with internal semicolons, terminated with a standalone slash / +CREATE OR REPLACE TRIGGER ""SCOTT"".""TRG_PARCELS_BI"" +BEFORE INSERT ON ""SCOTT"".""PARCELS"" +FOR EACH ROW +BEGIN + IF :NEW.""ID"" IS NULL THEN + SELECT SEQ_PARCELS.NEXTVAL INTO :NEW.""ID"" FROM DUAL; + END IF; +END; +/ + +INSERT INTO ""SCOTT"".""PARCELS"" (""ID"", ""NAME"", ""SHAPE"") VALUES (1, 'Park', SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('0101000000000000000000F03F000000000000F03F'))); + +-- ==================================================================== +-- mySQLPunk Native Manifest +-- Manifest-Table: SCOTT.PARCELS | Rows: 1 +-- Manifest-Checksum-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +-- ==================================================================== +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + var scanner = new OracleImportPreflightScanner(); + var result = scanner.Scan(tempFile); + + Assert(result.CanProceed, "Valid Oracle script should pass preflight"); + Assert(result.HasSpatialFeatures, "Should detect Oracle Spatial keywords (sdo_geometry, sdo_util)"); + Assert(result.HasPlSqlBlocks, "Should detect PL/SQL trigger block"); + Assert(result.PlSqlBlocksCount == 1, "Expected 1 PL/SQL block, got " + result.PlSqlBlocksCount); + + // 語句計數：CREATE TABLE (1) + TRIGGER (2) + INSERT (3) = 3 條語句 (TRIGGER 內部的分號不可被誤切！) + Assert(result.StatementCount == 3, "StatementCount should be 3, semicolons inside PL/SQL block must NOT be split. Got: " + result.StatementCount); + + Assert(result.ContainsManifest, "Should detect Native Manifest footer"); + Assert(result.ManifestDeclaredTables.ContainsKey("SCOTT.PARCELS"), "Manifest should contain SCOTT.PARCELS"); + Assert(result.ManifestDeclaredTables["SCOTT.PARCELS"] == 1, "Declared rows should be 1"); + Assert(result.ManifestSha256 == "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "SHA256 should match"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void OracleImport_PreflightScanner_EmbeddedNul() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_ora_nul_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + byte[] corrupt = new byte[] { (byte)'S', (byte)'E', (byte)'L', (byte)'E', (byte)'C', (byte)'T', 0x00, (byte)';' }; + File.WriteAllBytes(tempFile, corrupt); + + var scanner = new OracleImportPreflightScanner(); + var result = scanner.Scan(tempFile); + + Assert(!result.CanProceed, "Embedded NUL byte must fail preflight"); + Assert(result.Issues.Count > 0, "Issues list should report corrupted NUL character"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void OracleImport_Runner_ExecutionAndManifestVerification() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_ora_run_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +CREATE TABLE ""SCOTT"".""PARCELS"" (""ID"" NUMBER, ""SHAPE"" MDSYS.SDO_GEOMETRY); +INSERT INTO ""SCOTT"".""PARCELS"" VALUES (1, SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('0101'))); +-- mySQLPunk Native Manifest +-- Manifest-Table: SCOTT.PARCELS | Rows: 1 +-- Manifest-Checksum-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + var mockDb = new MockOracleImportDatabase(); + var options = new OracleImportOptions + { + UseTransaction = true, + StopOnError = true + }; + var request = new OracleImportRequest(tempFile, "SCOTT", options); + var runner = new OracleImportRunner(mockDb, request); + + var result = runner.Run(); + + Assert(result.Success, "Runner execution should succeed"); + Assert(result.StatementsExecuted == 2, "Expected 2 statements executed, got " + result.StatementsExecuted); + Assert(result.TotalRowsAffected == 1, "Expected 1 row affected, got " + result.TotalRowsAffected); + Assert(result.ManifestMatched, "Manifest verification should pass (declared 1 == actual 1)"); + Assert(File.Exists(result.LogFilePath), "Independent log file must exist: " + result.LogFilePath); + + // 驗證交易包覆有正確 COMMIT + Assert(mockDb.ExecutedQueries.Contains("COMMIT;"), "Transaction COMMIT must be executed"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void OracleImport_Runner_ExistingObjectStrategies() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_ora_strat_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +CREATE TABLE ""SCOTT"".""ROADS"" (""ID"" NUMBER); +INSERT INTO ""SCOTT"".""ROADS"" VALUES (1); +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + // 1. KeepExisting 策略：遇 ORA-00955 (物件已存在) 時跳過且不中斷 + var mockDb1 = new MockOracleImportDatabase(); + mockDb1.SimulateExistingObjectError = true; + var opt1 = new OracleImportOptions { ConflictStrategy = OracleConflictStrategy.KeepExisting, StopOnError = true }; + var runner1 = new OracleImportRunner(mockDb1, new OracleImportRequest(tempFile, "SCOTT", opt1)); + var res1 = runner1.Run(); + Assert(res1.Success, "KeepExisting should treat ORA-00955 as skipped and succeed"); + + // 2. DropAndRecreate 策略：應在建表前預先執行 DROP TABLE ... CASCADE CONSTRAINTS + var mockDb2 = new MockOracleImportDatabase(); + var opt2 = new OracleImportOptions { ConflictStrategy = OracleConflictStrategy.DropAndRecreate }; + var runner2 = new OracleImportRunner(mockDb2, new OracleImportRequest(tempFile, "SCOTT", opt2)); + runner2.Run(); + + bool hasDrop = mockDb2.ExecutedQueries.Exists(q => q.Contains("DROP TABLE \"SCOTT\".\"ROADS\" CASCADE CONSTRAINTS")); + Assert(hasDrop, "DropAndRecreate should execute DROP TABLE CASCADE CONSTRAINTS"); + + // 3. TruncateAndInsert 策略：應在第一次 INSERT 前執行 TRUNCATE TABLE + var mockDb3 = new MockOracleImportDatabase(); + var opt3 = new OracleImportOptions { ConflictStrategy = OracleConflictStrategy.TruncateAndInsert }; + var runner3 = new OracleImportRunner(mockDb3, new OracleImportRequest(tempFile, "SCOTT", opt3)); + runner3.Run(); + + bool hasTruncate = mockDb3.ExecutedQueries.Exists(q => q.Contains("TRUNCATE TABLE \"SCOTT\".\"ROADS\";")); + Assert(hasTruncate, "TruncateAndInsert should execute TRUNCATE TABLE"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private static void OracleImport_Runner_TransactionRollbackOnFailure() + { + string tempFile = Path.Combine(Path.GetTempPath(), "mysqlpunk_ora_fail_test_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + string sql = @" +CREATE TABLE ""SCOTT"".""VALID"" (""ID"" NUMBER); +INVALID SQL SYNTAX HERE; +"; + File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); + + var mockDb = new MockOracleImportDatabase(); + mockDb.FailOnSql = "INVALID SQL"; + + var opt = new OracleImportOptions + { + UseTransaction = true, + StopOnError = true + }; + var request = new OracleImportRequest(tempFile, "SCOTT", opt); + var runner = new OracleImportRunner(mockDb, request); + + var result = runner.Run(); + + Assert(!result.Success, "Runner should report failure"); + Assert(mockDb.ExecutedQueries.Contains("ROLLBACK;"), "Transaction ROLLBACK must be executed upon statement failure"); + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + + private sealed class MockOracleImportDatabase : IDatabase + { + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "oracle"; + public List ExecutedQueries { get; } = new List(); + public string FailOnSql { get; set; } = null; + public bool SimulateExistingObjectError { get; set; } = false; + + public void Open() { } + public void Close() { } + public void Dispose() { } + + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + ExecutedQueries.Add(sql); + var dt = new DataTable(); + + if (sql.Contains("SELECT COUNT(*) FROM \"SCOTT\".\"PARCELS\"")) + { + dt.Columns.Add("COUNT", typeof(long)); + dt.Rows.Add(1L); + return dt; + } + + return dt; + } + + public Dictionary ExecSQL(string sql, Dictionary parameters = null) + { + ExecutedQueries.Add(sql); + + if (!string.IsNullOrWhiteSpace(FailOnSql) && sql.Contains(FailOnSql)) + { + throw new InvalidOperationException("Simulated syntax error: " + sql); + } + + if (SimulateExistingObjectError && sql.StartsWith("CREATE TABLE", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("ORA-00955: name is already used by an existing object"); + } + + long rows = sql.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase) ? 1 : 0; + return new Dictionary + { + { "status", "OK" }, + { "rowsAffected", rows.ToString() } + }; + } + + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + public List GetDatabases() => new List { "SCOTT" }; + public List GetTables(string databaseName) => new List(); + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) => new DataTable(); + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => ""; + public bool TableExists(string databaseName, string tableName) => false; + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string databaseName, string oldTableName, string newTableName) { } + public void RenameView(string databaseName, string oldViewName, string newViewName) { } + public long CountRows(string databaseName, string tableName) => 0; + public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); + public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); + public void SetConn(string connection) { } + public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } + public void DropTableForCopy(string databaseName, string tableName) { } + public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } + public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); + public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } + public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } + } +} diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index b1e50a3..507b22d 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -53,6 +53,13 @@ public static int Main(string[] args) Run("PostgreSQL import runner execution and manifest verification", PostgreSqlImport_Runner_ExecutionAndManifestVerification, ref passed); Run("PostgreSQL import runner existing object strategies", PostgreSqlImport_Runner_ExistingObjectStrategies, ref passed); Run("PostgreSQL import runner transaction rollback on failure", PostgreSqlImport_Runner_TransactionRollbackOnFailure, ref passed); + Run("Oracle export options and models defaults", OracleExport_OptionsAndModels_Defaults, ref passed); + Run("Oracle export service and manifest streaming and SDO_GEOMETRY", OracleExport_ServiceAndManifest_StreamingAndSdoGeometry, ref passed); + Run("Oracle import preflight scanner SDO_GEOMETRY and PL/SQL", OracleImport_PreflightScanner_SdoGeometryAndPlSql, ref passed); + Run("Oracle import preflight scanner embedded NUL", OracleImport_PreflightScanner_EmbeddedNul, ref passed); + Run("Oracle import runner execution and manifest verification", OracleImport_Runner_ExecutionAndManifestVerification, ref passed); + Run("Oracle import runner existing object strategies", OracleImport_Runner_ExistingObjectStrategies, ref passed); + Run("Oracle import runner transaction rollback on failure", OracleImport_Runner_TransactionRollbackOnFailure, ref passed); Run("MSSQL import Transfer guards", MssqlImportTransferGuards, ref passed); Run("MSSQL import Transfer trial", MssqlImportTransferTrial, ref passed); Run("MSSQL import worker published cleanup receipt", MssqlImportPublishedReleaseRace, ref passed); From 09a4a9eddf66b4e715d5f216a5f26062fc97cceb Mon Sep 17 00:00:00 2001 From: John Date: Fri, 16 Oct 2026 13:37:10 +0800 Subject: [PATCH 09/12] fix: repair MySQL SQLite and Oracle import export wizards --- CHANGELOG.md | 9 +- TODO.md | 6 +- .../db-wizard-repairs/mysql-repair-report.md | 56 ++ .../db-wizard-repairs/mysql-scoped-review.md | 50 + .../db-wizard-repairs/oracle-repair-report.md | 75 ++ .../db-wizard-repairs/oracle-scoped-review.md | 78 ++ docs/design/db-wizard-repairs/plan.md | 51 + .../db-wizard-repairs/sqlite-repair-report.md | 77 ++ .../db-wizard-repairs/sqlite-scoped-review.md | 55 ++ docs/design/db-wizard-repairs/verification.md | 42 + mySQLPunk/Form1.MySqlExport.cs | 12 +- mySQLPunk/Form1.MySqlImport.cs | 3 +- mySQLPunk/Form1.OracleExport.cs | 4 +- mySQLPunk/Form1.OracleImport.cs | 3 +- mySQLPunk/Form1.SqliteExport.cs | 5 +- mySQLPunk/Form1.SqliteImport.cs | 3 +- mySQLPunk/Form1.cs | 4 +- mySQLPunk/MySqlExportWizardForm.cs | 66 +- mySQLPunk/MySqlImportWizardForm.cs | 17 +- mySQLPunk/OracleExportWizardForm.cs | 48 +- mySQLPunk/OracleImportWizardForm.cs | 31 +- mySQLPunk/SqliteExportWizardForm.cs | 70 +- mySQLPunk/SqliteImportWizardForm.cs | 14 +- mySQLPunk/lib/ConnectionOpenService.cs | 23 +- .../lib/MySqlExport/MySqlSqlExportService.cs | 290 ++++-- mySQLPunk/lib/MySqlImport/MySqlImportLog.cs | 2 +- .../MySqlImportPreflightScanner.cs | 140 +-- .../lib/MySqlImport/MySqlImportRunner.cs | 344 ++++--- .../lib/MySqlImport/MySqlScriptReader.cs | 177 ++++ .../lib/OracleExport/OracleExportModels.cs | 2 +- .../OracleExport/OracleSqlExportService.cs | 915 ++++++++++-------- mySQLPunk/lib/OracleImport/OracleImportLog.cs | 5 +- .../OracleImportPreflightScanner.cs | 298 +----- .../lib/OracleImport/OracleImportRunner.cs | 805 +++++++-------- mySQLPunk/lib/SqlExportPublication.cs | 22 + .../SqliteExport/SqliteSqlExportService.cs | 897 ++++++----------- .../lib/SqliteImport/SqliteImportModels.cs | 17 +- .../SqliteImportPreflightScanner.cs | 455 ++++----- .../lib/SqliteImport/SqliteImportRunner.cs | 815 +++++----------- mySQLPunk/lib/WizardOwnedSession.cs | 57 ++ mySQLPunk/mySQLPunk.csproj | 3 + tests/Run-SmokeTests.ps1 | 1 + tests/SmokeTests.MySqlExport.cs | 91 +- tests/SmokeTests.MySqlImport.cs | 141 ++- tests/SmokeTests.OracleExport.cs | 318 +++++- tests/SmokeTests.OracleImport.cs | 175 +++- tests/SmokeTests.SqliteExport.cs | 196 ++-- tests/SmokeTests.SqliteImport.cs | 353 ++++++- tests/SmokeTests.WizardRepairs.cs | 122 +++ tests/SmokeTests.cs | 26 + 50 files changed, 4526 insertions(+), 2943 deletions(-) create mode 100644 docs/design/db-wizard-repairs/mysql-repair-report.md create mode 100644 docs/design/db-wizard-repairs/mysql-scoped-review.md create mode 100644 docs/design/db-wizard-repairs/oracle-repair-report.md create mode 100644 docs/design/db-wizard-repairs/oracle-scoped-review.md create mode 100644 docs/design/db-wizard-repairs/plan.md create mode 100644 docs/design/db-wizard-repairs/sqlite-repair-report.md create mode 100644 docs/design/db-wizard-repairs/sqlite-scoped-review.md create mode 100644 docs/design/db-wizard-repairs/verification.md create mode 100644 mySQLPunk/lib/MySqlImport/MySqlScriptReader.cs create mode 100644 mySQLPunk/lib/SqlExportPublication.cs create mode 100644 mySQLPunk/lib/WizardOwnedSession.cs create mode 100644 tests/SmokeTests.WizardRepairs.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b31606..7111950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,14 @@ ## [Unreleased] ### 🚀 新增功能 -- **MySQL / MariaDB 四步 SQL 備份（傾印）與匯入精靈（對齊 MSSQL 標竿規格）**： - - **四步匯出精靈 (`MySqlExportWizardForm`)**：提供「模式與目標」、「物件選取（資料表／檢視／常式／觸發器）」、「選項確認（單一交易快照 `REPEATABLE READ`、外鍵約束開關、DROP 語句、移除 DEFINER）」與「即時進度與看板娘 Punky 動畫」完整四步體驗。底層改採串流直接寫入暫存檔 (`.staging.xxx.tmp`)，完成後原子移動，杜絕大型資料庫傾印時的 OOM 記憶體溢位問題。 - - **Native Manifest 元數據校驗**：傾印檔末端自動附加原生 Manifest 區塊與 SHA-256 數位簽章，包含資料庫名稱、資料表清單、列數統計與校驗碼。 +- **MySQL / MariaDB 四步 SQL 備份（傾印）與匯入精靈**： + - **四步匯出精靈 (`MySqlExportWizardForm`)**：提供「模式與目標」、「物件選取（資料表／檢視／常式／觸發器）」、「選項確認（單一交易快照 `REPEATABLE READ`、外鍵約束開關、DROP 語句、移除 DEFINER）」與「即時進度與看板娘 Punky 動畫」完整四步體驗。底層改採串流直接寫入暫存檔 (`.staging.xxx.tmp`)，完成後原子替換。來源頁面與輸出批次有明確大小限制；較大 LOB 與實機效能邊界見修整驗證文件。 + - **Native Manifest 元數據校驗**：傾印檔末端自動附加原生 Manifest 區塊與 SHA-256 校驗碼，包含資料庫名稱、資料表清單、列數統計與校驗碼。 - **四步匯入精靈 (`MySqlImportWizardForm`)**：提供「檔案選取與目標」、「Preflight 串流預檢（編碼判定、DELIMITER 切分、單批次 32 MiB 上限、Manifest 讀取）」、「執行選項（錯誤繼續／遇錯即停、物件存在處理策略）」與「執行回報與日誌檢視」。 - - **獨立日誌與即時審計**：所有匯入執行步驟與錯誤皆獨立記錄至 `%LOCALAPPDATA%\mySQLPunk\ImportLogs`，並於匯入完成後自動比對 Native Manifest 目標列數，確認還原完整性。 + - **獨立日誌與即時審計**：所有匯入執行步驟與錯誤皆獨立記錄至 `%LOCALAPPDATA%\mySQLPunk\ImportLogs`，並於匯入完成後自動比對 Native Manifest 目標列數；筆數符合不代表全部資料值與結構已驗收。 ### 🛠️ 問題修正與優化 +- **MySQL／SQLite／Oracle 匯出匯入修整**：修正衝突策略誤刪／漏匯、SQL 字串誤改、provider 失敗被當成功、交易與取消清理；改用獨立工作連線與保留舊檔的原子發布。SQLite 補上 trigger、generated 欄位、rowid／序號及文字 bytes 往返；Oracle 補來源快照、逐列 reader、SRID／metadata 與 bounded LOB 分段。單表入口預設不包含不相關物件。支援上限與尚未驗收項目見 [驗證文件](docs/design/db-wizard-repairs/verification.md)。 - **MySQL 匯出與匯入精靈版面與體驗優化**： - **修復版面跑版坍塌**：修正步驟 2 與步驟 3 之設定 GroupBox 在 FlowLayoutPanel 內因 AutoSize 導致寬度坍塌至邊框寬度 (~15px) 造成文字嚴重擠壓折疊之跑版問題，改採標準固定寬度 (740px) 與適應性內部流式排版。 - **新增 MySQL 專屬高速還原與約束優化選項**：新增「匯入時關閉唯一性檢查 (`SET UNIQUE_CHECKS = 0`)，啟用 Insert Buffer 提升大量匯入速度 2~5 倍」與「防止主鍵 0 被 AUTO_INCREMENT 覆蓋 (`SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO'`)」開關，兼顧高速傾印與匯入資料正確性。 diff --git a/TODO.md b/TODO.md index a736e06..6458e51 100644 --- a/TODO.md +++ b/TODO.md @@ -32,10 +32,10 @@ | 資料庫 Provider | SQL 傾印 (匯出) 現況 | SQL 檔案匯入現況 | 主要缺口與風險 | | :--- | :--- | :--- | :--- | -| **MySQL / MariaDB** | ✅ 已具備完整四步精靈 (`MySqlExportWizardForm`)，支援選表/View/Routine/Trigger、一致性快照、Native Manifest。 | ✅ 已具備完整四步精靈 (`MySqlImportWizardForm`)，支援串流預檢、4種衝突策略、Session鎖定、終端機滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格）。 | -| **SQLite** | ✅ 已具備完整四步精靈 (`SqliteExportWizardForm`)，支援選表、SpatiaLite/GeoPackage 空間幾何無損 Hex 傾印、排除虛擬表影子表、原子替換與 Native Manifest。 | ✅ 已具備完整四步精靈 (`SqliteImportWizardForm`)，支援串流狀態機預檢、SpatiaLite 動態模組自動加載、4種衝突策略、滑動視窗終端機日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS 空間特性）。 | +| **MySQL / MariaDB** | 四步精靈、選表／模式、獨立連線、InnoDB 快照、generated 欄位投影與原子發布已補強。 | 共用預檢／執行 lexer、識別字限定的目標處理、衝突策略、交易／清理結果與錯誤日誌已補強。 | 本機回歸後仍需實機往返；來源單列目前限 1 MiB、INSERT 限 8 MiB。更大 LOB 分段與主動中斷執行中命令尚待擴充。 | +| **SQLite** | 原生逐列 reader、來源快照、generated 欄位投影、rowid／序號、文字／BLOB bytes 與原子發布已補強。 | 策略值綁定、DROP 保護、trigger 解析、嚴格 UTF-8、交易／PRAGMA 清理與日誌已補強。 | 私有 SQLite 往返驗收；單列目前限 8 MiB。SpatiaLite／GeoPackage 擴充模組與多 GiB、實體 UI 仍需另外驗收，不能只靠筆數宣稱無損。 | | **PostgreSQL** | ✅ 已具備完整四步精靈 (`PostgreSqlExportWizardForm`)，支援選表/View、PostGIS 空間幾何無損 Hex EWKB 傾印、排除 spatial_ref_sys、REPEATABLE READ 快照與 Native Manifest。 | ✅ 已具備完整四步精靈 (`PostgreSqlImportWizardForm`)，支援狀態機 Lexer (精準解析 Dollar Quotes 與巢狀註解)、PostGIS 自動安裝開關、4種衝突策略、2,000行滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS PostGIS 空間特性）。 | -| **Oracle** | ✅ 已具備完整四步精靈 (`OracleExportWizardForm`)，支援選表/View、Oracle Spatial (`SDO_GEOMETRY`) 無損 WKB 傾印、排除 R-Tree 內部影子表 (`MDRT_*`)、`USER_SDO_GEOM_METADATA` 註冊、`SET TRANSACTION READ ONLY;` 快照與 Native Manifest。 | ✅ 已具備完整四步精靈 (`OracleImportWizardForm`)，支援串流狀態機預檢 (精準解析 PL/SQL 區塊與獨立斜線 `/`)、4種衝突策略、2,000行滑動視窗終端機日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS Oracle Spatial 與 PL/SQL 特長）。 | +| **Oracle** | 獨立連線、來源 READ ONLY、原生逐列 reader、Spatial 原始 metadata／SRID、bounded LOB 與原子發布修整。 | 檢查 provider 結果、目標 owner、交易、共用 lexer、日誌與進度 bytes 修整。 | 仍待 Oracle／Spatial 實機往返；大型值、identity、LRS／曲線及跨 owner 腳本有明確支援邊界，詳見修整驗證文件。尚未宣稱完全對齊 MSSQL。 | --- diff --git a/docs/design/db-wizard-repairs/mysql-repair-report.md b/docs/design/db-wizard-repairs/mysql-repair-report.md new file mode 100644 index 0000000..b1369cd --- /dev/null +++ b/docs/design/db-wizard-repairs/mysql-repair-report.md @@ -0,0 +1,56 @@ +# MySQL wizard repair report + +Baseline: d41825cead810bc67f263753bb5f986a7f69c65c. Review input: C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/mysql-review.md. User-authorized provider repair; no live database, saved connection, SSH tunnel or production test was performed. + +## Changes + +- `lib/MySqlImport/MySqlScriptReader.cs` is the shared preflight/execution lexer. It preserves original literal characters/newlines and ordinary comment text, splits multiple statements per line, recognizes DELIMITER only outside SQL text, and rejects unterminated syntax, executable MySQL/MariaDB version comments, embedded NUL, excessive tokens and statements beyond 32 MiB UTF-8. Executable version comments require the native client path; they are not silently dropped. +- `MySqlImportRunner.cs` redirects comment-prefixed USE and recognized schema-qualified object positions even without prior USE. Identifier token replacement preserves string/JSON/comment content and longer identifiers. Ambiguous qualifiers and dynamic execution/CALL/client commands fail explicitly; database-wide destructive commands fail rather than writing outside the chosen target. This is a conservative logical-dump restore boundary, not an arbitrary SQL sandbox. +- Initial target-table catalog is separate from the live object cache. SkipExisting protects initial objects while fresh CREATE plus repeated INSERT batches execute. Comment-prefixed DROP cannot bypass protected strategies; synthetic DROP checks actual status=NO results. Session initialization and cleanup validate execution results. Uncommitted explicit transactions, failed COMMIT, and failed cleanup cannot produce Completed. Logs auto-flush each statement. Cancellation is checked after returned SQL, including the final statement. +- `MySqlImportPreflightScanner.cs` uses the same parser and verifies the native SHA-256 footer against the exact byte prefix. The footer lookup operates on raw bytes, including non-ASCII input near the final buffer boundary. +- `MySqlSqlExportService.cs` validates snapshot setup, requires verified InnoDB selected tables for SingleTransaction, uses UTC session/output time zone, validates snapshot rollback before publication, and never commits the source session. Missing table DDL/metadata/bounds/pages or source row-count discrepancies fail explicitly. Generated columns retain DDL but are omitted from INSERT columns and values. Unsupported spatial values and unsupported runtime value types fail before publication. TIME microseconds are preserved. +- Export has a 1 MiB source-row limit, at most 16 rows per source page, and an 8 MiB UTF-8 INSERT limit. Server-side OCTET_LENGTH preflight and CASE-projected pages prevent oversized LOB materialization, including an enlargement after the initial bound query when SingleTransaction is disabled. Oversized values fail explicitly; no all-size/streaming-LOB claim is made. +- Export uses the shared `SqlExportPublication.Publish` helper. Replacement failure keeps the completed staging file and reports its path; failure/cancellation before publication preserves the previous destination. Hashing is cancellable and final byte count includes the footer. +- `MySqlExportWizardForm.cs` applies supplied initial mode, table selection and option flags. Catalog loading remains busy until awaited work finishes; close requests wait for cleanup. Both forms restore the cancel button and dispose per-operation cancellation sources after awaited completion. Shared Form1 owned-session routing is integrated by the root implementer. + +## Changed files and registration + +Provider-owned changes: `mySQLPunk/lib/MySqlImport/{MySqlScriptReader,MySqlImportRunner,MySqlImportPreflightScanner,MySqlImportLog}.cs`, `mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs`, both MySQL wizard forms, `tests/SmokeTests.MySqlImport.cs`, `tests/SmokeTests.MySqlExport.cs`. + +New Compile entry: `lib\MySqlImport\MySqlScriptReader.cs` (root-owned project integration). + +New test registrations: + +- MySqlImport_Regression_TargetLiteralsAndStrategies +- MySqlImport_Regression_CancelTransactionsAndPreflight +- MySqlExport_Regression_GeneratedBoundsAndFailures + +Existing touched files retain their original UTF-8 BOM state and CRLF endings (byte-checked against HEAD). The new lexer uses UTF-8 without BOM and CRLF. + +## Verification evidence + +An isolated Roslyn C#/.NET Framework harness compiled the changed services/models, shared publication helper and both existing smoke files against the pinned baseline executable's IDatabase interface. It ran all 11 original/new MySQL test methods successfully; the root owns full application builds and the shared smoke suite. + +Harness: `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/mysql-repair-harness`. + +Durable coverage includes comment-prefixed/inline USE, same-line splitting, explicit qualified INSERT without USE, exact multiline/JSON/escaped literal and comment preservation, longer identifier boundaries, mixed new/existing SkipExisting, protected commented DROP and database-wide DROP rejection, last-command cancellation, uncommitted script/failed COMMIT/failed cleanup, executable comments/unterminated literals/digest mismatch, generated INSERT omission, round-trip preflight/digest compatibility, failed snapshot/rollback/non-InnoDB/large-row handling, concurrent enlargement marker, cancelled final source page, and Windows replacement-lock failure with both old output and complete verified staging preserved. + +## Limits and unverified acceptance + +- No MySQL/MariaDB server round trip, two-database source-preservation fixture, actual generated-column insertion, SQL CASE type inference, zero dates, server max_allowed_packet, real permissions, active editor transaction or live SSH disposal was tested. Recording provider SQL/status evidence is distinct from server acceptance. +- IDatabase exposes no cancellable active command. Stop is observed when the current command/read returns or times out; it does not promise immediate server command cancellation. The forms keep the owned job alive while awaiting cleanup. +- Imports may have applied earlier autocommitted DDL/DML before a later rejected statement; failure logs and counters remain authoritative and no all-or-nothing restore claim is made. Explicit uncommitted transactions are rolled back and reported failed. +- Concurrent source DDL and source pagination without a snapshot remain outside a consistency guarantee. SingleTransaction is restricted to verified InnoDB tables; disabling it permits ordinary non-snapshot reads, with bounded pages and row-count discrepancy detection. +- Executable version comments, dynamic SQL/CALL, ambiguous dotted identifiers, noncanonical SQL_MODE changes, AUTOCOMMIT changes, complex transaction-control modifiers, spatial data values, oversized source rows, and unknown runtime value types require the native-client path. Logical import is intentionally conservative about unsupported constructs. +- Full supported-type fidelity remains pending real-server fixtures; in particular timestamp/zero-date/provider type mapping and MariaDB variants are not certified by these recording tests. Shared owned-session and routing/publication test results belong to the root report. + +## Scoped review correction wave + +The focused MySQL reviewer identified two provider gaps after the initial repair. They are corrected in `MySqlImportRunner.cs` with one additional durable test method in `tests/SmokeTests.MySqlImport.cs`. + +- INSERT/REPLACE target extraction now uses lexical tokens, covers LOW_PRIORITY/HIGH_PRIORITY/DELAYED/IGNORE and optional INTO, and is shared by target redirection and SkipExisting. Protected initial-table writes cannot bypass the policy using these accepted forms. Unidentifiable INSERT/REPLACE targets fail explicitly under SkipExisting. +- NEW/OLD pseudo-row aliases are accepted only inside a recognized CREATE TRIGGER definition. Trigger ON object positions are redirected to the selected target; ordinary non-trigger NEW/OLD qualifiers retain the conservative rejection behavior. Literals containing NEW/OLD or source schema text remain unchanged. + +New root registration: `MySqlImport_Regression_ModifiedInsertProtectionAndTriggerPseudoRows`. + +Fresh isolated compilation and all 12 MySQL provider smoke methods pass. The new test exercises eight INSERT/REPLACE forms against protected old and fresh tables, repeats all forms in a data-only fixture, covers quoted/qualified targets without USE, then imports one native DELIMITER trigger containing NEW/OLD, real database-qualified objects and source-looking literals. It also verifies the pseudo-row exemption does not apply outside trigger DDL. BOM/CRLF rechecked against HEAD and provider diff whitespace checks remain clean. No real database or shared full build was run by this worker. The third reviewer finding about single-table related-object defaults is assigned to the root implementer. diff --git a/docs/design/db-wizard-repairs/mysql-scoped-review.md b/docs/design/db-wizard-repairs/mysql-scoped-review.md new file mode 100644 index 0000000..75de50b --- /dev/null +++ b/docs/design/db-wizard-repairs/mysql-scoped-review.md @@ -0,0 +1,50 @@ +# Scoped MySQL repair review + +Reviewed 2026-10-16 against the supplied mysql-repair-diff.patch and current MySQL/shared files in the repair worktree. Scope: plan Task 1 and common ownership/publication/form-lifetime constraints. Oracle/SQLite provider changes were excluded. This was a static review only: no build, test rerun, database, saved connection, SSH session, or user dump was opened. The implementer/root test results remain their evidence, not a new result from this reviewer. + +## Decision + +Changes requested for the three focused issues below. The original comment/literal redirection, fresh-table SkipExisting, status=NO, generated-column, bounded-page, owned-session and atomic-publication repairs are materially present. No additional blocker was found in the common raw-provider/SSH ownership or publication helper during this pass. + +## Findings + +### P1 - SkipExisting still writes existing data for accepted INSERT forms + +Location: mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs:33-35 and 394-407. + +The policy recognizes only INSERT/REPLACE [IGNORE] INTO. With an initial table old, SkipExisting, and `INSERT LOW_PRIORITY INTO old VALUES (1);` or `INSERT old VALUES (1);`, this regex does not match. Neither the lexer nor target validation rejects these forms, so execution reaches ExecSQL at line 411 and writes the existing table, while the job can report Completed. This contradicts the visible "Skip existing objects and their data" policy. The separate initial catalog repairs the original fresh-table omission but does not close this bypass. MySQL's documented INSERT grammar permits priority modifiers and optional INTO: [official INSERT syntax](https://dev.mysql.com/doc/refman/8.4/en/insert.html). + +Fix locally by extracting the write target from tokens for accepted INSERT/REPLACE forms, or reject unrecognized write forms under the protected strategy before execution. Add durable data-only cases with initial old plus a fresh table, modifiers, omitted INTO, and quoted/qualified identifiers; assert no protected-table write reaches the provider. + +### P2 - Native trigger NEW/OLD qualifiers are rejected as database names + +Location: mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs:638-670, especially the alias construction at 643-658 and throw at 669-670. Export emits trigger DDL at mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs:298-316. + +`CREATE TRIGGER tr BEFORE INSERT ON t FOR EACH ROW SET NEW.id = 1;` passes shared preflight but EnforceTargetQualifiers does not recognize trigger pseudo-rows. NEW is neither a registered table alias nor an object prefix, so execution fails with "Ambiguous database qualifier 'NEW'". OLD behaves the same. The exporter includes selected trigger DDL and defaults IncludeTriggers to true; its existing test fixture uses an empty BEGIN END trigger, so it misses this ordinary native round-trip case. [MySQL trigger pseudo-row semantics](https://dev.mysql.com/doc/refman/8.4/en/trigger-syntax.html). + +Recognize NEW/OLD only within a recognized trigger definition without weakening ordinary database qualifier enforcement. Cover a native-style DELIMITER trigger with NEW/OLD references, one real database-qualified object, and a literal containing the same text; preserve pseudo-row/literal tokens and redirect only the database object. + +### P2 - Single-table export still selects unrelated database objects + +Location: mySQLPunk/Form1.MySqlExport.cs:29-30; mySQLPunk/MySqlExportWizardForm.cs:634-640. Defaults: mySQLPunk/lib/MySqlExport/MySqlExportModels.cs:53-55. + +The new single-table route restricts SelectedTables, but its options retain IncludeViews/IncludeRoutines/IncludeTriggers=true. The form then supplies every loaded view, routine and trigger to the service. A schema-and-data export of one table therefore also emits DROP/CREATE for unrelated database objects, including triggers whose tables were not exported. This leaves the accepted single-table route repair incomplete and can modify unrelated objects on restore. DataOnly suppresses these blocks, so the current routing test (DataOnly only) does not expose it. + +For the explicit single-table entry point, default related-object flags off or constrain them to verified dependencies/owning-table triggers. Retain full-database defaults on the database entry point. Extend actual route/form coverage to SchemaAndData with unrelated objects and confirm the constructed export options contain only the authorized scope. + +## Reviewed safety boundaries + +- WizardOwnedSession keeps the wrapper as owner and exposes NativeDatabase separately; ConnectionOpenService passes the raw provider while SecuredDatabase.Dispose releases provider then SSH lease in finally. MySQL pooling is disabled for the owned job. The shared copy omits cached runtime provider/connection-string fields and sets an explicit target without modifying the source dictionary. +- Both native MySQL forms keep busy=true until awaited work returns and defer close while busy. Cancellation sources are disposed after await. No provider work survives ShowDialog return through those routes. +- SqlExportPublication uses same-directory Replace/Move, and export retains the completed staging artifact when publication was attempted and failed. Earlier failure/cancellation preserves the previous destination. The source snapshot setup and successful rollback are checked before publication; generated fields are omitted from INSERT and source pages are bounded explicitly. +- Real MySQL/MariaDB value fidelity, generated/trigger server round trips, source preservation with two actual databases, CASE provider type inference, SSH disposal and command cancellation remain unverified. No claim of full provider parity or production acceptance is made. + +## Scoped correction re-review + +Reviewed the supplied mysql-corrections-diff.patch and current source once, limited to the three findings above and direct fix breakage. No build, test rerun, database, saved configuration or SSH operation was performed. Oracle/SQLite changes were excluded. + +- Finding 1, P1 SkipExisting INSERT forms: **addressed**. MySqlImportRunner.cs:641-651 extracts INSERT/REPLACE targets from tokens with priority modifiers, IGNORE and optional INTO. The protected policy uses that result at 390-405, and target redirection uses the same extraction at 662-667. Unidentifiable write targets fail under SkipExisting instead of falling through. The added MySqlImport_Regression_ModifiedInsertProtectionAndTriggerPseudoRows at tests/SmokeTests.MySqlImport.cs:454-514 checks eight forms against mixed initial/fresh tables, a data-only initial-table fixture, qualified targets without USE and literal preservation. +- Finding 2, P2 trigger NEW/OLD: **addressed** for the reported native trigger path. MySqlImportRunner.cs:659-661 permits pseudo-row qualifiers only after identifying a CREATE TRIGGER definition; real object positions still take precedence during redirection at 696-699. Trigger ON table positions are recognized at 674-676. The added regression covers a DELIMITER trigger with NEW/OLD, a real qualified trigger table/body INSERT and source-looking literals, plus rejection of the pseudo-row exemption outside trigger DDL (tests/SmokeTests.MySqlImport.cs:496-511). +- Finding 3, P2 single-table related-object scope: **addressed**. Form1.MySqlExport.cs:30-35 disables views/routines/triggers only for the explicit selected-table route. The form consumes those flags at MySqlExportWizardForm.cs:114-116, while database-wide defaults remain intact. WizardRepairs_MySqlTableRouting now checks both actual table handlers in SchemaAndData and DataOnly and asserts all three unrelated-object flags are false (tests/SmokeTests.WizardRepairs.cs:12-23). + +**Scoped verdict: approved.** No material direct regression was found in these three corrections during this static pass. The implementer's appended report records a fresh isolated compilation and 12 passing MySQL provider methods; that remains implementer evidence. Root's final application builds/full smoke suite and all previously stated real-server, SSH and production boundaries remain pending or unverified by this reviewer. diff --git a/docs/design/db-wizard-repairs/oracle-repair-report.md b/docs/design/db-wizard-repairs/oracle-repair-report.md new file mode 100644 index 0000000..f023cb8 --- /dev/null +++ b/docs/design/db-wizard-repairs/oracle-repair-report.md @@ -0,0 +1,75 @@ +# Oracle wizard repair report + +Scope: Task 3 against d41825c. Oracle sources/forms and existing Oracle smoke files only. Root owns shared session/publication helpers, Form1, project/test registration and final full builds. No Oracle server, external database, credentials, saved connection files or account/session state was accessed. No commit or push by this worker. + +## Execution repairs + +Every recording-provider ExecSQL response requires status=OK and retains status=NO reasons. Statement, DROP, TRUNCATE, transaction start, COMMIT and ROLLBACK failures cannot report success. KeepExisting only skips recognized existing-object errors on CREATE/ALTER. + +Native my_oracle uses OracleTransaction and cancellable OracleCommand. Error/cancellation cleans pending DML, including script-created transactions with the UI option unchecked. COMMIT failure fails the operation and rollback failure stays in the result. Oracle DDL/TRUNCATE implicit commits are explicit: whole-file rollback is not promised. + +A complete parser/target prepass precedes execution. CURRENT_SCHEMA is pinned to the selected owner. Cross-owner object targets, session/privilege changes, unsupported dynamic SQL and ambiguous stored/anonymous PL/SQL reject before execution. The checked canonical DROP wrapper and generated temporary-LOB INSERT blocks are supported. Literals/comments are never remapped. Manifests use owner-qualified identities and count the same owner. CURRENT_SCHEMA is not a security sandbox for arbitrary SQL/dependency side effects. + +USER_SDO_GEOM_METADATA restore requires SESSION_USER equal to selected owner; CURRENT_SCHEMA cannot redirect that USER view. Disabling registration skips script metadata mutations and never invents missing metadata. Both forms wait for worker cleanup before close, including catalog reads/file preflight. Root supplies isolated owned sessions. + +## Parser and logging + +One bounded parser serves preflight and execution: ordinary/alternative quotes, quoted identifiers, comments, ordinary SQL semicolon removal and slash-terminated PL/SQL retaining its final semicolon. It records source-encoding byte counts and original start lines, rejects NUL/unterminated syntax and limits statements/physical lines to 32 MiB. Comments become whitespace; source bytes remain counted. Text preflight does not validate server syntax/permissions. + +Logs use Windows-safe owner text, GUID names, CreateNew and AutoFlush. Concurrent jobs do not overwrite second-resolution filenames. + +## Export behavior and exact limits + +- READ ONLY executes on the dedicated source transaction before reads and rolls back on success/failure/cancel. Restore SQL contains no source READ ONLY. Native SYS snapshot sessions reject explicitly. +- Native table rows use OracleCommand.ExecuteReader(CommandBehavior.SequentialAccess); no full-table DataTable or SELECT-star fallback exists. Catalog/dictionary metadata is materialized separately. Recording providers expose a forward-reader contract. +- Total row values are bounded to 8 MiB; emitted row SQL/PLSQL to 28 MiB. BLOB/WKB uses temporary BLOBs and <=2000-byte HEXTORAW chunks. Larger CLOBs use temporary CLOBs with small escaped text chunks. Newlines survive; temporary LOBs free on success and exception. +- Small WKB/BLOB uses explicit TO_BLOB. EMPTY_BLOB/EMPTY_CLOB remains distinct from NULL. Scalar hex text <=4000 bytes and escaped UTF-8 scalar text <=4000 bytes. RAW >2000 bytes, oversized non-CLOB text, NUL or over-budget values fail before publication. Native text sources explicitly require AL32UTF8. NCLOB and unverified codecs reject. +- Geometry preserves each actual SRID, including NULL, via FROM_WKBGEOMETRY's second argument. Native preflight rejects LRS, unsupported dimensions/types, curved/compound element types and non-linear interpretations. Supported WKB semantics do not claim every Oracle internal SDO representation is preserved. +- Actual ALL_SDO_GEOM_METADATA/DIMINFO supplies dimension names, bounds, tolerances, NULL DIMINFO/SRID and metadata absence. No 4326, invented bounds/tolerances or fabricated SIDX names remain. Original spatial index DDL emits once after metadata/data. +- Native table/view/index DDL comes from DBMS_METADATA instead of the inherited copy type mapper. Bounded native CREATE OR REPLACE / FORCE or NOFORCE / EDITIONABLE or NONEDITIONABLE VIEW grammar is accepted, with actual view/reference owners checked. Independent comment/regular-index/spatial-index option gates apply. +- Virtual/hidden columns are omitted. Native column queries avoid the inherited collation fallback losing generated flags. Identity data rejects until sequence/state restoration is verified; SchemaOnly remains subject to dependencies. Timestamp precision above .NET ticks, timezone timestamps, unsupported user types, non-finite numerics and signed-zero binary floats reject explicitly. Finite BINARY_FLOAT/BINARY_DOUBLE use G9f/G17d typed literals. +- Schema FK parents are ordered first using every FK definition, including disabled definitions. Missing/unselected parents, cross-owner references and selected cycles/self references reject before publication. DataOnly considers enabled FKs only. Native schema export rejects recorded external sequence/function/package/type dependencies, which this wizard does not bundle. This is a bounded logical table export, not a universal Oracle backup. +- Checkbox edits update tagged selections, filters retain them, initial owner-qualified selection/options apply, and StartExport passes qualified names. DataOnly excludes views. +- Unique staging uses root's SqlExportPublication.Publish. The old target is never deleted first. Publication failure keeps staging; earlier failure removes staging only. + +Restrictions are visible in the options UI and dump header, not just implementation comments. + +## Local evidence + +`MYSQLPUNK_SMOKE_FILTER=Oracle ./tests/Run-SmokeTests.ps1`: **15/15 Oracle groups and AutoReleasePolicy passed after the scoped correction wave**, including product compilation. Evidence: `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/repair-oracle-scoped-smoke.log`. Root runs the final full suite and Debug/Release builds. + +New durable registrations supplied to root: + +- OracleImport_ReturnedNoAndTransactionControls +- OracleImport_TargetOwnerPreflightAndCancellation +- OracleImport_ParserBytesCommentsAndAlternativeQuotes +- OracleExport_SpatialMetadataAndOptionGates +- OracleExport_BoundedLobsSnapshotFailureAndCancel + +Coverage: returned-NO statement/commit/drop/truncate/rollback; transaction option and cancellation; owner ON/REFERENCES/COLUMN validation; comments, alternative quotes, UTF-8/CRLF bytes; source snapshot commands/cleanup; actual projected 3D metadata and per-value projected/NULL SRIDs; all DDL option gates; virtual projection and identity rejection; 3KB WKB/5000 Chinese-character CLOB chunks; empty LOBs; first flushed output while reader is active; >8MiB failure preserving old output; FK cycle/missing parent rejection; and actual exporter LOB script passed through the real runner with a recording provider. + +Nine existing Oracle files retain baseline BOM and uniform checkout CRLF (Git stores normalized LF). Scoped git diff --check passes. The unchecked spatial-registration and script-created transaction cleanup assertions are included in the passing Oracle run. + +## Pending acceptance + +No Oracle server roundtrip, concurrent-update snapshot, native geometry/DIMINFO order, PL/SQL LOB execution, native FK/default dependency/version compatibility, two-owner account behavior, cancellation latency, memory stress or interactive form acceptance was run. Recording fixtures prove control flow and emitted limits, not storage equivalence. Dependencies invisible to the account's Oracle dictionary remain outside the contract. Native-fidelity/server-version acceptance is pending; browser/IIS/production are outside this desktop repair. + + +## Scoped correction wave (six review findings) + +The six material findings in oracle-scoped-review.md were corrected together. The existing architecture and other providers were not changed. + +1. Direct DML targets share a token-derived identity, recognizing DELETE with optional FROM. CREATE UNIQUE/BITMAP INDEX owner positions are parsed independently of modifiers. Database-link tokens reject before session/mutation. Unquoted ONLY target wrappers explicitly reject rather than being mistaken for local object names. Same-owner variants and owner/@ text inside literals/comments remain safe. +2. Spatial metadata classification uses that same parsed identity, including quoted uppercase USER_SDO_GEOM_METADATA and optional FROM. Both SESSION_USER checking and disabled-registration skipping apply to all admitted target spellings. +3. A validated generated DECLARE LOB block has exactly one INSERT target. TruncateAndInsert uses it before execution, and qualified/unqualified identities share one canonical truncate record. Recording regressions start with nine rows and end with exactly two for both all-large and LOB-first/mixed dumps without manifests. +4. Finite binary floats emit G9 with f suffix and finite binary doubles emit G17 with d suffix, including high exponents and subnormals. .NET bit-level parsing regressions verify tokens roundtrip locally. Signed zero explicitly rejects before publication until native bit-preserving storage is verified. The recording reader preserves the negative-zero bit pattern (DataTable storage alone normalizes it). +5. Native FORCE/EDITIONABLE view forms pass through the actual exporter and runner. ON owner-position checking applies to CREATE INDEX object targets, not view JOIN alias predicates. Joined same-owner views pass; changed view/reference owners reject before any mutation. +6. Schema dependency ordering includes disabled FK definitions. Disabled parent-before-child, missing parent, cross-owner and cycle fixtures exercise the gates and preservation of prior output. Data-only retains its enabled-FK enforcement filter. + +Additional durable registrations supplied to root: + +- OracleImport_ScopedTargetVariantsMetadataAndLobTruncate +- OracleExport_ScopedBinaryFloatRoundTrip +- OracleExport_ScopedNativeViewsAndDisabledForeignKeys + +Final scoped evidence is the 15-group Oracle-filter run in repair-oracle-scoped-smoke.log plus scoped diff whitespace/BOM/CRLF audits. Root owns the final independent scoped review, full suite and Debug/Release builds. All previously listed native-server/storage/version/cancellation/UI acceptance boundaries remain pending. diff --git a/docs/design/db-wizard-repairs/oracle-scoped-review.md b/docs/design/db-wizard-repairs/oracle-scoped-review.md new file mode 100644 index 0000000..5be689f --- /dev/null +++ b/docs/design/db-wizard-repairs/oracle-scoped-review.md @@ -0,0 +1,78 @@ +# Oracle scoped repair review + +Date: 2026-10-16. Decision: changes requested; six material findings remain. + +Reviewed the stable Oracle repair source and `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/oracle-repair-diff.patch` against Task 3 and the acceptance constraints in `plan.md`, plus `oracle-repair-report.md`. This was source/documentation review only: no build, smoke test, saved configuration, credentials, live database, or other-provider source was accessed. The worker's 12/12 Oracle and AutoReleasePolicy results are reported evidence, not independently rerun evidence. Root's final full suite remains pending. + +## 1. [P1] Accepted statement variants bypass direct owner enforcement + +Location: [OracleImportRunner.cs:349](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleImport/OracleImportRunner.cs:349), through line 356; accepted forms are listed at line 326. + +For selected owner `B`, `DELETE A.T WHERE ID=1` passes the statement allowlist, but `DELETE` is absent from the target-keyword set. Because Oracle permits omission of `FROM`, no token triggers validation of `A.T`. Likewise, `CREATE UNIQUE INDEX A.I ON B.T(ID)` and `CREATE BITMAP INDEX A.I ON B.T(ID)` are admitted, but `INDEX` follows `UNIQUE`/`BITMAP`, so the index owner is not checked. An adequately privileged session executes the original SQL against owner A after CURRENT_SCHEMA is pinned to B. An explicit database-link target such as `INSERT INTO B.T@REMOTE VALUES (1)` also remains admitted because `@` is never rejected; it is a direct remote target, not a dependency side effect. [Oracle DELETE syntax/examples](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/DELETE.html), [Oracle CREATE INDEX prerequisites](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/CREATE-INDEX.html). + +Required correction: parse each supported statement's object position, including optional `FROM` and index modifiers, and reject database links under this local-owner contract. Add recording-provider regressions proving these statements fail before ALTER SESSION or any mutation. Keep same-owner accepted variants and owner-like literals/comments unchanged. + +## 2. [P1] Quoted spatial metadata targets bypass the SESSION_USER and option gates + +Location: [OracleImportRunner.cs:296](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleImport/OracleImportRunner.cs:296); callers are lines 74-80 and 95-99. + +`IsSpatialMetadataMutation` only matches unquoted `INSERT INTO`, `DELETE FROM`, or `UPDATE USER_SDO_GEOM_METADATA`. Valid `INSERT INTO "USER_SDO_GEOM_METADATA" (...) VALUES (...)` and `DELETE "USER_SDO_GEOM_METADATA" WHERE ...` are therefore ordinary accepted statements to the runner. With selected owner B and connected session user SCOTT, they do not request the SESSION_USER check and can mutate SCOTT's USER metadata view. With AutoRegisterSpatialMetadata=false, they still execute instead of being skipped. The newly added option test covers only the plain unquoted spelling. + +Required correction: derive metadata mutation classification from the same parsed target identity as owner validation, recognizing quoted uppercase identifiers and optional FROM. Apply both gates to all admitted spellings, or reject unsupported spellings before execution. Add separate regressions for the quoted spelling, optional FROM, session-owner mismatch and disabled registration. + +## 3. [P1] TruncateAndInsert misses generated large-LOB rows and can delete imported data + +Location: [OracleImportRunner.cs:284](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleImport/OracleImportRunner.cs:284), through line 289; exporter emits the accepted blocks at [OracleSqlExportService.cs:543](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs:543). + +The conflict strategy recognizes only statements beginning `INSERT INTO`. Supported large BLOB/CLOB/WKB rows begin `DECLARE`, so an all-large-LOB DataOnly dump never truncates an existing table. Worse, a generated LOB block followed by a small ordinary INSERT first appends the LOB row, then truncates the table immediately before the second row, deleting the row just imported. With no manifest, this can return Success=true. A manifest can detect the final count mismatch, but it does not prevent the destructive ordering. + +Required correction: extract the single INSERT target from validated generated LOB blocks and apply the per-table truncate policy before their first execution. Normalize table identity so qualified/unqualified equivalent targets share one truncate record. Regress all-LOB and LOB-first/mixed-row DataOnly scripts, including no-manifest success/count behavior. + +## 4. [P1] Supported binary floating-point values lose precision in emitted literals + +Location: [OracleSqlExportService.cs:637](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs:637), through line 638; BINARY_FLOAT/BINARY_DOUBLE are explicitly accepted at line 441. + +Finite float/double values use `Convert.ToString(value, InvariantCulture)`. On this .NET Framework target that uses general formatting with too few significant digits for guaranteed round trips. For example, a double such as 1.2345678901234567 is emitted with only 15 significant digits. Export can publish successfully and manifest row counts cannot detect the changed value. [Microsoft Double formatting documentation](https://learn.microsoft.com/en-us/dotnet/api/system.double.tostring) recommends an explicit precision sufficient for round trips. + +Required correction: use type-aware G9/G17 formatting and Oracle BINARY_FLOAT/BINARY_DOUBLE literal typing (`f`/`d` suffixes or an equivalently exact native codec); untyped large exponents can otherwise be interpreted as NUMBER and overflow. Preserve signed zero explicitly or reject it with a stated unsupported gate until verified. Add formatter/recording-reader cases that assert exact emitted tokens and round-trip values, including a finite high-exponent double. Real Oracle storage equivalence remains pending. + +## 5. [P2] Native view DDL is exported but rejected by the import prepass + +Location: [OracleImportRunner.cs:326](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleImport/OracleImportRunner.cs:326); native export route is [OracleSqlExportService.cs:271](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs:271), through line 273. + +The exporter writes DBMS_METADATA view DDL unchanged, while the allowlist accepts only `CREATE VIEW`. Native forms such as `CREATE OR REPLACE FORCE EDITIONABLE VIEW "SCOTT"."V" AS SELECT ...` fail the complete prepass before execution even with a correct owner and bundled tables. IncludeViews remains an advertised/default option. The recording fixture returns no view DDL, so current successful tests do not cover this route. This is a source-level producer/consumer mismatch; no claim is made that every server version's view DDL was exercised. + +Required correction: support the bounded native view modifiers while checking the actual view owner and referenced owners, or explicitly reject views at export before publication until that grammar is supported. Add a representative native view DDL fixture through exporter and runner, plus a cross-owner variant that still fails before execution. + +## 6. [P2] Disabled FK definitions are omitted from schema dependency ordering/gates + +Location: [OracleSqlExportService.cs:323](C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs:323); native table DDL is fetched unchanged at line 649. + +The dependency query filters `c.STATUS='ENABLED'`, while native table DDL also retains disabled FK definitions. Consequently a disabled FK to an unselected/cross-owner parent escapes the documented rejection, and a selected child can be emitted before its disabled-FK parent. Disabling enforcement does not remove the schema dependency: Oracle still requires the referenced primary/unique key to be defined. A successful export can therefore publish a schema dump that cannot be restored under the selected-owner/parent-first contract. [Oracle foreign-key restrictions and disabled constraints](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/constraint.html). + +Required correction: include all FK definitions for schema ordering and parent/owner gates; data-only enforcement may be handled separately if needed. Add disabled-FK fixtures for parent-first ordering, missing/cross-owner parent and cycle rejection, checking preservation of the previous output on refusal. + +## Other scoped conclusions and limits + +The original returned-NO false-success path is addressed by checked execution responses. Source READ ONLY is confined to the dedicated source transaction, source cleanup precedes publication, rows use a sequential reader, explicit row/SQL/LOB limits stop publication, and unsupported identity/type/geometry cases have explicit gates. Actual per-value SRID and spatial metadata are emitted rather than fabricated defaults. Selection tags/initial options and close-after-worker-cleanup paths are present in both forms. No additional material finding was identified in those inspected paths. + +These conclusions are static review, not native-server acceptance. Oracle LOB execution, geometry/DIMINFO order, native provider numeric/timestamp behavior, snapshot concurrency, server-version compatibility, cancellation latency and interactive forms remain unverified. Shared owned-session/publication helpers and other providers are root/other reviewers' scope. Stop this review here and correct the six bounded issues in one wave before final verification. + +## Scoped re-review after the six-finding correction wave + +Date: 2026-10-16. Decision: the six findings are addressed in the frozen source. Specification and static code-quality review approved for this bounded correction wave; no remaining material finding or direct correction regression was identified in this scope. + +Reviewed package: `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/oracle-corrections-final-diff.patch`, current frozen Oracle source, and the three added regression definitions. This is the requested single scoped re-review, including the final ONLY-wrapper guard and joined-view ON correction. No build/test, saved configuration, live database, other-provider source or subagent was used by this reviewer. + +| Original finding | Status | Source and regression evidence inspected | +| --- | --- | --- | +| 1. Direct owner enforcement variants | Addressed | OracleImportRunner.cs lines 314-365 parse DELETE with optional FROM, modified CREATE INDEX owner positions and canonical targets; lines 378-387 reject database-link tokens and cross-owner targets. Line 316 explicitly rejects unquoted ONLY wrappers instead of treating them as object names. OracleImport_ScopedTargetVariantsMetadataAndLobTruncate includes zero-execution DELETE/UPDATE ONLY cases, original cross-owner/index/link cases, and accepted same-owner/literal cases. | +| 2. Spatial metadata spelling gates | Addressed | OracleImportRunner.cs lines 300-304 classify the parsed mutation identity; existing preflight SESSION_USER and disabled-registration paths consume that classification. The import regression covers quoted INSERT/UPDATE, DELETE with optional FROM, wrong/matching SESSION_USER, and disabled registration without a metadata mutation or USER-gate query. | +| 3. Generated LOB truncate ordering | Addressed | OracleImportRunner.cs lines 282-293 use the one parsed INSERT target from generated DECLARE blocks and canonical owner-qualified identity before execution. Lines 327-353 reject multiple INSERT targets. No-manifest recording cases for all-large and LOB-first/mixed rows verify one truncate before the first LOB and two final rows from nine initial rows, including qualified/unqualified equivalent targets. | +| 4. Binary float/double literal fidelity | Addressed within the stated bound | OracleSqlExportService.cs lines 635-650 emit G9f/G17d tokens, reject non-finite values and explicitly reject negative zero before publication. OracleExport_ScopedBinaryFloatRoundTrip checks local bit equality after parsing, high exponents/subnormals, the recording-reader pipeline, and preservation of previous output on negative-zero rejection. The reader wrapper preserves the negative-zero bit pattern that DataTable storage would normalize. Native Oracle storage equality remains unverified. | +| 5. Native view grammar | Addressed | OracleImportRunner.cs lines 356-365 and 402 accept the bounded native OR REPLACE/FORCE/EDITIONABLE view prefix while retaining actual owner checks. Lines 426-433 apply ON object-position checking only to CREATE INDEX, so joined view alias predicates do not become owner targets. OracleExport_ScopedNativeViewsAndDisabledForeignKeys covers exporter-to-runner native view DDL, same-owner joined views, changed view owners, and changed FROM/JOIN owners with zero mutation. | +| 6. Disabled FK schema dependencies | Addressed | OracleSqlExportService.cs line 323 includes all FK definitions for schema-bearing modes and retains the enabled-only filter for DataOnly. The scoped view/FK regression covers disabled parent-before-child ordering and refusal of missing, cross-owner and cyclic disabled references while preserving existing output. | + +The fixes preserve the accepted architecture and fail explicitly for the added unsupported ONLY and signed-zero boundaries. The review does not expand supported Oracle types, server versions, PL/SQL, or account behavior. No additional review loop is requested. + +Execution evidence is owned by root/implementer. At this review handoff, the implementer reported 15 Oracle groups plus AutoReleasePolicy passing before the last ONLY guard; the final 15-group run including ONLY was still running. This reviewer did not rerun or independently certify those results. Root must record the final targeted/full-suite and Debug/Release outputs. All previously listed real Oracle, native storage/snapshot/version/cancellation/UI boundaries remain pending. diff --git a/docs/design/db-wizard-repairs/plan.md b/docs/design/db-wizard-repairs/plan.md new file mode 100644 index 0000000..9bd5cba --- /dev/null +++ b/docs/design/db-wizard-repairs/plan.md @@ -0,0 +1,51 @@ +# MySQL / SQLite / Oracle wizard repair plan + +Approved scope: the user requested repairs to the 2026-10-16 review findings. Baseline d41825cead810bc67f263753bb5f986a7f69c65c. Retain existing wizard architecture and defaults except unsafe behavior; no push/merge, real user databases, saved connection files or bulk restore. + +## Acceptance and constraints + +- Preserve each existing file's encoding/BOM/newlines. Keep unrelated PostgreSQL and MSSQL behavior unchanged. +- Use actual SQLite tiny private fixtures; MySQL/Oracle recording providers must follow real status=NO contracts. No live server acceptance claim without real evidence. +- Preserve old backup files on publication failure. Jobs use owned sessions, never cached editor sessions. No silent truncation, target redirection through string replacement, ignored commit failures, or fabricated spatial metadata. +- Review evidence: C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/review-summary.md and provider reports. Existing runtime reproductions are the baseline red evidence; add durable regression tests for repairs. +- Large unsupported values/features must stop explicitly before final publication, preserving originals; do not advertise universal type coverage. + +## Task 1: MySQL correctness (independent provider files) +- [x] Regression: SkipExisting new table data, protected DROP, literal/comment-safe USE and qualifiers, generated columns and large values. +- [x] Token-based target enforcement, separate initially existing objects from current cache, preserve logical statement bytes/comments. +- [x] Apply requested initial table/mode selection in form; owned job session contract and bounded source behavior remain enforced. +- [x] Add tests in existing MySQL smoke files and report exact coverage/limits. + +## Task 2: SQLite correctness (independent provider files) +- [x] Regression: every visible conflict policy, native DROP preserve/abort, trigger bodies, failed COMMIT and script transaction cleanup. +- [x] Generated columns, rowid/sequence, multiline/NUL/binary values, query-error handling and comments sidecar roundtrip. +- [x] Preserve text byte content and owned connection semantics; test tiny real SQLite roundtrips. +- [x] Add tests in existing SQLite smoke files and report coverage/limits. + +## Task 3: Oracle correctness (independent provider files) +- [x] Check every execution result; implement transaction option/cleanup, token-safe owner enforcement and accurate bytes/parser behavior. +- [x] Dedicated source read-only transaction; bounded row/LOB writing with explicit safe limit until verified chunk codecs exist. +- [x] Preserve actual geometry SRID and spatial metadata; reject unsupported cases instead of writing 4326 defaults. Honor generated/identity/selection/options. +- [x] Add tests in existing Oracle smoke files and report coverage/limits; no Oracle server available assumed. + +## Task 4: Integration and file/session protection (root) +- [x] Provide common owned job session and atomic publication helpers; integrate only MySQL/SQLite/Oracle native forms and relevant Form1 routes. +- [x] Preserve single-table/data-only routes and wait for worker cleanup on close/cancel. +- [x] Debug/Release builds, full smoke suite, provider regressions and focused independent final review. +- [x] Update truthful support documentation and append history with evidence and unverified boundaries. Retain local repair branch for user review. + +## Execution decisions + +The accepted review already supplies the design and root causes; no additional design approval is necessary. Independent provider edits are delegated in parallel with disjoint ownership; root alone edits shared routing/project/test registration. Each worker returns a report for scoped review. Native worktree creation failed to resolve this repository from the projectless chat, so git fallback created C:/Users/johnho/Documents/mySQLPunk-worktrees/db-wizard-repairs on codex/db-wizard-repairs. Original feature branches remain intact. + +## Execution ledger + +- Baseline: Debug application build, all 268 smoke groups and AutoReleasePolicy passed before repairs. Original five recording-provider and eight private SQLite reproductions remain red evidence outside Git. +- Common helpers: new tests observed missing old behavior against the original assembly, then actual-product publication-lock/owned-session rollback/routing tests passed. Single-table defaults were subsequently tightened and covered in both export modes and the SQLite/Oracle routes. +- MySQL: twelve provider groups pass isolated source tests. Independent review found INSERT modifiers, trigger pseudo-row qualifiers and unrelated single-table objects; one correction wave and one scoped re-review addressed all three. Final full suite passed with the integrated source. +- SQLite: sixteen provider/form groups pass isolated tests; fresh actual-product nineteen-group SQLite run plus AutoReleasePolicy passed. Independent review's five transaction/metadata/lifetime/options findings addressed in one wave; scoped re-review approved. +- Oracle: twelve actual-product recording-provider groups plus AutoReleasePolicy passed before final review. Six concrete owner/metadata/LOB-policy/float/view/FK findings addressed in one bounded wave and scoped re-review approved; final targeted Oracle15 and full-suite runs passed. +- Ruling: finite provider row/statement/type gates explicitly reject unverified large/unsupported values before publication rather than truncate or fabricate data. Larger theoretical type maxima and real-server fidelity remain pending; documented caps are visible in errors/UI/support notes. +- Ruling: keep native MySQL source snapshots limited to verified InnoDB, keep Oracle SYS read-only and cross-session-user spatial metadata restrictions explicit, and reject ambiguous SQL scope. Conservative failures can require the native client but preserve data rather than pretending general SQL confinement. + +- Final root gate: all 294 smoke groups plus AutoReleasePolicy passed; explicit Debug and Release application builds passed. Optional live tests were skipped. All three independent scoped reviews/re-reviews approved. Existing-file BOM/newlines and diff whitespace were audited; no code edits after final gate. diff --git a/docs/design/db-wizard-repairs/sqlite-repair-report.md b/docs/design/db-wizard-repairs/sqlite-repair-report.md new file mode 100644 index 0000000..d3c348b --- /dev/null +++ b/docs/design/db-wizard-repairs/sqlite-repair-report.md @@ -0,0 +1,77 @@ +# SQLite wizard repair report + +Baseline: d41825cead810bc67f263753bb5f986a7f69c65c. Scope implements Task 2 of the approved repair plan. No user database, saved connection configuration, account state, extension module, commit, push or merge was accessed/performed by this worker. + +## Changed files + +- `mySQLPunk/lib/SqliteImport/SqliteImportModels.cs`: explicit visible conflict-choice mapping; synchronous OFF is no longer an implicit import default because the import wizard has no corresponding safety control. +- `mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs`: one incremental strict UTF-8 lexer shared by preflight/execution, exact CR/LF characters, escaped quotes/backticks/brackets/comments, trigger BEGIN/END and nested CASE handling, 32 MiB decoded UTF-8 statement bound, bounded comment metadata, encoded manifest table names and verified payload checksum before target writes. +- `mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs`: initial versus current object tracking, coherent DROP/CREATE/DML/dependent-index/trigger policies, checked provider status for wrapper/drop/cleanup operations, script transaction tracking and rollback, native interrupt support, checked original FK/synchronous restoration, failed tail/BEGIN/COMMIT/ROLLBACK and continued errors cannot produce Completed. A supplied native connection must initially be in autocommit mode. Shared sidecar CREATE IF NOT EXISTS remains idempotent and preserves unrelated comments. +- `mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs`: checked source snapshot/query/count/DDL operations, table_xinfo writable-column projection, table_list rowid metadata, implicit rowid and sqlite_sequence restoration, source-side exact TEXT/BLOB literals, filtered trigger dependencies, one sidecar-comments path, bounded native sequential reads, accurate delivered-file hash/length and shared atomic publication helper. +- `mySQLPunk/SqliteImportWizardForm.cs`: confirmation and execution use the same explicit conflict-choice mapping. +- `mySQLPunk/SqliteExportWizardForm.cs`: applies the requested initial table selection; subset exports omit the database-wide view list because dependencies cannot be inferred safely from that UI. Triggers are filtered against selected owners by the service. +- `tests/SmokeTests.SqliteImport.cs`, `tests/SmokeTests.SqliteExport.cs`: durable private SQLite fixtures and regression assertions; existing mocks now follow the actual catalog/PRAGMA/leading-whitespace contract. + +The root owns dedicated raw job-session construction, routing, worker close/cancel integration, shared publication implementation and test registration. Provider code invokes `SqlExportPublication.Publish(staging,target)`, retains recoverable staging after publication failure, and polls cancellation after hashing before publication. + +## Findings addressed + +- S1/S2: all four visible choices map correctly. Native DROP statements cannot erase protected objects under Skip/Keep/Abort; dependent indexes/triggers and Unicode/qualified names are covered. Abort conflicts are detected before writes. Skip protects existing table/view DML; Keep preserves structures and executes the script's data statements. +- S3/S11/S12: preflight and runner use the same trigger-aware reader, preserve literal line separators and reject malformed UTF-8, raw NUL, incomplete lexical states and oversized complete statements. TEXT export uses `CAST(X'...' AS TEXT)`, retaining LF/CR/CRLF/Unicode/quotes/NUL bytes in UTF-8 databases. BLOBs use `X'...'`. +- S4/S5/S13: script and wrapper transactions are cleaned up on stop/cancel/failure; rejected COMMIT is failure and pending work is rolled back. Continued SQL errors withhold COMMIT. Every wrapper/rollback/PRAGMA result is inspected. Native AutoCommit confirms cleanup; original FK/synchronous settings are restored and re-read. The package documents SQLiteCommand.Cancel as unimplemented, so cancellation uses the documented thread-safe SQLiteConnection.Cancel. +- S6/S9: generated/hidden columns are excluded from writable projections, ordinary accessible implicit rowids are retained, WITHOUT ROWID metadata is read from table_list rather than DDL text, and AUTOINCREMENT deleted/empty high-water marks are restored for selected tables. +- S7/S8/S10/S14/S15: missing DDL, QueryError and negative counts stop publication; native source reads use a single read transaction. Comments are exported once and only for selected tables. Initial table selection is applied. Delivered SHA-256 and length describe the actual final file; the separate payload checksum footer is verified before import writes. Base64 manifest names allow whitespace and pipe characters. Existing outputs are published using the root's atomic helper. + +## Verification evidence + +Worker verification compiled the exact modified provider/model/lexer/log source with Roslyn into a separate scratch executable; it used System.Data.SQLite 1.0.119.0 and private raw connections, with a minimal my_sqlite/Localization shim only to isolate compilation from the product UI. Six groups use actual SQLite connections; other groups cover model/mock/preflight contracts. No SpatiaLite extension loading occurred. + +Latest isolated run: 12/12 methods passed. Log: `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/sqlite-repair-verify/results.log`. + +New registration names (root owns registry): + +- `SqliteImport_NativeConflictPolicies_ProtectDropAndDependencies` +- `SqliteImport_NativeLexer_TriggerAndExactText` +- `SqliteImport_NativeTransactions_FailureCancelAndPragmas` +- `SqliteImport_StatementLimit_MultibyteAndTerminated` +- `SqliteExport_NativeRoundtrip_GeneratedRowidSequenceTextComments` +- `SqliteExport_Failures_PreserveOutputAndRejectTamper` + +Actual assertions include all four native conflict policies, protected dependent objects and shared sidecar metadata; trigger CASE/comment/semicolon grammar and firing; exact mixed newline literal text; malformed encoding and unterminated lexical input; failed tail SQL, deferred FK COMMIT rejection and provider NO for BEGIN/COMMIT/ROLLBACK; cancellation interrupt under five seconds; checked PRAGMA restoration; a private file-backed cancellation followed by a successful write from a separate connection; generated VIRTUAL/STORED columns; sparse negative/near-Int64-max rowids; deleted and empty AUTOINCREMENT history; WITHOUT ROWID with trailing DDL comments; text/BLOB/comment NUL bytes; subset sidecar and trigger scope; whitespace/pipe manifest names; external full-file hash/length; one-byte tampering; missing/source-query-error output preservation; oversized row and inaccessible rowid rejection. + +A complete terminated multibyte single-line statement over 32 MiB is rejected during incremental preflight. Native export is O(n) through one sequential SQLiteDataReader per table, with size as the first ordinal and CASE guards preventing SQL hex rendering for rows beyond the safe limit. Native comment reads use the same bounded sequential pattern. The generic IDatabase fallback is bounded one-row paging; the actual native provider does not use that fallback. + +`git diff --check` passed. Existing UTF-8 BOM presence and CRLF were checked against the baseline with Git checkout filters; all eight changed existing files retain their original encoding/newline convention. + +The root reported a fresh integrated 15-test SQLite run passed before the final table_list metadata adjustment; root owns authoritative product Debug/Release/full-suite evidence and should rerun its focused group against that final edit. + +## Explicit support/acceptance boundaries + +- Native data export supports ordinary UTF-8 SQLite tables with table_xinfo/table_list metadata. Virtual modules (including FTS/RTree), inaccessible implicit rowid, unsupported metadata and non-UTF-8 database text encodings fail explicitly. Geometry BLOB bytes are preserved for ordinary tables; this is not evidence for a complete SpatiaLite/GeoPackage reconstruction. +- The safe export limit is 8 MiB of source value bytes per row/comment and 32 MiB per serialized SQL statement/DDL. Larger values stop before publication, preserving the old dump. The importer limits each SQL statement to 32 MiB and comment metadata to 1 MiB. No large-production-LOB memory/throughput benchmark was performed. +- Attach/detach, temporary-schema imports and savepoint scripts are explicitly unsupported. CTE statements under SkipExisting with pre-existing objects are rejected because their target cannot be safely classified by this importer. +- Script-supplied transactions are tracked and any active transaction is rolled back on failure/cancel. Statements already committed by an earlier explicit script COMMIT, or executed with wrapping disabled, cannot be undone; the runner does not claim whole-file atomicity for arbitrary transaction scripts. +- Row manifests establish row counts only. They do not assert all-value, spatial, production or application equivalence. Skip/Keep can intentionally yield a row-count mismatch. +- Full product build, root shared-helper fault injection, actual displayed WinForms/DPI flows, real accounts, live source databases, IIS/production and SpatiaLite/GeoPackage runtime validation remain root/user acceptance boundaries. This worker did not run a product/shared-output build. + + +## Scoped review correction wave + +Read all five findings in `sqlite-scoped-review.md`; each was reproduced in regression coverage and corrected within SQLite-owned files. + +1. Managed BEGIN now precedes the policy-generated DROP. Real SQLite tests verify both a later SQL failure and an invalid replacement CREATE restore the original populated table. +2. Native AutoCommit is reconciled at the statement-error boundary and before further statements. If OR ROLLBACK or RAISE(ROLLBACK) ends the transaction, import stops even with ContinueOnError, preventing later statements from leaking into autocommit. Actual managed/explicit transaction and trigger rollback fixtures preserve the initial table and leave no leaked table. +3. Both forms cancel every close event while busy; the closing flag only suppresses duplicate cancellation requests. Import preflight and export catalog loading honor pending close in their finally paths after clearing busy. Export catalog loading now participates in the same busy guard, protecting the owned session during its existing asynchronous catalog queries too. WinForms event tests invoke two close attempts and confirm both are cancelled, the form remains undisposed and the provider remains open; pending preflight/catalog completion then closes the form. +4. Under Skip/Keep with existing owners, sequence/sidecar DML accepts only the native exporter's exact single-owner forms. Unscoped DELETE/UPDATE, multi-owner/ambiguous inserts and broadened WHERE predicates are skipped safely. Actual SQLite tests retain deleted AUTOINCREMENT high-water marks and unrelated comments while permitting scoped restoration for newly imported owners. +5. Supplied IncludeViews/IncludeTriggers values now initialize the actual controls. StartExportAsync uses a small BuildExportOptions method so the regression can inspect the exact service options derived from those controls. A one-table database fixture with a view and a trigger referring to an omitted table verifies flags remain false and the actual export contains neither object. + +Latest isolated run after this correction wave: 16/16 SQLite methods passed. It compiles the exact modified SQLite provider and wizard form sources with a minimal styling/mascot/localization shim (no production UI dependency or saved configuration); the WinForms control/event/async-finally logic is the actual modified source. Log: `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/sqlite-repair-verify/review-wave-results.log`. + +Additional registration names: + +- `SqliteImport_ReviewRegression_ManagedDropAndImplicitRollback` +- `SqliteImport_ReviewRegression_MetadataOwnerPreservation` +- `SqliteImport_ReviewRegression_RepeatedCloseAndPreflight` +- `SqliteExport_ReviewRegression_RepeatedCloseAndInitialOptions` + +Scoped diff-check passes; baseline checkout BOM/CRLF remain preserved in all eight existing changed SQLite files. No shared output build, user database, saved configuration, account access, commit or push occurred. Root still owns the fresh complete-product build/full-suite and visual DPI/real-world acceptance boundaries. diff --git a/docs/design/db-wizard-repairs/sqlite-scoped-review.md b/docs/design/db-wizard-repairs/sqlite-scoped-review.md new file mode 100644 index 0000000..7e44c94 --- /dev/null +++ b/docs/design/db-wizard-repairs/sqlite-scoped-review.md @@ -0,0 +1,55 @@ +# SQLite repair scoped review + +Review date: 2026-10-16. Reviewed Task 2, `sqlite-repair-report.md`, the supplied SQLite repair patch, and the current scoped SQLite source. Static review only: no builds, test reruns, live databases, saved configuration, or other provider source were accessed. Existing private-fixture results were supplied by the implementer/root and are not independent runtime evidence from this review. + +Decision: changes requested. The documented unsupported modules, encodings, size limits, savepoints, attached/temp schemas, arbitrary explicit-transaction atomicity, and spatial reconstruction limits were treated as acceptance boundaries, not findings. + +## Material findings + +### 1. [P1] Put strategy-generated DROP inside the managed transaction + +Location: `mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs:118-131` (DROP at 123; BEGIN at 131). + +With an existing populated `t`, `DropAndRecreate`, `WrapInTransaction=true`, and a CREATE-only dump such as `CREATE TABLE t(x); INSERT INTO t VALUES(1); FAILING_SQL;`, the strategy executes its generated `DROP TABLE` while the connection is still in autocommit. Only afterward does it start `BEGIN IMMEDIATE`. A later failure rolls back the replacement table but leaves the original table and its rows permanently deleted. Begin the managed transaction before any policy-generated mutation. Add a private SQLite regression that retains the original table and data after a later failure, including a failing replacement CREATE. + +### 2. [P1] Stop when a continued statement error has already rolled back the transaction + +Location: `mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs:129-148` (failure continuation at 142-143). + +The runner keeps `txOpen=true` after every ordinary execution error. SQLite `INSERT OR ROLLBACK`/`RAISE(ROLLBACK,...)` can roll back the entire transaction. With wrapping and ContinueOnError enabled, `CREATE TABLE t(x UNIQUE); INSERT INTO t VALUES(1); INSERT OR ROLLBACK INTO t VALUES(1); CREATE TABLE leaked(x); INSERT INTO leaked VALUES(2);` rolls back `t`, but subsequent statements run in autocommit because the stale flag suppresses a new BEGIN. The import reports Failed while `leaked` persists, outside the documented earlier-explicit-COMMIT/wrapping-disabled boundary. Check native AutoCommit after a statement error and stop if the tracked transaction has disappeared; also reconcile state before executing further statements. Cover this with an actual private SQLite fixture. + +### 3. [P1] Reject every close attempt while the worker is busy + +Locations: `mySQLPunk/SqliteImportWizardForm.cs:129-136`, `mySQLPunk/SqliteExportWizardForm.cs:127-134`; newly owned-session callers at `mySQLPunk/Form1.SqliteImport.cs:33-38` and `mySQLPunk/Form1.SqliteExport.cs:46-51`. + +The first window-close attempt sets `closing=true` and cancels that close, but a second close attempt while cleanup is still running bypasses `if (busy && !closing)` and closes the dialog. The new caller-owned session is then disposed while the background task can still execute SQL/rollback or post progress. Always set `e.Cancel=true` while busy; use `closing` only to avoid sending cancellation twice, and allow the completion/finally path to close after busy becomes false. This must cover both import and export. Import preflight also uses busy, so its finally path needs to honor a pending close after the scan ends. + +### 4. [P2] Fail closed for unscoped sequence/comment mutations under preserving policies + +Location: `mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs:218-225`. + +SkipExisting exempts the entire `sqlite_sequence` and comments sidecar from ordinary protected-table DML checks, then protects them only if a narrow literal-owner regex matches. A standard SQLite dump's `DELETE FROM sqlite_sequence;` therefore executes even when an existing AUTOINCREMENT table is being skipped; its restoring INSERT is then skipped because that INSERT names the protected table. For an existing table whose deleted high-water mark was 100 but whose current maximum id is 1, a subsequent insert can reuse 2 instead of preserving 101. Unscoped sidecar DELETE has the same ownership hole for unrelated comments. Reject or safely skip ambiguous mutations when protected owners exist, while retaining the exporter's narrowly scoped owner operations. Add a private fixture with a deleted sequence high-water mark and an unscoped DELETE. + +### 5. [P2] Apply the new single-table initial options to the actual wizard controls + +Locations: `mySQLPunk/SqliteExportWizardForm.cs:209`, `:219`, `:641-648`; caller at `mySQLPunk/Form1.SqliteExport.cs:34-36`. + +The single-table route now sets IncludeViews/IncludeTriggers=false, but BuildPage0 still initializes both controls to true, and StartExportAsync rebuilds options from those controls. Consequently the actual dialog ignores the new safe trigger default and exports selected-owner triggers even when their bodies reference omitted tables. If the requested table is the database's only table, the view list is also re-enabled because catalog.All is true. Apply the supplied booleans to the controls so the normal dialog agrees with the injected-runner route. Verify the initial single-table control values and resulting service options. + +## Scope completion + +No additional material findings were identified in the scoped pass for the shared lexer, strict UTF-8/NUL representation, native bounded sequential row reads, generated-column projection, rowid/table_list metadata, sequence emission, checked source queries, snapshot/publication flow, or comments roundtrip beyond the preserving-policy hole above. This is a static review conclusion, not a new product build, UI, or production acceptance claim. The root owns shared-helper review and the authoritative final suite after the repair wave. + +## Correction re-review verdict (2026-10-16) + +Decision: all five findings are addressed; approve this scoped correction wave for specification and code quality. This verdict supersedes the earlier changes-requested decision for those five items. One static re-review was performed against `sqlite-corrections-diff.patch`, the current corrected source, appended implementer report, and the four regression test definitions. No builds, tests, live databases, or saved configuration were accessed during re-review. No broader new review loop was undertaken. + +| Finding | Verdict | Current source evidence | +| --- | --- | --- | +| 1. Managed DROP ordering | Addressed | `SqliteImportRunner.cs:122-132` starts BEGIN IMMEDIATE before the generated DROP. The regression covers both a later failure and an invalid replacement CREATE while asserting the original populated table remains. | +| 2. Continued error after full rollback | Addressed | `SqliteImportRunner.cs:113-116` reconciles native AutoCommit before subsequent statements; `:145-148` stops immediately when an execution error ended the tracked transaction. Managed, explicit BEGIN, and RAISE(ROLLBACK) regression cases assert no later leaked table. | +| 3. Repeated close and pending asynchronous work | Addressed | Both forms' `:129-139` close handlers cancel every close while busy and request cancellation once. Import preflight `SqliteImportWizardForm.cs:569-576`, import execution `:692-699`, export catalog `SqliteExportWizardForm.cs:508-532`, and export execution `:714-721` release busy and honor the pending close after awaited work. Export catalog loading now holds the busy guard across all three provider queries. | +| 4. Unscoped protected metadata mutation | Addressed | `SqliteImportRunner.cs:232-269` permits only complete single-owner native-exporter sequence/comment forms when existing owners are present, skips ambiguous forms, and preserves existing owners. The regression covers broad DELETE, broadened WHERE, deleted high-water marks, unrelated comments, and allowed new-owner metadata restoration under both Skip and Keep. | +| 5. Single-table supplied flags | Addressed | `SqliteExportWizardForm.cs:96-98` applies the supplied IncludeViews/IncludeTriggers values to the actual controls. `:638-655` derives service options from those controls, and `:668` uses that method in execution. The one-table regression includes a view and a trigger with an omitted dependency and asserts neither is exported. | + +No material direct breakage from these five corrections was found in this bounded pass. Runtime evidence remains attributed to its owners: the implementer reports isolated 16/16 against exact SQLite/form source, and the root reports a fresh actual-product build plus integrated SQLite 19 groups and AutoReleasePolicy passing (`repair-sqlite-final-tests.log`). This reviewer inspected the regression assertions but did not rerun them. Displayed WinForms/DPI interaction, live databases, and production acceptance remain outside this approval. diff --git a/docs/design/db-wizard-repairs/verification.md b/docs/design/db-wizard-repairs/verification.md new file mode 100644 index 0000000..b6d9b61 --- /dev/null +++ b/docs/design/db-wizard-repairs/verification.md @@ -0,0 +1,42 @@ +# Wizard repairs: verification and support boundaries + +Baseline: `d41825cead810bc67f263753bb5f986a7f69c65c`. Local repair branch: `codex/db-wizard-repairs`. The three feature branches are preserved. Their stack also includes PostgreSQL; this repair does not certify that provider. + +## What changed + +- MySQL: one bounded lexical reader for preflight/execution; rewrite identifiers without changing literal/comment bytes; enforce the chosen target; separate pre-existing objects from newly created tables; protect conflict strategies; project generated columns correctly; validate source snapshot/cleanup; preserve initial table/mode and await worker cleanup. +- SQLite: bind strategy values rather than cast indexes; protect native DROP/dependencies; parse trigger bodies and exact UTF-8 text; check all transaction results and actual native transaction state; restore PRAGMAs; preserve generated columns, implicit rowid, sqlite_sequence, NUL/multiline/BLOB bytes and comment sidecars. Native exports use one sequential reader with bounded SQL-side projection. +- Oracle: check provider NO results, owner/transaction/cleanup, parser bytes/comments/alternative quotes and UI selection. Native source snapshot and sequential readers replace full-table DataTable loading. Spatial uses original SRID/DIMINFO/tolerance and option gates. BLOB/CLOB/large WKB use bounded temporary-LOB PL/SQL blocks rather than one oversized literal. +- Common: the six native wizard entries open an owned provider session with its own SSH transport lifetime, never borrowing the editor connection. MySQL/Oracle wizard sessions disable pooling. Export publication uses same-directory File.Replace or first-file File.Move, retaining old destination and complete staging when publication fails. + +## Current deliberate limits + +| Provider | Limit / unsupported cases | +| --- | --- | +| MySQL | Source row <=1 MiB; 16-row source pages; INSERT <=8 MiB. Snapshot requires verified InnoDB. Executable version comments, dynamic SQL/CALL, ambiguous scope and native spatial values are explicitly unsupported. Bigger LOB chunking remains future work. | +| SQLite | Source row <=8 MiB; strict UTF-8 SQL and <=32 MiB decoded statement. Virtual modules and inaccessible row identities fail explicitly. SpatiaLite/GeoPackage extension-dependent acceptance is pending. | +| Oracle | Row values <=8 MiB, generated statement <=28 MiB, input statement <=32 MiB. BLOB chunks <=2000 bytes. Native export currently requires AL32UTF8; identity data, signed zero, LRS/curved/compound spatial cases and unverified types fail explicitly. Cross-owner targets and general stored/dynamic PL/SQL are rejected; generated bounded LOB and DROP wrappers are supported. | + +Unsupported data aborts the job before final publication; no truncation, fabricated SRID or silent approximation is used. These are finite budgets, not claims to cover theoretical provider type maxima. Counts/manifest success alone do not prove value or schema fidelity. + +## Evidence + +- Original baseline: full smoke 268 groups plus AutoReleasePolicy passed. +- Common regression: actual locked-file publication failure preserves both original/candidate; actual private SQLite owned-session disposal rolls back job data and leaves editor connection open; both MySQL table entry handlers preserve raw identity and DataOnly. +- MySQL/Oracle use recording-provider contracts including status=NO, rather than live engines. SQLite uses tiny private actual System.Data.SQLite fixtures; no saved user databases/dumps or account settings were read. +- Provider reports: [MySQL](mysql-repair-report.md), [SQLite](sqlite-repair-report.md), [Oracle](oracle-repair-report.md). +- Final actual-product gate: **294 smoke groups and AutoReleasePolicy passed**, followed by successful explicit **Debug and Release** application builds. Optional live groups printed SKIP; no live DB acceptance is implied. +- Final focused evidence: actual-product SQLite19 and Oracle15 passed. MySQL correction harness12 passed and the final full suite executed all registered MySQL regressions. Common four groups passed within the final full run. +- Independent scoped code/spec reviews and one bounded correction re-review per provider approved: [MySQL](mysql-scoped-review.md), [SQLite](sqlite-scoped-review.md), [Oracle](oracle-scoped-review.md). Original findings remain in those reports as history; appended/final verdicts supersede initial changes-requested decisions. +- Logs: repair-full-smoke-final.log, repair-debug-final.log, repair-release-final.log, repair-sqlite-final-tests.log and repair-oracle-scoped-smoke.log in the local evidence directory. +- Source and test files were frozen before this gate; only documentation/history was finalized afterward. Existing-file encoding/BOM/CRLF audit and git diff --check passed. + +To repeat: run `tests/Run-SmokeTests.ps1` from this checkout using Windows PowerShell/PowerShell and installed MSBuild. Optional `MYSQLPUNK_SMOKE_FILTER` values: `Wizard repairs`, `MySQL`, `SQLite`, `Oracle`. Do not run quarantined ReportExportTests. + +Local baseline, red reproductions, focused test logs and final build logs are retained outside Git at `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/`. + +## Pending acceptance + +Real MySQL/MariaDB and Oracle/Spatial roundtrips, live SSH/permissions/source-editor concurrency, multi-GiB memory/performance, native UI/DPI/audio and production are not verified. MySQL stops after the current command returns/times out; immediate server-command cancellation is not promised. A failed import can retain earlier committed DDL/DML; no automatic cleanup/retry or all-or-nothing restore guarantee is introduced. + +Oracle reference semantics used during review: [ODP.NET transactions](https://docs.oracle.com/en/database/oracle/oracle-database/19/odpnt/featOraCommand.html), [SET TRANSACTION](https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/SET-TRANSACTION.html), [SDO_UTIL WKB and SRID](https://docs.oracle.com/en/database/oracle/oracle-database/19/spatl/SDO_UTIL-reference.html). These references are design evidence, not a live acceptance result. diff --git a/mySQLPunk/Form1.MySqlExport.cs b/mySQLPunk/Form1.MySqlExport.cs index 40d46b4..4c675cd 100644 --- a/mySQLPunk/Form1.MySqlExport.cs +++ b/mySQLPunk/Form1.MySqlExport.cs @@ -19,7 +19,7 @@ private void SetMySqlExportStatus(string message) else MySqlExportStatusSink(message); } - private void OpenMySqlExportWizard(TreeDatabaseTarget target, MySqlExportMode mode = MySqlExportMode.SchemaAndData) + private void OpenMySqlExportWizard(TreeDatabaseTarget target, MySqlExportMode mode = MySqlExportMode.SchemaAndData, string specificTable = null) { if (target == null) return; try @@ -27,6 +27,13 @@ private void OpenMySqlExportWizard(TreeDatabaseTarget target, MySqlExportMode mo string defaultFileName = target.DatabaseName + "_dump_" + DateTime.Now.ToString("yyyyMMdd_HHmmss") + ".sql"; string targetPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), defaultFileName); var options = new MySqlExportWizardOptions(mode, targetPath); + if (!string.IsNullOrWhiteSpace(specificTable)) + { + options.SelectedTables.Add(specificTable); + options.IncludeViews = false; + options.IncludeRoutines = false; + options.IncludeTriggers = false; + } MySqlExportResult result; if (MySqlExportWizardRunner != null) @@ -35,7 +42,8 @@ private void OpenMySqlExportWizard(TreeDatabaseTarget target, MySqlExportMode mo } else { - using (var wizard = new MySqlExportWizardForm(target.Database, target.DatabaseName, options)) + using (var session = WizardOwnedSession.Open(target.ConnectionInfo, target.ProviderName, target.DatabaseName)) + using (var wizard = new MySqlExportWizardForm(session.Database, target.DatabaseName, options)) { wizard.ShowDialog(this); result = wizard.Result; diff --git a/mySQLPunk/Form1.MySqlImport.cs b/mySQLPunk/Form1.MySqlImport.cs index 1d23379..f1a9248 100644 --- a/mySQLPunk/Form1.MySqlImport.cs +++ b/mySQLPunk/Form1.MySqlImport.cs @@ -30,7 +30,8 @@ private void OpenMySqlImportWizard(TreeDatabaseTarget target) } else { - using (var wizard = new MySqlImportWizardForm(target.Database, target.DatabaseName)) + using (var session = WizardOwnedSession.Open(target.ConnectionInfo, target.ProviderName, target.DatabaseName)) + using (var wizard = new MySqlImportWizardForm(session.Database, target.DatabaseName)) { wizard.ShowDialog(this); result = wizard.Result; diff --git a/mySQLPunk/Form1.OracleExport.cs b/mySQLPunk/Form1.OracleExport.cs index 4e37b8d..bd565e4 100644 --- a/mySQLPunk/Form1.OracleExport.cs +++ b/mySQLPunk/Form1.OracleExport.cs @@ -30,6 +30,7 @@ private void OpenOracleExportWizard(TreeDatabaseTarget target, OracleExportMode if (!string.IsNullOrWhiteSpace(specificTable)) { options.SelectedTables.Add(specificTable); + options.IncludeViews = false; } OracleExportResult result; @@ -39,7 +40,8 @@ private void OpenOracleExportWizard(TreeDatabaseTarget target, OracleExportMode } else { - using (var wizard = new OracleExportWizardForm(target.Database, target.DatabaseName, options)) + using (var session = WizardOwnedSession.Open(target.ConnectionInfo, target.ProviderName, target.DatabaseName)) + using (var wizard = new OracleExportWizardForm(session.Database, target.DatabaseName, options)) { wizard.ShowDialog(this); result = wizard.Result; diff --git a/mySQLPunk/Form1.OracleImport.cs b/mySQLPunk/Form1.OracleImport.cs index 0975571..c1c8d12 100644 --- a/mySQLPunk/Form1.OracleImport.cs +++ b/mySQLPunk/Form1.OracleImport.cs @@ -29,7 +29,8 @@ private void OpenOracleImportWizard(TreeDatabaseTarget target) } else { - using (var wizard = new OracleImportWizardForm(target.Database, target.DatabaseName)) + using (var session = WizardOwnedSession.Open(target.ConnectionInfo, target.ProviderName, target.DatabaseName)) + using (var wizard = new OracleImportWizardForm(session.Database, target.DatabaseName)) { wizard.ShowDialog(this); result = wizard.Result; diff --git a/mySQLPunk/Form1.SqliteExport.cs b/mySQLPunk/Form1.SqliteExport.cs index 148a3b2..e0d1097 100644 --- a/mySQLPunk/Form1.SqliteExport.cs +++ b/mySQLPunk/Form1.SqliteExport.cs @@ -32,6 +32,8 @@ private void OpenSqliteExportWizard(TreeDatabaseTarget target, SqliteExportMode if (!string.IsNullOrWhiteSpace(specificTable)) { options.SelectedTables.Add(specificTable); + options.IncludeViews = false; + options.IncludeTriggers = false; } SqliteExportResult result; @@ -41,7 +43,8 @@ private void OpenSqliteExportWizard(TreeDatabaseTarget target, SqliteExportMode } else { - using (var wizard = new SqliteExportWizardForm(target.Database, target.DatabaseName, options)) + using (var session = WizardOwnedSession.Open(target.ConnectionInfo, target.ProviderName, target.DatabaseName)) + using (var wizard = new SqliteExportWizardForm(session.Database, target.DatabaseName, options)) { wizard.ShowDialog(this); result = wizard.Result; diff --git a/mySQLPunk/Form1.SqliteImport.cs b/mySQLPunk/Form1.SqliteImport.cs index 5677040..e9b5044 100644 --- a/mySQLPunk/Form1.SqliteImport.cs +++ b/mySQLPunk/Form1.SqliteImport.cs @@ -30,7 +30,8 @@ private void OpenSqliteImportWizard(TreeDatabaseTarget target) } else { - using (var wizard = new SqliteImportWizardForm(target.Database, target.DatabaseName)) + using (var session = WizardOwnedSession.Open(target.ConnectionInfo, target.ProviderName, target.DatabaseName)) + using (var wizard = new SqliteImportWizardForm(session.Database, target.DatabaseName)) { wizard.ShowDialog(this); result = wizard.Result; diff --git a/mySQLPunk/Form1.cs b/mySQLPunk/Form1.cs index 5deee2c..d20d500 100644 --- a/mySQLPunk/Form1.cs +++ b/mySQLPunk/Form1.cs @@ -6677,7 +6677,7 @@ private void DumpSelectedTableSql(bool dataOnly) } if (IsMySqlTarget(target)) { - OpenMySqlExportWizard(target, dataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData); + OpenMySqlExportWizard(target, dataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData, tableName); return; } if (IsSqliteTarget(target)) @@ -6952,7 +6952,7 @@ private void DumpCurrentSelectionSqlWithDialog(bool dataOnlyForTable) if (IsMySqlTarget(target) && (selection == null || selection.GroupName == "Tables")) { bool tableDataOnly = selection != null && dataOnlyForTable; - OpenMySqlExportWizard(target, tableDataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData); + OpenMySqlExportWizard(target, tableDataOnly ? lib.MySqlExport.MySqlExportMode.DataOnly : lib.MySqlExport.MySqlExportMode.SchemaAndData, selection == null ? null : selection.ObjectName); return; } if (IsSqliteTarget(target) && (selection == null || selection.GroupName == "Tables")) diff --git a/mySQLPunk/MySqlExportWizardForm.cs b/mySQLPunk/MySqlExportWizardForm.cs index f43f82a..a85d68b 100644 --- a/mySQLPunk/MySqlExportWizardForm.cs +++ b/mySQLPunk/MySqlExportWizardForm.cs @@ -18,6 +18,7 @@ internal sealed class MySqlExportWizardForm : Form private readonly IDatabase db; private readonly string databaseName; private readonly MySqlSqlExportService service; + private readonly MySqlExportWizardOptions initialOptions; private readonly Panel[] pages = new Panel[4]; private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); private readonly DataGridView tableGrid = new DataGridView(); @@ -55,6 +56,7 @@ internal MySqlExportWizardForm(IDatabase db, string databaseName, MySqlExportWiz this.db = db ?? throw new ArgumentNullException(nameof(db)); this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); this.service = new MySqlSqlExportService(db, databaseName); + this.initialOptions = initialOptions; Name = "mySqlExportWizard"; Text = T("匯出資料庫 · MySQL / MariaDB", "Export database · MySQL / MariaDB"); @@ -97,6 +99,22 @@ internal MySqlExportWizardForm(IDatabase db, string databaseName, MySqlExportWiz BuildPage1(initialOptions); BuildPage2(); BuildPage3(); + if (initialOptions != null) + { + schemaData.Checked = initialOptions.Mode == MySqlExportMode.SchemaAndData; + schemaOnly.Checked = initialOptions.Mode == MySqlExportMode.SchemaOnly; + dataOnly.Checked = initialOptions.Mode == MySqlExportMode.DataOnly; + singleTransaction.Checked = initialOptions.SingleTransaction; + disableFk.Checked = initialOptions.DisableForeignKeyChecks; + disableUnique.Checked = initialOptions.DisableUniqueChecks; + noAutoValueOnZero.Checked = initialOptions.SetNoAutoValueOnZero; + includeDrop.Checked = initialOptions.IncludeDropStatements; + includeCreateDb.Checked = initialOptions.IncludeCreateDatabase; + removeDefiner.Checked = initialOptions.RemoveDefiner; + includeViews.Checked = initialOptions.IncludeViews; + includeRoutines.Checked = initialOptions.IncludeRoutines; + includeTriggers.Checked = initialOptions.IncludeTriggers; + } statusLabel.AutoSize = true; statusLabel.Margin = new Padding(0, 8, 0, 6); @@ -455,20 +473,39 @@ private void BuildPage3() private async Task LoadCatalogAsync() { - statusLabel.Text = T("正在載入資料庫結構中...", "Loading database catalog..."); - catalog = await Task.Run(() => service.LoadTableCatalog()); - views = await Task.Run(() => service.LoadViews()); - routines = await Task.Run(() => service.LoadRoutines()); - triggers = await Task.Run(() => service.LoadTriggers()); - - tableGrid.Rows.Clear(); - foreach (var item in catalog) + busy = true; + nextButton.Enabled = false; + cancellation = new CancellationTokenSource(); + try { - tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0")); + statusLabel.Text = T("正在載入資料庫結構中...", "Loading database catalog..."); + catalog = await Task.Run(() => service.LoadTableCatalog()); + if (closing || cancellation.IsCancellationRequested) return; + views = await Task.Run(() => service.LoadViews()); + if (closing || cancellation.IsCancellationRequested) return; + routines = await Task.Run(() => service.LoadRoutines()); + if (closing || cancellation.IsCancellationRequested) return; + triggers = await Task.Run(() => service.LoadTriggers()); + if (closing || cancellation.IsCancellationRequested) return; + if (initialOptions != null && initialOptions.SelectedTables != null && initialOptions.SelectedTables.Count > 0) + { + var selected = new HashSet(initialOptions.SelectedTables, StringComparer.OrdinalIgnoreCase); + foreach (var item in catalog) item.IsSelected = selected.Contains(item.Name); + } + tableGrid.Rows.Clear(); + foreach (var item in catalog) tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0")); + statusLabel.Text = ""; + UpdateSelectionSummary(); + } + catch (Exception ex) { statusLabel.Text = ex.Message; } + finally + { + busy = false; + cancelButton.Enabled = true; + cancellation.Dispose(); + cancellation = null; + if (closing) Close(); } - - statusLabel.Text = ""; - UpdateSelectionSummary(); } private void ApplySearchFilter() @@ -591,6 +628,8 @@ private async Task StartExportAsync() SetNoAutoValueOnZero = noAutoValueOnZero.Checked, IncludeDropStatements = includeDrop.Checked, IncludeCreateDatabase = includeCreateDb.Checked, + IncludeUseDatabase = initialOptions?.IncludeUseDatabase ?? true, + InsertBatchSize = initialOptions?.InsertBatchSize ?? 1000, RemoveDefiner = removeDefiner.Checked, IncludeViews = includeViews.Checked, IncludeRoutines = includeRoutines.Checked, @@ -650,6 +689,9 @@ private async Task StartExportAsync() busy = false; exporting = false; cancelButton.Text = T("關閉", "Close"); + cancelButton.Enabled = true; + cancellation.Dispose(); + cancellation = null; nextButton.Enabled = true; if (closing) Close(); } diff --git a/mySQLPunk/MySqlImportWizardForm.cs b/mySQLPunk/MySqlImportWizardForm.cs index 85d1fe3..d7920ec 100644 --- a/mySQLPunk/MySqlImportWizardForm.cs +++ b/mySQLPunk/MySqlImportWizardForm.cs @@ -504,7 +504,19 @@ private async Task RunPreflightAsync() string path = sourcePathBox.Text.Trim(); cancellation = new CancellationTokenSource(); - preflight = await Task.Run(() => MySqlImportPreflightScanner.Scan(path, cancellation.Token)); + try + { + preflight = await Task.Run(() => MySqlImportPreflightScanner.Scan(path, cancellation.Token)); + } + finally + { + busy = false; + cancelButton.Enabled = true; + cancellation.Dispose(); + cancellation = null; + if (closing) Close(); + } + if (closing || IsDisposed) return; scanBar.Style = ProgressBarStyle.Blocks; scanBar.Value = preflight.Passed ? 100 : 0; @@ -604,6 +616,9 @@ private async Task StartImportAsync() busy = false; executing = false; cancelButton.Text = T("關閉", "Close"); + cancelButton.Enabled = true; + cancellation.Dispose(); + cancellation = null; nextButton.Enabled = true; if (closing) Close(); } diff --git a/mySQLPunk/OracleExportWizardForm.cs b/mySQLPunk/OracleExportWizardForm.cs index dba8609..41f0d2e 100644 --- a/mySQLPunk/OracleExportWizardForm.cs +++ b/mySQLPunk/OracleExportWizardForm.cs @@ -19,6 +19,7 @@ internal sealed class OracleExportWizardForm : Form private readonly IDatabase db; private readonly string databaseName; // Owner / Schema private readonly OracleSqlExportService service; + private readonly List initialSelection = new List(); private readonly Panel[] pages = new Panel[4]; private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); private readonly DataGridView tableGrid = new DataGridView(); @@ -52,9 +53,10 @@ internal OracleExportWizardForm(IDatabase db, string databaseName, OracleExportW this.db = db ?? throw new ArgumentNullException(nameof(db)); this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : "USER"; this.service = new OracleSqlExportService(db, databaseName); + if (initialOptions != null) initialSelection.AddRange(initialOptions.SelectedTables); Name = "oracleExportWizard"; - Text = T("匯出資料庫 · Oracle (支援 SDO_GEOMETRY 空間幾何無損備份)", "Export database · Oracle (with SDO_GEOMETRY spatial support)"); + Text = T("匯出資料庫 · Oracle (支援有界 SDO_GEOMETRY 備份)", "Export database · Oracle (with SDO_GEOMETRY spatial support)"); AutoScaleMode = AutoScaleMode.Dpi; Font = UiKit.Body; ClientSize = new Size(1120, 740); @@ -116,8 +118,10 @@ internal OracleExportWizardForm(IDatabase db, string databaseName, OracleExportW timer.Tick += (s, e) => UpdateLiveTicker(); FormClosing += (s, e) => { - if (exporting && !closing) + if (exporting || busy) { + e.Cancel = true; + if (closing) return; if (MessageBox.Show(T("正在匯出 Oracle 資料庫中，確定要中止嗎？", "Export is currently running. Cancel it?"), T("確認", "Confirm"), MessageBoxButtons.YesNo, MessageBoxIcon.Question) != DialogResult.Yes) { e.Cancel = true; @@ -125,6 +129,7 @@ internal OracleExportWizardForm(IDatabase db, string databaseName, OracleExportW } closing = true; cancellation?.Cancel(); + return; // Wait for worker reader/transaction cleanup. } timer.Stop(); clock.Stop(); @@ -172,6 +177,18 @@ private void BuildPage0() tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "rows", HeaderText = T("列數預估", "Rows (est.)"), Width = 110, ReadOnly = true }); tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "bytes", HeaderText = T("大小 (Bytes)", "Bytes"), Width = 110, ReadOnly = true }); + tableGrid.CurrentCellDirtyStateChanged += (s, e) => + { + if (tableGrid.IsCurrentCellDirty) tableGrid.CommitEdit(DataGridViewDataErrorContexts.Commit); + }; + tableGrid.CellValueChanged += (s, e) => + { + if (e.RowIndex < 0 || e.ColumnIndex != 0) return; + var row = tableGrid.Rows[e.RowIndex]; + var item = row.Tag as OracleTableExportItem; + if (item != null) item.Selected = Convert.ToBoolean(row.Cells[0].Value ?? false); + UpdateSelectionSummary(); + }; tableGrid.CellContentClick += (s, e) => { if (e.RowIndex >= 0 && e.ColumnIndex == 0) @@ -293,6 +310,16 @@ private void BuildPage1(OracleExportWizardOptions opt) if (opt.Mode == OracleExportMode.DataOnly) dataOnly.Checked = true; else if (opt.Mode == OracleExportMode.SchemaOnly) schemaOnly.Checked = true; if (!string.IsNullOrWhiteSpace(opt.TargetPath)) targetPathBox.Text = opt.TargetPath; + readOnlySnapshot.Checked = opt.UseTransactionSnapshot; + excludeShadowTables.Checked = opt.ExcludeSpatialIndexInternalTables; + includeSpatialMetadata.Checked = opt.IncludeSpatialMetadata; + includeSpatialIndexes.Checked = opt.IncludeSpatialIndexes; + includeDrop.Checked = opt.DropTableIfExists; + truncateBefore.Checked = opt.TruncateBeforeInsert; + includeViews.Checked = opt.IncludeViews; + includeIndexes.Checked = opt.IncludeIndexes; + includeComments.Checked = opt.IncludeComments; + includeManifest.Checked = opt.IncludeManifest; } if (string.IsNullOrWhiteSpace(targetPathBox.Text)) { @@ -300,6 +327,7 @@ private void BuildPage1(OracleExportWizardOptions opt) string.Format("oracle_{0}_dump_{1:yyyyMMdd_HHmmss}.sql", databaseName, DateTime.Now)); } + host.Controls.Add(new Label { AutoSize = true, MaximumSize = new Size(900, 0), Text = T("支援 AL32UTF8；每列值最多 8 MiB／SQL 區塊 28 MiB。BLOB／CLOB 分段；identity、NCLOB、LRS／曲線／複合幾何等明確拒絕。唯讀快照需非 SYS 帳號；空間 metadata 還原帳號須等於 owner。FK 父表必須一併選取，循環 FK／外部 sequence、function 等相依尚不支援。", "AL32UTF8 only; row values <=8 MiB / SQL block <=28 MiB. BLOB/CLOB chunks supported; identity, NCLOB, LRS/curves/compound geometry rejected. Non-SYS snapshot; metadata restore session user must equal owner. Include all FK parents; cyclic FKs/external sequences/functions are unsupported.") }); p.Controls.Add(host); } @@ -403,10 +431,16 @@ private async Task LoadCatalogAsync() { if (busy) return; busy = true; + cancellation = new CancellationTokenSource(); statusLabel.Text = T("正在探測 Oracle 資料庫物件與空間幾何欄位...", "Discovering Oracle tables and spatial features..."); try { - catalog = await service.GetTableCatalogAsync().ConfigureAwait(true); + catalog = await service.GetTableCatalogAsync(cancellation.Token).ConfigureAwait(true); + if (initialSelection.Count > 0) + { + var selected = new HashSet(initialSelection.Select(name => mySQLPunk.lib.OracleImport.OracleImportRunner.ResolveManifestTarget(name, databaseName)), StringComparer.Ordinal); + foreach (var item in catalog) item.Selected = selected.Contains(item.FullQualifiedName); + } ApplyFilter(); statusLabel.Text = string.Format(T("探測完成：共找到 {0} 個資料表與檢視。", "Catalog loaded: {0} tables/views found."), catalog.Count); } @@ -417,6 +451,9 @@ private async Task LoadCatalogAsync() finally { busy = false; + cancellation?.Dispose(); + cancellation = null; + if (closing && !IsDisposed) BeginInvoke(new Action(Close)); } } @@ -605,7 +642,7 @@ private async Task StartExportAsync() foreach (var it in catalog) { - if (it.Selected) opt.SelectedTables.Add(it.Name); + if (it.Selected) opt.SelectedTables.Add(it.FullQualifiedName); } cancellation = new CancellationTokenSource(); @@ -653,6 +690,9 @@ private async Task StartExportAsync() finally { exporting = false; + cancellation?.Dispose(); + cancellation = null; + if (closing && !IsDisposed) BeginInvoke(new Action(Close)); } } diff --git a/mySQLPunk/OracleImportWizardForm.cs b/mySQLPunk/OracleImportWizardForm.cs index 4b78941..3510f93 100644 --- a/mySQLPunk/OracleImportWizardForm.cs +++ b/mySQLPunk/OracleImportWizardForm.cs @@ -116,8 +116,10 @@ internal OracleImportWizardForm(IDatabase db, string databaseName, string initia timer.Tick += (s, e) => UpdateLiveTicker(); FormClosing += (s, e) => { - if (executing && !closing) + if (executing) { + e.Cancel = true; + if (closing) return; if (MessageBox.Show(T("正在匯入 Oracle SQL 腳本中，確定要中途停止嗎？", "Import is currently executing. Cancel it?"), T("確認", "Confirm"), MessageBoxButtons.YesNo, MessageBoxIcon.Question) != DialogResult.Yes) { e.Cancel = true; @@ -125,6 +127,7 @@ internal OracleImportWizardForm(IDatabase db, string databaseName, string initia } closing = true; cancellation?.Cancel(); + return; // Wait for worker reader/transaction cleanup. } timer.Stop(); clock.Stop(); @@ -225,7 +228,7 @@ private void BuildPage2() var optBox = new GroupBox { Text = T("執行保護與 Oracle Spatial 選項", "Execution protection & Spatial settings"), AutoSize = true, Padding = new Padding(12), Margin = new Padding(0, 0, 0, 8) }; var optFlow = Stack(); - autoRegisterSpatial.Text = T("自動補全 USER_SDO_GEOM_METADATA 空間中繼字典 (若腳本內未包含)", "Auto-register USER_SDO_GEOM_METADATA if missing"); + autoRegisterSpatial.Text = T("執行腳本內的 USER_SDO_GEOM_METADATA 註冊 (不推測缺少的 metadata)", "Execute script spatial metadata registration (never infer missing metadata)"); autoRegisterSpatial.Checked = true; autoRegisterSpatial.AutoSize = true; @@ -398,6 +401,7 @@ private void MoveStep(int delta) private async Task OnNextAsync() { + if (executing || closing) return; if (step == 0) { string path = filePathBox.Text.Trim(); @@ -408,7 +412,8 @@ private async Task OnNextAsync() } await RunPreflightAsync(path); - if (preflight.CanProceed) + if (closing || IsDisposed) return; + if (preflight != null && preflight.CanProceed) { MoveStep(1); } @@ -430,10 +435,15 @@ private async Task OnNextAsync() private async Task RunPreflightAsync(string path) { + executing = true; + cancellation = new CancellationTokenSource(); + try + { statusLabel.Text = T("正在對 SQL 檔案進行狀態機預檢分析...", "Scanning SQL file with preflight lexer..."); var scanner = new OracleImportPreflightScanner(); - preflight = await Task.Run(() => scanner.Scan(path)).ConfigureAwait(true); + preflight = await Task.Run(() => scanner.Scan(path, cancellation.Token)).ConfigureAwait(true); + if (closing || IsDisposed) return; if (!preflight.CanProceed) { MessageBox.Show(string.Join("\n", preflight.Issues.ToArray()), T("預檢未通過", "Preflight Failed"), MessageBoxButtons.OK, MessageBoxIcon.Error); @@ -459,10 +469,18 @@ private async Task RunPreflightAsync(string path) } else { - issuesList.Items.Add(T("無任何阻擋性異常，腳本結構完整且符合 Oracle 語法標準。", "No issues found. Script is ready for execution.")); + issuesList.Items.Add(T("文字預檢通過；尚未驗證 Oracle 語法／權限。一般預存 PL/SQL 不支援；DDL 會隱含提交，rollback 僅保護尚未提交的 DML。", "Text preflight passed; Oracle syntax/permissions remain unverified. Stored PL/SQL is unsupported. DDL implicitly commits; rollback protects pending DML only.")); } statusLabel.Text = T("預檢完成，請確認設定。", "Preflight complete."); + } + finally + { + executing = false; + cancellation?.Dispose(); + cancellation = null; + if (closing && !IsDisposed) BeginInvoke(new Action(Close)); + } } private async Task StartImportAsync() @@ -548,6 +566,9 @@ private async Task StartImportAsync() finally { executing = false; + cancellation?.Dispose(); + cancellation = null; + if (closing && !IsDisposed) BeginInvoke(new Action(Close)); } } diff --git a/mySQLPunk/SqliteExportWizardForm.cs b/mySQLPunk/SqliteExportWizardForm.cs index 8ad7d7b..26d9fea 100644 --- a/mySQLPunk/SqliteExportWizardForm.cs +++ b/mySQLPunk/SqliteExportWizardForm.cs @@ -18,6 +18,7 @@ internal sealed class SqliteExportWizardForm : Form private readonly IDatabase db; private readonly string databaseName; private readonly SqliteSqlExportService service; + private readonly HashSet initialSelection; private readonly Panel[] pages = new Panel[4]; private readonly TextBox search = new TextBox(), targetPathBox = new TextBox(); private readonly DataGridView tableGrid = new DataGridView(); @@ -53,6 +54,7 @@ internal SqliteExportWizardForm(IDatabase db, string databaseName, SqliteExportW this.db = db ?? throw new ArgumentNullException(nameof(db)); this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); this.service = new SqliteSqlExportService(db, databaseName); + initialSelection = initialOptions?.SelectedTables?.Count > 0 ? new HashSet(initialOptions.SelectedTables, StringComparer.OrdinalIgnoreCase) : null; Name = "sqliteExportWizard"; Text = T("匯出資料庫 · SQLite (支援 SpatiaLite / GeoPackage GIS)", "Export database · SQLite (with SpatiaLite / GeoPackage GIS)"); @@ -92,6 +94,8 @@ internal SqliteExportWizardForm(IDatabase db, string databaseName, SqliteExportW } BuildPage0(); + includeViews.Checked = initialOptions?.IncludeViews ?? true; + includeTriggers.Checked = initialOptions?.IncludeTriggers ?? true; BuildPage1(initialOptions); BuildPage2(); BuildPage3(); @@ -124,11 +128,14 @@ internal SqliteExportWizardForm(IDatabase db, string databaseName, SqliteExportW Shown += async (s, e) => await LoadCatalogAsync(); FormClosing += (s, e) => { - if (busy && !closing) + if (busy) { e.Cancel = true; - closing = true; - RequestCancel(); + if (!closing) + { + closing = true; + RequestCancel(); + } } }; } @@ -500,19 +507,29 @@ private void BuildPage3() private async Task LoadCatalogAsync() { - statusLabel.Text = T("正在載入 SQLite 資料庫目錄與空間中繼資訊...", "Loading SQLite database catalog and spatial metadata..."); - catalog = await Task.Run(() => service.LoadTableCatalog()); - views = await Task.Run(() => service.LoadViews()); - triggers = await Task.Run(() => service.LoadTriggers()); + busy = true; + try + { + statusLabel.Text = T("正在載入 SQLite 資料庫目錄與空間中繼資訊...", "Loading SQLite database catalog and spatial metadata..."); + catalog = await Task.Run(() => service.LoadTableCatalog()); + views = await Task.Run(() => service.LoadViews()); + triggers = await Task.Run(() => service.LoadTriggers()); - tableGrid.Rows.Clear(); - foreach (var item in catalog) + tableGrid.Rows.Clear(); + foreach (var item in catalog) + { + if (initialSelection != null) item.IsSelected = initialSelection.Contains(item.Name); + tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0"), item.SpatialInfo); + } + + statusLabel.Text = ""; + UpdateSelectionSummary(); + } + finally { - tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0"), item.SpatialInfo); + busy = false; + if (closing) Close(); } - - statusLabel.Text = ""; - UpdateSelectionSummary(); } private void ApplySearchFilter() @@ -618,18 +635,10 @@ private async Task OnNextAsync() } } - private async Task StartExportAsync() + private SqliteExportWizardOptions BuildExportOptions() { - busy = true; - exporting = true; - cancellation = new CancellationTokenSource(); - nextButton.Enabled = false; - backButton.Enabled = false; - cancelButton.Text = T("停止匯出", "Stop Export"); - logList.Items.Clear(); - var mode = schemaData.Checked ? SqliteExportMode.SchemaAndData : schemaOnly.Checked ? SqliteExportMode.SchemaOnly : SqliteExportMode.DataOnly; - var opt = new SqliteExportWizardOptions(mode, targetPathBox.Text.Trim()) + return new SqliteExportWizardOptions(mode, targetPathBox.Text.Trim()) { SingleTransaction = singleTransaction.Checked, PragmaForeignKeysOff = pragmaFk.Checked, @@ -641,9 +650,22 @@ private async Task StartExportAsync() IncludeComments = includeComments.Checked, ExcludeSpatialMetadata = excludeSpatialMetadata.Checked, SelectedTables = catalog.Where(t => t.IsSelected).Select(t => t.Name).ToList(), - SelectedViews = views, + SelectedViews = catalog.All(t => t.IsSelected) ? views : new List(), SelectedTriggers = triggers }; + } + + private async Task StartExportAsync() + { + busy = true; + exporting = true; + cancellation = new CancellationTokenSource(); + nextButton.Enabled = false; + backButton.Enabled = false; + cancelButton.Text = T("停止匯出", "Stop Export"); + logList.Items.Clear(); + + var opt = BuildExportOptions(); var progress = new Progress(p => { diff --git a/mySQLPunk/SqliteImportWizardForm.cs b/mySQLPunk/SqliteImportWizardForm.cs index 6a2e6be..99d1880 100644 --- a/mySQLPunk/SqliteImportWizardForm.cs +++ b/mySQLPunk/SqliteImportWizardForm.cs @@ -128,11 +128,14 @@ internal SqliteImportWizardForm(IDatabase db, string databaseName, string initia FormClosing += (s, e) => { - if (busy && !closing) + if (busy) { e.Cancel = true; - closing = true; - RequestCancel(); + if (!closing) + { + closing = true; + RequestCancel(); + } } }; } @@ -570,12 +573,13 @@ private async Task RunPreflightAsync() recheckButton.Enabled = true; backButton.Enabled = true; nextButton.Enabled = (preflight != null && preflight.Passed); + if (closing) Close(); } } private void UpdateConfirmSummary() { - var strat = (SqliteExistingObjectStrategy)strategyCombo.SelectedIndex; + var strat = SqliteExistingObjectSelection.FromIndex(strategyCombo.SelectedIndex); confirmSummary.Text = $"● {T("目標 SQLite 資料庫：", "Target Database: ")} {databaseName}\n" + $"● {T("來源 SQL 檔案：", "Source SQL: ")} {request.SourcePath} ({new FileInfo(request.SourcePath).Length / 1024.0 / 1024.0:N2} MB)\n" + @@ -631,7 +635,7 @@ private async Task StartImportAsync() var opt = new SqliteImportOptions { ContinueOnError = continueOnError.Checked, - ExistingObjectStrategy = (SqliteExistingObjectStrategy)strategyCombo.SelectedIndex, + ExistingObjectStrategy = SqliteExistingObjectSelection.FromIndex(strategyCombo.SelectedIndex), WrapInTransaction = wrapInTransaction.Checked, AutoLoadSpatiaLite = autoLoadSpatiaLite.Checked }; diff --git a/mySQLPunk/lib/ConnectionOpenService.cs b/mySQLPunk/lib/ConnectionOpenService.cs index eebb584..f2a5bb2 100644 --- a/mySQLPunk/lib/ConnectionOpenService.cs +++ b/mySQLPunk/lib/ConnectionOpenService.cs @@ -14,15 +14,17 @@ public sealed class ConnectionOpenResult public string ConnectionString { get; private set; } internal SqlConnection NativeMssqlConnection { get; private set; } + internal IDatabase NativeDatabase { get; private set; } public ConnectionOpenResult(IDatabase database, List databases, string connectionString = null) : this(database, databases, connectionString, null) { } - internal ConnectionOpenResult(IDatabase database, List databases, string connectionString, SqlConnection nativeMssqlConnection) + internal ConnectionOpenResult(IDatabase database, List databases, string connectionString, SqlConnection nativeMssqlConnection, IDatabase nativeDatabase = null) { NativeMssqlConnection = nativeMssqlConnection; + NativeDatabase = nativeDatabase ?? database; Database = database; Databases = databases ?? new List(); ConnectionString = connectionString ?? string.Empty; @@ -56,7 +58,12 @@ public static ConnectionOpenResult Open(Dictionary connection, b return Open(connection, loadDatabases, false); } - private static ConnectionOpenResult Open(Dictionary connection, bool loadDatabases, bool importOnly) + internal static ConnectionOpenResult OpenWizardConnection(Dictionary connection) + { + return Open(connection, false, false, true); + } + + private static ConnectionOpenResult Open(Dictionary connection, bool loadDatabases, bool importOnly, bool wizardOnly = false) { if (connection == null) throw new ArgumentNullException("connection"); Dictionary effective = new Dictionary(connection); @@ -79,13 +86,23 @@ private static ConnectionOpenResult Open(Dictionary connection, string connectionString = importOnly ? ConnectionConfigurationService.BuildMssqlImportConnectionString(effective) : ConnectionConfigurationService.BuildConnectionString(effective); + if (wizardOnly && provider == "mysql") + { + var builder = new MySqlConnectionStringBuilder(connectionString) { Pooling = false }; + connectionString = builder.ConnectionString; + } + if (wizardOnly && provider == "oracle") + { + var builder = new Oracle.ManagedDataAccess.Client.OracleConnectionStringBuilder(connectionString) { Pooling = false }; + connectionString = builder.ConnectionString; + } IDatabase rawDatabase = ConnectionConfigurationService.CreateDatabase(provider); database = tunnel == null ? rawDatabase : new SecuredDatabase(rawDatabase, tunnel); if (tunnel != null) tunnel = null; database.SetConn(connectionString); database.Open(); List databases = loadDatabases ? database.GetDatabases() : new List(); - return new ConnectionOpenResult(database, databases, connectionString, (rawDatabase as my_mssql)?.MCT); + return new ConnectionOpenResult(database, databases, connectionString, (rawDatabase as my_mssql)?.MCT, rawDatabase); } catch { diff --git a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs index f328895..784a8ff 100644 --- a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs +++ b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs @@ -4,6 +4,8 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Globalization; +using mySQLPunk.lib.MySqlImport; using System.Security.Cryptography; using System.Text; using System.Threading; @@ -59,7 +61,7 @@ public Task ExportAsync( IProgress progress, CancellationToken cancellation) { - return Task.Run(() => ExportInternal(options, progress, cancellation), cancellation); + return Task.Run(() => ExportInternal(options, progress, cancellation)); } private MySqlExportResult ExportInternal( @@ -90,35 +92,21 @@ private MySqlExportResult ExportInternal( var selectedRoutines = options.IncludeRoutines ? (options.SelectedRoutines?.ToList() ?? new List()) : new List(); var selectedTriggers = options.IncludeTriggers ? (options.SelectedTriggers?.ToList() ?? new List()) : new List(); - // 預估總列數 - if (options.Mode != MySqlExportMode.SchemaOnly) - { - foreach (var table in selectedTables) - { - try { estimatedTotalRows += db.CountRows(databaseName, table); } catch { } - } - } - bool inTransaction = false; + bool publicationAttempted = false; try { cancellation.ThrowIfCancellationRequested(); - - // 啟動可選一致性快照 (single-transaction) + RequireExecution("SET TIME_ZONE = '+00:00';"); if (options.SingleTransaction && options.Mode != MySqlExportMode.SchemaOnly) { - try - { - db.ExecSQL("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); - db.ExecSQL("START TRANSACTION WITH CONSISTENT SNAPSHOT;"); - inTransaction = true; - } - catch - { - // 若無足夠權限或引擎不支援一致性快照，安全降級為普通讀取 - inTransaction = false; - } + RequireTransactionalTables(selectedTables); + RequireExecution("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); + RequireExecution("START TRANSACTION WITH CONSISTENT SNAPSHOT;"); + inTransaction = true; } + if (options.Mode != MySqlExportMode.SchemaOnly) + foreach (var table in selectedTables) estimatedTotalRows += db.CountRows(databaseName, table); using (var fileStream = new FileStream(stagingPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 65536)) using (var writer = new StreamWriter(fileStream, new UTF8Encoding(false))) @@ -132,6 +120,7 @@ private MySqlExportResult ExportInternal( writer.WriteLine("-- ========================================================"); writer.WriteLine(); writer.WriteLine("SET NAMES utf8mb4;"); + writer.WriteLine("SET TIME_ZONE = '+00:00';"); if (options.SetNoAutoValueOnZero) writer.WriteLine("SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO';"); if (options.DisableForeignKeyChecks) @@ -143,24 +132,24 @@ private MySqlExportResult ExportInternal( var info = db.GetDatabaseInfo(databaseName) ?? new Dictionary(); string charset = info.ContainsKey("character_set") ? info["character_set"] : "utf8mb4"; string collation = info.ContainsKey("collation") ? info["collation"] : "utf8mb4_general_ci"; - writer.WriteLine($"CREATE DATABASE IF NOT EXISTS `{databaseName}` DEFAULT CHARACTER SET {charset} COLLATE {collation};"); + writer.WriteLine($"CREATE DATABASE IF NOT EXISTS {MySqlImportRunner.QuoteIdentifier(databaseName)} DEFAULT CHARACTER SET {charset} COLLATE {collation};"); } if (options.IncludeUseDatabase) - writer.WriteLine($"USE `{databaseName}`;"); + writer.WriteLine($"USE {MySqlImportRunner.QuoteIdentifier(databaseName)};"); writer.WriteLine(); // 1. DROP 語句 if (options.IncludeDropStatements && options.Mode != MySqlExportMode.DataOnly) { - foreach (var tr in selectedTriggers) writer.WriteLine($"DROP TRIGGER IF EXISTS `{tr}`;"); + foreach (var tr in selectedTriggers) writer.WriteLine($"DROP TRIGGER IF EXISTS {MySqlImportRunner.QuoteIdentifier(tr)};"); foreach (var r in selectedRoutines) { string type = r.StartsWith("PROCEDURE:", StringComparison.OrdinalIgnoreCase) ? "PROCEDURE" : "FUNCTION"; string name = r.Contains(":") ? r.Substring(r.IndexOf(':') + 1) : r; - writer.WriteLine($"DROP {type} IF EXISTS `{name}`;"); + writer.WriteLine($"DROP {type} IF EXISTS {MySqlImportRunner.QuoteIdentifier(name)};"); } - foreach (var v in selectedViews) writer.WriteLine($"DROP VIEW IF EXISTS `{v}`;"); - foreach (var t in selectedTables) writer.WriteLine($"DROP TABLE IF EXISTS `{t}`;"); + foreach (var v in selectedViews) writer.WriteLine($"DROP VIEW IF EXISTS {MySqlImportRunner.QuoteIdentifier(v)};"); + foreach (var t in selectedTables) writer.WriteLine($"DROP TABLE IF EXISTS {MySqlImportRunner.QuoteIdentifier(t)};"); writer.WriteLine(); } @@ -173,6 +162,7 @@ private MySqlExportResult ExportInternal( { cancellation.ThrowIfCancellationRequested(); string ddl = db.GetTableCreateStatement(databaseName, table); + if (string.IsNullOrWhiteSpace(ddl)) throw new InvalidOperationException("Missing table DDL: " + table); if (!string.IsNullOrWhiteSpace(ddl)) { writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); @@ -184,52 +174,62 @@ private MySqlExportResult ExportInternal( // 3. 資料列 (Data) if (options.Mode != MySqlExportMode.SchemaOnly) { - int batchSize = options.InsertBatchSize > 0 ? options.InsertBatchSize : 1000; + // IDatabase materializes a page; bound bytes before fetching. + int batchSize = Math.Min(options.InsertBatchSize > 0 ? options.InsertBatchSize : 16, 16); for (int i = 0; i < selectedTables.Count; i++) { cancellation.ThrowIfCancellationRequested(); string table = selectedTables[i]; long tableRows = 0; - long totalTableRows = 0; - try { totalTableRows = db.CountRows(databaseName, table); } catch { } + long totalTableRows = db.CountRows(databaseName, table); + var shape = LoadGeneratedColumnsAndCheckRowBound(table, cancellation); ReportProgress(progress, MySqlExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); while (tableRows < totalTableRows || totalTableRows == 0) { cancellation.ThrowIfCancellationRequested(); - DataTable page = db.SelectTablePage(databaseName, table, tableRows, batchSize); - if (page == null || page.Rows.Count == 0) break; - - writer.Write("INSERT INTO `"); - writer.Write(table.Replace("`", "``")); - writer.Write("` ("); - for (int c = 0; c < page.Columns.Count; c++) - { - if (c > 0) writer.Write(", "); - writer.Write("`"); - writer.Write(page.Columns[c].ColumnName.Replace("`", "``")); - writer.Write("`"); - } - writer.WriteLine(") VALUES"); - - for (int r = 0; r < page.Rows.Count; r++) + DataTable page = ReadBoundedPage(table, shape, tableRows, batchSize); + if (page == null) throw new InvalidOperationException("Source row query returned no result: " + table); + cancellation.ThrowIfCancellationRequested(); + if (page.Rows.Count == 0) break; + + var writableColumns = page.Columns.Cast().Where(c => !shape.Generated.Contains(c.ColumnName)).ToList(); + if (writableColumns.Count == 0) throw new NotSupportedException("Tables containing only generated columns require a native client: " + table); + string prefix = "INSERT INTO " + MySqlImportRunner.QuoteIdentifier(table) + " (" + + string.Join(", ", writableColumns.Select(c => MySqlImportRunner.QuoteIdentifier(c.ColumnName))) + ") VALUES\r\n"; + var insert = new StringBuilder(prefix); + foreach (DataRow row in page.Rows) { - writer.Write("("); - for (int c = 0; c < page.Columns.Count; c++) + cancellation.ThrowIfCancellationRequested(); + var rowText = new StringBuilder("("); + for (int c = 0; c < writableColumns.Count; c++) { - if (c > 0) writer.Write(", "); - writer.Write(MySqlExportService.ToMySqlLiteral(page.Rows[r][c])); + if (c > 0) rowText.Append(", "); + rowText.Append(ToBoundedLiteral(row[writableColumns[c]])); + if (rowText.Length > MaxInsertBytes) throw new NotSupportedException("Single row exceeds safe logical-export limit: " + table); } - writer.Write(r == page.Rows.Count - 1 ? ");" : "),"); - writer.WriteLine(); + rowText.Append(")"); + string rowSql = rowText.ToString(); + if (Encoding.UTF8.GetByteCount(prefix) + Encoding.UTF8.GetByteCount(rowSql) + 4 > MaxInsertBytes) + throw new NotSupportedException("Single row exceeds 8 MiB SQL limit: " + table); + if (Encoding.UTF8.GetByteCount(insert.ToString()) + Encoding.UTF8.GetByteCount(rowSql) + 4 > MaxInsertBytes) + { + writer.Write(insert.ToString()); + writer.WriteLine(";"); + insert.Clear().Append(prefix); + } + if (insert.Length > prefix.Length) insert.Append(",\r\n"); + insert.Append(rowSql); } + if (insert.Length > prefix.Length) { writer.Write(insert.ToString()); writer.WriteLine(";"); } tableRows += page.Rows.Count; totalProcessedRows += page.Rows.Count; ReportProgress(progress, MySqlExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); } + if (tableRows != totalTableRows) throw new InvalidOperationException("Source row count changed or a page was lost: " + table); if (tableRows > 0) writer.WriteLine(); tableCount++; tableManifest.Add(Tuple.Create(table, tableRows)); @@ -250,7 +250,7 @@ private MySqlExportResult ExportInternal( string ddl = db.GetViewCreateStatement(databaseName, view); if (!string.IsNullOrWhiteSpace(ddl)) { - ddl = MySqlExportService.NormalizeObjectCreateSql(ddl, options.RemoveDefiner); + ddl = NormalizeCreateSql(ddl, options.RemoveDefiner); writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); writer.WriteLine(); } @@ -270,7 +270,7 @@ private MySqlExportResult ExportInternal( (r.Type + ":" + r.Name).Equals(rName, StringComparison.OrdinalIgnoreCase)); if (routine != null) { - string sql = $"SHOW CREATE {routine.Type} `{databaseName}`.`{routine.Name}`;"; + string sql = $"SHOW CREATE {routine.Type} {MySqlImportRunner.QuoteIdentifier(databaseName)}.{MySqlImportRunner.QuoteIdentifier(routine.Name)};"; var dt = db.SelectSQL(sql, new Dictionary()); if (dt != null && dt.Rows.Count > 0) { @@ -278,7 +278,7 @@ private MySqlExportResult ExportInternal( string ddl = dt.Columns.Contains(createKey) ? dt.Rows[0][createKey]?.ToString() : dt.Rows[0][2]?.ToString(); if (!string.IsNullOrWhiteSpace(ddl)) { - ddl = MySqlExportService.NormalizeObjectCreateSql(ddl, options.RemoveDefiner); + ddl = NormalizeCreateSql(ddl, options.RemoveDefiner); writer.WriteLine("DELIMITER ;;"); writer.WriteLine(ddl.TrimEnd().TrimEnd(';')); writer.WriteLine(";;"); @@ -298,14 +298,14 @@ private MySqlExportResult ExportInternal( foreach (var triggerName in selectedTriggers) { cancellation.ThrowIfCancellationRequested(); - string sql = $"SHOW CREATE TRIGGER `{databaseName}`.`{triggerName}`;"; + string sql = $"SHOW CREATE TRIGGER {MySqlImportRunner.QuoteIdentifier(databaseName)}.{MySqlImportRunner.QuoteIdentifier(triggerName)};"; var dt = db.SelectSQL(sql, new Dictionary()); if (dt != null && dt.Rows.Count > 0) { string ddl = dt.Columns.Contains("SQL Original Statement") ? dt.Rows[0]["SQL Original Statement"]?.ToString() : dt.Rows[0][2]?.ToString(); if (!string.IsNullOrWhiteSpace(ddl)) { - ddl = MySqlExportService.NormalizeObjectCreateSql(ddl, options.RemoveDefiner); + ddl = NormalizeCreateSql(ddl, options.RemoveDefiner); writer.WriteLine("DELIMITER ;;"); writer.WriteLine(ddl.TrimEnd().TrimEnd(';')); writer.WriteLine(";;"); @@ -338,6 +338,13 @@ private MySqlExportResult ExportInternal( writer.Flush(); } + cancellation.ThrowIfCancellationRequested(); + if (inTransaction) + { + RequireExecution("ROLLBACK;"); + inTransaction = false; + } + // 計算產出檔案的 SHA-256 string sha256Hex; long finalLength; @@ -346,8 +353,15 @@ private MySqlExportResult ExportInternal( finalLength = fs.Length; using (var sha = SHA256.Create()) { - byte[] hash = sha.ComputeHash(fs); - sha256Hex = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); + var buffer = new byte[65536]; + int read; + while ((read = fs.Read(buffer, 0, buffer.Length)) > 0) + { + cancellation.ThrowIfCancellationRequested(); + sha.TransformBlock(buffer, 0, read, buffer, 0); + } + sha.TransformFinalBlock(new byte[0], 0, 0); + sha256Hex = BitConverter.ToString(sha.Hash).Replace("-", "").ToLowerInvariant(); } } @@ -355,8 +369,10 @@ private MySqlExportResult ExportInternal( File.AppendAllText(stagingPath, $"-- Manifest-SHA256: {sha256Hex}{Environment.NewLine}", Encoding.UTF8); // 原子替換到目標檔案 - if (File.Exists(targetPath)) File.Delete(targetPath); - File.Move(stagingPath, targetPath); + cancellation.ThrowIfCancellationRequested(); + finalLength = new FileInfo(stagingPath).Length; + publicationAttempted = true; + SqlExportPublication.Publish(stagingPath, targetPath); clock.Stop(); ReportProgress(progress, MySqlExportStage.Completed, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, finalLength, clock); @@ -375,7 +391,7 @@ private MySqlExportResult ExportInternal( } catch (OperationCanceledException) { - CleanStaging(stagingPath); + if (!publicationAttempted) CleanStaging(stagingPath); clock.Stop(); return new MySqlExportResult( MySqlExportStage.Cancelled, @@ -392,7 +408,7 @@ private MySqlExportResult ExportInternal( } catch (Exception ex) { - CleanStaging(stagingPath); + if (!publicationAttempted) CleanStaging(stagingPath); clock.Stop(); return new MySqlExportResult( MySqlExportStage.Failed, @@ -405,15 +421,155 @@ private MySqlExportResult ExportInternal( 0, null, clock.Elapsed, - ex.Message); + ex.Message + (publicationAttempted ? " Completed staging file retained at: " + stagingPath : "")); } finally { if (inTransaction) { - try { db.ExecSQL("COMMIT;"); } catch { } + try { db.ExecSQL("ROLLBACK;"); } catch { } + } + } + } + + private static string NormalizeCreateSql(string sql, bool removeDefiner) + { + if (!removeDefiner) return sql; + var tokens = MySqlScriptReader.Tokens(sql); + // DEFINER is only a CREATE header clause. Never run replacement on + // string constants or on the body of a stored routine. + for (int i = 1; i < tokens.Count; i++) + { + string value = tokens[i].Value.ToUpperInvariant(); + if (!tokens[i].Quoted && (value == "VIEW" || value == "TRIGGER" || value == "PROCEDURE" || value == "FUNCTION")) break; + if (!tokens[i].Quoted && value == "DEFINER" && i + 4 < tokens.Count && tokens[i + 1].Value == "=") + { + int end = i + 3; + if (tokens[end].Value == "@" && end + 1 < tokens.Count) end += 2; + if (end < tokens.Count) sql = sql.Remove(tokens[i].Start, tokens[end].Start - tokens[i].Start); + break; } } + tokens = MySqlScriptReader.Tokens(sql); + for (int i = tokens.Count - 3; i >= 0; i--) + if (!tokens[i].Quoted && !tokens[i + 1].Quoted && !tokens[i + 2].Quoted && + tokens[i].Value.Equals("SQL", StringComparison.OrdinalIgnoreCase) && + tokens[i + 1].Value.Equals("SECURITY", StringComparison.OrdinalIgnoreCase) && + tokens[i + 2].Value.Equals("DEFINER", StringComparison.OrdinalIgnoreCase)) + sql = MySqlScriptReader.ReplaceToken(sql, tokens[i + 2], "INVOKER"); + return sql; + } + + private void RequireTransactionalTables(List selectedTables) + { + DataTable status = db.GetTableStatus(databaseName); + if (status == null || !status.Columns.Contains("Name") || !status.Columns.Contains("Engine")) + throw new InvalidOperationException("Cannot verify table engines for a consistent snapshot."); + foreach (string table in selectedTables) + { + DataRow entry = status.Rows.Cast().FirstOrDefault(r => string.Equals(Convert.ToString(r["Name"]), table, StringComparison.OrdinalIgnoreCase)); + if (entry == null || !string.Equals(Convert.ToString(entry["Engine"]), "InnoDB", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException("Consistent snapshot requires verified InnoDB tables: " + table); + } + } + + private const int MaxSourceRowBytes = 1024 * 1024; + private const int MaxInsertBytes = 8 * 1024 * 1024; + + private void RequireExecution(string sql) + { + var result = db.ExecSQL(sql); + if (!MySqlImportRunner.IsExecutionSuccess(result)) + throw new InvalidOperationException(DatabaseExecutionResultService.GetFailureReason(result)); + } + + private TableShape LoadGeneratedColumnsAndCheckRowBound(string table, CancellationToken cancellation) + { + DataTable columns = db.GetColumns(databaseName, table); + if (columns == null || columns.Rows.Count == 0) throw new InvalidOperationException("Missing column metadata: " + table); + string nameColumn = columns.Columns.Contains("Field") ? "Field" : "COLUMN_NAME"; + string extraColumn = columns.Columns.Contains("Extra") ? "Extra" : "EXTRA"; + if (!columns.Columns.Contains(nameColumn) || !columns.Columns.Contains(extraColumn)) + throw new NotSupportedException("Generated-column metadata unavailable: " + table); + var shape = new TableShape(); + var generated = shape.Generated; + var sizes = new List(); + foreach (DataRow column in columns.Rows) + { + string name = Convert.ToString(column[nameColumn]); + string extra = Convert.ToString(column[extraColumn]); + string expression = columns.Columns.Contains("GenerationExpression") ? Convert.ToString(column["GenerationExpression"]) : ""; + if (extra.IndexOf("VIRTUAL", StringComparison.OrdinalIgnoreCase) >= 0 || + extra.IndexOf("STORED", StringComparison.OrdinalIgnoreCase) >= 0 || !string.IsNullOrWhiteSpace(expression)) generated.Add(name); + string type = columns.Columns.Contains("Type") ? Convert.ToString(column["Type"]) : + columns.Columns.Contains("DATA_TYPE") ? Convert.ToString(column["DATA_TYPE"]) : ""; + if (type.IndexOf("geometry", StringComparison.OrdinalIgnoreCase) >= 0 || type.IndexOf("point", StringComparison.OrdinalIgnoreCase) >= 0 || + type.IndexOf("polygon", StringComparison.OrdinalIgnoreCase) >= 0 || type.IndexOf("linestring", StringComparison.OrdinalIgnoreCase) >= 0) + throw new NotSupportedException("Spatial column values require a native MySQL export: " + table + "." + name); + shape.Names.Add(name); + sizes.Add("COALESCE(OCTET_LENGTH(" + MySqlImportRunner.QuoteIdentifier(name) + "),0)"); + } + cancellation.ThrowIfCancellationRequested(); + string sql = "SELECT COALESCE(MAX(" + string.Join(" + ", sizes) + "),0) AS MaxRowBytes FROM " + + MySqlImportRunner.QuoteIdentifier(databaseName) + "." + MySqlImportRunner.QuoteIdentifier(table) + ";"; + DataTable bounds = db.SelectSQL(sql); + if (bounds == null || bounds.Rows.Count != 1 || !bounds.Columns.Contains("MaxRowBytes") || bounds.Rows[0]["MaxRowBytes"] == DBNull.Value) + throw new InvalidOperationException("Cannot establish a safe source row-size bound: " + table); + long maxBytes = Convert.ToInt64(bounds.Rows[0]["MaxRowBytes"], CultureInfo.InvariantCulture); + if (maxBytes < 0 || maxBytes > MaxSourceRowBytes) + throw new NotSupportedException("Source row exceeds 1 MiB materialization limit; use a native MySQL export: " + table); + shape.ByteExpression = string.Join(" + ", sizes); + return shape; + } + + private sealed class TableShape + { + public readonly List Names = new List(); + public readonly HashSet Generated = new HashSet(StringComparer.OrdinalIgnoreCase); + public string ByteExpression; + } + + private DataTable ReadBoundedPage(string table, TableShape shape, long offset, int limit) + { + string marker = "__mysqlpunk_row_bytes"; + while (shape.Names.Contains(marker)) marker += "_"; + // Server-side CASE prevents a concurrently enlarged LOB from being + // materialized even when the user disabled the snapshot option. + string predicate = "(" + shape.ByteExpression + ") <= " + MaxSourceRowBytes; + string projection = string.Join(", ", shape.Names.Select(name => "CASE WHEN " + predicate + " THEN " + + MySqlImportRunner.QuoteIdentifier(name) + " ELSE NULL END AS " + MySqlImportRunner.QuoteIdentifier(name))); + DataTable page = db.SelectSQL("SELECT " + projection + ", (" + shape.ByteExpression + ") AS " + marker + + " FROM " + MySqlImportRunner.QuoteIdentifier(databaseName) + "." + MySqlImportRunner.QuoteIdentifier(table) + + " LIMIT " + offset.ToString(CultureInfo.InvariantCulture) + ", " + limit.ToString(CultureInfo.InvariantCulture) + ";"); + if (page == null || !page.Columns.Contains(marker)) throw new InvalidOperationException("Bounded source page query failed: " + table); + foreach (DataRow row in page.Rows) + if (row[marker] == DBNull.Value || Convert.ToInt64(row[marker], CultureInfo.InvariantCulture) > MaxSourceRowBytes) + throw new NotSupportedException("Source row exceeds 1 MiB materialization limit: " + table); + page.Columns.Remove(marker); + return page; + } + + private static string ToBoundedLiteral(object value) + { + if (value is byte[] && ((byte[])value).Length > MaxSourceRowBytes || + value is string && Encoding.UTF8.GetByteCount((string)value) > MaxSourceRowBytes) + throw new NotSupportedException("Source value exceeds 1 MiB logical-export limit."); + if (value is TimeSpan) + { + TimeSpan span = (TimeSpan)value; + if (span.Ticks % 10 != 0) throw new NotSupportedException("TIME precision exceeds MySQL microseconds."); + bool negative = span < TimeSpan.Zero; + TimeSpan abs = span.Duration(); + return "'" + (negative ? "-" : "") + ((long)abs.TotalHours).ToString("00", CultureInfo.InvariantCulture) + + ":" + abs.Minutes.ToString("00") + ":" + abs.Seconds.ToString("00") + + "." + ((abs.Ticks % TimeSpan.TicksPerSecond) / 10).ToString("000000") + "'"; + } + if (value != null && value != DBNull.Value && !(value is string) && !(value is char) && !(value is byte[]) && + !(value is bool) && !(value is DateTime) && !(value is decimal) && !(value is double) && !(value is float) && + !(value is byte) && !(value is sbyte) && !(value is short) && !(value is ushort) && !(value is int) && !(value is uint) && + !(value is long) && !(value is ulong) && value.GetType().FullName != "MySqlConnector.MySqlDateTime") + throw new NotSupportedException("Unsupported MySQL value type: " + value.GetType().FullName); + return MySqlExportService.ToMySqlLiteral(value); } private static void CleanStaging(string path) diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs b/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs index 8c0a4a0..2de03be 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportLog.cs @@ -28,7 +28,7 @@ public MySqlImportLog(Guid jobId, string directory = null) Path = System.IO.Path.Combine(directory, stem + ".log"); stream = new FileStream(Path, FileMode.Create, FileAccess.Write, FileShare.Read); - writer = new StreamWriter(stream, Encoding.UTF8); + writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true }; WriteHeader(jobId); } diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs index 086bbb9..c50f044 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportPreflightScanner.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Security.Cryptography; using System.Text.RegularExpressions; using System.Threading; @@ -33,97 +34,21 @@ public static MySqlImportPreflight Scan(string filePath, CancellationToken cance try { using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) - using (var reader = new StreamReader(fs, Encoding.UTF8, true)) + using (var reader = new StreamReader(fs, new UTF8Encoding(false, true), true)) { - string delimiter = ";"; - var currentBatch = new StringBuilder(); - int lineIndex = 0; - string line; - - while ((line = reader.ReadLine()) != null) - { - cancellation.ThrowIfCancellationRequested(); - lineIndex++; - - // 檢查 embedded NUL (非法字元或編碼錯誤) - if (line.IndexOf('\0') >= 0) - { - issues.Add($"Line {lineIndex}: Contains Embedded NUL character (possible malformed encoding or binary data)."); - break; - } - - string trimmed = line.Trim(); - - // 檢查 Manifest - if (trimmed.StartsWith("--")) - { - if (trimmed.Contains("mySQLPunk Native Manifest")) - { - hasManifest = true; - } - else if (hasManifest) - { - var mTables = ManifestTablesRegex.Match(trimmed); - if (mTables.Success && int.TryParse(mTables.Groups[1].Value, out int mt)) - { - manifestTables = mt; - continue; - } - var mRows = ManifestRowsRegex.Match(trimmed); - if (mRows.Success && long.TryParse(mRows.Groups[1].Value, out long mr)) - { - manifestRows = mr; - continue; - } - var mTable = ManifestTableRegex.Match(trimmed); - if (mTable.Success) - { - if (manifestTables == 0) manifestTables++; - if (long.TryParse(mTable.Groups[2].Value, out long r) && manifestRows == 0) manifestRows += r; - continue; - } - var mSha = ManifestSha256Regex.Match(trimmed); - if (mSha.Success) - { - manifestSha = mSha.Groups[1].Value; - } - } - continue; - } - - // DELIMITER 宣告 - if (trimmed.StartsWith("DELIMITER", StringComparison.OrdinalIgnoreCase)) - { - string newDelim = trimmed.Substring(9).Trim(); - if (!string.IsNullOrWhiteSpace(newDelim)) - { - delimiter = newDelim; - currentBatch.Clear(); - continue; - } - } - - currentBatch.AppendLine(line); - - // 單批次大小檢查 (32 MiB) - if (currentBatch.Length > MaxBatchBytes) - { - issues.Add($"Line {lineIndex}: Single SQL batch exceeds safe limit of 32 MiB."); - break; - } - - // 檢查批次終止 - if (trimmed.EndsWith(delimiter, StringComparison.Ordinal)) - { - statementCount++; - currentBatch.Clear(); - } - } - - if (currentBatch.Length > 0 && !string.IsNullOrWhiteSpace(currentBatch.ToString())) + foreach (var statement in MySqlScriptReader.Read(reader, cancellation, comment => { - statementCount++; - } + if (comment.Contains("mySQLPunk Native Manifest")) hasManifest = true; + if (!hasManifest) return; + var tables = ManifestTablesRegex.Match(comment); + if (tables.Success && int.TryParse(tables.Groups[1].Value, out int mt)) manifestTables = mt; + var rows = ManifestRowsRegex.Match(comment); + if (rows.Success && long.TryParse(rows.Groups[1].Value, out long mr)) manifestRows = mr; + var sha = ManifestSha256Regex.Match(comment); + if (sha.Success) manifestSha = sha.Groups[1].Value; + })) statementCount++; + if (manifestSha != null && !VerifyHash(filePath, manifestSha, cancellation)) + issues.Add("Native manifest SHA-256 does not match the SQL file contents."); } } catch (OperationCanceledException) @@ -146,5 +71,42 @@ public static MySqlImportPreflight Scan(string filePath, CancellationToken cance manifestSha, issues); } + private static bool VerifyHash(string path, string expected, CancellationToken cancellation) + { + using (var stream = File.OpenRead(path)) + using (var sha = SHA256.Create()) + { + int tailSize = (int)Math.Min(1024, stream.Length); + stream.Position = stream.Length - tailSize; + var tail = new byte[tailSize]; + int read = stream.Read(tail, 0, tail.Length); + byte[] marker = Encoding.ASCII.GetBytes("-- Manifest-SHA256: " + expected); + int index = -1; + for (int i = read - marker.Length; i >= 0; i--) + { + bool match = true; + for (int j = 0; j < marker.Length; j++) + if (char.ToUpperInvariant((char)tail[i + j]) != char.ToUpperInvariant((char)marker[j])) { match = false; break; } + if (match) { index = i; break; } + } + if (index < 0 || (index > 0 && tail[index - 1] != 10 && tail[index - 1] != 13)) return false; + for (int i = index + marker.Length; i < read; i++) + if (tail[i] != 10 && tail[i] != 13 && tail[i] != 32 && tail[i] != 9) return false; + long remaining = stream.Length - tailSize + index; + stream.Position = 0; + var buffer = new byte[65536]; + while (remaining > 0) + { + cancellation.ThrowIfCancellationRequested(); + int count = stream.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); + if (count == 0) return false; + sha.TransformBlock(buffer, 0, count, buffer, 0); + remaining -= count; + } + sha.TransformFinalBlock(new byte[0], 0, 0); + return string.Equals(BitConverter.ToString(sha.Hash).Replace("-", ""), expected, StringComparison.OrdinalIgnoreCase); + } + } + } } diff --git a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs index 2125e14..30ab5d9 100644 --- a/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs +++ b/mySQLPunk/lib/MySqlImport/MySqlImportRunner.cs @@ -30,10 +30,6 @@ public sealed class MySqlImportRunner @"^\s*CREATE\b[\s\S]*?\b(?TABLE|VIEW|FUNCTION|PROCEDURE|TRIGGER)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex InsertTargetRegex = new Regex( - @"^\s*(?:INSERT|REPLACE)\s+(?:IGNORE\s+)?INTO\s+(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - private readonly IDatabase db; private readonly string databaseName; @@ -49,7 +45,7 @@ public Task RunAsync( IProgress progress, CancellationToken cancellation) { - return Task.Run(() => RunInternal(request, options, progress, cancellation), cancellation); + return Task.Run(() => RunInternal(request, options, progress, cancellation)); } private MySqlImportResult RunInternal( @@ -73,8 +69,10 @@ private MySqlImportResult RunInternal( long totalRowsAffected = 0; var errors = new List(); var manifestTables = new List>(); - bool readingManifest = false; bool haltedOnError = false; + bool scriptTransaction = false; + bool sessionTouched = false; + bool sessionCleaned = false; string safeTargetDb = QuoteIdentifier(databaseName); string activeSourceDb = null; @@ -82,10 +80,17 @@ private MySqlImportResult RunInternal( var existingTables = new HashSet(StringComparer.OrdinalIgnoreCase); var existingViews = new HashSet(StringComparer.OrdinalIgnoreCase); var skippedTables = new HashSet(StringComparer.OrdinalIgnoreCase); + var initiallyExistingTables = new HashSet(StringComparer.OrdinalIgnoreCase); try { + cancellation.ThrowIfCancellationRequested(); + var preflight = MySqlImportPreflightScanner.Scan(request.SourcePath, cancellation); + cancellation.ThrowIfCancellationRequested(); + if (!preflight.Passed) throw new InvalidDataException(string.Join("; ", preflight.Issues)); + // 1. 切換至目標資料庫並鎖定 Session + sessionTouched = true; var useRes = db.ExecSQL($"USE {safeTargetDb};"); if (!IsExecutionSuccess(useRes)) { @@ -94,10 +99,10 @@ private MySqlImportResult RunInternal( } // 2. 初始化 Session 環境變數以加快匯入並確保相容性 - try { db.ExecSQL("SET NAMES utf8mb4;"); } catch { } - try { db.ExecSQL("SET FOREIGN_KEY_CHECKS = 0;"); } catch { } - try { db.ExecSQL("SET UNIQUE_CHECKS = 0;"); } catch { } - try { db.ExecSQL("SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO';"); } catch { } + RequireExecution("SET NAMES utf8mb4;"); + RequireExecution("SET FOREIGN_KEY_CHECKS = 0;"); + RequireExecution("SET UNIQUE_CHECKS = 0;"); + RequireExecution("SET SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO';"); // 3. 快取目標資料庫中既有物件，避免頻繁發送 SHOW TABLES try @@ -105,7 +110,8 @@ private MySqlImportResult RunInternal( var tList = db.GetTables(databaseName); if (tList != null) existingTables.UnionWith(tList); } - catch { } + catch { throw new InvalidOperationException("Cannot read initial target table catalog safely."); } + initiallyExistingTables.UnionWith(existingTables); try { @@ -115,135 +121,39 @@ private MySqlImportResult RunInternal( catch { } using (var fs = new FileStream(request.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) - using (var reader = new StreamReader(fs, Encoding.UTF8, true)) + using (var reader = new StreamReader(fs, new UTF8Encoding(false, true), true)) { - string delimiter = ";"; - var currentBatch = new StringBuilder(); - int batchStartLine = 1; - int currentLine = 0; - string line; - long lastReportMs = 0; - - while ((line = reader.ReadLine()) != null) + foreach (var statement in MySqlScriptReader.Read(reader, cancellation, comment => { - cancellation.ThrowIfCancellationRequested(); - currentLine++; - - string trimmed = line.Trim(); - - // 檢查 Native Manifest - if (trimmed.StartsWith("--")) - { - if (trimmed.Contains("mySQLPunk Native Manifest") && !readingManifest) - { - readingManifest = true; - log.LogInfo("Detected Native Manifest in SQL script footer."); - } - - var match = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); - if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) - { - manifestTables.Add(Tuple.Create(match.Groups[1].Value, r)); - } - continue; - } - - // DELIMITER 切換 - if (trimmed.StartsWith("DELIMITER", StringComparison.OrdinalIgnoreCase)) - { - string newDelim = trimmed.Substring(9).Trim(); - if (!string.IsNullOrWhiteSpace(newDelim)) - { - delimiter = newDelim; - currentBatch.Clear(); - batchStartLine = currentLine + 1; - continue; - } - } - - if (currentBatch.Length == 0) batchStartLine = currentLine; - currentBatch.AppendLine(line); - - // 判斷是否為批次結尾 - if (trimmed.EndsWith(delimiter, StringComparison.Ordinal)) - { - string sql = currentBatch.ToString().TrimEnd(); - if (sql.EndsWith(delimiter, StringComparison.Ordinal)) - sql = sql.Substring(0, sql.Length - delimiter.Length).Trim(); - - if (!string.IsNullOrWhiteSpace(sql)) - { - string headline; - bool shouldContinue = ProcessAndExecuteStatement( - sql, - batchStartLine, - options, - log, - safeTargetDb, - ref activeSourceDb, - existingTables, - existingViews, - skippedTables, - ref executed, - ref failed, - ref skipped, - ref totalRowsAffected, - errors, - out headline); - - if (!shouldContinue && !options.ContinueOnError) - { - haltedOnError = true; - log.LogInfo("Execution stopped on error as requested by options."); - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, totalRowsAffected, clock, headline); - break; - } - - long now = clock.ElapsedMilliseconds; - bool isDdl = IsSignificantStatement(headline); - if (!shouldContinue || isDdl || (now - lastReportMs >= 60)) - { - lastReportMs = now; - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, totalRowsAffected, clock, headline); - } - } - - currentBatch.Clear(); - batchStartLine = currentLine + 1; - } - } - - // 處理尾端剩餘語句 - if (currentBatch.Length > 0 && !haltedOnError && (failed == 0 || options.ContinueOnError)) + var match = Regex.Match(comment, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); + if (match.Success && long.TryParse(match.Groups[2].Value, out long rows)) + manifestTables.Add(Tuple.Create(match.Groups[1].Value, rows)); + })) { - string remainingSql = currentBatch.ToString().Trim(); - if (!string.IsNullOrWhiteSpace(remainingSql)) + cancellation.ThrowIfCancellationRequested(); + string headline; + bool ok = ProcessAndExecuteStatement(statement.Sql, statement.Line, options, log, + safeTargetDb, ref activeSourceDb, existingTables, existingViews, skippedTables, + initiallyExistingTables, ref executed, ref failed, ref skipped, ref totalRowsAffected, + errors, out headline); + cancellation.ThrowIfCancellationRequested(); + if (ok) { - string headline; - bool ok = ProcessAndExecuteStatement( - remainingSql, - batchStartLine, - options, - log, - safeTargetDb, - ref activeSourceDb, - existingTables, - existingViews, - skippedTables, - ref executed, - ref failed, - ref skipped, - ref totalRowsAffected, - errors, - out headline); - if (!ok && !options.ContinueOnError) - { - haltedOnError = true; - } - ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, executed, failed, skipped, totalRowsAffected, clock, headline); + var tokens = MySqlScriptReader.Tokens(statement.Sql); + if (Keyword(tokens[0], "BEGIN") || Keyword(tokens[0], "START")) scriptTransaction = true; + if (Keyword(tokens[0], "COMMIT") || Keyword(tokens[0], "ROLLBACK")) scriptTransaction = false; } + ReportProgress(progress, request.JobId, MySqlImportStage.Executing, fileBytes, fs.Position, + executed, failed, skipped, totalRowsAffected, clock, headline); + if (!ok && !options.ContinueOnError) { haltedOnError = true; break; } } } + cancellation.ThrowIfCancellationRequested(); + + if (scriptTransaction) throw new InvalidDataException("Script ended with an uncommitted transaction; pending rows were rolled back."); + sessionCleaned = true; + if (!CleanupSession(log, errors)) throw new InvalidOperationException("Import session cleanup failed; completion cannot be confirmed."); + cancellation.ThrowIfCancellationRequested(); // 4. 若有 Native Manifest，自動驗證目標資料庫實際列數 bool manifestMismatch = false; @@ -322,13 +232,13 @@ private MySqlImportResult RunInternal( errors); log.WriteEnd(finalResult); - log.Dispose(); ReportProgress(progress, request.JobId, MySqlImportStage.Finished, fileBytes, fileBytes, executed, failed, skipped, totalRowsAffected, clock, summary); return finalResult; } catch (OperationCanceledException) { + if (sessionTouched && !sessionCleaned) { sessionCleaned = true; CleanupSession(log, errors); } clock.Stop(); log.LogInfo("Execution cancelled by user."); var cancelResult = new MySqlImportResult( @@ -344,11 +254,11 @@ private MySqlImportResult RunInternal( "Import was cancelled by user.", errors); log.WriteEnd(cancelResult); - log.Dispose(); return cancelResult; } catch (Exception ex) { + if (sessionTouched && !sessionCleaned) { sessionCleaned = true; CleanupSession(log, errors); } clock.Stop(); log.LogInfo("Fatal error: " + ex.Message); errors.Add(ex.Message); @@ -365,15 +275,20 @@ private MySqlImportResult RunInternal( ex.Message, errors); log.WriteEnd(failResult); - log.Dispose(); return failResult; } - finally + finally { log.Dispose(); } + } + + private bool CleanupSession(MySqlImportLog log, List errors) + { + bool success = true; + foreach (string sql in new[] { "ROLLBACK;", "SET FOREIGN_KEY_CHECKS = 1;", "SET UNIQUE_CHECKS = 1;" }) { - // 還原外鍵與唯一性約束檢查 - try { db.ExecSQL("SET FOREIGN_KEY_CHECKS = 1;"); } catch { } - try { db.ExecSQL("SET UNIQUE_CHECKS = 1;"); } catch { } + try { RequireExecution(sql); } + catch (Exception ex) { success = false; errors.Add("Session cleanup: " + ex.Message); log.LogInfo("Session cleanup failed: " + ex.Message); } } + return success; } private bool ProcessAndExecuteStatement( @@ -386,6 +301,7 @@ private bool ProcessAndExecuteStatement( HashSet existingTables, HashSet existingViews, HashSet skippedTables, + HashSet initiallyExistingTables, ref int executed, ref int failed, ref int skipped, @@ -394,9 +310,11 @@ private bool ProcessAndExecuteStatement( out string headline) { string cleanSql = SanitizeSqlHeadline(sql); + string code = MySqlScriptReader.CodeOnly(sql); + ValidateTargetSyntax(sql); // A. USE 語句重定向 - Match useMatch = UseDatabaseRegex.Match(sql); + Match useMatch = UseDatabaseRegex.Match(code); if (useMatch.Success) { string targetInSql = UnquoteIdentifier(useMatch.Groups["database"].Value); @@ -405,28 +323,26 @@ private bool ProcessAndExecuteStatement( activeSourceDb = targetInSql; log.LogInfo($"Redirected statement from 'USE `{targetInSql}`' to 'USE {safeTargetDb}'"); } - sql = $"USE {safeTargetDb};"; + var useTokens = MySqlScriptReader.Tokens(sql); + sql = MySqlScriptReader.ReplaceToken(sql, useTokens[1], safeTargetDb); return ExecuteBatchStatement(sql, line, options, log, ref executed, ref failed, ref skipped, ref totalRowsAffected, existingTables, existingViews, errors, out headline); } // B. CREATE DATABASE / SCHEMA 重定向 - if (CreateDatabaseRegex.IsMatch(sql)) + if (CreateDatabaseRegex.IsMatch(code)) { log.LogInfo($"Redirected CREATE DATABASE to ensure target database {safeTargetDb} exists"); sql = $"CREATE DATABASE IF NOT EXISTS {safeTargetDb};"; return ExecuteBatchStatement(sql, line, options, log, ref executed, ref failed, ref skipped, ref totalRowsAffected, existingTables, existingViews, errors, out headline); } - // 若先前偵測到來源資料庫名稱，且與目標不同，將顯式資料庫限定名稱改寫至目標庫 - if (!string.IsNullOrEmpty(activeSourceDb) && !string.Equals(activeSourceDb, databaseName, StringComparison.OrdinalIgnoreCase)) - { - sql = RewriteDatabaseQualifier(sql, activeSourceDb, databaseName); - } + sql = EnforceTargetQualifiers(sql, activeSourceDb, databaseName); + code = MySqlScriptReader.CodeOnly(sql); // C. DROP 語句策略判定 if ((options.ExistingObjectStrategy == MySqlExistingObjectStrategy.CreateIfMissing || options.ExistingObjectStrategy == MySqlExistingObjectStrategy.SkipExisting) && - DropObjectRegex.IsMatch(sql)) + DropObjectRegex.IsMatch(code)) { skipped++; log.LogStatement(line, cleanSql, true, null, "SKIP_DROP"); @@ -435,7 +351,7 @@ private bool ProcessAndExecuteStatement( } // D. CREATE 物件策略判定 - Match createMatch = CreateObjectRegex.Match(sql); + Match createMatch = CreateObjectRegex.Match(code); if (createMatch.Success) { string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); @@ -463,7 +379,7 @@ private bool ProcessAndExecuteStatement( if (exists && options.ExistingObjectStrategy == MySqlExistingObjectStrategy.DropAndRecreate) { string dropSql = $"DROP {objType} IF EXISTS {safeTargetDb}.{QuoteIdentifier(objName)};"; - db.ExecSQL(dropSql); + if (!ExecuteBatchStatement(dropSql, line, options, log, ref executed, ref failed, ref skipped, ref totalRowsAffected, existingTables, existingViews, errors, out headline)) return false; existingTables.Remove(objName); existingViews.Remove(objName); log.LogStatement(line, $"DROP {objType} IF EXISTS `{objName}`", true, null, "DROP_BEFORE_CREATE"); @@ -473,11 +389,15 @@ private bool ProcessAndExecuteStatement( // E. INSERT / REPLACE 資料寫入策略判定 if (options.ExistingObjectStrategy == MySqlExistingObjectStrategy.SkipExisting) { - Match insertMatch = InsertTargetRegex.Match(sql); - if (insertMatch.Success) + var writeTokens = MySqlScriptReader.Tokens(sql); + int insertTarget = FindInsertTargetIndex(writeTokens); + if (insertTarget == -2) throw new InvalidDataException("Cannot identify INSERT/REPLACE target safely under SkipExisting."); + if (insertTarget >= 0) { - string insTable = UnquoteIdentifier(insertMatch.Groups["name"].Value); - if (skippedTables.Contains(insTable) || existingTables.Contains(insTable)) + int name = insertTarget; + if (name + 2 < writeTokens.Count && writeTokens[name + 1].Value == ".") name += 2; + string insTable = writeTokens[name].Value; + if (skippedTables.Contains(insTable) || initiallyExistingTables.Contains(insTable)) { skipped++; log.LogStatement(line, cleanSql, true, null, "SKIP_DATA"); @@ -577,7 +497,8 @@ private static void UpdateObjectCacheOnSuccess(string sql, HashSet exist { if (string.IsNullOrWhiteSpace(sql)) return; - Match createMatch = CreateObjectRegex.Match(sql); + string code = MySqlScriptReader.CodeOnly(sql); + Match createMatch = CreateObjectRegex.Match(code); if (createMatch.Success) { string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); @@ -593,7 +514,7 @@ private static void UpdateObjectCacheOnSuccess(string sql, HashSet exist return; } - Match dropMatch = DropObjectRegex.Match(sql); + Match dropMatch = DropObjectRegex.Match(code); if (dropMatch.Success) { string objType = dropMatch.Groups["type"].Value.ToUpperInvariant(); @@ -672,11 +593,112 @@ public static string RewriteDatabaseQualifier(string sql, string fromDb, string if (string.IsNullOrWhiteSpace(sql) || string.IsNullOrWhiteSpace(fromDb) || string.IsNullOrWhiteSpace(toDb)) return sql; - string unquotedFrom = UnquoteIdentifier(fromDb); - string safeTo = QuoteIdentifier(toDb); + var tokens = MySqlScriptReader.Tokens(sql); + for (int i = tokens.Count - 3; i >= 0; i--) + if (tokens[i].Identifier && tokens[i + 1].Value == "." && tokens[i + 2].Identifier && + string.Equals(tokens[i].Value, UnquoteIdentifier(fromDb), StringComparison.OrdinalIgnoreCase)) + sql = MySqlScriptReader.ReplaceToken(sql, tokens[i], QuoteIdentifier(toDb)); + return sql; + } + + private void RequireExecution(string sql) + { + var result = db.ExecSQL(sql); + if (!IsExecutionSuccess(result)) throw new InvalidOperationException(DatabaseExecutionResultService.GetFailureReason(result)); + } + + private static bool Keyword(MySqlScriptReader.Token token, string value) + { return !token.Quoted && string.Equals(token.Value, value, StringComparison.OrdinalIgnoreCase); } + + private static void ValidateTargetSyntax(string sql) + { + var tokens = MySqlScriptReader.Tokens(sql); + if (tokens.Count > 1 && (Keyword(tokens[0], "COMMIT") || Keyword(tokens[0], "ROLLBACK") || Keyword(tokens[0], "BEGIN"))) + throw new InvalidDataException("Only plain BEGIN, COMMIT, and ROLLBACK are supported."); + for (int i = 0; i < tokens.Count; i++) + { + if (Keyword(tokens[i], "AUTOCOMMIT")) throw new InvalidDataException("SET AUTOCOMMIT is unsupported; use explicit BEGIN/COMMIT."); + if (Keyword(tokens[i], "PREPARE") || Keyword(tokens[i], "EXECUTE") || Keyword(tokens[i], "CALL") || + Keyword(tokens[i], "SOURCE") || Keyword(tokens[i], "CONNECT")) + throw new InvalidDataException("Dynamic SQL, CALL, and client commands cannot be safely redirected to the selected target."); + if (i > 0 && Keyword(tokens[i], "USE")) + throw new InvalidDataException("Nested USE cannot be safely redirected."); + if ((Keyword(tokens[i], "DATABASE") || Keyword(tokens[i], "SCHEMA")) && + (i == 0 || !Keyword(tokens[i - 1], "CREATE"))) + throw new InvalidDataException("Only CREATE DATABASE is supported; database-wide destructive commands are rejected."); + if (Keyword(tokens[i], "SQL_MODE")) + { + if (tokens.Count != 4 || !Keyword(tokens[0], "SET") || i != 1 || tokens[2].Value != "=" || + sql.Substring(tokens[3].Start, tokens[3].Length) != "'NO_AUTO_VALUE_ON_ZERO'") + throw new InvalidDataException("Changing SQL_MODE is unsupported by the logical import reader."); + } + } + } + + // INSERT accepts priority modifiers, IGNORE and optional INTO; REPLACE + // accepts the corresponding priority/optional-INTO forms. Share target + // extraction between redirection and the SkipExisting policy. + private static int FindInsertTargetIndex(List tokens) + { + if (tokens.Count == 0 || (!Keyword(tokens[0], "INSERT") && !Keyword(tokens[0], "REPLACE"))) return -1; + int target = 1; + while (target < tokens.Count && (Keyword(tokens[target], "LOW_PRIORITY") || Keyword(tokens[target], "HIGH_PRIORITY") || + Keyword(tokens[target], "DELAYED") || Keyword(tokens[target], "IGNORE"))) target++; + if (target < tokens.Count && Keyword(tokens[target], "INTO")) target++; + if (target >= tokens.Count || !tokens[target].Identifier) return -2; + if (target + 1 < tokens.Count && tokens[target + 1].Value == "." && + (target + 2 >= tokens.Count || !tokens[target + 2].Identifier)) return -2; + return target; + } - string pattern = @"(`?" + Regex.Escape(unquotedFrom) + @"`?)\s*\.\s*(`?" + IdentifierPattern + @"`?)"; - return Regex.Replace(sql, pattern, safeTo + ".$2", RegexOptions.IgnoreCase); + private static string EnforceTargetQualifiers(string sql, string source, string target) + { + var tokens = MySqlScriptReader.Tokens(sql); + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); + var objectPrefixes = new HashSet(); + Match create = CreateObjectRegex.Match(MySqlScriptReader.CodeOnly(sql)); + bool triggerDefinition = create.Success && create.Groups["type"].Value.Equals("TRIGGER", StringComparison.OrdinalIgnoreCase); + if (triggerDefinition) { aliases.Add("NEW"); aliases.Add("OLD"); } + int insertTarget = FindInsertTargetIndex(tokens); + if (insertTarget >= 0) + { + int last = insertTarget; + if (last + 2 < tokens.Count && tokens[last + 1].Value == ".") { objectPrefixes.Add(last); last += 2; } + aliases.Add(tokens[last].Value); + } + string[] objectWords = { "INTO", "UPDATE", "TABLE", "VIEW", "FROM", "JOIN", "REFERENCES", "TRIGGER", "PROCEDURE", "FUNCTION", "TRUNCATE", "TO" }; + for (int i = 0; i < tokens.Count - 1; i++) + { + bool objectWord = false; + foreach (string word in objectWords) if (Keyword(tokens[i], word)) objectWord = true; + // ON names a table in a trigger header; it is not a general + // object keyword (JOIN predicates use ON for expressions). + if (triggerDefinition && Keyword(tokens[i], "ON")) objectWord = true; + if (!objectWord) continue; + int name = i + 1; + while (name < tokens.Count && (Keyword(tokens[name], "IF") || Keyword(tokens[name], "NOT") || Keyword(tokens[name], "EXISTS") || Keyword(tokens[name], "ONLY"))) name++; + if (name >= tokens.Count || !tokens[name].Identifier) continue; + int last = name; + if (name + 2 < tokens.Count && tokens[name + 1].Value == "." && tokens[name + 2].Identifier) + { objectPrefixes.Add(name); last = name + 2; } + aliases.Add(tokens[last].Value); + int alias = last + 1; + if (alias < tokens.Count && Keyword(tokens[alias], "AS")) alias++; + if (alias < tokens.Count && tokens[alias].Identifier) aliases.Add(tokens[alias].Value); + } + for (int i = tokens.Count - 3; i >= 0; i--) + { + if (!tokens[i].Identifier || tokens[i + 1].Value != "." || !tokens[i + 2].Identifier) continue; + if (i >= 2 && tokens[i - 1].Value == ".") continue; + bool threePart = i + 3 < tokens.Count && tokens[i + 3].Value == "."; + bool sourcePrefix = !string.IsNullOrEmpty(source) && string.Equals(tokens[i].Value, source, StringComparison.OrdinalIgnoreCase); + bool targetPrefix = string.Equals(tokens[i].Value, target, StringComparison.OrdinalIgnoreCase); + if (objectPrefixes.Contains(i) || threePart || ((sourcePrefix || targetPrefix) && !aliases.Contains(tokens[i].Value))) + sql = MySqlScriptReader.ReplaceToken(sql, tokens[i], QuoteIdentifier(target)); + else if (!aliases.Contains(tokens[i].Value)) + throw new InvalidDataException("Ambiguous database qualifier '" + tokens[i].Value + "'; cannot safely redirect SQL."); + } + return sql; } public static string SanitizeSqlHeadline(string sql, int maxLength = 130) diff --git a/mySQLPunk/lib/MySqlImport/MySqlScriptReader.cs b/mySQLPunk/lib/MySqlImport/MySqlScriptReader.cs new file mode 100644 index 0000000..66b1c06 --- /dev/null +++ b/mySQLPunk/lib/MySqlImport/MySqlScriptReader.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; + +namespace mySQLPunk.lib.MySqlImport +{ + // A shared reader for preflight and execution. Delimiters/comments are syntax + // only outside quoted text; original newline and literal characters survive. + public static class MySqlScriptReader + { + public const int MaxStatementBytes = 32 * 1024 * 1024; + public sealed class Statement + { + public string Sql { get; internal set; } + public int Line { get; internal set; } + } + + public sealed class Token + { + public string Value { get; internal set; } + public int Start { get; internal set; } + public int Length { get; internal set; } + public bool Identifier { get; internal set; } + public bool Quoted { get; internal set; } + } + + public static IEnumerable Read(TextReader reader, CancellationToken cancellation, Action commentLine = null) + { + var batch = new StringBuilder(); + string delimiter = ";"; + char quote = '\0'; + bool block = false, escaped = false; + int lineNumber = 0, startLine = 1; + string line; + while ((line = ReadLineWithEnding(reader)) != null) + { + cancellation.ThrowIfCancellationRequested(); + lineNumber++; + string trimmed = line.Trim(); + if (quote == '\0' && !block) + { + if (trimmed.StartsWith("--", StringComparison.Ordinal) && commentLine != null) commentLine(trimmed); + var command = Regex.Match(trimmed, @"^DELIMITER\s+(\S+)\s*$", RegexOptions.IgnoreCase); + if (command.Success && Tokens(batch.ToString()).Count == 0) + { + delimiter = command.Groups[1].Value; + if (delimiter.Length > 16 || delimiter.IndexOfAny(new[] { '\'', '"', '`', '\\' }) >= 0) + throw new InvalidDataException("Unsupported DELIMITER at line " + lineNumber); + batch.Clear(); + startLine = lineNumber + 1; + continue; + } + } + for (int i = 0; i < line.Length; i++) + { + char c = line[i]; + if (quote != '\0') + { + batch.Append(c); + if (escaped) { escaped = false; continue; } + if (c == '\\' && quote != '`') { escaped = true; continue; } + if (c == quote) + { + if (i + 1 < line.Length && line[i + 1] == quote) { batch.Append(line[++i]); continue; } + quote = '\0'; + } + continue; + } + if (block) + { + batch.Append(c); + if (c == '*' && i + 1 < line.Length && line[i + 1] == '/') { batch.Append(line[++i]); block = false; } + continue; + } + if (c == '#' || (c == '-' && i + 2 < line.Length && line[i + 1] == '-' && char.IsWhiteSpace(line[i + 2]))) + { + batch.Append(line, i, line.Length - i); + break; + } + if (c == '/' && i + 1 < line.Length && line[i + 1] == '*') + { + if (i + 2 < line.Length && (line[i + 2] == '!' || (line[i + 2] == 'M' && i + 3 < line.Length && line[i + 3] == '!'))) + throw new InvalidDataException("Executable version comments require a native MySQL client; line " + lineNumber); + block = true; + batch.Append(c).Append(line[++i]); + continue; + } + if (c == '\'' || c == '"' || c == '`') { quote = c; batch.Append(c); continue; } + if (i + delimiter.Length <= line.Length && string.CompareOrdinal(line, i, delimiter, 0, delimiter.Length) == 0) + { + string sql = batch.ToString().Trim(); + CheckSize(sql); + if (Tokens(sql).Count > 0) yield return new Statement { Sql = sql, Line = startLine }; + batch.Clear(); + startLine = lineNumber; + i += delimiter.Length - 1; + } + else batch.Append(c); + if (batch.Length > MaxStatementBytes) throw new InvalidDataException("Single SQL statement exceeds 32 MiB safe limit."); + } + if (batch.Length > MaxStatementBytes) throw new InvalidDataException("Single SQL statement exceeds 32 MiB safe limit."); + } + cancellation.ThrowIfCancellationRequested(); + if (quote != '\0' || block) throw new InvalidDataException("Unterminated SQL literal, identifier, or block comment."); + string remaining = batch.ToString().Trim(); + CheckSize(remaining); + if (Tokens(remaining).Count > 0) yield return new Statement { Sql = remaining, Line = startLine }; + } + + private static string ReadLineWithEnding(TextReader reader) + { + var line = new StringBuilder(); + int c; + while ((c = reader.Read()) >= 0) + { + line.Append((char)c); + if (line.Length > MaxStatementBytes) throw new InvalidDataException("Single SQL line exceeds 32 MiB safe limit."); + if (c == '\n') break; + if (c == '\r') { if (reader.Peek() == '\n') line.Append((char)reader.Read()); break; } + } + return line.Length == 0 ? null : line.ToString(); + } + + private static void CheckSize(string sql) + { + if (Encoding.UTF8.GetByteCount(sql) > MaxStatementBytes) throw new InvalidDataException("Single SQL statement exceeds 32 MiB UTF-8 safe limit."); + if (sql.IndexOf('\0') >= 0) throw new InvalidDataException("SQL contains an embedded NUL; use an escaped literal."); + } + + public static List Tokens(string sql) + { + var tokens = new List(); + for (int i = 0; i < sql.Length;) + { + char c = sql[i]; + if (char.IsWhiteSpace(c)) { i++; continue; } + if (c == '#' || (c == '-' && i + 2 < sql.Length && sql[i + 1] == '-' && char.IsWhiteSpace(sql[i + 2]))) + { while (i < sql.Length && sql[i] != '\r' && sql[i] != '\n') i++; continue; } + if (c == '/' && i + 1 < sql.Length && sql[i + 1] == '*') + { int end = sql.IndexOf("*/", i + 2, StringComparison.Ordinal); i = end < 0 ? sql.Length : end + 2; continue; } + int start = i++; + if (c == '\'' || c == '"' || c == '`') + { + while (i < sql.Length) + { + if (sql[i] == '\\' && c != '`') { i = Math.Min(sql.Length, i + 2); continue; } + if (sql[i++] == c) { if (i < sql.Length && sql[i] == c) { i++; continue; } break; } + } + if (c == '`') tokens.Add(new Token { Start = start, Length = i - start, Value = MySqlImportRunner.UnquoteIdentifier(sql.Substring(start, i - start)), Identifier = true, Quoted = true }); + else tokens.Add(new Token { Start = start, Length = i - start, Value = "" }); + } + else + { + bool identifier = char.IsLetterOrDigit(c) || c == '_' || c == '$'; + if (identifier) while (i < sql.Length && (char.IsLetterOrDigit(sql[i]) || sql[i] == '_' || sql[i] == '$')) i++; + tokens.Add(new Token { Start = start, Length = i - start, Value = sql.Substring(start, i - start), Identifier = identifier }); + } + if (tokens.Count > 250000) throw new InvalidDataException("SQL statement token count exceeds safe limit."); + } + return tokens; + } + + public static string CodeOnly(string sql) + { + var code = new string(' ', sql.Length).ToCharArray(); + foreach (var token in Tokens(sql)) + if (token.Value != "") sql.CopyTo(token.Start, code, token.Start, token.Length); + return new string(code); + } + + public static string ReplaceToken(string sql, Token token, string replacement) + { return sql.Substring(0, token.Start) + replacement + sql.Substring(token.Start + token.Length); } + } +} diff --git a/mySQLPunk/lib/OracleExport/OracleExportModels.cs b/mySQLPunk/lib/OracleExport/OracleExportModels.cs index 2596aee..d91cb68 100644 --- a/mySQLPunk/lib/OracleExport/OracleExportModels.cs +++ b/mySQLPunk/lib/OracleExport/OracleExportModels.cs @@ -26,7 +26,7 @@ public sealed class OracleTableExportItem { public string Owner { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; - public string FullQualifiedName => "\"" + (string.IsNullOrWhiteSpace(Owner) ? "SYSTEM" : Owner) + "\".\"" + Name + "\""; + public string FullQualifiedName => "\"" + Owner.Replace("\"", "\"\"") + "\".\"" + Name.Replace("\"", "\"\"") + "\""; public string DisplayName => string.IsNullOrWhiteSpace(Owner) ? Name : Owner + "." + Name; public long RowCount { get; set; } public long EstimatedBytes { get; set; } diff --git a/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs b/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs index 259e161..71fef19 100644 --- a/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs +++ b/mySQLPunk/lib/OracleExport/OracleSqlExportService.cs @@ -8,13 +8,22 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Oracle.ManagedDataAccess.Client; +using mySQLPunk.lib.OracleImport; namespace mySQLPunk.lib.OracleExport { + // Optional recording-provider contract; production uses native SequentialAccess. + public interface IOracleExportReaderSource + { + IDataReader OpenExportReader(string sql, CancellationToken cancellationToken); + } + public sealed class OracleSqlExportService { private const int BufferSize = 65536; // 64KB private readonly IDatabase db; + private OracleTransaction sourceTransaction; private readonly string databaseName; // In Oracle, this represents the Schema / Owner private static readonly HashSet SpatialIndexPrefixes = new HashSet(StringComparer.OrdinalIgnoreCase) @@ -36,7 +45,7 @@ public OracleSqlExportService(IDatabase db, string databaseName) public List GetTableCatalog(CancellationToken cancellationToken = default(CancellationToken)) { var list = new List(); - string owner = string.IsNullOrWhiteSpace(databaseName) ? "USER" : databaseName.Trim().ToUpperInvariant(); + string owner = string.IsNullOrWhiteSpace(databaseName) ? "USER" : databaseName.Trim(); // 1. 取得所有一般資料表 string tablesSql = @" @@ -46,21 +55,12 @@ public OracleSqlExportService(IDatabase db, string databaseName) COALESCE(t.NUM_ROWS, 0) AS ROW_COUNT, COALESCE(t.BLOCKS * 8192, 0) AS ESTIMATED_BYTES FROM ALL_TABLES t - WHERE (:owner = 'USER' OR t.OWNER = :owner) + WHERE t.OWNER = :owner AND t.TABLE_NAME NOT LIKE 'BIN$%' ORDER BY t.OWNER, t.TABLE_NAME"; var p = new Dictionary { { "owner", owner } }; - DataTable dtTables; - try - { - dtTables = db.SelectSQL(tablesSql, p); - } - catch - { - // Fallback 若無 bind 支援 - dtTables = db.SelectSQL("SELECT OWNER, TABLE_NAME, COALESCE(NUM_ROWS, 0) AS ROW_COUNT, 0 AS ESTIMATED_BYTES FROM ALL_TABLES WHERE TABLE_NAME NOT LIKE 'BIN$%' ORDER BY OWNER, TABLE_NAME"); - } + DataTable dtTables = QueryMetadata(tablesSql, p, cancellationToken); var itemMap = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -69,6 +69,7 @@ AND t.TABLE_NAME NOT LIKE 'BIN$%' cancellationToken.ThrowIfCancellationRequested(); string tblOwner = row["OWNER"]?.ToString() ?? ""; string name = row["TABLE_NAME"]?.ToString() ?? ""; + if (!string.Equals(tblOwner, owner, StringComparison.Ordinal)) continue; long rows = 0; long bytes = 0; if (row["ROW_COUNT"] != DBNull.Value) long.TryParse(row["ROW_COUNT"].ToString(), out rows); @@ -105,15 +106,16 @@ AND t.TABLE_NAME NOT LIKE 'BIN$%' string viewsSql = @" SELECT OWNER, VIEW_NAME FROM ALL_VIEWS - WHERE (:owner = 'USER' OR OWNER = :owner) + WHERE OWNER = :owner ORDER BY OWNER, VIEW_NAME"; - DataTable dtViews = db.SelectSQL(viewsSql, p); + DataTable dtViews = QueryMetadata(viewsSql, p, cancellationToken); foreach (DataRow row in dtViews.Rows) { cancellationToken.ThrowIfCancellationRequested(); string vOwner = row["OWNER"]?.ToString() ?? ""; string vName = row["VIEW_NAME"]?.ToString() ?? ""; + if (!string.Equals(vOwner, owner, StringComparison.Ordinal)) continue; var item = new OracleTableExportItem { @@ -130,7 +132,7 @@ FROM ALL_VIEWS itemMap[vOwner + "." + vName] = item; } } - catch { } + catch { throw; } // 3. 探測 Oracle Spatial (SDO_GEOMETRY) 空間欄位 try @@ -138,10 +140,10 @@ FROM ALL_VIEWS string spatialSql = @" SELECT OWNER, TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM ALL_TAB_COLUMNS - WHERE (:owner = 'USER' OR OWNER = :owner) + WHERE OWNER = :owner AND (DATA_TYPE = 'SDO_GEOMETRY' OR (DATA_TYPE_OWNER = 'MDSYS' AND DATA_TYPE = 'SDO_GEOMETRY'))"; - DataTable dtSpatial = db.SelectSQL(spatialSql, p); + DataTable dtSpatial = QueryMetadata(spatialSql, p, cancellationToken); foreach (DataRow row in dtSpatial.Rows) { string sOwner = row["OWNER"]?.ToString() ?? ""; @@ -160,516 +162,593 @@ FROM ALL_TAB_COLUMNS } } } - catch { } + catch { throw; } return list; } - public async Task ExportAsync( - OracleExportWizardOptions options, - IProgress progress = null, - CancellationToken cancellationToken = default(CancellationToken)) + // STANDARD SQL bounds until a verified streaming LOB codec exists. + public const int MaxBinaryLiteralBytes = 2000; + public const int MaxTextLiteralBytes = 4000; + public const int MaxRowValueBytes = 8 * 1024 * 1024; + public const int MaxRowSqlBytes = 28 * 1024 * 1024; + private const int MaxMetadataChars = 1024 * 1024; + + public async Task ExportAsync(OracleExportWizardOptions options, IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) { if (options == null) throw new ArgumentNullException(nameof(options)); - if (string.IsNullOrWhiteSpace(options.TargetPath)) throw new ArgumentException("TargetPath is required.", nameof(options)); - + if (string.IsNullOrWhiteSpace(options.TargetPath)) throw new ArgumentException("TargetPath is required."); + if (string.IsNullOrWhiteSpace(databaseName) || databaseName == "USER") throw new InvalidOperationException("An explicit Oracle owner is required."); + if (options.BatchSize <= 0) throw new ArgumentException("BatchSize must be positive."); var result = new OracleExportResult { FilePath = options.TargetPath }; - var stopwatch = Stopwatch.StartNew(); - - string dir = Path.GetDirectoryName(options.TargetPath); - if (!string.IsNullOrWhiteSpace(dir) && !Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - string stagingPath = options.TargetPath + ".writing"; - if (File.Exists(stagingPath)) - { - try { File.Delete(stagingPath); } catch { } - } - + var clock = Stopwatch.StartNew(); + string directory = Path.GetDirectoryName(options.TargetPath); + if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory); + string stagingPath = options.TargetPath + "." + Guid.NewGuid().ToString("N") + ".writing"; + bool publicationAttempted = false, snapshotActive = false; try { - // 1. 取得完整目標清單 - progress?.Report(new OracleExportProgress - { - Stage = OracleExportStage.Preflight, - Message = "正在探測 Oracle 綱要與空間物件結構..." - }); - - var catalog = await GetTableCatalogAsync(cancellationToken).ConfigureAwait(false); - var selectedSet = new HashSet(options.SelectedTables, StringComparer.OrdinalIgnoreCase); - - var itemsToExport = catalog.FindAll(i => - (options.SelectedTables.Count == 0 && i.Selected) || - selectedSet.Contains(i.Name) || - selectedSet.Contains(i.DisplayName) || - selectedSet.Contains(i.FullQualifiedName)); - - // 排除影子表 - if (options.ExcludeSpatialIndexInternalTables) + if (options.UseTransactionSnapshot) { - itemsToExport.RemoveAll(i => i.IsSpatialMetadata); + var native = db as my_oracle; + if (native != null) + { + var session = QueryMetadata("SELECT SYS_CONTEXT('USERENV','SESSION_USER') FROM DUAL", null, cancellationToken); + if (session == null || session.Rows.Count != 1 || Convert.ToString(session.Rows[0][0]) == "SYS") + throw new NotSupportedException("Oracle SYS sessions do not provide READ ONLY snapshot guarantees; use a dedicated non-SYS source session."); + sourceTransaction = native.MCT.BeginTransaction(); + } + snapshotActive = true; + ExecuteSourceChecked("SET TRANSACTION READ ONLY", cancellationToken); } - - int totalItems = itemsToExport.Count; - long totalRows = 0; - foreach (var it in itemsToExport) + if (db is my_oracle) { - if (!it.IsView) totalRows += it.RowCount; + var charset = QueryMetadata("SELECT VALUE FROM NLS_DATABASE_PARAMETERS WHERE PARAMETER='NLS_CHARACTERSET'", null, cancellationToken); + if (charset == null || charset.Rows.Count != 1 || Convert.ToString(charset.Rows[0][0]) != "AL32UTF8") + throw new NotSupportedException("The bounded text codec is verified structurally for AL32UTF8 source databases only."); } - - int itemsProcessed = 0; - long rowsExportedTotal = 0; - - using (var fs = new FileStream(stagingPath, FileMode.Create, FileAccess.Write, FileShare.None, BufferSize)) - using (var sw = new StreamWriter(fs, new UTF8Encoding(false), BufferSize)) + var catalog = await GetTableCatalogAsync(cancellationToken).ConfigureAwait(false); + var selection = new HashSet(StringComparer.Ordinal); + foreach (string name in options.SelectedTables) selection.Add(OracleImportRunner.ResolveManifestTarget(name, databaseName)); + var items = catalog.FindAll(item => (selection.Count == 0 ? item.Selected : selection.Contains(item.FullQualifiedName)) && + (!options.ExcludeSpatialIndexInternalTables || !item.IsSpatialMetadata) && (!item.IsView || (options.IncludeViews && options.Mode != OracleExportMode.DataOnly))); + if (items.Count == 0) throw new InvalidOperationException("No Oracle objects selected."); + items = OrderSchemaDependencies(items, options.Mode, cancellationToken); + var columns = new Dictionary>(StringComparer.Ordinal); + foreach (var item in items) + if (!item.IsView && options.Mode != OracleExportMode.SchemaOnly) columns[item.FullQualifiedName] = GetWritableColumns(item, cancellationToken); + long totalRows = 0; + foreach (var item in items) if (!item.IsView) totalRows += item.RowCount; + var deferred = new List(); + using (var stream = new FileStream(stagingPath, FileMode.CreateNew, FileAccess.Write, FileShare.Read, BufferSize)) + using (var writer = new StreamWriter(stream, new UTF8Encoding(false), BufferSize)) { - // 寫入 Header 註解 - sw.WriteLine("-- ===================================================================="); - sw.WriteLine("-- mySQLPunk Oracle Logical Dump"); - sw.WriteLine("-- Host Database: " + (string.IsNullOrWhiteSpace(databaseName) ? "Current Schema" : databaseName)); - sw.WriteLine("-- Created: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); - sw.WriteLine("-- Architecture: Streaming StreamWriter (64KB Buffer, Zero Memory Bloat)"); - sw.WriteLine("-- Oracle Spatial: MDSYS.SDO_GEOMETRY with WKB / Hex literal support"); - sw.WriteLine("-- ===================================================================="); - sw.WriteLine(); - - // 一致性快照保護 (SET TRANSACTION READ ONLY) - if (options.UseTransactionSnapshot) + writer.WriteLine("-- mySQLPunk Oracle Logical Dump"); + writer.WriteLine("-- Owner: " + databaseName); + writer.WriteLine("-- Bounded v1: literal chunks <= 2000 binary / 4000 UTF-8 text bytes; total row values <= 8 MiB; block SQL <= 28 MiB."); + writer.WriteLine("-- Native source: AL32UTF8, non-SYS snapshot sessions. Identity data, NCLOB, LRS/curves/compound geometry, unsupported types and over-budget rows fail before publication."); + writer.WriteLine("-- Oracle DDL implicitly commits. Spatial metadata restore requires SESSION_USER equal to the dump owner."); + // READ ONLY belongs exclusively to the source session and is never restore SQL. + foreach (var item in items) { - sw.WriteLine("-- 啟用交易一致性唯讀快照 (防範外鍵與空間關聯導出時不一致)"); - sw.WriteLine("SET TRANSACTION READ ONLY;"); - sw.WriteLine(); - } - - // 2. 結構 DDL 階段 (CREATE TABLE, COMMENTS, SPATIAL METADATA) - if (options.Mode != OracleExportMode.DataOnly) - { - progress?.Report(new OracleExportProgress - { - Stage = OracleExportStage.ExportingSchema, - TotalTables = totalItems, - TotalRows = totalRows, - Message = "正在產生 Oracle 資料表與空間中繼結構 DDL..." - }); - - foreach (var item in itemsToExport) + cancellationToken.ThrowIfCancellationRequested(); + if (item.IsView) continue; + if (options.Mode != OracleExportMode.DataOnly) { - cancellationToken.ThrowIfCancellationRequested(); - if (item.IsView) continue; // View 於最後匯出 - - sw.WriteLine("-- --------------------------------------------------"); - sw.WriteLine(string.Format("-- Table structure for: {0}", item.FullQualifiedName)); - if (item.IsSpatial) - { - sw.WriteLine(string.Format("-- Spatial Feature: {0} ({1})", item.SpatialFeatureDescription, string.Join(", ", item.SpatialColumnNames.ToArray()))); - } - sw.WriteLine("-- --------------------------------------------------"); - + progress?.Report(new OracleExportProgress { Stage = OracleExportStage.ExportingSchema, CurrentTable = item.Name, Message = "Reading native DDL: " + item.DisplayName }); if (options.DropTableIfExists) { - sw.WriteLine("BEGIN"); - sw.WriteLine(string.Format(" EXECUTE IMMEDIATE 'DROP TABLE {0} CASCADE CONSTRAINTS';", item.FullQualifiedName)); - sw.WriteLine("EXCEPTION WHEN OTHERS THEN NULL;"); - sw.WriteLine("END;"); - sw.WriteLine("/"); - } - - string ddl = db.GetTableCreateStatement(item.Owner, item.Name); - if (!string.IsNullOrWhiteSpace(ddl)) - { - sw.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + writer.WriteLine("BEGIN EXECUTE IMMEDIATE 'DROP TABLE " + item.FullQualifiedName.Replace("'", "''") + " CASCADE CONSTRAINTS'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF; END;"); + writer.WriteLine("/"); } - - // 空間中繼表註冊 (USER_SDO_GEOM_METADATA) - if (options.IncludeSpatialMetadata && item.IsSpatial && item.SpatialColumnNames.Count > 0) - { - foreach (var geomCol in item.SpatialColumnNames) - { - sw.WriteLine(string.Format( - "DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME = '{0}' AND COLUMN_NAME = '{1}';", - item.Name.ToUpperInvariant(), geomCol.ToUpperInvariant())); - sw.WriteLine(string.Format( - "INSERT INTO USER_SDO_GEOM_METADATA (TABLE_NAME, COLUMN_NAME, DIMINFO, SRID) " + - "VALUES ('{0}', '{1}', MDSYS.SDO_DIM_ARRAY(MDSYS.SDO_DIM_ELEMENT('X', -180, 180, 0.005), MDSYS.SDO_DIM_ELEMENT('Y', -90, 90, 0.005)), 4326);", - item.Name.ToUpperInvariant(), geomCol.ToUpperInvariant())); - } - } - - sw.WriteLine(); + WriteTableDdl(item, options, writer, deferred, cancellationToken); + if (item.IsSpatial && options.IncludeSpatialMetadata) WriteSpatialMetadata(item, writer, cancellationToken); } - } - - // 3. 資料傾印階段 (INSERT STATEMENTS) - if (options.Mode != OracleExportMode.SchemaOnly) - { - foreach (var item in itemsToExport) + if (options.Mode != OracleExportMode.SchemaOnly) { - cancellationToken.ThrowIfCancellationRequested(); - if (item.IsView) continue; - - progress?.Report(new OracleExportProgress - { - Stage = OracleExportStage.ExportingData, - CurrentTable = item.Name, - CurrentOwner = item.Owner, - TablesCompleted = itemsProcessed, - TotalTables = totalItems, - RowsExported = rowsExportedTotal, - TotalRows = totalRows, - BytesExported = fs.Position, - Message = string.Format("正在串流傾印資料表: {0}...", item.DisplayName) - }); - - sw.WriteLine("-- --------------------------------------------------"); - sw.WriteLine(string.Format("-- Dumping data for: {0}", item.FullQualifiedName)); - sw.WriteLine("-- --------------------------------------------------"); - - if (options.TruncateBeforeInsert) - { - sw.WriteLine(string.Format("TRUNCATE TABLE {0};", item.FullQualifiedName)); - } - - long tableRowsExported = ExportTableDataStreaming(db, item, sw, options, cancellationToken); - rowsExportedTotal += tableRowsExported; - result.ManifestTables[item.Name] = tableRowsExported; - result.TablesExported++; - - sw.WriteLine("COMMIT;"); - sw.WriteLine(); - - itemsProcessed++; + if (options.TruncateBeforeInsert) writer.WriteLine("TRUNCATE TABLE " + item.FullQualifiedName + ";"); + long rows = ExportTableDataStreaming(item, columns[item.FullQualifiedName], writer, options, cancellationToken, + rowCount => progress?.Report(new OracleExportProgress + { + Stage = OracleExportStage.ExportingData, CurrentOwner = item.Owner, CurrentTable = item.Name, + TotalTables = items.Count, TablesCompleted = result.TablesExported, + TotalRows = totalRows, RowsExported = result.TotalRowsExported + rowCount, BytesExported = stream.Position, + Message = "Streaming " + item.DisplayName + ": " + rowCount + " rows" + })); + result.TotalRowsExported += rows; + result.ManifestTables[item.FullQualifiedName] = rows; + writer.WriteLine("COMMIT;"); } + result.TablesExported++; } - - // 4. 檢視與空間索引階段 if (options.Mode != OracleExportMode.DataOnly) { - // 建立檢視 (Views) - if (options.IncludeViews) + foreach (string ddl in deferred) WriteStatement(writer, ddl); + foreach (var item in items) { - foreach (var item in itemsToExport) - { - if (!item.IsView) continue; - cancellationToken.ThrowIfCancellationRequested(); - - sw.WriteLine("-- --------------------------------------------------"); - sw.WriteLine(string.Format("-- View structure for: {0}", item.FullQualifiedName)); - sw.WriteLine("-- --------------------------------------------------"); - - string viewDdl = db.GetViewCreateStatement(item.Owner, item.Name); - if (!string.IsNullOrWhiteSpace(viewDdl)) - { - sw.WriteLine(viewDdl.TrimEnd().TrimEnd(';') + ";"); - sw.WriteLine(); - } - result.ViewsExported++; - } - } - - // 建立空間索引 (MDSYS.SPATIAL_INDEX) - if (options.IncludeSpatialIndexes) - { - foreach (var item in itemsToExport) - { - if (item.IsView || !item.IsSpatial) continue; - - foreach (var col in item.SpatialColumnNames) - { - string idxName = string.Format("SIDX_{0}_{1}", item.Name, col); - if (idxName.Length > 30) idxName = idxName.Substring(0, 30); - - sw.WriteLine(string.Format("-- Oracle Spatial Domain Index for {0}.{1}", item.Name, col)); - sw.WriteLine(string.Format( - "CREATE INDEX \"{0}\".\"{1}\" ON {2} (\"{3}\") INDEXTYPE IS MDSYS.SPATIAL_INDEX;", - item.Owner, idxName, item.FullQualifiedName, col)); - sw.WriteLine(); - } - } + if (!item.IsView) continue; + cancellationToken.ThrowIfCancellationRequested(); + string ddl = db is my_oracle ? GetNativeDdl("VIEW", item.Owner, item.Name, cancellationToken) : db.GetViewCreateStatement(item.Owner, item.Name); + if (string.IsNullOrWhiteSpace(ddl)) throw new InvalidOperationException("View DDL unavailable: " + item.DisplayName); + WriteStatement(writer, ddl); + result.ViewsExported++; } } - - // 5. 結尾交易完成與 Native Manifest - sw.WriteLine("COMMIT;"); - sw.WriteLine(); - + writer.WriteLine("COMMIT;"); if (options.IncludeManifest) { - sw.WriteLine("-- ===================================================================="); - sw.WriteLine("-- mySQLPunk Native Manifest"); - sw.WriteLine(string.Format("-- Manifest-Provider: oracle")); - sw.WriteLine(string.Format("-- Manifest-Database: {0}", databaseName)); - sw.WriteLine(string.Format("-- Manifest-Generated: {0}", DateTime.UtcNow.ToString("o"))); - foreach (var kvp in result.ManifestTables) - { - sw.WriteLine(string.Format("-- Manifest-Table: {0} | Rows: {1}", kvp.Key, kvp.Value)); - } - sw.WriteLine("-- ===================================================================="); + writer.WriteLine("-- mySQLPunk Native Manifest"); + writer.WriteLine("-- Manifest-Provider: oracle"); + writer.WriteLine("-- Manifest-Database: " + databaseName); + foreach (var pair in result.ManifestTables) writer.WriteLine("-- Manifest-Table: " + pair.Key + " | Rows: " + pair.Value); } - - sw.Flush(); } - - // 6. 計算 SHA-256 Checksum 並原子替換檔案 - result.FileSizeBytes = new FileInfo(stagingPath).Length; + cancellationToken.ThrowIfCancellationRequested(); + if (snapshotActive) { EndSnapshot(); snapshotActive = false; } result.Sha256Checksum = ComputeSha256(stagingPath); - - // 在檔案末尾附加 Checksum if (options.IncludeManifest) - { - using (var sw = new StreamWriter(stagingPath, true, Encoding.UTF8)) - { - sw.WriteLine(string.Format("-- Manifest-Checksum-SHA256: {0}", result.Sha256Checksum)); - sw.WriteLine("-- ===================================================================="); - } - result.FileSizeBytes = new FileInfo(stagingPath).Length; - } - - if (File.Exists(options.TargetPath)) - { - File.Delete(options.TargetPath); - } - File.Move(stagingPath, options.TargetPath); - - stopwatch.Stop(); + using (var writer = new StreamWriter(stagingPath, true, new UTF8Encoding(false))) writer.WriteLine("-- Manifest-Checksum-SHA256: " + result.Sha256Checksum); + result.FileSizeBytes = new FileInfo(stagingPath).Length; + cancellationToken.ThrowIfCancellationRequested(); + publicationAttempted = true; + SqlExportPublication.Publish(stagingPath, options.TargetPath); result.Success = true; - result.Elapsed = stopwatch.Elapsed; - result.TotalRowsExported = rowsExportedTotal; - - progress?.Report(new OracleExportProgress - { - Stage = OracleExportStage.Completed, - TablesCompleted = totalItems, - TotalTables = totalItems, - RowsExported = rowsExportedTotal, - TotalRows = totalRows, - BytesExported = result.FileSizeBytes, - Message = string.Format("Oracle 匯出完成！共 {0} 張資料表，{1:N0} 列，耗時 {2:hh\\:mm\\:ss}。", result.TablesExported, rowsExportedTotal, result.Elapsed) - }); + result.Elapsed = clock.Elapsed; + progress?.Report(new OracleExportProgress { Stage = OracleExportStage.Completed, TablesCompleted = items.Count, TotalTables = items.Count, RowsExported = result.TotalRowsExported, BytesExported = result.FileSizeBytes, Message = "Oracle export completed." }); + return result; } catch (Exception ex) { - stopwatch.Stop(); - result.Success = false; result.ErrorMessage = ex.Message; - result.Elapsed = stopwatch.Elapsed; - - if (File.Exists(stagingPath)) + progress?.Report(new OracleExportProgress { Stage = ex is OperationCanceledException ? OracleExportStage.Cancelled : OracleExportStage.Failed, Message = "Oracle export failed: " + ex.Message }); + throw; + } + finally + { + try { if (snapshotActive) EndSnapshot(); } + finally { - try { File.Delete(stagingPath); } catch { } + sourceTransaction?.Dispose(); sourceTransaction = null; + if (!publicationAttempted && File.Exists(stagingPath)) File.Delete(stagingPath); } - - progress?.Report(new OracleExportProgress - { - Stage = OracleExportStage.Failed, - Message = "Oracle 匯出失敗: " + ex.Message - }); - throw; } - - return result; } - private static long ExportTableDataStreaming( - IDatabase db, - OracleTableExportItem item, - StreamWriter sw, - OracleExportWizardOptions options, - CancellationToken cancellationToken) + private List OrderSchemaDependencies(List items, OracleExportMode mode, CancellationToken cancellationToken) { - DataTable colsMeta; - try + var selected = new Dictionary(StringComparer.Ordinal); + var dependencies = new Dictionary>(StringComparer.Ordinal); + foreach (var item in items) + if (!item.IsView) { selected[item.Name] = item; dependencies[item.Name] = new HashSet(StringComparer.Ordinal); } + var metadata = QueryMetadata("SELECT c.TABLE_NAME AS CHILD_TABLE, p.OWNER AS PARENT_OWNER, p.TABLE_NAME AS PARENT_TABLE FROM ALL_CONSTRAINTS c JOIN ALL_CONSTRAINTS p ON p.OWNER=c.R_OWNER AND p.CONSTRAINT_NAME=c.R_CONSTRAINT_NAME WHERE c.OWNER=:owner AND c.CONSTRAINT_TYPE='R'" + (mode == OracleExportMode.DataOnly ? " AND c.STATUS='ENABLED'" : ""), new Dictionary { { "owner", databaseName } }, cancellationToken); + foreach (DataRow row in metadata.Rows) { - colsMeta = db.GetColumns(item.Owner, item.Name); + string child = MetadataValue(row, "CHILD_TABLE"), parent = MetadataValue(row, "PARENT_TABLE"), owner = MetadataValue(row, "PARENT_OWNER"); + if (!selected.ContainsKey(child)) continue; + if (owner != databaseName) throw new NotSupportedException("Cross-owner FK dependency cannot be restored under this owner's enforced scope: " + child + " -> " + owner + "." + parent); + if (!selected.ContainsKey(parent)) throw new NotSupportedException("Selected table has an unselected FK parent. Include all required parent tables: " + child + " -> " + parent); + dependencies[child].Add(parent); } - catch + if (mode != OracleExportMode.DataOnly && db is my_oracle) { - colsMeta = null; - } - - // 空間欄位集合 - var spatialSet = new HashSet(item.SpatialColumnNames, StringComparer.OrdinalIgnoreCase); - - // 建構查詢 SELECT 語句 - // 若包含 SDO_GEOMETRY 欄位，轉換為 WKB 以避免 ODP.NET UDT 映射錯誤 - string selectSql; - if (colsMeta != null && colsMeta.Rows.Count > 0) - { - var selectParts = new List(); - foreach (DataRow r in colsMeta.Rows) + foreach (var item in selected.Values) { - string colName = r["COLUMN_NAME"]?.ToString() ?? ""; - if (spatialSet.Contains(colName)) - { - // 轉換為 WKB 二進位 (BLOB) - selectParts.Add(string.Format("CASE WHEN \"{0}\" IS NOT NULL THEN SDO_UTIL.TO_WKBGEOMETRY(\"{0}\") ELSE NULL END AS \"{0}\"", colName)); - } - else + var references = QueryMetadata("SELECT REFERENCED_OWNER, REFERENCED_NAME, REFERENCED_TYPE FROM ALL_DEPENDENCIES WHERE OWNER=:owner AND NAME=:tableName AND TYPE='TABLE' AND REFERENCED_TYPE IN ('SEQUENCE','FUNCTION','PROCEDURE','PACKAGE','TYPE')", new Dictionary { { "owner", item.Owner }, { "tableName", item.Name } }, cancellationToken); + foreach (DataRow reference in references.Rows) { - selectParts.Add(string.Format("\"{0}\"", colName)); + string owner = MetadataValue(reference, "REFERENCED_OWNER"); + if (owner == "SYS" || owner == "MDSYS") continue; + throw new NotSupportedException("External table dependency is not bundled by this wizard: " + item.DisplayName + " -> " + owner + "." + MetadataValue(reference, "REFERENCED_NAME") + " (" + MetadataValue(reference, "REFERENCED_TYPE") + ")."); } } - selectSql = string.Format("SELECT {0} FROM {1}", string.Join(", ", selectParts.ToArray()), item.FullQualifiedName); } - else + var ordered = new List(); + var emitted = new HashSet(StringComparer.Ordinal); + while (emitted.Count < selected.Count) { - selectSql = string.Format("SELECT * FROM {0}", item.FullQualifiedName); + bool progress = false; + foreach (var item in items) + { + if (item.IsView || emitted.Contains(item.Name)) continue; + bool ready = true; + foreach (string parent in dependencies[item.Name]) if (!emitted.Contains(parent)) { ready = false; break; } + if (!ready) continue; + emitted.Add(item.Name); ordered.Add(item); progress = true; + } + if (!progress) throw new NotSupportedException("Selected cyclic/self-referencing foreign keys require a deferred-constraint restoration phase; this export is rejected before publication."); } + foreach (var item in items) if (item.IsView) ordered.Add(item); + return ordered; + } - long rowCount = 0; - int batchCount = 0; - - DataTable dt; - try + private DataTable QueryMetadata(string sql, Dictionary parameters, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var native = db as my_oracle; + if (native == null) return db.SelectSQL(sql, parameters); + using (var command = new OracleCommand(sql, native.MCT)) { - dt = db.SelectSQL(selectSql); + command.BindByName = true; + command.Transaction = sourceTransaction; + if (parameters != null) foreach (var parameter in parameters) command.Parameters.Add(new OracleParameter(parameter.Key, parameter.Value ?? DBNull.Value)); + using (cancellationToken.Register(() => { try { command.Cancel(); } catch { } })) + using (var reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) + { + // Dictionary/catalog metadata is bounded by the selected owner; table data uses WriteRows exclusively. + var table = new DataTable(); table.Load(reader); return table; + } } - catch + } + + private void ExecuteSourceChecked(string sql, CancellationToken cancellationToken) + { + var native = db as my_oracle; + if (native != null) { - // Fallback 直接 SELECT * - dt = db.SelectSQL(string.Format("SELECT * FROM {0}", item.FullQualifiedName)); + using (var command = new OracleCommand(sql, native.MCT)) + { + command.Transaction = sourceTransaction; + using (cancellationToken.Register(() => { try { command.Cancel(); } catch { } })) { cancellationToken.ThrowIfCancellationRequested(); command.ExecuteNonQuery(); } + } + return; } + var response = db.ExecSQL(sql); + if (response == null || !response.ContainsKey("status") || !string.Equals(response["status"], "OK", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException(DatabaseExecutionResultService.GetFailureReason(response)); + } + private void EndSnapshot() + { + if (sourceTransaction != null) { sourceTransaction.Rollback(); sourceTransaction.Dispose(); sourceTransaction = null; } + else ExecuteSourceChecked("ROLLBACK", CancellationToken.None); + } - if (dt == null || dt.Rows.Count == 0) return 0; - - int colCount = dt.Columns.Count; - var colNames = new string[colCount]; - var isSpatialCol = new bool[colCount]; + private sealed class ExportColumn + { + public string Name, DataType; + public bool Spatial; + } - for (int i = 0; i < colCount; i++) + private List GetWritableColumns(OracleTableExportItem item, CancellationToken cancellationToken) + { + DataTable metadata; + if (db is my_oracle) { - colNames[i] = "\"" + dt.Columns[i].ColumnName + "\""; - isSpatialCol[i] = spatialSet.Contains(dt.Columns[i].ColumnName); + var parameters = new Dictionary { { "owner", item.Owner }, { "tableName", item.Name } }; + try { metadata = QueryMetadata("SELECT COLUMN_NAME, DATA_TYPE, VIRTUAL_COLUMN, HIDDEN_COLUMN, IDENTITY_COLUMN FROM ALL_TAB_COLS WHERE OWNER=:owner AND TABLE_NAME=:tableName ORDER BY COLUMN_ID", parameters, cancellationToken); } + catch (OracleException ex) + { + if (ex.Number != 904) throw; + metadata = QueryMetadata("SELECT COLUMN_NAME, DATA_TYPE, VIRTUAL_COLUMN, HIDDEN_COLUMN FROM ALL_TAB_COLS WHERE OWNER=:owner AND TABLE_NAME=:tableName ORDER BY COLUMN_ID", parameters, cancellationToken); + } } - - string insertPrefix = string.Format("INSERT INTO {0} ({1}) VALUES (", item.FullQualifiedName, string.Join(", ", colNames)); - - foreach (DataRow row in dt.Rows) + else metadata = db.GetColumns(item.Owner, item.Name); + if (metadata == null || metadata.Rows.Count == 0) throw new InvalidOperationException("Column metadata unavailable: " + item.DisplayName + ". SELECT * fallback is disabled."); + var columns = new List(); + foreach (DataRow row in metadata.Rows) { - cancellationToken.ThrowIfCancellationRequested(); - rowCount++; - batchCount++; - - sw.Write(insertPrefix); - for (int c = 0; c < colCount; c++) + string name = MetadataValue(row, "COLUMN_NAME"); + string type = MetadataValue(row, "DATA_TYPE").ToUpperInvariant(); + if (MetadataValue(row, "VIRTUAL_COLUMN").Equals("YES", StringComparison.OrdinalIgnoreCase) || MetadataValue(row, "HIDDEN_COLUMN").Equals("YES", StringComparison.OrdinalIgnoreCase)) continue; + var timestamp = System.Text.RegularExpressions.Regex.Match(type, @"^TIMESTAMP\((\d+)\)$"); + if (timestamp.Success) { - if (c > 0) sw.Write(", "); - object val = row[c]; - bool spatial = isSpatialCol[c]; - - sw.Write(FormatOracleLiteral(val, spatial)); + if (int.Parse(timestamp.Groups[1].Value, CultureInfo.InvariantCulture) > 7) throw new NotSupportedException("Oracle timestamp precision above .NET ticks requires a native timestamp codec."); + type = "TIMESTAMP"; } - sw.WriteLine(");"); + if (MetadataValue(row, "IDENTITY_COLUMN").Equals("YES", StringComparison.OrdinalIgnoreCase) || !string.IsNullOrWhiteSpace(MetadataValue(row, "IDENTITY_GENERATION"))) + throw new NotSupportedException("Identity data export requires a verified identity sequence-state restore policy: " + item.DisplayName + "." + name + ". SchemaOnly remains available."); + bool spatial = item.SpatialColumnNames.Contains(name) || type == "SDO_GEOMETRY"; + if (!spatial && Array.IndexOf(new[] { "NUMBER", "FLOAT", "BINARY_FLOAT", "BINARY_DOUBLE", "VARCHAR2", "NVARCHAR2", "CHAR", "NCHAR", "CLOB", "RAW", "BLOB", "DATE", "TIMESTAMP" }, type) < 0) + throw new NotSupportedException("Unsupported Oracle data type (no implicit string conversion): " + item.DisplayName + "." + name + " " + type); + columns.Add(new ExportColumn { Name = name, DataType = type, Spatial = spatial }); + } + if (columns.Count == 0) throw new NotSupportedException("No writable Oracle columns: " + item.DisplayName); + return columns; + } + private static string MetadataValue(DataRow row, string name) => row.Table.Columns.Contains(name) && row[name] != DBNull.Value ? Convert.ToString(row[name], CultureInfo.InvariantCulture) : ""; - if (batchCount >= options.BatchSize) + private long ExportTableDataStreaming(OracleTableExportItem item, List columns, StreamWriter writer, OracleExportWizardOptions options, CancellationToken cancellationToken, Action onRow) + { + var projection = new List(); + var names = new List(); + foreach (var column in columns) + { + string quoted = "t." + OracleImportRunner.QuoteIdentifier(column.Name); + if (column.Spatial && db is my_oracle) ValidateNativeGeometry(item, column, cancellationToken); + names.Add(OracleImportRunner.QuoteIdentifier(column.Name)); + if (column.Spatial) { - sw.WriteLine("COMMIT;"); - batchCount = 0; + projection.Add("CASE WHEN " + quoted + " IS NOT NULL THEN SDO_UTIL.TO_WKBGEOMETRY(" + quoted + ") ELSE NULL END"); + projection.Add(quoted + ".SDO_SRID"); + projection.Add(quoted + ".SDO_GTYPE"); } + else projection.Add(quoted); } - - return rowCount; + string sql = "SELECT " + string.Join(", ", projection) + " FROM " + item.FullQualifiedName + " t"; + string prefix = "INSERT INTO " + item.FullQualifiedName + " (" + string.Join(", ", names) + ") VALUES ("; + var native = db as my_oracle; + if (native != null) + { + using (var command = new OracleCommand(sql, native.MCT)) + { + command.Transaction = sourceTransaction; + using (cancellationToken.Register(() => { try { command.Cancel(); } catch { } })) + using (var reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) + return WriteRows(reader, columns, writer, prefix, options.BatchSize, cancellationToken, onRow); + } + } + var source = db as IOracleExportReaderSource; + if (source == null) throw new NotSupportedException("Oracle export requires a forward-only reader source; full-table DataTable fallback is disabled."); + using (var reader = source.OpenExportReader(sql, cancellationToken)) + return WriteRows(reader, columns, writer, prefix, options.BatchSize, cancellationToken, onRow); } - public static string FormatOracleLiteral(object val, bool isSpatial) + private void ValidateNativeGeometry(OracleTableExportItem item, ExportColumn column, CancellationToken cancellationToken) { - if (val == null || val == DBNull.Value) return "NULL"; + string geometry = "t." + OracleImportRunner.QuoteIdentifier(column.Name); + string sql = "SELECT COUNT(*) FROM " + item.FullQualifiedName + " t WHERE " + geometry + " IS NOT NULL AND (" + + "MOD(TRUNC(" + geometry + ".SDO_GTYPE/100),10) != 0 OR TRUNC(" + geometry + ".SDO_GTYPE/1000) NOT IN (2,3) OR " + + "MOD(" + geometry + ".SDO_GTYPE,1000) NOT BETWEEN 1 AND 7 OR EXISTS (SELECT 1 FROM " + + "(SELECT COLUMN_VALUE AS V, ROWNUM AS N FROM TABLE(" + geometry + ".SDO_ELEM_INFO)) e " + + "WHERE (MOD(e.N,3)=2 AND e.V NOT IN (1,2,1003,2003)) OR (MOD(e.N,3)=0 AND e.V != 1)))"; + var native = (my_oracle)db; + using (var command = new OracleCommand(sql, native.MCT)) + { + command.Transaction = sourceTransaction; + using (cancellationToken.Register(() => { try { command.Cancel(); } catch { } })) + if (Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture) != 0) + throw new NotSupportedException("Curved, compound, non-linear interpretation, LRS or unsupported geometry is not representable by this bounded WKB codec: " + item.DisplayName + "." + column.Name); + } + } - // SDO_GEOMETRY 空間幾何無損還原 - if (isSpatial) + private static long WriteRows(IDataReader reader, List columns, StreamWriter writer, string prefix, int batchSize, CancellationToken cancellationToken, Action onRow) + { + long rows = 0; + while (true) { - if (val is byte[] geomBytes) + cancellationToken.ThrowIfCancellationRequested(); + if (!reader.Read()) break; + cancellationToken.ThrowIfCancellationRequested(); + var insert = new StringBuilder(prefix); + var declarations = new StringBuilder(); + var initialize = new StringBuilder(); + var cleanup = new StringBuilder(); + int ordinal = 0, valueBytes = 0; + foreach (var column in columns) { - var hexSb = new StringBuilder(geomBytes.Length * 2); - for (int i = 0; i < geomBytes.Length; i++) + cancellationToken.ThrowIfCancellationRequested(); + if (ordinal > 0) insert.Append(", "); + object value = ReadBoundedValue(reader, ordinal++, column); + valueBytes += value is byte[] ? ((byte[])value).Length : value is string ? Encoding.UTF8.GetByteCount((string)value) : 32; + if (valueBytes > MaxRowValueBytes) throw new NotSupportedException("Oracle row values exceed the 8 MiB budget."); + string expression = ValueExpression(value, column, ordinal, declarations, initialize, cleanup); + if (column.Spatial) { - hexSb.Append(geomBytes[i].ToString("X2")); + object srid = reader.IsDBNull(ordinal) ? null : reader.GetValue(ordinal); ordinal++; + object gtype = reader.IsDBNull(ordinal) ? null : reader.GetValue(ordinal); ordinal++; + if (value == null || value == DBNull.Value) insert.Append("NULL"); + else + { + if (gtype == null) throw new NotSupportedException("Geometry GTYPE is required for WKB export."); + int type = Convert.ToInt32(gtype, CultureInfo.InvariantCulture); + if ((type / 100) % 10 != 0 || (type / 1000 != 2 && type / 1000 != 3) || type % 1000 < 1 || type % 1000 > 7) + throw new NotSupportedException("LRS/unsupported geometry requires a native codec; no geometry is approximated."); + string sridSql = FormatOracleLiteral(srid, false); + insert.Append("SDO_UTIL.FROM_WKBGEOMETRY(").Append(expression).Append(", ").Append(sridSql).Append(")"); + } } - return string.Format("SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('{0}'))", hexSb.ToString()); + else insert.Append(expression); } + insert.Append(");"); + string sql = declarations.Length == 0 ? insert.ToString() : "DECLARE\n" + declarations + "BEGIN\n" + initialize + insert + "\n" + cleanup + "EXCEPTION WHEN OTHERS THEN\n" + cleanup + "RAISE;\nEND;\n/"; + if (Encoding.UTF8.GetByteCount(sql) > MaxRowSqlBytes) throw new NotSupportedException("Oracle row SQL exceeds the 28 MiB statement budget."); + writer.WriteLine(sql); rows++; + if (rows % batchSize == 0) { writer.WriteLine("COMMIT;"); writer.Flush(); } + onRow?.Invoke(rows); + } + return rows; + } - string strVal = val.ToString().Trim(); - if (strVal.StartsWith("SDO_GEOMETRY", StringComparison.OrdinalIgnoreCase) || - strVal.StartsWith("MDSYS.SDO_GEOMETRY", StringComparison.OrdinalIgnoreCase)) - { - return strVal; - } - if (strVal.StartsWith("POINT", StringComparison.OrdinalIgnoreCase) || - strVal.StartsWith("POLYGON", StringComparison.OrdinalIgnoreCase) || - strVal.StartsWith("LINESTRING", StringComparison.OrdinalIgnoreCase) || - strVal.StartsWith("MULTIPOINT", StringComparison.OrdinalIgnoreCase) || - strVal.StartsWith("MULTILINESTRING", StringComparison.OrdinalIgnoreCase) || - strVal.StartsWith("MULTIPOLYGON", StringComparison.OrdinalIgnoreCase)) + private static string ValueExpression(object value, ExportColumn column, int ordinal, StringBuilder declarations, StringBuilder initialize, StringBuilder cleanup) + { + bool blob = value is byte[] && ((byte[])value).Length > MaxBinaryLiteralBytes; + bool clob = value is string && Encoding.UTF8.GetByteCount(((string)value).Replace("'", "''")) > MaxTextLiteralBytes; + if (value is byte[] && ((byte[])value).Length == 0) + { + if (column.Spatial) throw new NotSupportedException("Non-null geometry has empty WKB."); + if (column.DataType == "BLOB") return "EMPTY_BLOB()"; + } + if (value is string && ((string)value).Length == 0 && column.DataType == "CLOB") return "EMPTY_CLOB()"; + if (!blob && !clob) + { + string literal = FormatOracleLiteral(value, false); + return value is byte[] && (column.Spatial || column.DataType == "BLOB") ? "TO_BLOB(" + literal + ")" : literal; + } + if (blob && !column.Spatial && column.DataType != "BLOB") throw new NotSupportedException("RAW values over 2000 bytes require a verified extended-RAW codec."); + if (clob && column.DataType != "CLOB") throw new NotSupportedException("Oversized non-CLOB text cannot use a temporary CLOB without changing the type."); + string variable = "mp_lob_" + ordinal; + declarations.Append(" ").Append(variable).Append(blob ? " BLOB;\n" : " CLOB;\n"); + initialize.Append(" DBMS_LOB.CREATETEMPORARY(").Append(variable).Append(", TRUE);\n"); + if (blob) + { + byte[] bytes = (byte[])value; + for (int offset = 0; offset < bytes.Length; offset += MaxBinaryLiteralBytes) { - return string.Format("SDO_GEOMETRY('{0}', 4326)", strVal.Replace("'", "''")); + int count = Math.Min(MaxBinaryLiteralBytes, bytes.Length - offset); + var chunk = new byte[count]; Array.Copy(bytes, offset, chunk, 0, count); + initialize.Append(" DBMS_LOB.WRITEAPPEND(").Append(variable).Append(", ").Append(count).Append(", ").Append(FormatOracleLiteral(chunk, false)).Append(");\n"); } } - - // 二進位 (RAW / BLOB) - if (val is byte[] bytes) + else { - var hex = new StringBuilder(bytes.Length * 2); - for (int i = 0; i < bytes.Length; i++) + string text = (string)value; + if (text.IndexOf('\0') >= 0) throw new NotSupportedException("NUL CLOB text requires a native character codec."); + for (int offset = 0; offset < text.Length;) { - hex.Append(bytes[i].ToString("X2")); + int count = Math.Min(500, text.Length - offset); // <= 3000 UTF-8 bytes even when every quote doubles. + if (offset + count < text.Length && char.IsHighSurrogate(text[offset + count - 1])) count--; + string literal = FormatOracleLiteral(text.Substring(offset, count), false); + initialize.Append(" DBMS_LOB.WRITEAPPEND(").Append(variable).Append(", LENGTH(").Append(literal).Append("), ").Append(literal).Append(");\n"); + offset += count; } - return string.Format("HEXTORAW('{0}')", hex.ToString()); } + cleanup.Append(" IF DBMS_LOB.ISTEMPORARY(").Append(variable).Append(") = 1 THEN DBMS_LOB.FREETEMPORARY(").Append(variable).Append("); END IF;\n"); + return variable; + } - if (val is bool b) + private static object ReadBoundedValue(IDataReader reader, int ordinal, ExportColumn column) + { + if (reader.IsDBNull(ordinal)) return DBNull.Value; + if (column.Spatial || column.DataType == "BLOB" || column.DataType == "RAW") { - return b ? "1" : "0"; + long length = reader.GetBytes(ordinal, 0, null, 0, 0); + if (length > MaxRowValueBytes) throw new NotSupportedException("Oracle BLOB/RAW/WKB exceeds the 8 MiB row value budget."); + var buffer = new byte[(int)length]; + int position = 0; + while (position < buffer.Length) { int count = (int)reader.GetBytes(ordinal, position, buffer, position, buffer.Length - position); if (count == 0) throw new EndOfStreamException("Incomplete Oracle binary value."); position += count; } + return buffer; } - - if (val is sbyte || val is byte || val is short || val is ushort || - val is int || val is uint || val is long || val is ulong || - val is float || val is double || val is decimal) + if (column.DataType.Contains("CHAR") || column.DataType == "CLOB" || column.DataType == "NCLOB") { - return Convert.ToString(val, CultureInfo.InvariantCulture); + long length = reader.GetChars(ordinal, 0, null, 0, 0); + if (length > MaxRowValueBytes / 2) throw new NotSupportedException("Oracle CLOB/text exceeds the 8 MiB row value budget."); + var buffer = new char[(int)length]; + int position = 0; + while (position < buffer.Length) { int count = (int)reader.GetChars(ordinal, position, buffer, position, buffer.Length - position); if (count == 0) throw new EndOfStreamException("Incomplete Oracle text value."); position += count; } + return new string(buffer); } + return reader.GetValue(ordinal); + } - if (val is DateTime dt) + public static string FormatOracleLiteral(object value, bool isSpatial) + { + if (value == null || value == DBNull.Value) return "NULL"; + if (isSpatial) throw new NotSupportedException("Geometry export requires per-value SRID/GTYPE; use the native reader projection."); + if (value is byte[]) + { + var bytes = (byte[])value; + if (bytes.Length > MaxBinaryLiteralBytes) throw new NotSupportedException("Oracle binary literal exceeds 2000 bytes."); + return "HEXTORAW('" + BitConverter.ToString(bytes).Replace("-", "") + "')"; + } + if (value is DateTime) return "TO_TIMESTAMP('" + ((DateTime)value).ToString("yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.InvariantCulture) + "', 'YYYY-MM-DD HH24:MI:SS.FF7')"; + if (value is bool) return (bool)value ? "1" : "0"; + if (value is double && (double.IsNaN((double)value) || double.IsInfinity((double)value)) || value is float && (float.IsNaN((float)value) || float.IsInfinity((float)value))) + throw new NotSupportedException("Non-finite Oracle floating values require a native codec."); + if (value is double) + { + double number = (double)value; + if (number == 0d && BitConverter.DoubleToInt64Bits(number) < 0) throw new NotSupportedException("Signed-zero BINARY_DOUBLE requires a verified native bit-preserving codec."); + return number.ToString("G17", CultureInfo.InvariantCulture) + "d"; + } + if (value is float) { - return string.Format( - "TO_TIMESTAMP('{0}', 'YYYY-MM-DD HH24:MI:SS.FF7')", - dt.ToString("yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.InvariantCulture)); + float number = (float)value; + if (number == 0f && BitConverter.ToInt32(BitConverter.GetBytes(number), 0) < 0) throw new NotSupportedException("Signed-zero BINARY_FLOAT requires a verified native bit-preserving codec."); + return number.ToString("G9", CultureInfo.InvariantCulture) + "f"; } + if (value is byte || value is sbyte || value is short || value is ushort || value is int || value is uint || value is long || value is ulong || value is decimal) + return Convert.ToString(value, CultureInfo.InvariantCulture); + if (!(value is string)) throw new NotSupportedException("Unsupported Oracle provider value: " + value.GetType().FullName); + string text = (string)value; + if (text.IndexOf('\0') >= 0) throw new NotSupportedException("Oracle NUL text requires a verified native codec."); + string escaped = text.Replace("'", "''"); + if (Encoding.UTF8.GetByteCount(escaped) > MaxTextLiteralBytes) throw new NotSupportedException("Escaped Oracle UTF-8 text literal exceeds 4000 bytes."); + return "'" + escaped + "'"; + } - string s = val.ToString(); - // CLOB 超長字串支援 - if (s.Length > 4000) + private void WriteTableDdl(OracleTableExportItem item, OracleExportWizardOptions options, StreamWriter writer, List deferred, CancellationToken cancellationToken) + { + string ddl = db is my_oracle ? GetNativeDdl("TABLE", item.Owner, item.Name, cancellationToken) : db.GetTableCreateStatement(item.Owner, item.Name); + if (string.IsNullOrWhiteSpace(ddl)) throw new InvalidOperationException("Native table DDL unavailable: " + item.DisplayName); + using (var reader = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(ddl)), Encoding.UTF8)) + { + foreach (var statement in OracleImportRunner.EnumerateStatements(reader, cancellationToken)) + { + string sql = statement.Sql.Trim(); + bool index = System.Text.RegularExpressions.Regex.IsMatch(sql, @"^CREATE\s+(UNIQUE\s+|BITMAP\s+)?INDEX\b", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + bool spatial = sql.IndexOf("MDSYS.SPATIAL_INDEX", StringComparison.OrdinalIgnoreCase) >= 0; + bool comment = sql.StartsWith("COMMENT ", StringComparison.OrdinalIgnoreCase); + if (index) { if (spatial ? options.IncludeSpatialIndexes : options.IncludeIndexes) deferred.Add(sql); } + else if (!comment || options.IncludeComments) WriteStatement(writer, sql); + } + } + if (!(db is my_oracle)) return; + var parameters = new Dictionary { { "owner", item.Owner }, { "tableName", item.Name } }; + if (options.IncludeComments) + { + var comments = QueryMetadata("SELECT COLUMN_NAME, COMMENTS FROM ALL_COL_COMMENTS WHERE OWNER=:owner AND TABLE_NAME=:tableName AND COMMENTS IS NOT NULL UNION ALL SELECT NULL, COMMENTS FROM ALL_TAB_COMMENTS WHERE OWNER=:owner AND TABLE_NAME=:tableName AND COMMENTS IS NOT NULL", parameters, cancellationToken); + foreach (DataRow row in comments.Rows) + { + string column = MetadataValue(row, "COLUMN_NAME"); + writer.WriteLine("COMMENT ON " + (column.Length == 0 ? "TABLE " + item.FullQualifiedName : "COLUMN " + item.FullQualifiedName + "." + OracleImportRunner.QuoteIdentifier(column)) + " IS " + FormatOracleLiteral(row["COMMENTS"], false) + ";"); + } + } + if (options.IncludeIndexes || options.IncludeSpatialIndexes) { - // 超過 4000 字元切段為 TO_CLOB(...) || TO_CLOB(...) - var sb = new StringBuilder(); - int idx = 0; - while (idx < s.Length) + var indexes = QueryMetadata("SELECT i.OWNER, i.INDEX_NAME, i.INDEX_TYPE, i.ITYP_OWNER, i.ITYP_NAME FROM ALL_INDEXES i WHERE i.TABLE_OWNER=:owner AND i.TABLE_NAME=:tableName AND NOT EXISTS (SELECT 1 FROM ALL_CONSTRAINTS c WHERE c.OWNER=i.OWNER AND c.INDEX_NAME=i.INDEX_NAME AND c.CONSTRAINT_TYPE IN ('P','U')) ORDER BY i.INDEX_NAME", parameters, cancellationToken); + foreach (DataRow row in indexes.Rows) { - int chunkLen = Math.Min(2000, s.Length - idx); - string chunk = s.Substring(idx, chunkLen); - if (sb.Length > 0) sb.Append(" || "); - sb.Append("TO_CLOB('").Append(chunk.Replace("'", "''")).Append("')"); - idx += chunkLen; + bool spatial = MetadataValue(row, "ITYP_OWNER") == "MDSYS" && MetadataValue(row, "ITYP_NAME").StartsWith("SPATIAL_INDEX", StringComparison.OrdinalIgnoreCase); + if (spatial ? options.IncludeSpatialIndexes : options.IncludeIndexes) + deferred.Add(GetNativeDdl("INDEX", MetadataValue(row, "OWNER"), MetadataValue(row, "INDEX_NAME"), cancellationToken)); } - return sb.ToString(); } + } - return "'" + s.Replace("'", "''") + "'"; + private string GetNativeDdl(string kind, string owner, string name, CancellationToken cancellationToken) + { + var native = (my_oracle)db; + using (var command = new OracleCommand("SELECT DBMS_METADATA.GET_DDL(:kind,:name,:owner) FROM DUAL", native.MCT)) + { + command.BindByName = true; command.Transaction = sourceTransaction; + command.Parameters.Add("kind", OracleDbType.Varchar2).Value = kind; + command.Parameters.Add("name", OracleDbType.Varchar2).Value = name; + command.Parameters.Add("owner", OracleDbType.Varchar2).Value = owner; + using (cancellationToken.Register(() => { try { command.Cancel(); } catch { } })) + using (var reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) + { + if (!reader.Read() || reader.IsDBNull(0)) throw new InvalidOperationException("DBMS_METADATA returned no DDL: " + owner + "." + name); + long count = reader.GetChars(0, 0, null, 0, 0); + if (count > MaxMetadataChars) throw new NotSupportedException("Oracle DDL exceeds 1 MiB metadata bound."); + var chars = new char[(int)count]; int position = 0; + while (position < chars.Length) { int fetched = (int)reader.GetChars(0, position, chars, position, chars.Length - position); if (fetched == 0) throw new EndOfStreamException(); position += fetched; } + return new string(chars); + } + } } - private static string ComputeSha256(string filePath) + private void WriteSpatialMetadata(OracleTableExportItem item, StreamWriter writer, CancellationToken cancellationToken) { - using (var sha = SHA256.Create()) - using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize)) + foreach (string column in item.SpatialColumnNames) { - byte[] hash = sha.ComputeHash(fs); - var sb = new StringBuilder(hash.Length * 2); - for (int i = 0; i < hash.Length; i++) + var parameters = new Dictionary { { "owner", item.Owner }, { "tableName", item.Name }, { "columnName", column } }; + var metadata = QueryMetadata("SELECT SRID, CASE WHEN DIMINFO IS NULL THEN 1 ELSE 0 END AS DIMINFO_NULL FROM ALL_SDO_GEOM_METADATA WHERE OWNER=:owner AND TABLE_NAME=:tableName AND COLUMN_NAME=:columnName", parameters, cancellationToken); + if (metadata == null || metadata.Rows.Count > 1) throw new InvalidOperationException("Invalid Oracle spatial metadata: " + item.DisplayName + "." + column); + writer.WriteLine("DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME=" + FormatOracleLiteral(item.Name, false) + " AND COLUMN_NAME=" + FormatOracleLiteral(column, false) + ";"); + if (metadata.Rows.Count == 0) continue; + var row = metadata.Rows[0]; + string dimensionsSql = "NULL"; + if (Convert.ToInt32(row["DIMINFO_NULL"], CultureInfo.InvariantCulture) == 0) { - sb.Append(hash[i].ToString("x2")); + var dimensions = QueryMetadata("SELECT ROWNUM AS DIM_ORD, d.SDO_DIMNAME AS DIM_NAME, d.SDO_LB AS LOWER_BOUND, d.SDO_UB AS UPPER_BOUND, d.SDO_TOLERANCE AS TOLERANCE FROM ALL_SDO_GEOM_METADATA m, TABLE(m.DIMINFO) d WHERE m.OWNER=:owner AND m.TABLE_NAME=:tableName AND m.COLUMN_NAME=:columnName ORDER BY DIM_ORD", parameters, cancellationToken); + if (dimensions == null || dimensions.Rows.Count > 4) throw new InvalidOperationException("Incomplete spatial DIMINFO metadata."); + var parts = new List(); + foreach (DataRow dimension in dimensions.Rows) + parts.Add("MDSYS.SDO_DIM_ELEMENT(" + FormatOracleLiteral(dimension["DIM_NAME"], false) + ", " + FormatOracleLiteral(dimension["LOWER_BOUND"], false) + ", " + FormatOracleLiteral(dimension["UPPER_BOUND"], false) + ", " + FormatOracleLiteral(dimension["TOLERANCE"], false) + ")"); + dimensionsSql = "MDSYS.SDO_DIM_ARRAY(" + string.Join(", ", parts) + ")"; } - return sb.ToString(); + writer.WriteLine("INSERT INTO USER_SDO_GEOM_METADATA (TABLE_NAME,COLUMN_NAME,DIMINFO,SRID) VALUES (" + FormatOracleLiteral(item.Name, false) + ", " + FormatOracleLiteral(column, false) + ", " + dimensionsSql + ", " + FormatOracleLiteral(row["SRID"], false) + ");"); } } + private static void WriteStatement(StreamWriter writer, string sql) + { + writer.WriteLine(sql.Trim().TrimEnd(';') + ";"); + if (System.Text.RegularExpressions.Regex.IsMatch(sql.Trim(), @"^(BEGIN|DECLARE|CREATE\s+(OR\s+REPLACE\s+)?(TRIGGER|PROCEDURE|FUNCTION|PACKAGE|TYPE))\b", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) writer.WriteLine("/"); + } + private static string ComputeSha256(string filePath) + { + using (var hash = SHA256.Create()) + using (var stream = File.OpenRead(filePath)) return BitConverter.ToString(hash.ComputeHash(stream)).Replace("-", "").ToLowerInvariant(); + } } } diff --git a/mySQLPunk/lib/OracleImport/OracleImportLog.cs b/mySQLPunk/lib/OracleImport/OracleImportLog.cs index 04bd1c6..651970c 100644 --- a/mySQLPunk/lib/OracleImport/OracleImportLog.cs +++ b/mySQLPunk/lib/OracleImport/OracleImportLog.cs @@ -22,10 +22,11 @@ public OracleImportLog(string targetDatabase) } string dbSanitized = string.IsNullOrWhiteSpace(targetDatabase) ? "default" : targetDatabase.Replace(':', '_').Replace('/', '_').Replace('\\', '_'); - string fileName = string.Format("import-oracle-{0}-{1:yyyyMMdd_HHmmss}.log", dbSanitized, DateTime.Now); + foreach (char invalid in Path.GetInvalidFileNameChars()) dbSanitized = dbSanitized.Replace(invalid, '_'); + string fileName = string.Format("import-oracle-{0}-{1:yyyyMMdd_HHmmss}.log", dbSanitized, DateTime.Now).Replace(".log", "-" + Guid.NewGuid().ToString("N") + ".log"); logPath = Path.Combine(logDir, fileName); - writer = new StreamWriter(new FileStream(logPath, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8); + writer = new StreamWriter(new FileStream(logPath, FileMode.CreateNew, FileAccess.Write, FileShare.Read), Encoding.UTF8) { AutoFlush = true }; writer.WriteLine("===================================================================="); writer.WriteLine(" mySQLPunk Oracle Import Execution Log"); writer.WriteLine(string.Format(" Target Schema/Owner : {0}", targetDatabase)); diff --git a/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs b/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs index 5ef8399..7c145c5 100644 --- a/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs +++ b/mySQLPunk/lib/OracleImport/OracleImportPreflightScanner.cs @@ -9,295 +9,57 @@ namespace mySQLPunk.lib.OracleImport { public sealed class OracleImportPreflightScanner { - private const int MaxBatchSizeBytes = 32 * 1024 * 1024; // 32 MiB - - private static readonly string[] SpatialKeywords = new[] - { - "sdo_geometry", "mdsys.sdo_geometry", "sdo_util", "user_sdo_geom_metadata", - "spatial_index", "sdo_dim_array", "sdo_dim_element", "sdo_point_type" - }; - - private static readonly Regex PlSqlStartRegex = new Regex( - @"^\s*(DECLARE\b|BEGIN\b|CREATE\s+(OR\s+REPLACE\s+)?(PROCEDURE|FUNCTION|PACKAGE|TRIGGER|TYPE)\b)", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - public OracleImportPreflight Scan(string filePath, CancellationToken cancellationToken = default(CancellationToken)) { - if (string.IsNullOrWhiteSpace(filePath)) - return new OracleImportPreflight(false, "未指定匯入檔案路徑。"); - - if (!File.Exists(filePath)) - return new OracleImportPreflight(false, string.Format("找不到指定的 SQL 檔案: {0}", filePath)); - - var fi = new FileInfo(filePath); - var result = new OracleImportPreflight - { - FileSizeBytes = fi.Length, - CanProceed = true - }; - - Encoding detectedEncoding = DetectEncoding(filePath); - result.DetectedEncoding = detectedEncoding; - + if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath)) return new OracleImportPreflight(false, "SQL file not found: " + filePath); + var result = new OracleImportPreflight { FileSizeBytes = new FileInfo(filePath).Length, DetectedEncoding = DetectEncoding(filePath) }; try { - using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) - using (var reader = new StreamReader(fs, detectedEncoding, true, 65536)) + Action comment = line => { - int statementCount = 0; - long currentStatementBytes = 0; - bool hasStatementContent = false; - bool inSingleQuote = false; - bool inDoubleQuote = false; - bool inLineComment = false; - int blockCommentDepth = 0; - bool inPlSqlBlock = false; - - var spatialFound = new HashSet(StringComparer.OrdinalIgnoreCase); - - string line; - int lineNumber = 0; - - while ((line = reader.ReadLine()) != null) + if (line.Contains("mySQLPunk Native Manifest")) result.ContainsManifest = true; + var table = Regex.Match(line, @"^--\s*Manifest-Table:\s*(.+?)\s*\|\s*Rows:\s*(\d+)\s*$", RegexOptions.IgnoreCase); + long rows; + if (table.Success && long.TryParse(table.Groups[2].Value, out rows)) result.ManifestDeclaredTables[table.Groups[1].Value] = rows; + var hash = Regex.Match(line, @"^--\s*Manifest-Checksum-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase); + if (hash.Success) result.ManifestSha256 = hash.Groups[1].Value.ToLowerInvariant(); + }; + using (var reader = new StreamReader(filePath, result.DetectedEncoding, true, 65536)) + { + foreach (var item in OracleImportRunner.EnumerateStatements(reader, cancellationToken, comment)) { - cancellationToken.ThrowIfCancellationRequested(); - lineNumber++; - - // NUL byte 損毀快速檢查 - if (line.IndexOf('\0') >= 0) + result.StatementCount++; + if (Regex.IsMatch(item.Sql, @"^\s*(DECLARE|BEGIN|CREATE\s+(OR\s+REPLACE\s+)?(PROCEDURE|FUNCTION|PACKAGE|TRIGGER|TYPE))\b", RegexOptions.IgnoreCase)) { - result.CanProceed = false; - result.Issues.Add(string.Format("第 {0} 行包含二進位 NUL 空字元，檔案可能已損毀或非純文字 SQL 腳本。", lineNumber)); - return result; + result.HasPlSqlBlocks = true; + result.PlSqlBlocksCount++; } - - // 檢查 Native Manifest 標記 - string trimmed = line.Trim(); - if (trimmed.StartsWith("--", StringComparison.Ordinal)) - { - if (trimmed.Contains("mySQLPunk Native Manifest")) - { - result.ContainsManifest = true; - } - - var matchTable = Regex.Match(trimmed, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); - if (matchTable.Success && long.TryParse(matchTable.Groups[2].Value, out long rows)) - { - result.ManifestDeclaredTables[matchTable.Groups[1].Value] = rows; - } - - var matchSha = Regex.Match(trimmed, @"^--\s*Manifest-Checksum-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase); - if (matchSha.Success) - { - result.ManifestSha256 = matchSha.Groups[1].Value.ToLowerInvariant(); - } - } - - // 探測 Oracle Spatial 空間關鍵字 - string lowerLine = line.ToLowerInvariant(); - foreach (var kw in SpatialKeywords) - { - if (lowerLine.Contains(kw)) + foreach (string keyword in new[] { "sdo_geometry", "sdo_util", "user_sdo_geom_metadata", "spatial_index", "sdo_dim_array", "sdo_dim_element" }) + if (item.Sql.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) >= 0) { result.HasSpatialFeatures = true; - if (spatialFound.Add(kw)) - { - result.DetectedSpatialKeywords.Add(kw); - } - } - } - - // 檢查是否開展 PL/SQL 區塊 (DECLARE / BEGIN / CREATE PROCEDURE 等) - if (!inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) - { - if (PlSqlStartRegex.IsMatch(line)) - { - inPlSqlBlock = true; - result.HasPlSqlBlocks = true; - result.PlSqlBlocksCount++; - } - } - - // 檢查 PL/SQL 區塊結束符：獨立行斜線 / - if (inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) - { - if (trimmed == "/") - { - inPlSqlBlock = false; - if (hasStatementContent) - { - statementCount++; - hasStatementContent = false; - currentStatementBytes = 0; - } - continue; - } - } - - int len = line.Length; - for (int i = 0; i < len; i++) - { - char c = line[i]; - - // 1. 單行註解 - if (inLineComment) - { - continue; + if (!result.DetectedSpatialKeywords.Contains(keyword)) result.DetectedSpatialKeywords.Add(keyword); } - - // 2. 塊註解 /* ... */ - if (blockCommentDepth > 0) - { - if (c == '*' && i + 1 < len && line[i + 1] == '/') - { - blockCommentDepth--; - i++; - } - continue; - } - - // 3. 單引號字串 '...' - if (inSingleQuote) - { - hasStatementContent = true; - currentStatementBytes += (c > 127) ? 2 : 1; - if (c == '\'') - { - if (i + 1 < len && line[i + 1] == '\'') - { - i++; // 跳過 '' - currentStatementBytes++; - } - else - { - inSingleQuote = false; - } - } - continue; - } - - // 4. 雙引號識別字 "..." - if (inDoubleQuote) - { - hasStatementContent = true; - currentStatementBytes += (c > 127) ? 2 : 1; - if (c == '"') - { - if (i + 1 < len && line[i + 1] == '"') - { - i++; // 跳過 "" - currentStatementBytes++; - } - else - { - inDoubleQuote = false; - } - } - continue; - } - - // 5. 符號開始判定 - if (c == '-' && i + 1 < len && line[i + 1] == '-') - { - inLineComment = true; - i++; - continue; - } - - if (c == '/' && i + 1 < len && line[i + 1] == '*') - { - blockCommentDepth = 1; - i++; - continue; - } - - if (c == '\'') - { - inSingleQuote = true; - hasStatementContent = true; - currentStatementBytes += 1; - continue; - } - - if (c == '"') - { - inDoubleQuote = true; - hasStatementContent = true; - currentStatementBytes += 1; - continue; - } - - // 6. 語句結束判定 - if (c == ';' && !inPlSqlBlock) - { - if (hasStatementContent) - { - statementCount++; - hasStatementContent = false; - currentStatementBytes = 0; - } - continue; - } - - if (!char.IsWhiteSpace(c)) - { - hasStatementContent = true; - } - - if (hasStatementContent) - { - currentStatementBytes += (c > 127) ? 2 : 1; - if (currentStatementBytes > MaxBatchSizeBytes) - { - result.CanProceed = false; - result.Issues.Add(string.Format("第 {0} 行單一語句大小已超過 32 MiB 安全上限。", lineNumber)); - return result; - } - } - } - - inLineComment = false; // 換行結束單行註解 - } - - if (hasStatementContent) - { - statementCount++; - } - - result.StatementCount = statementCount; - - if (statementCount == 0) - { - result.Issues.Add("SQL 腳本中未發現任何有效的執行語句。"); } } + if (result.StatementCount == 0) { result.CanProceed = false; result.Issues.Add("No executable SQL statements found."); } } - catch (Exception ex) - { - result.CanProceed = false; - result.Issues.Add("預檢掃描檔案時發生例外錯誤: " + ex.Message); - } - + catch (Exception ex) { result.CanProceed = false; result.Issues.Add(ex.Message); } return result; } private static Encoding DetectEncoding(string filePath) { - byte[] bom = new byte[4]; - using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) + var bom = new byte[4]; + using (var stream = File.OpenRead(filePath)) { - int read = fs.Read(bom, 0, 4); - if (read >= 3 && bom[0] == 0xEF && bom[1] == 0xBB && bom[2] == 0xBF) - return Encoding.UTF8; - if (read >= 2 && bom[0] == 0xFF && bom[1] == 0xFE) - return Encoding.Unicode; // UTF-16LE - if (read >= 2 && bom[0] == 0xFE && bom[1] == 0xFF) - return Encoding.BigEndianUnicode; // UTF-16BE - if (read >= 4 && bom[0] == 0 && bom[1] == 0 && bom[2] == 0xFE && bom[3] == 0xFF) - return Encoding.UTF32; + int length = stream.Read(bom, 0, bom.Length); + if (length >= 4 && bom[0] == 0 && bom[1] == 0 && bom[2] == 0xfe && bom[3] == 0xff) return new UTF32Encoding(true, true, true); + if (length >= 4 && bom[0] == 0xff && bom[1] == 0xfe && bom[2] == 0 && bom[3] == 0) return new UTF32Encoding(false, true, true); + if (length >= 2 && bom[0] == 0xff && bom[1] == 0xfe) return new UnicodeEncoding(false, true, true); + if (length >= 2 && bom[0] == 0xfe && bom[1] == 0xff) return new UnicodeEncoding(true, true, true); + return new UTF8Encoding(false, true); } - - return new UTF8Encoding(false); } } } diff --git a/mySQLPunk/lib/OracleImport/OracleImportRunner.cs b/mySQLPunk/lib/OracleImport/OracleImportRunner.cs index 55c0ca8..6f33928 100644 --- a/mySQLPunk/lib/OracleImport/OracleImportRunner.cs +++ b/mySQLPunk/lib/OracleImport/OracleImportRunner.cs @@ -7,16 +7,20 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Oracle.ManagedDataAccess.Client; namespace mySQLPunk.lib.OracleImport { public sealed class OracleImportRunner { private const int BufferSize = 65536; + private const int MaxStatementBytes = 32 * 1024 * 1024; private readonly IDatabase db; private readonly OracleImportRequest request; private readonly Action terminalSink; - + private OracleTransaction transaction; + private bool transactionActive; + private CancellationToken currentToken; private static readonly Regex PlSqlStartRegex = new Regex( @"^\s*(DECLARE\b|BEGIN\b|CREATE\s+(OR\s+REPLACE\s+)?(PROCEDURE|FUNCTION|PACKAGE|TRIGGER|TYPE)\b)", RegexOptions.IgnoreCase | RegexOptions.Compiled); @@ -33,490 +37,521 @@ public OracleImportRunner(IDatabase db, OracleImportRequest request, Action RunAsync(IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) + public Task RunAsync(IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) { - return await Task.Run(() => RunInternal(progress, cancellationToken), cancellationToken).ConfigureAwait(false); + return Task.Run(() => RunInternal(progress, cancellationToken), cancellationToken); } - private OracleImportResult RunInternal(IProgress progress = null, CancellationToken cancellationToken = default(CancellationToken)) + private OracleImportResult RunInternal(IProgress progress, CancellationToken cancellationToken) { var clock = Stopwatch.StartNew(); - var log = new OracleImportLog(request.DatabaseName); - var result = new OracleImportResult { LogFilePath = log.LogPath }; - - Action LogAndTerminal = msg => + var result = new OracleImportResult(); + currentToken = cancellationToken; + using (var log = new OracleImportLog(request.DatabaseName)) { - log.LogInfo(msg); - terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] {1}", DateTime.Now, msg)); - }; - - LogAndTerminal(string.Format("正在初始化 Oracle 匯入任務: {0}", Path.GetFileName(request.FilePath))); - LogAndTerminal(string.Format("目標綱要 (Owner): {0}", string.IsNullOrWhiteSpace(request.DatabaseName) ? "Current Schema" : request.DatabaseName)); - - // 1. 預檢掃描 - var scanner = new OracleImportPreflightScanner(); - var preflight = scanner.Scan(request.FilePath, cancellationToken); - if (!preflight.CanProceed) - { - clock.Stop(); - result.Success = false; - result.ErrorMessage = string.Join("; ", preflight.Issues.ToArray()); - LogAndTerminal("❌ 預檢未通過：" + result.ErrorMessage); - log.LogSummary(0, 0, 0, clock.Elapsed, false); - log.Flush(); - log.Dispose(); - return result; - } - - LogAndTerminal(string.Format("預檢通過：共探測到約 {0:N0} 條語句 (檔案大小: {1:N0} 位元組)。", preflight.StatementCount, preflight.FileSizeBytes)); - if (preflight.HasSpatialFeatures) - { - LogAndTerminal(string.Format("偵測到 Oracle Spatial (SDO_GEOMETRY) 空間特性關鍵字: {0}", string.Join(", ", preflight.DetectedSpatialKeywords.ToArray()))); - } - if (preflight.HasPlSqlBlocks) - { - LogAndTerminal(string.Format("偵測到 PL/SQL 預存區塊或觸發程序 (約 {0} 個區塊)。", preflight.PlSqlBlocksCount)); - } - - long totalBytes = preflight.FileSizeBytes; - int totalStatements = preflight.StatementCount; - - int currentStmtIndex = 0; - int failedStatements = 0; - long totalRowsAffected = 0; - long bytesRead = 0; - var truncatedTables = new HashSet(StringComparer.OrdinalIgnoreCase); - - var expectedManifestTables = new Dictionary(preflight.ManifestDeclaredTables, StringComparer.OrdinalIgnoreCase); - bool readingManifest = false; - - Action checkManifestLine = comment => - { - if (comment.Contains("mySQLPunk Native Manifest") && !readingManifest) - { - readingManifest = true; - log.LogInfo("Detected Native Manifest in SQL script footer."); - } - - var match = Regex.Match(comment, @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase); - if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) + result.LogFilePath = log.LogPath; + Action info = message => { - expectedManifestTables[match.Groups[1].Value] = r; - } - }; - - try - { - using (var fs = new FileStream(request.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize)) - using (var reader = new StreamReader(fs, preflight.DetectedEncoding, true, BufferSize)) + log.LogInfo(message); + terminalSink?.Invoke(message); + }; + int attempted = 0, failed = 0; + long affected = 0, bytes = 0; + var truncated = new HashSet(StringComparer.Ordinal); + try { - string statement; - int lineNum = 0; - - foreach (var stmtInfo in EnumerateStatements(reader, cancellationToken, checkManifestLine)) - { - statement = stmtInfo.Sql.Trim(); - lineNum = stmtInfo.LineNumber; - bytesRead += stmtInfo.ByteLength; - - if (string.IsNullOrWhiteSpace(statement)) continue; - - currentStmtIndex++; - - // 套用衝突處理策略 - string effectiveSql = ApplyConflictStrategy(statement, request.Options.ConflictStrategy, truncatedTables, db, LogAndTerminal); - - // 去除獨立行結尾斜線（若有） - if (effectiveSql.EndsWith("/") && effectiveSql.Length > 1) + var preflight = new OracleImportPreflightScanner().Scan(request.FilePath, cancellationToken); + if (!preflight.CanProceed) throw new InvalidOperationException(string.Join("; ", preflight.Issues.ToArray())); + string owner = request.DatabaseName; + if (string.IsNullOrWhiteSpace(owner)) + throw new InvalidOperationException("An explicit target Oracle owner is required."); + // Validate the complete script before changing a session or executing any statement. + bool spatialMetadata = false; + using (var reader = new StreamReader(request.FilePath, preflight.DetectedEncoding, true, BufferSize)) + foreach (var item in EnumerateStatements(reader, cancellationToken)) { - effectiveSql = effectiveSql.Substring(0, effectiveSql.Length - 1).TrimEnd(); + ValidateTarget(item.Sql, owner); + if (IsSpatialMetadataMutation(item.Sql)) spatialMetadata = true; } - - string preview = effectiveSql.Length > 80 ? effectiveSql.Substring(0, 80).Replace("\r", " ").Replace("\n", " ") + "..." : effectiveSql.Replace("\r", " ").Replace("\n", " "); - - try + if (spatialMetadata && request.Options.AutoRegisterSpatialMetadata) + { + var session = db.SelectSQL("SELECT SYS_CONTEXT('USERENV','SESSION_USER') FROM DUAL"); + if (session == null || session.Rows.Count != 1 || !string.Equals(Convert.ToString(session.Rows[0][0]), owner, StringComparison.Ordinal)) + throw new NotSupportedException("USER_SDO_GEOM_METADATA requires the connected session user to equal the selected owner. Cross-owner spatial metadata restore is unsupported."); + } + foreach (var pair in preflight.ManifestDeclaredTables) ResolveManifestTarget(pair.Key, owner); + ExecuteChecked("ALTER SESSION SET CURRENT_SCHEMA = " + QuoteIdentifier(owner)); + info("Target owner pinned to " + QuoteIdentifier(owner) + ". Source-qualified objects in another owner are rejected."); + info("Oracle DDL and TRUNCATE implicitly commit. Rollback protection applies to pending DML only."); + if (request.Options.UseTransaction) BeginTransaction(); + using (var reader = new StreamReader(request.FilePath, preflight.DetectedEncoding, true, BufferSize)) + { + foreach (var item in EnumerateStatements(reader, cancellationToken)) { - var execRes = db.ExecSQL(effectiveSql); - long affected = 0; - if (execRes != null && execRes.ContainsKey("rowsAffected")) + cancellationToken.ThrowIfCancellationRequested(); + bytes += item.ByteLength; + string sql = item.Sql; + string normalized = RemoveComments(sql).Trim(); + if (!request.Options.AutoRegisterSpatialMetadata && IsSpatialMetadataMutation(normalized)) { - long.TryParse(execRes["rowsAffected"], out affected); + info("[SKIP] Spatial metadata registration disabled."); + continue; } - - if (affected > 0) totalRowsAffected += affected; - - log.LogStatement(lineNum, preview, true); - terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [OK] Line {1}: {2} (Rows: {3})", DateTime.Now, lineNum, preview, affected)); - } - catch (Exception ex) - { - // 若衝突策略為 KeepExisting 且為物件已存在錯誤 (ORA-00955 / ORA-01430 / ORA-02260 等) - if (request.Options.ConflictStrategy == OracleConflictStrategy.KeepExisting && IsObjectExistsError(ex.Message)) + attempted++; + string preview = sql.Replace("\r", " ").Replace("\n", " "); + if (preview.Length > 120) preview = preview.Substring(0, 120); + try { - log.LogStatement(lineNum, preview, true, "物件已存在，依 KeepExisting 策略略過。"); - terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [SKIP] Line {1}: {2} (物件已存在)", DateTime.Now, lineNum, preview)); + if (Regex.IsMatch(normalized, @"^(COMMIT|ROLLBACK)\b", RegexOptions.IgnoreCase)) + { + if (!Regex.IsMatch(normalized, @"^(COMMIT|ROLLBACK)(\s+WORK)?$", RegexOptions.IgnoreCase)) + throw new InvalidOperationException("Only plain COMMIT/ROLLBACK transaction statements are supported."); + EndTransaction(normalized.StartsWith("COMMIT", StringComparison.OrdinalIgnoreCase)); + } + else if (Regex.IsMatch(normalized, @"^SET\s+TRANSACTION\b", RegexOptions.IgnoreCase)) + { + if (!Regex.IsMatch(normalized, @"^SET\s+TRANSACTION\s+READ\s+WRITE$", RegexOptions.IgnoreCase)) + throw new InvalidOperationException("Import supports only SET TRANSACTION READ WRITE."); + if (!transactionActive) BeginTransaction(); + } + else + { + ApplyConflictStrategy(normalized, truncated, info); + if (request.Options.UseTransaction && !transactionActive) BeginTransaction(); + long rows = ExecuteChecked(sql); + if (rows > 0) affected += rows; + if (IsDdl(normalized)) ForgetImplicitlyCommittedTransaction(); + } + log.LogStatement(item.LineNumber, preview, true); + terminalSink?.Invoke("[OK] Line " + item.LineNumber + ": " + preview); } - else + catch (Exception ex) { - failedStatements++; - log.LogStatement(lineNum, preview, false, ex.Message); - terminalSink?.Invoke(string.Format("[{0:HH:mm:ss}] [ERROR] Line {1}: {2} -> {3}", DateTime.Now, lineNum, preview, ex.Message)); - - if (request.Options.StopOnError) + if (request.Options.ConflictStrategy == OracleConflictStrategy.KeepExisting && + Regex.IsMatch(normalized, @"^(CREATE|ALTER)\b", RegexOptions.IgnoreCase) && IsObjectExistsError(ex.Message)) { - try { db.ExecSQL("ROLLBACK;"); LogAndTerminal("遇錯停止，已執行 ROLLBACK。"); } catch { } - throw new InvalidOperationException(string.Format("第 {0} 行執行失敗：{1}", lineNum, ex.Message), ex); + ForgetImplicitlyCommittedTransaction(); + info("[SKIP] " + ex.Message); + } + else + { + failed++; + log.LogStatement(item.LineNumber, preview, false, ex.Message); + terminalSink?.Invoke("[ERROR] Line " + item.LineNumber + ": " + ex.Message); + result.ErrorMessage = ex.Message; + if (request.Options.StopOnError || ex is OperationCanceledException) throw; } } - } - - if (currentStmtIndex % 50 == 0 || currentStmtIndex == totalStatements) - { progress?.Report(new OracleImportProgress { - Stage = "Executing", - CurrentStatement = currentStmtIndex, - TotalStatements = totalStatements, - BytesRead = bytesRead, - TotalBytes = totalBytes, - RowsAffected = totalRowsAffected, - Message = string.Format("正在執行語句: {0}/{1} (已影響 {2:N0} 列)", currentStmtIndex, totalStatements, totalRowsAffected) + Stage = "Executing", CurrentStatement = attempted, TotalStatements = preflight.StatementCount, + BytesRead = bytes, TotalBytes = preflight.FileSizeBytes, RowsAffected = affected, + Message = "Executing Oracle statement " + attempted + "/" + preflight.StatementCount }); } } - } - - // 結尾提交交易 - try - { - db.ExecSQL("COMMIT;"); - LogAndTerminal("交易已成功提交 (COMMIT)。"); + cancellationToken.ThrowIfCancellationRequested(); + if (failed != 0) EndTransaction(false); + else if (transactionActive) EndTransaction(true); + result.Success = failed == 0; + if (result.Success) + { + foreach (var pair in preflight.ManifestDeclaredTables) + { + cancellationToken.ThrowIfCancellationRequested(); + string target = ResolveManifestTarget(pair.Key, owner); + var count = db.SelectSQL("SELECT COUNT(*) FROM " + target); + if (count == null || count.Rows.Count != 1) throw new InvalidOperationException("Manifest count query did not return one row: " + target); + long actual = Convert.ToInt64(count.Rows[0][0]); + result.RestoredTables[pair.Key] = actual; + if (actual != pair.Value) result.ManifestMatched = false; + } + if (!result.ManifestMatched) + { + result.Success = false; + result.ErrorMessage = "Restored table counts do not match the native manifest."; + } + if (preflight.ManifestDeclaredTables.Count > 0) + log.LogManifest(preflight.ManifestDeclaredTables, result.RestoredTables, result.ManifestMatched); + } } catch (Exception ex) { - LogAndTerminal("提交交易時發生警告: " + ex.Message); + result.Success = false; + result.ErrorMessage = ex.Message; + if (failed == 0) failed++; + info("Import failed: " + ex.Message); } - - result.Success = (failedStatements == 0); - } - catch (Exception ex) - { - result.Success = false; - result.ErrorMessage = ex.Message; - LogAndTerminal("匯入過程中斷: " + ex.Message); - } - - clock.Stop(); - result.Elapsed = clock.Elapsed; - result.StatementsExecuted = currentStmtIndex; - result.StatementsFailed = failedStatements; - result.TotalRowsAffected = totalRowsAffected; - - // 7. 核對 Native Manifest - if (expectedManifestTables.Count > 0) - { - LogAndTerminal(string.Format("正在核對腳本結尾 Manifest 清單 (共 {0} 張資料表)...", expectedManifestTables.Count)); - bool allMatch = true; - - foreach (var kvp in expectedManifestTables) + finally { - string tableName = kvp.Key; - long expectedRows = kvp.Value; - long actualRows = -1; - - try + if (transactionActive) { - string countSql = tableName.Contains(".") - ? string.Format("SELECT COUNT(*) FROM \"{0}\".\"{1}\"", tableName.Split('.')[0], tableName.Split('.')[1]) - : string.Format("SELECT COUNT(*) FROM \"{0}\"", tableName); - - DataTable dt = db.SelectSQL(countSql); - if (dt != null && dt.Rows.Count > 0) + try { EndTransaction(false); info("Pending Oracle transaction rolled back."); } + catch (Exception ex) { - actualRows = Convert.ToInt64(dt.Rows[0][0]); + result.Success = false; + result.ErrorMessage += "; rollback failed: " + ex.Message; + info("Rollback failed: " + ex.Message); } } - catch - { - actualRows = -1; - } + transaction?.Dispose(); + transaction = null; + transactionActive = false; + clock.Stop(); + result.Elapsed = clock.Elapsed; + result.StatementsExecuted = attempted; + result.StatementsFailed = failed; + result.TotalRowsAffected = affected; + log.LogSummary(attempted, failed, affected, result.Elapsed, result.Success); + log.Flush(); + } + } + return result; + } - result.RestoredTables[tableName] = actualRows; - if (actualRows != expectedRows) - { - allMatch = false; - LogAndTerminal(string.Format("⚠️ 資料表 [{0}] 筆數不符：Manifest 宣告 {1} 列，還原後實際 {2} 列。", tableName, expectedRows, actualRows)); - } - else + private long ExecuteChecked(string sql) + { + var native = db as my_oracle; + if (native != null) + { + using (var command = new OracleCommand(sql, native.MCT)) + { + command.Transaction = transaction; + using (currentToken.Register(() => { try { command.Cancel(); } catch { } })) { - LogAndTerminal(string.Format("✅ 資料表 [{0}] 筆數相符：{1} 列。", tableName, actualRows)); + currentToken.ThrowIfCancellationRequested(); + return command.ExecuteNonQuery(); } } - - result.ManifestMatched = allMatch; - log.LogManifest(expectedManifestTables, result.RestoredTables, allMatch); } + var response = db.ExecSQL(sql); + if (response == null || !response.ContainsKey("status") || !string.Equals(response["status"], "OK", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException(DatabaseExecutionResultService.GetFailureReason(response)); + long rows; + return response.ContainsKey("rowsAffected") && long.TryParse(response["rowsAffected"], out rows) ? rows : 0; + } - LogAndTerminal(string.Format("Oracle 匯入作業完成，耗時 {0:hh\\:mm\\:ss\\.fff}，詳細日誌已寫入: {1}", result.Elapsed, result.LogFilePath)); - log.LogSummary(currentStmtIndex, failedStatements, totalRowsAffected, result.Elapsed, result.Success); - log.Flush(); - log.Dispose(); + private void BeginTransaction() + { + var native = db as my_oracle; + if (native != null) transaction = native.MCT.BeginTransaction(); + else ExecuteChecked("SET TRANSACTION READ WRITE"); + transactionActive = true; + } - return result; + private void EndTransaction(bool commit) + { + // Cleanup must still be possible after cancellation. It is checked for failures, but not cancelled. + CancellationToken old = currentToken; + currentToken = CancellationToken.None; + try + { + if (transaction != null) + { + if (commit) transaction.Commit(); else transaction.Rollback(); + transaction.Dispose(); + transaction = null; + } + else ExecuteChecked(commit ? "COMMIT" : "ROLLBACK"); + transactionActive = false; + } + finally { currentToken = old; } } - private static bool IsObjectExistsError(string message) + private void ForgetImplicitlyCommittedTransaction() { - if (string.IsNullOrWhiteSpace(message)) return false; - // ORA-00955: name is already used by an existing object - // ORA-02260: table can have only one primary key - // ORA-01430: column being added already exists in table - return message.IndexOf("ORA-00955", StringComparison.OrdinalIgnoreCase) >= 0 || - message.IndexOf("ORA-02260", StringComparison.OrdinalIgnoreCase) >= 0 || - message.IndexOf("ORA-01430", StringComparison.OrdinalIgnoreCase) >= 0; + transaction?.Dispose(); + transaction = null; + transactionActive = false; } - private static string ApplyConflictStrategy( - string sql, - OracleConflictStrategy strategy, - HashSet truncatedTables, - IDatabase db, - Action logAction) + private void ApplyConflictStrategy(string sql, HashSet truncated, Action info) { - if (strategy == OracleConflictStrategy.DropAndRecreate || strategy == OracleConflictStrategy.ReplaceAll) + string identifier = @"(?:""(?:[^""]|"""")+""|[A-Za-z_][\w$#]*)(?:\s*\.\s*(?:""(?:[^""]|"""")+""|[A-Za-z_][\w$#]*))?"; + if (request.Options.ConflictStrategy == OracleConflictStrategy.DropAndRecreate || request.Options.ConflictStrategy == OracleConflictStrategy.ReplaceAll) { - // 在 CREATE TABLE 前執行 DROP TABLE ... CASCADE CONSTRAINTS - var match = Regex.Match(sql, @"^CREATE\s+TABLE\s+([""\w\.]+)", RegexOptions.IgnoreCase); + var match = Regex.Match(sql, @"^CREATE\s+TABLE\s+(" + identifier + ")", RegexOptions.IgnoreCase); if (match.Success) { - string target = match.Groups[1].Value; - string dropSql = string.Format("BEGIN EXECUTE IMMEDIATE 'DROP TABLE {0} CASCADE CONSTRAINTS'; EXCEPTION WHEN OTHERS THEN NULL; END;", target); - try - { - db.ExecSQL(dropSql); - logAction(string.Format("[衝突處理] 已預先清除資料表: {0}", target)); - } - catch { } + try { ExecuteChecked("DROP TABLE " + match.Groups[1].Value + " CASCADE CONSTRAINTS"); } + catch (Exception ex) { if (ex.Message.IndexOf("ORA-00942", StringComparison.OrdinalIgnoreCase) < 0) throw; } + ForgetImplicitlyCommittedTransaction(); + info("[Conflict] DROP checked: " + match.Groups[1].Value); } } - else if (strategy == OracleConflictStrategy.TruncateAndInsert) + else if (request.Options.ConflictStrategy == OracleConflictStrategy.TruncateAndInsert) { - // 當第一次遇到該表的 INSERT 時，先執行 TRUNCATE - var match = Regex.Match(sql, @"^INSERT\s+INTO\s+([""\w\.]+)", RegexOptions.IgnoreCase); - if (match.Success) + var target = GetMutationTarget(Tokens(sql)); + if (target != null && target.Operation == "INSERT") { - string target = match.Groups[1].Value; - if (truncatedTables.Add(target)) + string identity = target.QualifiedName(request.DatabaseName); + if (truncated.Add(identity)) { - string truncSql = string.Format("TRUNCATE TABLE {0};", target); - try - { - db.ExecSQL(truncSql); - logAction(string.Format("[衝突處理] 已清空資料表內容: {0}", target)); - } - catch { } + ExecuteChecked("TRUNCATE TABLE " + identity); + ForgetImplicitlyCommittedTransaction(); + info("[Conflict] TRUNCATE checked: " + identity); } } } - - return sql; } - private struct StatementInfo + private static bool IsDdl(string sql) => Regex.IsMatch(sql, @"^(CREATE|ALTER|DROP|TRUNCATE|COMMENT|GRANT|REVOKE)\b", RegexOptions.IgnoreCase) || Regex.IsMatch(sql, @"^BEGIN\s+EXECUTE\s+IMMEDIATE", RegexOptions.IgnoreCase); + private static bool IsObjectExistsError(string text) => text.IndexOf("ORA-00955", StringComparison.OrdinalIgnoreCase) >= 0 || text.IndexOf("ORA-02260", StringComparison.OrdinalIgnoreCase) >= 0 || text.IndexOf("ORA-01430", StringComparison.OrdinalIgnoreCase) >= 0; + private static bool IsSpatialMetadataMutation(string sql) { - public string Sql; - public int LineNumber; - public int ByteLength; + var target = GetMutationTarget(Tokens(sql)); + return target != null && target.Name == "USER_SDO_GEOM_METADATA"; } - private static IEnumerable EnumerateStatements(StreamReader reader, CancellationToken cancellationToken, Action onCommentLine = null) + private sealed class ObjectTarget { - var sb = new StringBuilder(); - int startLine = 1; - int currentLine = 1; - int byteLen = 0; + public string Operation, Owner, Name; + public string QualifiedName(string selectedOwner) => QuoteIdentifier(Owner ?? selectedOwner) + "." + QuoteIdentifier(Name); + } - bool inSingleQuote = false; - bool inDoubleQuote = false; - bool inLineComment = false; - int blockCommentDepth = 0; - bool inPlSqlBlock = false; + private static bool Word(List tokens, int index, string expected) => index >= 0 && index < tokens.Count && string.Equals(tokens[index], expected, StringComparison.OrdinalIgnoreCase); - string line; - while ((line = reader.ReadLine()) != null) + private static ObjectTarget ReadObjectTarget(List tokens, int index, string operation) + { + if (Word(tokens, index, "ONLY")) throw new NotSupportedException("Oracle ONLY target wrappers are unsupported under the direct local-owner target contract."); + if (index >= tokens.Count || !IsIdentifier(tokens[index])) throw new NotSupportedException("Oracle object target could not be parsed."); + var target = new ObjectTarget { Operation = operation, Name = IdentifierValue(tokens[index]) }; + if (index + 1 < tokens.Count && tokens[index + 1] == ".") { - cancellationToken.ThrowIfCancellationRequested(); - byteLen += line.Length + 1; - - string trimmed = line.Trim(); + if (index + 2 >= tokens.Count || !IsIdentifier(tokens[index + 2])) throw new NotSupportedException("Oracle qualified object target could not be parsed."); + target.Owner = target.Name; + target.Name = IdentifierValue(tokens[index + 2]); + } + return target; + } - // 獨立行註解不作為執行語句，僅觸發註解回呼 (如 Manifest 解析) - if (sb.Length == 0 && trimmed.StartsWith("--", StringComparison.Ordinal)) + private static ObjectTarget GetMutationTarget(List tokens) + { + bool block = Word(tokens, 0, "DECLARE"); + ObjectTarget target = null; + for (int i = 0; i < tokens.Count; i++) + { + if (i != 0 && !block) break; + string operation = tokens[i].ToUpperInvariant(); + int objectIndex; + if (operation == "INSERT") { - onCommentLine?.Invoke(trimmed); - currentLine++; - startLine = currentLine; - continue; + if (!Word(tokens, i + 1, "INTO")) throw new NotSupportedException("Only INSERT INTO is supported by this Oracle target contract."); + objectIndex = i + 2; + } + else if (!block && operation == "DELETE") objectIndex = Word(tokens, i + 1, "FROM") ? i + 2 : i + 1; + else if (!block && operation == "UPDATE") objectIndex = i + 1; + else if (!block && operation == "MERGE") + { + if (!Word(tokens, i + 1, "INTO")) throw new NotSupportedException("Only MERGE INTO is supported."); + objectIndex = i + 2; } + else continue; + if (target != null) throw new NotSupportedException("A generated LOB block must contain exactly one INSERT target."); + target = ReadObjectTarget(tokens, objectIndex, operation); + } + return target; + } - if (sb.Length == 0) startLine = currentLine; + private static ObjectTarget GetCreateTarget(List tokens) + { + if (!Word(tokens, 0, "CREATE")) return null; + int index = 1; + if (Word(tokens, index, "OR") && Word(tokens, index + 1, "REPLACE")) index += 2; + if (Word(tokens, index, "FORCE") || Word(tokens, index, "NOFORCE")) index++; + if (Word(tokens, index, "EDITIONABLE") || Word(tokens, index, "NONEDITIONABLE")) index++; + if (Word(tokens, index, "UNIQUE") || Word(tokens, index, "BITMAP")) index++; + if (Word(tokens, index, "TABLE") || Word(tokens, index, "INDEX") || Word(tokens, index, "VIEW")) return ReadObjectTarget(tokens, index + 1, "CREATE " + tokens[index].ToUpperInvariant()); + return null; + } + internal static string QuoteIdentifier(string name) => "\"" + name.Replace("\"", "\"\"") + "\""; + + internal static string ResolveManifestTarget(string name, string owner) + { + var tokens = Tokens(name); + if (tokens.Count == 1 && IsIdentifier(tokens[0])) return QuoteIdentifier(owner) + "." + QuoteIdentifier(IdentifierValue(tokens[0])); + if (tokens.Count == 3 && tokens[1] == "." && IsIdentifier(tokens[0]) && IsIdentifier(tokens[2]) && string.Equals(IdentifierValue(tokens[0]), owner, StringComparison.Ordinal)) + return QuoteIdentifier(owner) + "." + QuoteIdentifier(IdentifierValue(tokens[2])); + throw new InvalidOperationException("Manifest target is invalid or belongs to a different owner: " + name); + } - // 檢查是否開展 PL/SQL 區塊 - if (!inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) + internal static void ValidateTarget(string sql, string owner) + { + var tokens = Tokens(sql); + if (tokens.Count == 0) return; + if (tokens.Contains("@")) throw new NotSupportedException("Oracle database-link targets are outside the local-owner import contract."); + var mutationTarget = GetMutationTarget(tokens); + var createTarget = GetCreateTarget(tokens); + foreach (var target in new[] { mutationTarget, createTarget }) + if (target != null && target.Owner != null && target.Owner != owner) + throw new InvalidOperationException("Cross-owner object rejected before execution: " + target.QualifiedName(owner)); + string normalized = RemoveComments(sql).Trim(); + if (Regex.IsMatch(normalized, @"^SET\s+TRANSACTION\b", RegexOptions.IgnoreCase) && !Regex.IsMatch(normalized, @"^SET\s+TRANSACTION\s+READ\s+WRITE$", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Import accepts only SET TRANSACTION READ WRITE."); + string tokenText = string.Join(" ", tokens); + // Dynamic SQL cannot be safely redirected. Permit only the exact bounded DROP wrapper emitted by this exporter. + if (Regex.IsMatch(tokenText, @"\b(EXECUTE\s+IMMEDIATE|DBMS_SQL|DBMS_UTILITY)\b", RegexOptions.IgnoreCase)) + { + var drop = Regex.Match(normalized, @"^BEGIN\s+EXECUTE\s+IMMEDIATE\s+'DROP TABLE ((?:[^']|'')+) CASCADE CONSTRAINTS';\s*EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF;\s*END;$", RegexOptions.IgnoreCase); + if (!drop.Success) throw new InvalidOperationException("Dynamic Oracle SQL is unsupported because target-owner enforcement cannot be guaranteed."); + ResolveManifestTarget(drop.Groups[1].Value.Replace("''", "'"), owner); + return; + } + if (Regex.IsMatch(normalized, @"^(ALTER\s+SESSION|CONNECT\b|DISCONNECT\b|GRANT\b|REVOKE\b|CREATE\s+(USER|DATABASE|SCHEMA|SYNONYM|DATABASE\s+LINK)\b|ALTER\s+USER|DROP\s+USER)", RegexOptions.IgnoreCase)) + throw new InvalidOperationException("The script attempts to change session/owner scope or unsupported privileges."); + if (!Regex.IsMatch(tokenText, @"^(INSERT\b|UPDATE\b|DELETE\b|MERGE\b|CREATE ((OR REPLACE )?(FORCE |NOFORCE )?(EDITIONABLE |NONEDITIONABLE )?VIEW|TABLE|INDEX|UNIQUE INDEX|BITMAP INDEX)\b|ALTER (TABLE|INDEX)\b|DROP (TABLE|VIEW|INDEX)\b|TRUNCATE TABLE\b|COMMENT ON\b|COMMIT\b|ROLLBACK\b|SET TRANSACTION\b|BEGIN\b|DECLARE\b)", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Unsupported Oracle statement. This wizard accepts bounded native dump DDL/DML and explicit transaction control."); + bool block = Regex.IsMatch(normalized, @"^(BEGIN|DECLARE)\b", RegexOptions.IgnoreCase); + if (Regex.IsMatch(normalized, @"^CREATE\s+(OR\s+REPLACE\s+)?(TRIGGER|PROCEDURE|FUNCTION|PACKAGE|TYPE)\b", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Stored PL/SQL import requires an explicit owner/dependency policy and is not supported by this wizard."); + if (block) + { + // Only the bounded temporary-LOB blocks generated by this exporter are admitted. + if (!Regex.IsMatch(normalized, @"^DECLARE\s+(mp_lob_\d+\s+(BLOB|CLOB);\s*)+BEGIN\b", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Anonymous PL/SQL is unsupported except generated bounded temporary-LOB INSERT blocks."); + if (mutationTarget == null || mutationTarget.Operation != "INSERT") throw new NotSupportedException("A generated LOB block must contain one INSERT target."); + for (int i = 0; i + 2 < tokens.Count; i++) { - if (PlSqlStartRegex.IsMatch(line)) - { - inPlSqlBlock = true; - } + if (tokens[i + 1] != ".") continue; + string qualifier = IdentifierValue(tokens[i]); + if (qualifier != owner && qualifier != "DBMS_LOB" && qualifier != "SDO_UTIL") + throw new NotSupportedException("Unsupported qualified call in Oracle LOB block: " + qualifier); + if (qualifier == "DBMS_LOB" && Array.IndexOf(new[] { "CREATETEMPORARY", "WRITEAPPEND", "ISTEMPORARY", "FREETEMPORARY" }, tokens[i + 2].ToUpperInvariant()) < 0) + throw new NotSupportedException("Unsupported DBMS_LOB operation in Oracle import."); } + } + for (int i = 0; i < tokens.Count; i++) + { + string token = tokens[i].ToUpperInvariant(); + bool target = token == "INTO" || token == "UPDATE" || token == "FROM" || token == "JOIN" || token == "USING" || token == "REFERENCES" || + (token == "ON" && createTarget != null && createTarget.Operation == "CREATE INDEX") || token == "COLUMN" || token == "TABLE" || + ((token == "TABLE" || token == "VIEW" || token == "INDEX" || token == "TRIGGER" || token == "PROCEDURE" || token == "FUNCTION" || token == "SEQUENCE" || token == "PACKAGE" || token == "TYPE") && i > 0 && + (tokens[i - 1].Equals("CREATE", StringComparison.OrdinalIgnoreCase) || tokens[i - 1].Equals("ALTER", StringComparison.OrdinalIgnoreCase) || tokens[i - 1].Equals("DROP", StringComparison.OrdinalIgnoreCase) || tokens[i - 1].Equals("TRUNCATE", StringComparison.OrdinalIgnoreCase) || tokens[i - 1].Equals("REPLACE", StringComparison.OrdinalIgnoreCase) || tokens[i - 1].Equals("ON", StringComparison.OrdinalIgnoreCase))); + if (!target || i + 1 >= tokens.Count || !IsIdentifier(tokens[i + 1])) continue; + if (i + 3 < tokens.Count && tokens[i + 2] == "." && IsIdentifier(tokens[i + 3]) && + !string.Equals(IdentifierValue(tokens[i + 1]), owner, StringComparison.Ordinal)) + throw new InvalidOperationException("Cross-owner object rejected before execution: " + tokens[i + 1] + "." + tokens[i + 3] + ". Selected owner: " + owner); + } + } - // 檢查 PL/SQL 結束斜線 / - if (inPlSqlBlock && !inSingleQuote && !inDoubleQuote && blockCommentDepth == 0) + private static bool IsIdentifier(string token) => token.Length > 0 && (token[0] == '"' || char.IsLetter(token[0]) || token[0] == '_'); + private static string IdentifierValue(string token) => token[0] == '"' ? token.Substring(1, token.Length - 2).Replace("\"\"", "\"") : token.ToUpperInvariant(); + private static List Tokens(string sql) + { + var tokens = new List(); + foreach (Match match in Regex.Matches(sql, @"--[^\r\n]*|/\*[\s\S]*?\*/|[qQ]'(?:\[[\s\S]*?\]|\{[\s\S]*?\}|\([\s\S]*?\)|<[\s\S]*?>)'|[qQ]'(?[^\s])[\s\S]*?\k'|'(?:[^']|'')*'|""(?:[^""]|"""")*""|[A-Za-z_][\w$#]*|[^\s]")) + { + string value = match.Value; + if (value.StartsWith("--") || value.StartsWith("/*") || value[0] == '\'' || (value.Length > 1 && (value[0] == 'q' || value[0] == 'Q') && value[1] == '\'')) continue; + tokens.Add(value); + } + return tokens; + } + private static string RemoveComments(string sql) + { + // Preserve literals and quoted identifiers while masking comments for classification only. + return Regex.Replace(sql, @"'(?:[^']|'')*'|""(?:[^""]|"""")*""|--[^\r\n]*|/\*[\s\S]*?\*/", m => m.Value.StartsWith("--") || m.Value.StartsWith("/*") ? " " : m.Value); + } + + internal struct StatementInfo + { + public string Sql; + public int LineNumber; + public int ByteLength; + } + + internal static IEnumerable EnumerateStatements(StreamReader reader, CancellationToken cancellationToken, Action onCommentLine = null) + { + var buffer = new StringBuilder(); + int lineNumber = 1, startLine = 1, sourceBytes = 0; + bool single = false, quoted = false, blockComment = false, plsql = false, hasContent = false; + char alternateEnd = '\0'; + string line; + while ((line = ReadPhysicalLine(reader)) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + if (line.IndexOf('\0') >= 0) throw new InvalidOperationException("SQL script contains a NUL character at line " + lineNumber + "."); + if (!single && !quoted && !blockComment && alternateEnd == '\0' && line.Trim() == "/") { - if (trimmed == "/") + sourceBytes += reader.CurrentEncoding.GetByteCount(line); + if (hasContent) { - inPlSqlBlock = false; - string plsql = sb.ToString().Trim(); - sb.Clear(); - byteLen = 0; - startLine = currentLine + 1; - - if (!string.IsNullOrWhiteSpace(plsql)) - { - yield return new StatementInfo - { - Sql = plsql, - LineNumber = startLine, - ByteLength = byteLen - }; - } - currentLine++; - continue; + yield return new StatementInfo { Sql = buffer.ToString().Trim(), LineNumber = startLine, ByteLength = sourceBytes }; + buffer.Clear(); sourceBytes = 0; hasContent = false; plsql = false; } + lineNumber++; + continue; } - - int len = line.Length; - for (int i = 0; i < len; i++) + if (!single && !quoted && !blockComment && alternateEnd == '\0' && line.TrimStart().StartsWith("--")) onCommentLine?.Invoke(line.Trim()); + bool lineComment = false; + for (int i = 0; i < line.Length; i++) { char c = line[i]; - - if (inLineComment) continue; - - if (blockCommentDepth > 0) + int count = char.IsHighSurrogate(c) && i + 1 < line.Length && char.IsLowSurrogate(line[i + 1]) ? 2 : 1; + sourceBytes += reader.CurrentEncoding.GetByteCount(line.Substring(i, count)); + if (sourceBytes > MaxStatementBytes) throw new InvalidOperationException("A SQL statement exceeds the 32 MiB safety limit."); + if (lineComment) { if (c == '\n') buffer.Append('\n'); continue; } + if (blockComment) { - if (c == '*' && i + 1 < len && line[i + 1] == '/') { blockCommentDepth--; i++; } - sb.Append(c); + // Comments are intentionally replaced with whitespace; SQL delimiter bytes remain counted. + if (c == '*' && i + 1 < line.Length && line[i + 1] == '/') { blockComment = false; i++; sourceBytes += reader.CurrentEncoding.GetByteCount("/"); buffer.Append(' '); } + else if (c == '\n') buffer.Append('\n'); continue; } - - if (inSingleQuote) + if (alternateEnd != '\0') { - sb.Append(c); - if (c == '\'') - { - if (i + 1 < len && line[i + 1] == '\'') - { - sb.Append(line[i + 1]); - i++; - } - else - { - inSingleQuote = false; - } - } + buffer.Append(c); + if (c == alternateEnd && i + 1 < line.Length && line[i + 1] == '\'') { buffer.Append('\''); i++; sourceBytes += reader.CurrentEncoding.GetByteCount("'"); alternateEnd = '\0'; } + else if (count == 2) { buffer.Append(line[++i]); } continue; } - - if (inDoubleQuote) + if (single || quoted) { - sb.Append(c); - if (c == '"') + buffer.Append(c); + char end = single ? '\'' : '"'; + if (c == end) { - if (i + 1 < len && line[i + 1] == '"') - { - sb.Append(line[i + 1]); - i++; - } - else - { - inDoubleQuote = false; - } + if (i + 1 < line.Length && line[i + 1] == end) { buffer.Append(line[++i]); sourceBytes += reader.CurrentEncoding.GetByteCount(end.ToString()); } + else { single = false; quoted = false; } } + else if (count == 2) buffer.Append(line[++i]); continue; } - - // 符號檢查 - if (c == '-' && i + 1 < len && line[i + 1] == '-') + if (c == '-' && i + 1 < line.Length && line[i + 1] == '-') { lineComment = true; buffer.Append(' '); continue; } + if (c == '/' && i + 1 < line.Length && line[i + 1] == '*') { blockComment = true; buffer.Append(' '); i++; sourceBytes += reader.CurrentEncoding.GetByteCount("*"); continue; } + if (!hasContent && !char.IsWhiteSpace(c)) { hasContent = true; startLine = lineNumber; } + if ((c == 'q' || c == 'Q') && i + 2 < line.Length && line[i + 1] == '\'') { - inLineComment = true; - i++; - continue; + char opening = line[i + 2]; + alternateEnd = opening == '[' ? ']' : opening == '{' ? '}' : opening == '(' ? ')' : opening == '<' ? '>' : opening; + buffer.Append(c).Append('\'').Append(opening); i += 2; sourceBytes += reader.CurrentEncoding.GetByteCount("'" + opening); continue; } - - if (c == '/' && i + 1 < len && line[i + 1] == '*') + if (c == '\'') single = true; + if (c == '"') quoted = true; + if (c == ';') { - blockCommentDepth = 1; - sb.Append("/*"); - i++; - continue; - } - - if (c == '\'') - { - inSingleQuote = true; - sb.Append(c); - continue; - } - - if (c == '"') - { - inDoubleQuote = true; - sb.Append(c); - continue; - } - - // 非 PL/SQL 區塊下的分號終止符 - if (c == ';' && !inPlSqlBlock) - { - sb.Append(';'); - string sql = sb.ToString().Trim(); - sb.Clear(); - byteLen = 0; - startLine = currentLine; - - if (!string.IsNullOrWhiteSpace(sql)) + plsql = plsql || PlSqlStartRegex.IsMatch(buffer.ToString()); + if (!plsql) { - yield return new StatementInfo - { - Sql = sql, - LineNumber = startLine, - ByteLength = byteLen - }; + if (hasContent) yield return new StatementInfo { Sql = buffer.ToString().Trim(), LineNumber = startLine, ByteLength = sourceBytes }; + buffer.Clear(); sourceBytes = 0; hasContent = false; continue; } - continue; } - - sb.Append(c); + buffer.Append(c); + if (count == 2) buffer.Append(line[++i]); } - - inLineComment = false; - if (sb.Length > 0) sb.Append("\r\n"); - currentLine++; + lineNumber++; } + if (single || quoted || blockComment || alternateEnd != '\0') throw new InvalidOperationException("Unterminated Oracle literal, identifier or comment."); + if (hasContent) yield return new StatementInfo { Sql = buffer.ToString().Trim(), LineNumber = startLine, ByteLength = sourceBytes }; + } - if (sb.Length > 0) - { - string rem = sb.ToString().Trim(); - if (!string.IsNullOrWhiteSpace(rem) && !rem.StartsWith("--", StringComparison.Ordinal)) - { - yield return new StatementInfo - { - Sql = rem, - LineNumber = startLine, - ByteLength = byteLen - }; - } - } + private static string ReadPhysicalLine(StreamReader reader) + { + var line = new StringBuilder(); + int value; + while ((value = reader.Read()) >= 0) { line.Append((char)value); if (line.Length > MaxStatementBytes) throw new InvalidOperationException("A SQL line exceeds the 32 MiB parser safety limit."); if (value == '\n') break; } + return line.Length == 0 ? null : line.ToString(); } } } diff --git a/mySQLPunk/lib/SqlExportPublication.cs b/mySQLPunk/lib/SqlExportPublication.cs new file mode 100644 index 0000000..a9a91ff --- /dev/null +++ b/mySQLPunk/lib/SqlExportPublication.cs @@ -0,0 +1,22 @@ +using System; +using System.IO; + +namespace mySQLPunk.lib +{ + internal static class SqlExportPublication + { + // The candidate must be closed and verified before entering this non-cancellable commit. + // Never emulate replacement with Delete+Move: both old and candidate survive failures. + internal static void Publish(string stagingPath, string targetPath) + { + string source = Path.GetFullPath(stagingPath); + string target = Path.GetFullPath(targetPath); + if (string.Equals(source, target, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("The staging file must differ from the destination."); + if (!string.Equals(Path.GetDirectoryName(source), Path.GetDirectoryName(target), StringComparison.OrdinalIgnoreCase)) + throw new IOException("Atomic publication requires staging beside the destination."); + if (File.Exists(target)) File.Replace(source, target, null); + else File.Move(source, target); + } + } +} diff --git a/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs b/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs index 859b064..2b7ad8d 100644 --- a/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs +++ b/mySQLPunk/lib/SqliteExport/SqliteSqlExportService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Data.SQLite; using System.Diagnostics; using System.Globalization; using System.IO; @@ -45,668 +46,386 @@ public SqliteSqlExportService(IDatabase db, string databaseName) this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); } + private const long MaxRowBytes = 8 * 1024 * 1024; + public List LoadTableCatalog() { var list = new List(); - - var virtualTables = GetVirtualTableNames(); - var spatialTableMap = GetSpatialTableDetails(); - - // 取得所有一般與虛擬資料表（排除 sqlite_% 系統表） - DataTable dt = null; - try - { - dt = db.SelectSQL("SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;", new Dictionary()); - } - catch - { - dt = new DataTable(); + var virtuals = new HashSet(StringComparer.OrdinalIgnoreCase); + DataTable tables = Query("SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;"); + foreach (DataRow row in tables.Rows) + if (System.Text.RegularExpressions.Regex.IsMatch(SqliteImport.SqliteStatementReader.PolicySql(Convert.ToString(row["sql"])), @"^CREATE\s+VIRTUAL\b", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) virtuals.Add(Convert.ToString(row["name"])); + var spatial = GetSpatialTableDetails(); + foreach (DataRow row in tables.Rows) + { + string name = Convert.ToString(row["name"]); + if (name.Equals(my_sqlite.ColumnCommentTableName, StringComparison.OrdinalIgnoreCase) || IsShadowTable(name, virtuals)) continue; + bool metadata = KnownSpatialMetadataTables.Contains(name), geometry = spatial.ContainsKey(name), virt = virtuals.Contains(name); + list.Add(new SqliteTableExportItem(name, CountChecked(name), !virt && !(metadata && name.Equals("spatial_ref_sys", StringComparison.OrdinalIgnoreCase)), geometry, metadata, virt, + geometry ? spatial[name] : virt ? "Virtual table (native data export unsupported)" : metadata ? "Spatial Metadata" : "")); } - - foreach (DataRow row in dt.Rows) - { - string tableName = row["name"]?.ToString(); - if (string.IsNullOrWhiteSpace(tableName)) continue; - - // 排除影子表 (FTS / RTree 內部實體表，還原時由虛擬表自動生成，若重複建立會報錯) - if (IsShadowTable(tableName, virtualTables)) continue; - - bool isVirtual = virtualTables.Contains(tableName); - bool isSpatialMetadata = KnownSpatialMetadataTables.Contains(tableName); - bool isSpatial = spatialTableMap.ContainsKey(tableName); - string spatialInfo = isSpatial ? spatialTableMap[tableName] : (isSpatialMetadata ? "Spatial Metadata" : (isVirtual ? "Virtual Table" : "")); - - long count = 0; - try - { - count = db.CountRows(databaseName, tableName); - } - catch - { - count = 0; - } - - // 預設選取：除巨量空間中繼資料表（如 spatial_ref_sys）外預設勾選 - bool defaultSelect = !(isSpatialMetadata && string.Equals(tableName, "spatial_ref_sys", StringComparison.OrdinalIgnoreCase)); - - list.Add(new SqliteTableExportItem( - tableName, - count, - defaultSelect, - isSpatial, - isSpatialMetadata, - isVirtual, - spatialInfo)); - } - return list; } - - public List LoadViews() + public List LoadViews() { return LoadNames("view"); } + public List LoadTriggers() { return LoadNames("trigger"); } + private List LoadNames(string type) { - var views = new List(); - try - { - DataTable dt = db.SelectSQL("SELECT name FROM sqlite_master WHERE type='view' AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;", new Dictionary()); - foreach (DataRow row in dt.Rows) - { - string name = row[0]?.ToString(); - if (!string.IsNullOrWhiteSpace(name)) views.Add(name); - } - } - catch { } - return views; + return Query("SELECT name FROM sqlite_master WHERE type='" + type + "' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;") + .Rows.Cast().Select(r => Convert.ToString(r[0])).ToList(); } - - public List LoadTriggers() + public Task ExportAsync(SqliteExportWizardOptions options, IProgress progress, CancellationToken cancellation) { + return Task.Run(() => ExportInternal(options, progress, cancellation)); + } + private SqliteExportResult ExportInternal(SqliteExportWizardOptions options, IProgress progress, CancellationToken cancellation) + { + var clock = Stopwatch.StartNew(); + string target = options.TargetPath; + if (string.IsNullOrWhiteSpace(target)) throw new ArgumentException("Target path is required", nameof(options.TargetPath)); + target = Path.GetFullPath(target); + Directory.CreateDirectory(Path.GetDirectoryName(target)); + string staging = target + ".staging." + Guid.NewGuid().ToString("N") + ".tmp"; + bool publicationAttempted = false, snapshot = false; + int tableCount = 0, viewCount = 0, triggerCount = 0, indexCount = 0; + long rows = 0, estimated = 0; + var selected = (options.SelectedTables ?? new List()).Where(t => !t.Equals(my_sqlite.ColumnCommentTableName, StringComparison.OrdinalIgnoreCase)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + if (options.ExcludeSpatialMetadata) selected.RemoveAll(t => KnownSpatialMetadataTables.Contains(t)); + var views = options.IncludeViews ? (options.SelectedViews ?? new List()).Distinct(StringComparer.OrdinalIgnoreCase).ToList() : new List(); var triggers = new List(); + var sourceCancellation = db is my_sqlite nativeSource && nativeSource.MCT != null + ? cancellation.Register(() => { try { nativeSource.MCT.Cancel(); } catch { } }) + : default(CancellationTokenRegistration); try { - DataTable dt = db.SelectSQL("SELECT name FROM sqlite_master WHERE type='trigger' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY name ASC;", new Dictionary()); - foreach (DataRow row in dt.Rows) + cancellation.ThrowIfCancellationRequested(); + // This snapshot belongs to the dedicated wizard session; never reuse + // the editor connection or silently continue when BEGIN fails. + ExecChecked("BEGIN;"); snapshot = true; + var ddls = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (string table in selected) { - string name = row[0]?.ToString(); - if (!string.IsNullOrWhiteSpace(name)) triggers.Add(name); + cancellation.ThrowIfCancellationRequested(); + string ddl = GetDdl("table", table); + if (System.Text.RegularExpressions.Regex.IsMatch(SqliteImport.SqliteStatementReader.PolicySql(ddl), @"^CREATE\s+VIRTUAL\b", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + throw new NotSupportedException("Virtual table native export is not yet supported: " + table + ". Use a module-specific backup."); + ddls[table] = ddl; + if (options.Mode != SqliteExportMode.SchemaOnly) estimated += CountChecked(table); } - } - catch { } - return triggers; - } - - public Task ExportAsync( - SqliteExportWizardOptions options, - IProgress progress, - CancellationToken cancellation) - { - return Task.Run(() => ExportInternal(options, progress, cancellation), cancellation); - } - - private SqliteExportResult ExportInternal( - SqliteExportWizardOptions options, - IProgress progress, - CancellationToken cancellation) - { - var clock = Stopwatch.StartNew(); - if (string.IsNullOrWhiteSpace(options.TargetPath)) - throw new ArgumentException("Target path is required", nameof(options.TargetPath)); - - string targetPath = options.TargetPath; - string targetDir = Path.GetDirectoryName(targetPath); - if (!string.IsNullOrWhiteSpace(targetDir) && !Directory.Exists(targetDir)) - Directory.CreateDirectory(targetDir); - - string stagingPath = targetPath + ".staging." + Guid.NewGuid().ToString("N") + ".tmp"; - long totalProcessedRows = 0; - long estimatedTotalRows = 0; - int tableCount = 0; - int viewCount = 0; - int triggerCount = 0; - int indexCount = 0; - var tableManifest = new List>(); - - var selectedTables = options.SelectedTables?.ToList() ?? new List(); - var selectedViews = options.IncludeViews ? (options.SelectedViews?.ToList() ?? new List()) : new List(); - var selectedTriggers = options.IncludeTriggers ? (options.SelectedTriggers?.ToList() ?? new List()) : new List(); - - // 空間中繼資料表排除過濾 - if (options.ExcludeSpatialMetadata) - { - selectedTables.RemoveAll(t => KnownSpatialMetadataTables.Contains(t)); - } - - // 預估總列數 - if (options.Mode != SqliteExportMode.SchemaOnly) - { - foreach (var table in selectedTables) + if (options.Mode != SqliteExportMode.SchemaOnly) { - try { estimatedTotalRows += db.CountRows(databaseName, table); } catch { } + var encoding = Query("PRAGMA encoding;"); + if (encoding.Rows.Count != 1 || !Convert.ToString(encoding.Rows[0][0]).Equals("UTF-8", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException("Native text byte export currently requires a UTF-8 SQLite database."); } - } - - try - { - cancellation.ThrowIfCancellationRequested(); - - using (var fileStream = new FileStream(stagingPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 65536)) - using (var writer = new StreamWriter(fileStream, new UTF8Encoding(false), 65536)) + foreach (string view in views) GetDdl("view", view); + var selectedSet = new HashSet(selected.Concat(views), StringComparer.OrdinalIgnoreCase); + if (options.IncludeTriggers) { - writer.WriteLine("-- ========================================================"); - writer.WriteLine("-- mySQLPunk SQLite Logical Dump"); - writer.WriteLine($"-- Database: {databaseName}"); - writer.WriteLine($"-- Created: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); - writer.WriteLine($"-- Mode: {options.Mode}"); - writer.WriteLine($"-- SingleTransaction: {options.SingleTransaction}"); - writer.WriteLine("-- GIS / SpatiaLite / GeoPackage Binary Fidelity: Enabled"); - writer.WriteLine("-- ========================================================"); - writer.WriteLine(); - - // PRAGMA 設定 - if (options.PragmaForeignKeysOff) - writer.WriteLine("PRAGMA foreign_keys = OFF;"); - if (options.PragmaSynchronousOff) - writer.WriteLine("PRAGMA synchronous = OFF;"); - writer.WriteLine(); - - // 交易起始 - if (options.SingleTransaction) - { - writer.WriteLine("BEGIN TRANSACTION;"); - writer.WriteLine(); - } - - // 1. DROP 語句 - if (options.IncludeDropStatements && options.Mode != SqliteExportMode.DataOnly) + foreach (string name in (options.SelectedTriggers ?? new List()).Distinct(StringComparer.OrdinalIgnoreCase)) { - foreach (var tr in selectedTriggers) - writer.WriteLine($"DROP TRIGGER IF EXISTS {QuoteIdentifier(tr)};"); - foreach (var v in selectedViews) - writer.WriteLine($"DROP VIEW IF EXISTS {QuoteIdentifier(v)};"); - foreach (var t in selectedTables) - writer.WriteLine($"DROP TABLE IF EXISTS {QuoteIdentifier(t)};"); - writer.WriteLine(); + var trigger = Query("SELECT tbl_name, sql FROM sqlite_master WHERE type='trigger' AND name=" + TextLiteral(name) + ";"); + if (trigger.Rows.Count != 1 || trigger.Rows[0]["sql"] == DBNull.Value) throw new InvalidOperationException("Missing selected trigger DDL: " + name); + if (selectedSet.Contains(Convert.ToString(trigger.Rows[0]["tbl_name"]))) triggers.Add(name); } - - // 2. 結構 DDL (Tables) + } + var manifest = new List>(); + using (var stream = new FileStream(staging, FileMode.CreateNew, FileAccess.Write, FileShare.None, 65536)) + using (var writer = new StreamWriter(stream, new UTF8Encoding(false, true), 65536)) + { + writer.WriteLine("-- mySQLPunk SQLite Logical Dump"); + writer.WriteLine("-- Created: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)); + writer.WriteLine("-- Native payload checksum covers bytes preceding the checksum footer."); + writer.WriteLine("-- Data fidelity: ordinary UTF-8 SQLite tables; virtual modules require separate backup."); + if (options.PragmaForeignKeysOff) writer.WriteLine("PRAGMA foreign_keys = OFF;"); + if (options.PragmaSynchronousOff) writer.WriteLine("PRAGMA synchronous = OFF;"); + if (options.SingleTransaction) writer.WriteLine("BEGIN TRANSACTION;"); if (options.Mode != SqliteExportMode.DataOnly) { - ReportProgress(progress, SqliteExportStage.WritingSchema, "", 0, selectedTables.Count, 0, estimatedTotalRows, fileStream.Position, clock); - - foreach (var table in selectedTables) + if (options.IncludeDropStatements) { - cancellation.ThrowIfCancellationRequested(); - string ddl = GetTableDdl(table); - if (!string.IsNullOrWhiteSpace(ddl)) - { - writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); - writer.WriteLine(); - } - - // 側車欄位註解 DDL - if (options.IncludeComments) - { - string commentDdl = BuildColumnCommentsDump(table); - if (!string.IsNullOrWhiteSpace(commentDdl)) - { - writer.WriteLine(commentDdl.TrimEnd()); - writer.WriteLine(); - } - } + foreach (string tr in triggers) writer.WriteLine("DROP TRIGGER IF EXISTS " + QuoteIdentifier(tr) + ";"); + foreach (string view in views) writer.WriteLine("DROP VIEW IF EXISTS " + QuoteIdentifier(view) + ";"); + foreach (string table in selected) writer.WriteLine("DROP TABLE IF EXISTS " + QuoteIdentifier(table) + ";"); } + foreach (string table in selected) { cancellation.ThrowIfCancellationRequested(); WriteDdl(writer, ddls[table]); } } - - // 3. 資料列 (Data) - if (options.Mode != SqliteExportMode.SchemaOnly) + foreach (string table in selected) { - int batchSize = options.InsertBatchSize > 0 ? options.InsertBatchSize : 1000; - for (int i = 0; i < selectedTables.Count; i++) + cancellation.ThrowIfCancellationRequested(); + long tableRows = 0; + if (options.Mode != SqliteExportMode.SchemaOnly) { - cancellation.ThrowIfCancellationRequested(); - string table = selectedTables[i]; - long tableRows = 0; - long totalTableRows = 0; - try { totalTableRows = db.CountRows(databaseName, table); } catch { } - - ReportProgress(progress, SqliteExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); - - while (tableRows < totalTableRows || totalTableRows == 0) - { - cancellation.ThrowIfCancellationRequested(); - DataTable page = db.SelectTablePage(databaseName, table, tableRows, batchSize); - if (page == null || page.Rows.Count == 0) break; - - string insertPrefix = "INSERT INTO " + QuoteIdentifier(table) + " ("; - var colNames = new string[page.Columns.Count]; - for (int c = 0; c < page.Columns.Count; c++) - { - colNames[c] = QuoteIdentifier(page.Columns[c].ColumnName); - } - string colsJoined = string.Join(", ", colNames); - - for (int r = 0; r < page.Rows.Count; r++) - { - writer.Write(insertPrefix); - writer.Write(colsJoined); - writer.Write(") VALUES ("); - - DataRow row = page.Rows[r]; - for (int c = 0; c < page.Columns.Count; c++) - { - if (c > 0) writer.Write(", "); - WriteSqliteLiteral(writer, row[c]); - } - writer.WriteLine(");"); - } - - tableRows += page.Rows.Count; - totalProcessedRows += page.Rows.Count; - ReportProgress(progress, SqliteExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); - } - - if (tableRows > 0) writer.WriteLine(); - tableCount++; - tableManifest.Add(Tuple.Create(table, tableRows)); + tableRows = WriteTableData(writer, table, ddls[table], cancellation); + rows += tableRows; + manifest.Add(Tuple.Create(table, tableRows)); } + tableCount++; + ReportProgress(progress, SqliteExportStage.WritingData, table, tableCount, selected.Count, rows, estimated, stream.Position, clock); } - else - { - tableCount = selectedTables.Count; - } - - // 4. 檢視 (Views) - if (options.Mode != SqliteExportMode.DataOnly && selectedViews.Count > 0) - { - ReportProgress(progress, SqliteExportStage.WritingSchema, "Views", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); - foreach (var view in selectedViews) - { - cancellation.ThrowIfCancellationRequested(); - string ddl = GetViewDdl(view); - if (!string.IsNullOrWhiteSpace(ddl)) - { - writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); - writer.WriteLine(); - } - viewCount++; - } - } - - // 5. 索引 (Indexes) - if (options.Mode != SqliteExportMode.DataOnly && options.IncludeIndexes) - { - ReportProgress(progress, SqliteExportStage.WritingIndexes, "Indexes", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); - var indexes = GetIndexesForTables(selectedTables); - foreach (var idx in indexes) - { - cancellation.ThrowIfCancellationRequested(); - writer.WriteLine(idx.TrimEnd().TrimEnd(';') + ";"); - indexCount++; - } - if (indexes.Count > 0) writer.WriteLine(); - } - - // 6. 觸發器 (Triggers) - if (options.Mode != SqliteExportMode.DataOnly && selectedTriggers.Count > 0) + if (options.Mode != SqliteExportMode.SchemaOnly) WriteSequence(writer, selected, cancellation); + if (options.Mode != SqliteExportMode.DataOnly) { - ReportProgress(progress, SqliteExportStage.WritingTriggers, "Triggers", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); - foreach (var triggerName in selectedTriggers) + foreach (string view in views) { cancellation.ThrowIfCancellationRequested(); WriteDdl(writer, GetDdl("view", view)); viewCount++; } + if (options.IncludeIndexes) { - cancellation.ThrowIfCancellationRequested(); - string ddl = GetTriggerDdl(triggerName); - if (!string.IsNullOrWhiteSpace(ddl)) - { - writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); - writer.WriteLine(); - triggerCount++; - } + DataTable indexes = Query("SELECT sql, tbl_name FROM sqlite_master WHERE type='index' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%';"); + foreach (DataRow index in indexes.Rows) + if (selected.Contains(Convert.ToString(index["tbl_name"]), StringComparer.OrdinalIgnoreCase)) + { + cancellation.ThrowIfCancellationRequested(); + string ddl = Convert.ToString(index["sql"]); + if (string.IsNullOrWhiteSpace(ddl)) throw new InvalidOperationException("Missing selected index DDL."); + WriteDdl(writer, ddl); indexCount++; + } } + foreach (string trigger in triggers) { cancellation.ThrowIfCancellationRequested(); WriteDdl(writer, GetDdl("trigger", trigger)); triggerCount++; } } - - // 交易結束 - if (options.SingleTransaction) - { - writer.WriteLine("COMMIT;"); - writer.WriteLine(); - } - - // 恢復 foreign_keys - if (options.PragmaForeignKeysOff) - { - writer.WriteLine("PRAGMA foreign_keys = ON;"); - } - - writer.Flush(); - - // 7. 寫入 Native Manifest 摘要 - writer.WriteLine(); - writer.WriteLine("-- ========================================================"); + if (options.IncludeComments) WriteComments(writer, selected, cancellation); + if (options.SingleTransaction) writer.WriteLine("COMMIT;"); + // The runner restores its actual original PRAGMA values in finally. + // Standalone SQL explicitly re-enables FK checks after restoration. + if (options.PragmaForeignKeysOff) writer.WriteLine("PRAGMA foreign_keys = ON;"); writer.WriteLine("-- mySQLPunk Native Manifest"); - writer.WriteLine($"-- Tables: {tableCount}"); - writer.WriteLine($"-- TotalRows: {totalProcessedRows}"); - foreach (var pair in tableManifest) - { - writer.WriteLine($"-- Manifest-Table: {pair.Item1} | Rows: {pair.Item2}"); - } - writer.WriteLine("-- ========================================================"); - writer.Flush(); - } - - // 計算產出檔案的 SHA-256 - string sha256Hex; - long finalLength; - using (var fs = File.OpenRead(stagingPath)) - { - finalLength = fs.Length; - using (var sha = SHA256.Create()) - { - byte[] hash = sha.ComputeHash(fs); - sha256Hex = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); - } - } - - // 結尾附加 SHA256 到檔案中 - using (var appendWriter = new StreamWriter(stagingPath, true, new UTF8Encoding(false))) - { - appendWriter.WriteLine($"-- Manifest-SHA256: {sha256Hex}"); + writer.WriteLine("-- Tables: " + tableCount); + writer.WriteLine("-- TotalRows: " + rows); + foreach (var pair in manifest) + writer.WriteLine("-- Manifest-Table64: " + Convert.ToBase64String(Encoding.UTF8.GetBytes(pair.Item1)) + " | Rows: " + pair.Item2); } - - // 原子替換到目標路徑 - if (File.Exists(targetPath)) - { - File.Delete(targetPath); - } - File.Move(stagingPath, targetPath); - - clock.Stop(); - - ReportProgress(progress, SqliteExportStage.Completed, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, finalLength, clock); - - return new SqliteExportResult( - SqliteExportStage.Completed, - targetPath, - tableCount, - viewCount, - triggerCount, - indexCount, - totalProcessedRows, - finalLength, - sha256Hex, - clock.Elapsed); - } - catch (OperationCanceledException) - { + ExecChecked("COMMIT;"); snapshot = false; + sourceCancellation.Dispose(); + cancellation.ThrowIfCancellationRequested(); + string payloadHash = HashFile(staging, cancellation); + using (var writer = new StreamWriter(staging, true, new UTF8Encoding(false, true))) writer.WriteLine("-- Manifest-Payload-SHA256: " + payloadHash); + string deliveredHash = HashFile(staging, cancellation); + long deliveredBytes = new FileInfo(staging).Length; + cancellation.ThrowIfCancellationRequested(); + publicationAttempted = true; + SqlExportPublication.Publish(staging, target); clock.Stop(); - TryDeleteFile(stagingPath); - ReportProgress(progress, SqliteExportStage.Cancelled, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, 0, clock); - return new SqliteExportResult( - SqliteExportStage.Cancelled, - targetPath, - tableCount, - viewCount, - triggerCount, - indexCount, - totalProcessedRows, - 0, - null, - clock.Elapsed, - "Export cancelled by user."); + ReportProgress(progress, SqliteExportStage.Completed, "", tableCount, selected.Count, rows, estimated, deliveredBytes, clock); + return new SqliteExportResult(SqliteExportStage.Completed, target, tableCount, viewCount, triggerCount, indexCount, rows, deliveredBytes, deliveredHash, clock.Elapsed); } catch (Exception ex) { + sourceCancellation.Dispose(); + bool cancelled = ex is OperationCanceledException || cancellation.IsCancellationRequested; + string error = cancelled ? "Export cancelled by user." : ex.Message; + if (snapshot) + { + try { if (!(db is my_sqlite nativeCleanup) || nativeCleanup.MCT == null || !nativeCleanup.MCT.AutoCommit) ExecChecked("ROLLBACK;"); snapshot = false; } + catch (Exception cleanup) { error += " Source ROLLBACK failed: " + cleanup.Message; cancelled = false; } + } + if (!publicationAttempted) TryDeleteFile(staging); + else error += " Recoverable staging file: " + staging; clock.Stop(); - TryDeleteFile(stagingPath); - ReportProgress(progress, SqliteExportStage.Failed, "", tableCount, selectedTables.Count, totalProcessedRows, estimatedTotalRows, 0, clock, ex.Message); - return new SqliteExportResult( - SqliteExportStage.Failed, - targetPath, - tableCount, - viewCount, - triggerCount, - indexCount, - totalProcessedRows, - 0, - null, - clock.Elapsed, - ex.Message); + var stage = cancelled ? SqliteExportStage.Cancelled : SqliteExportStage.Failed; + ReportProgress(progress, stage, "", tableCount, selected.Count, rows, estimated, 0, clock, error); + return new SqliteExportResult(stage, target, tableCount, viewCount, triggerCount, indexCount, rows, 0, null, clock.Elapsed, error); } + finally { sourceCancellation.Dispose(); } } - - private static void WriteSqliteLiteral(StreamWriter writer, object value) + private long WriteTableData(StreamWriter writer, string table, string ddl, CancellationToken cancellation) { - if (value == null || value == DBNull.Value) - { - writer.Write("NULL"); - return; + DataTable metadata = Query("PRAGMA table_xinfo(" + QuoteIdentifier(table) + ");"); + if (metadata.Rows.Count == 0 || !metadata.Columns.Contains("hidden")) throw new NotSupportedException("Writable column metadata unavailable: " + table); + var all = metadata.Rows.Cast().Select(r => Convert.ToString(r["name"])).ToList(); + var columns = metadata.Rows.Cast().Where(r => Convert.ToInt32(r["hidden"]) == 0).Select(r => Convert.ToString(r["name"])).ToList(); + if (metadata.Rows.Cast().Any(r => Convert.ToInt32(r["hidden"]) == 1)) throw new NotSupportedException("Hidden virtual columns are not supported: " + table); + DataTable tableInfo = Query("PRAGMA table_list(" + QuoteIdentifier(table) + ");"); + if (tableInfo.Rows.Count != 1 || !tableInfo.Columns.Contains("wr")) throw new NotSupportedException("Rowid metadata unavailable: " + table); + bool withoutRowid = Convert.ToInt32(tableInfo.Rows[0]["wr"]) != 0; + string stableOrder = null; + if (!withoutRowid) + { + string alias = new[] { "rowid", "_rowid_", "oid" }.FirstOrDefault(n => !all.Contains(n, StringComparer.OrdinalIgnoreCase)); + if (alias != null) { columns.Insert(0, alias); stableOrder = QuoteIdentifier(alias); } + else + { + // An accessible INTEGER PRIMARY KEY aliases rowid only when no + // separate PK index exists (INTEGER PRIMARY KEY DESC is different). + DataTable indexes = Query("PRAGMA index_list(" + QuoteIdentifier(table) + ");"); + bool integerPk = metadata.Rows.Cast().Any(r => Convert.ToInt32(r["pk"]) == 1 && Convert.ToString(r["type"]).Equals("INTEGER", StringComparison.OrdinalIgnoreCase)); + bool pkIndex = indexes.Columns.Contains("origin") && indexes.Rows.Cast().Any(r => Convert.ToString(r["origin"]) == "pk"); + stableOrder = string.Join(", ", metadata.Rows.Cast().Where(r => Convert.ToInt32(r["pk"]) > 0).OrderBy(r => Convert.ToInt32(r["pk"])).Select(r => QuoteIdentifier(Convert.ToString(r["name"])))); + if (!integerPk || pkIndex) throw new NotSupportedException("All implicit rowid aliases are shadowed; cannot safely preserve row identities: " + table); + } } - - if (value is byte[] bytes) - { - // SpatiaLite / GeoPackage GIS 幾何與一般 BLOB 二進位無損輸出 - writer.Write("X'"); - char[] hexBuffer = new char[Math.Min(bytes.Length * 2, 8192)]; - int bufIdx = 0; - for (int i = 0; i < bytes.Length; i++) + if (columns.Count == 0) throw new NotSupportedException("No writable columns found: " + table); + long expected = CountChecked(table); + string names = string.Join(", ", columns.Select(QuoteIdentifier)); + string insert = "INSERT INTO " + QuoteIdentifier(table) + " (" + names + ") VALUES ("; + int insertBytes = Encoding.UTF8.GetByteCount(insert) + columns.Count * 2 + 3; + string from = " FROM " + QuoteIdentifier(table); + if (stableOrder == null) stableOrder = string.Join(", ", metadata.Rows.Cast().Where(r => Convert.ToInt32(r["pk"]) > 0).OrderBy(r => Convert.ToInt32(r["pk"])).Select(r => QuoteIdentifier(Convert.ToString(r["name"])))); + if (string.IsNullOrEmpty(stableOrder)) throw new NotSupportedException("Stable row identity unavailable: " + table); + string order = " ORDER BY " + stableOrder; + string size = string.Join(" + ", columns.Select(c => "coalesce(length(CAST(" + QuoteIdentifier(c) + " AS BLOB)),0)")); + string literals = string.Join(", ", columns.Select(c => SqlValueExpression(QuoteIdentifier(c)))); + if (db is my_sqlite native && native.MCT != null) + { + // One sequential reader for the native provider. Gating the SQL + // expressions avoids computing a huge hex literal before size is checked. + string guarded = string.Join(", ", columns.Select(c => "CASE WHEN (" + size + ") <= " + MaxRowBytes + " THEN " + SqlValueExpression(QuoteIdentifier(c)) + " ELSE NULL END")); + using (var command = new SQLiteCommand("SELECT " + size + ", " + guarded + from + order + ";", native.MCT)) + using (var reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) { - byte b = bytes[i]; - hexBuffer[bufIdx++] = HexLookup[b >> 4]; - hexBuffer[bufIdx++] = HexLookup[b & 0x0F]; - if (bufIdx >= hexBuffer.Length - 1) + long actual = 0; + while (reader.Read()) { - writer.Write(hexBuffer, 0, bufIdx); - bufIdx = 0; + cancellation.ThrowIfCancellationRequested(); + if (reader.GetInt64(0) > MaxRowBytes) throw new NotSupportedException("Row exceeds safe 8 MiB native export limit: " + table); + long statementBytes = insertBytes; + writer.Write(insert); + for (int c = 0; c < columns.Count; c++) + { + cancellation.ThrowIfCancellationRequested(); + if (c > 0) writer.Write(", "); + if (reader.IsDBNull(c + 1)) throw new InvalidOperationException("Source literal rendering failed: " + table); + string literal = reader.GetString(c + 1); + statementBytes += Encoding.UTF8.GetByteCount(literal); + if (statementBytes > SqliteImport.SqliteStatementReader.MaxStatementBytes) throw new NotSupportedException("Serialized row exceeds 32 MiB statement limit: " + table); + writer.Write(literal); + } + writer.WriteLine(");"); actual++; } + if (actual != expected) throw new InvalidOperationException("Source snapshot row count differs for " + table); + return actual; } - if (bufIdx > 0) - { - writer.Write(hexBuffer, 0, bufIdx); - } - writer.Write("'"); - return; } - - if (value is string s) - { - writer.Write("'"); - writer.Write(s.Replace("'", "''")); - writer.Write("'"); - return; - } - - if (value is bool bVal) - { - writer.Write(bVal ? "1" : "0"); - return; - } - - if (value is DateTime dt) + for (long offset = 0; offset < expected; offset++) { - writer.Write("'"); - writer.Write(dt.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture)); - writer.Write("'"); - return; - } - - if (value is float f) - { - writer.Write(f.ToString("R", CultureInfo.InvariantCulture)); - return; + cancellation.ThrowIfCancellationRequested(); + string page = from + order + " LIMIT 1 OFFSET " + offset.ToString(CultureInfo.InvariantCulture) + ";"; + DataTable sizes = Query("SELECT " + size + page); + if (sizes.Rows.Count != 1 || Convert.ToInt64(sizes.Rows[0][0]) > MaxRowBytes) + throw new NotSupportedException("Missing row or row exceeds the safe 8 MiB native export limit: " + table + ". Existing output was preserved."); + DataTable values = Query("SELECT " + literals + page); + if (values.Rows.Count != 1 || values.Columns.Count != columns.Count) throw new InvalidOperationException("Incomplete source page for " + table); + long statementBytes = insertBytes; + writer.Write(insert); + for (int c = 0; c < columns.Count; c++) + { + if (c > 0) writer.Write(", "); + string literal = Convert.ToString(values.Rows[0][c], CultureInfo.InvariantCulture); + statementBytes += Encoding.UTF8.GetByteCount(literal); + if (statementBytes > SqliteImport.SqliteStatementReader.MaxStatementBytes) throw new NotSupportedException("Serialized row exceeds 32 MiB statement limit: " + table); + writer.Write(literal); + } + writer.WriteLine(");"); } - - if (value is double d) + if (CountChecked(table) != expected) throw new InvalidOperationException("Source row count changed during snapshot: " + table); + return expected; + } + private void WriteSequence(StreamWriter writer, List selected, CancellationToken cancellation) + { + if (!ObjectExists("table", "sqlite_sequence")) return; + foreach (string table in selected) { - writer.Write(d.ToString("R", CultureInfo.InvariantCulture)); - return; + cancellation.ThrowIfCancellationRequested(); + DataTable sequence = Query("SELECT seq FROM sqlite_sequence WHERE name=" + TextLiteral(table) + ";"); + if (sequence.Rows.Count == 0) continue; + if (sequence.Rows.Count != 1) throw new InvalidOperationException("Ambiguous sqlite_sequence row for " + table); + writer.WriteLine("DELETE FROM sqlite_sequence WHERE name=" + TextLiteral(table) + ";"); + writer.WriteLine("INSERT INTO sqlite_sequence(name,seq) VALUES (" + TextLiteral(table) + "," + Convert.ToInt64(sequence.Rows[0][0]).ToString(CultureInfo.InvariantCulture) + ");"); } - - if (value is decimal dec) + } + private void WriteComments(StreamWriter writer, List selected, CancellationToken cancellation) + { + if (!ObjectExists("table", my_sqlite.ColumnCommentTableName)) return; + bool created = false; + foreach (string table in selected) { - writer.Write(dec.ToString(CultureInfo.InvariantCulture)); - return; + cancellation.ThrowIfCancellationRequested(); + string size = "coalesce(length(CAST(comment AS BLOB)),0)+coalesce(length(CAST(column_name AS BLOB)),0)"; + string query = "SELECT " + size + ", CASE WHEN (" + size + ") <= " + MaxRowBytes + " THEN column_name ELSE NULL END, CASE WHEN (" + size + ") <= " + MaxRowBytes + " THEN " + SqlValueExpression("comment") + " ELSE NULL END FROM " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + " WHERE table_name=" + TextLiteral(table); + if (db is my_sqlite native && native.MCT != null) + { + using (var command = new SQLiteCommand(query + ";", native.MCT)) + using (var reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) + while (reader.Read()) + { + cancellation.ThrowIfCancellationRequested(); + if (reader.GetInt64(0) > MaxRowBytes) throw new NotSupportedException("Column comment exceeds safe 8 MiB export limit: " + table); + WriteComment(writer, table, reader.GetString(1), reader.GetString(2), ref created); + } + } + else + { + for (long offset = 0; ; offset++) + { + cancellation.ThrowIfCancellationRequested(); + DataTable data = Query(query + " ORDER BY column_name LIMIT 1 OFFSET " + offset + ";"); + if (data.Rows.Count == 0) break; + if (Convert.ToInt64(data.Rows[0][0]) > MaxRowBytes) throw new NotSupportedException("Column comment exceeds safe 8 MiB export limit: " + table); + WriteComment(writer, table, Convert.ToString(data.Rows[0][1]), Convert.ToString(data.Rows[0][2]), ref created); + } + } } - - if (value is IFormattable formattable) + } + private static void WriteComment(StreamWriter writer, string table, string column, string literal, ref bool created) + { + if (!created) { - writer.Write(formattable.ToString(null, CultureInfo.InvariantCulture)); - return; + writer.WriteLine("CREATE TABLE IF NOT EXISTS " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + " (table_name TEXT NOT NULL,column_name TEXT NOT NULL,comment TEXT NOT NULL,PRIMARY KEY(table_name,column_name));"); + created = true; } - - writer.Write("'"); - writer.Write(value.ToString().Replace("'", "''")); - writer.Write("'"); + string sql = "INSERT OR REPLACE INTO " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + " (table_name,column_name,comment) VALUES (" + TextLiteral(table) + "," + TextLiteral(column) + "," + literal + ");"; + if (Encoding.UTF8.GetByteCount(sql) > SqliteImport.SqliteStatementReader.MaxStatementBytes) throw new NotSupportedException("Column comment SQL exceeds native statement size limit."); + writer.WriteLine(sql); } - - public static string QuoteIdentifier(string name) + private static string SqlValueExpression(string column) { - name = name ?? string.Empty; - return "\"" + name.Replace("\"", "\"\"") + "\""; + return "CASE typeof(" + column + ") WHEN 'text' THEN 'CAST(X'''||hex(CAST(" + column + " AS BLOB))||''' AS TEXT)' WHEN 'blob' THEN 'X'''||hex(" + column + ")||'''' ELSE quote(" + column + ") END"; } - - private string GetTableDdl(string tableName) + private DataTable Query(string sql) { - try - { - string safe = tableName.Replace("'", "''"); - DataTable dt = db.SelectSQL($"SELECT sql FROM sqlite_master WHERE type='table' AND name='{safe}';", new Dictionary()); - if (dt != null && dt.Rows.Count > 0 && dt.Rows[0][0] != DBNull.Value) - return dt.Rows[0][0].ToString(); - } - catch { } - return ""; + DataTable result = db.SelectSQL(sql, new Dictionary()); + if (result == null || result.ExtendedProperties.ContainsKey("QueryError")) throw new InvalidOperationException("SQLite source query failed: " + (result == null ? "no result" : Convert.ToString(result.ExtendedProperties["QueryError"]))); + return result; } - - private string GetViewDdl(string viewName) + private long CountChecked(string table) { - try - { - string safe = viewName.Replace("'", "''"); - DataTable dt = db.SelectSQL($"SELECT sql FROM sqlite_master WHERE type='view' AND name='{safe}';", new Dictionary()); - if (dt != null && dt.Rows.Count > 0 && dt.Rows[0][0] != DBNull.Value) - return dt.Rows[0][0].ToString(); - } - catch { } - return ""; + long count = db.CountRows(databaseName, table); + if (count < 0) throw new InvalidOperationException("SQLite row count failed: " + table); + return count; } - - private string GetTriggerDdl(string triggerName) + private bool ObjectExists(string type, string name) { - try - { - string safe = triggerName.Replace("'", "''"); - DataTable dt = db.SelectSQL($"SELECT sql FROM sqlite_master WHERE type='trigger' AND name='{safe}';", new Dictionary()); - if (dt != null && dt.Rows.Count > 0 && dt.Rows[0][0] != DBNull.Value) - return dt.Rows[0][0].ToString(); - } - catch { } - return ""; + return Query("SELECT sql FROM sqlite_master WHERE type='" + type + "' AND name=" + TextLiteral(name) + ";").Rows.Count == 1; } - - private List GetIndexesForTables(IEnumerable tableNames) + private string GetDdl(string type, string name) { - var result = new List(); - var set = new HashSet(tableNames, StringComparer.OrdinalIgnoreCase); - try - { - DataTable dt = db.SelectSQL("SELECT sql, tbl_name FROM sqlite_master WHERE type='index' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%';", new Dictionary()); - foreach (DataRow row in dt.Rows) - { - string tbl = row["tbl_name"]?.ToString(); - string sql = row["sql"]?.ToString(); - if (!string.IsNullOrWhiteSpace(tbl) && set.Contains(tbl) && !string.IsNullOrWhiteSpace(sql)) - { - result.Add(sql); - } - } - } - catch { } - return result; + DataTable ddl = Query("SELECT sql FROM sqlite_master WHERE type='" + type + "' AND name=" + TextLiteral(name) + ";"); + if (ddl.Rows.Count != 1 || ddl.Rows[0][0] == DBNull.Value || string.IsNullOrWhiteSpace(Convert.ToString(ddl.Rows[0][0]))) throw new InvalidOperationException("Missing selected " + type + " DDL: " + name); + return Convert.ToString(ddl.Rows[0][0]); } - - private string BuildColumnCommentsDump(string tableName) + private void ExecChecked(string sql) { - try - { - DataTable columns = db.GetColumns(databaseName, tableName); - if (columns == null || columns.Rows.Count == 0) return ""; - - var inserts = new List(); - foreach (DataRow row in columns.Rows) - { - string colName = null; - foreach (string key in new[] { "name", "Name", "COLUMN_NAME", "column_name", "Field" }) - { - if (row.Table.Columns.Contains(key) && row[key] != DBNull.Value) - { - colName = row[key].ToString(); - break; - } - } - string comment = null; - foreach (string key in new[] { "Comment", "comment" }) - { - if (row.Table.Columns.Contains(key) && row[key] != DBNull.Value) - { - comment = row[key].ToString(); - break; - } - } - - if (string.IsNullOrWhiteSpace(colName) || string.IsNullOrWhiteSpace(comment)) continue; - - inserts.Add("INSERT OR REPLACE INTO " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + - " (table_name, column_name, comment) VALUES (" + - "'" + tableName.Replace("'", "''") + "', " + - "'" + colName.Replace("'", "''") + "', " + - "'" + comment.Replace("'", "''") + "');"); - } - - if (inserts.Count == 0) return ""; - - var sb = new StringBuilder(); - sb.AppendLine("-- SQLite column comments sidecar metadata"); - sb.AppendLine("CREATE TABLE IF NOT EXISTS " + QuoteIdentifier(my_sqlite.ColumnCommentTableName) + " (" + - "table_name TEXT NOT NULL, " + - "column_name TEXT NOT NULL, " + - "comment TEXT NOT NULL, " + - "PRIMARY KEY (table_name, column_name));"); - foreach (var ins in inserts) sb.AppendLine(ins); - return sb.ToString(); - } - catch - { - return ""; - } + var result = db.ExecSQL(sql); + if (result == null || !result.TryGetValue("status", out string status) || !status.Equals("OK", StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException(sql + " " + DatabaseExecutionResultService.GetFailureReason(result)); } - - private HashSet GetVirtualTableNames() + private static string HashFile(string path, CancellationToken cancellation) { - var names = new HashSet(StringComparer.OrdinalIgnoreCase); - try + using (var stream = File.OpenRead(path)) using (var hash = SHA256.Create()) { - DataTable dt = db.SelectSQL("SELECT name FROM sqlite_master WHERE type='table' AND sql LIKE 'CREATE VIRTUAL%';", new Dictionary()); - foreach (DataRow row in dt.Rows) - { - string n = row[0]?.ToString(); - if (!string.IsNullOrWhiteSpace(n)) names.Add(n); - } + var buffer = new byte[65536]; int n; + while ((n = stream.Read(buffer, 0, buffer.Length)) > 0) { cancellation.ThrowIfCancellationRequested(); hash.TransformBlock(buffer, 0, n, null, 0); } + hash.TransformFinalBlock(new byte[0], 0, 0); + return BitConverter.ToString(hash.Hash).Replace("-", "").ToLowerInvariant(); } - catch { } - return names; } - + private static void WriteDdl(StreamWriter writer, string ddl) + { + if (Encoding.UTF8.GetByteCount(ddl) > SqliteImport.SqliteStatementReader.MaxStatementBytes - 2) throw new NotSupportedException("DDL exceeds native statement size limit."); + writer.WriteLine(ddl.TrimEnd().TrimEnd(';') + ";"); + } + private static string TextLiteral(string text) { return "'" + text.Replace("'", "''") + "'"; } + public static string QuoteIdentifier(string name) { return "\"" + (name ?? "").Replace("\"", "\"\"") + "\""; } private static bool IsShadowTable(string tableName, HashSet virtualNames) { - if (virtualNames == null || virtualNames.Count == 0) return false; - int idx = tableName.LastIndexOf('_'); - if (idx <= 0) return false; - string prefix = tableName.Substring(0, idx); - if (!virtualNames.Contains(prefix)) return false; - string suffix = tableName.Substring(idx + 1); - foreach (string known in ShadowTableSuffixes) - { - if (string.Equals(suffix, known, StringComparison.OrdinalIgnoreCase)) return true; - } - return false; + int i = tableName.LastIndexOf('_'); + return i > 0 && virtualNames.Contains(tableName.Substring(0, i)) && ShadowTableSuffixes.Contains(tableName.Substring(i + 1), StringComparer.OrdinalIgnoreCase); } - private Dictionary GetSpatialTableDetails() { var map = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -830,11 +549,7 @@ private static void ReportProgress( private static void TryDeleteFile(string path) { - try - { - if (File.Exists(path)) File.Delete(path); - } - catch { } + try { if (File.Exists(path)) File.Delete(path); } catch { } } } } diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs b/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs index 39137c2..7530e38 100644 --- a/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs +++ b/mySQLPunk/lib/SqliteImport/SqliteImportModels.cs @@ -20,6 +20,21 @@ public enum SqliteExistingObjectStrategy OverwriteKeepExisting } + public static class SqliteExistingObjectSelection + { + public static SqliteExistingObjectStrategy FromIndex(int index) + { + switch (index) + { + case 0: return SqliteExistingObjectStrategy.SkipExisting; + case 1: return SqliteExistingObjectStrategy.DropAndRecreate; + case 2: return SqliteExistingObjectStrategy.OverwriteKeepExisting; + case 3: return SqliteExistingObjectStrategy.AbortIfExists; + default: throw new ArgumentOutOfRangeException(nameof(index)); + } + } + } + public sealed class SqliteImportRequest { public Guid JobId { get; } @@ -41,7 +56,7 @@ public sealed class SqliteImportOptions public bool AutoLoadSpatiaLite { get; set; } = true; public bool WrapInTransaction { get; set; } = true; public bool PragmaForeignKeysOff { get; set; } = true; - public bool PragmaSynchronousOff { get; set; } = true; + public bool PragmaSynchronousOff { get; set; } = false; } public sealed class SqliteImportPreflight diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs b/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs index b2cd91b..4389ef6 100644 --- a/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs +++ b/mySQLPunk/lib/SqliteImport/SqliteImportPreflightScanner.cs @@ -1,243 +1,266 @@ using System; using System.Collections.Generic; using System.IO; +using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading; namespace mySQLPunk.lib.SqliteImport { - public static class SqliteImportPreflightScanner + // Preflight and execution use the same bounded lexer. It retains the original + // characters (including literal CR/LF), and treats a trigger body as one statement. + internal sealed class SqliteStatementReader : IDisposable { - private const int MaxBatchBytes = 32 * 1024 * 1024; // 32 MiB - private static readonly Regex ManifestTableRegex = new Regex(@"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex ManifestTablesRegex = new Regex(@"^--\s*Tables:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex ManifestRowsRegex = new Regex(@"^--\s*TotalRows:\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex ManifestSha256Regex = new Regex(@"^--\s*Manifest-SHA256:\s*([a-fA-F0-9]{64})", RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly string[] SpatialKeywords = new string[] + internal const int MaxStatementBytes = 32 * 1024 * 1024; + private readonly StreamReader reader; + private readonly CancellationToken cancellation; + private readonly Action comment; + private int line = 1; + private bool first = true; + internal int StatementLine { get; private set; } + internal SqliteStatementReader(string path, CancellationToken cancellation, Action comment = null) { - "InitSpatialMetaData", "AddGeometryColumn", "CreateSpatialIndex", - "mod_spatialite", "spatial_ref_sys", "geometry_columns", - "gpkg_spatial_ref_sys", "gpkg_geometry_columns", "gpkg_contents", - "GeomFromText", "ST_GeomFromText", "AsText", "ST_AsText", - "PointFromText", "LineFromText", "PolyFromText" - }; - - public static SqliteImportPreflight Scan(string filePath, CancellationToken cancellation) + reader = new StreamReader(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 65536), + new UTF8Encoding(false, true), false, 65536); + this.cancellation = cancellation; + this.comment = comment; + } + internal string Read() { - if (string.IsNullOrWhiteSpace(filePath)) - throw new ArgumentException("File path is required", nameof(filePath)); - if (!File.Exists(filePath)) - return new SqliteImportPreflight(false, 0, 0, false, 0, 0, null, false, new[] { "File not found: " + filePath }); - - long fileBytes = new FileInfo(filePath).Length; - var issues = new List(); - int statementCount = 0; - bool hasManifest = false; - int manifestTables = 0; - long manifestRows = 0; - string manifestSha = null; - bool detectedSpatiaLite = false; - - try + var sql = new StringBuilder(); + var word = new StringBuilder(); + var lineComment = new StringBuilder(); + int quote = 0, bytes = 0, caseDepth = 0, prefixCount = 0; + bool block = false, inline = false, meaningful = false, create = false, trigger = false, body = false, end = false; + StatementLine = line; + Action flushWord = () => { - using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) - using (var reader = new StreamReader(fs, Encoding.UTF8, true, 65536)) - { - bool inSingleQuote = false; - bool inDoubleQuote = false; - bool inBracket = false; - bool inBlockComment = false; - int currentBatchLength = 0; - bool hasStatementContent = false; - int lineIndex = 0; - string line; - - while ((line = reader.ReadLine()) != null) + if (word.Length == 0) return; + string token = word.ToString().ToUpperInvariant(); + word.Clear(); + if (prefixCount == 0) create = token == "CREATE"; + if (create && prefixCount < 4 && token == "TRIGGER") trigger = true; + prefixCount++; + if (trigger) + { + if (token == "BEGIN" && !body) body = true; + else if (body && token == "CASE") { caseDepth++; end = false; } + else if (body && token == "END") { - cancellation.ThrowIfCancellationRequested(); - lineIndex++; - - // 檢查 embedded NUL (非法字元或編碼毀損) - if (line.IndexOf('\0') >= 0) - { - issues.Add($"Line {lineIndex}: Contains Embedded NUL character (possible malformed encoding or corrupted binary file)."); - break; - } - - string trimmed = line.Trim(); - - // 檢查 Manifest - if (trimmed.StartsWith("--")) - { - if (trimmed.Contains("mySQLPunk Native Manifest")) - { - hasManifest = true; - } - else if (hasManifest) - { - var mTables = ManifestTablesRegex.Match(trimmed); - if (mTables.Success && int.TryParse(mTables.Groups[1].Value, out int mt)) - { - manifestTables = mt; - continue; - } - var mRows = ManifestRowsRegex.Match(trimmed); - if (mRows.Success && long.TryParse(mRows.Groups[1].Value, out long mr)) - { - manifestRows = mr; - continue; - } - var mTable = ManifestTableRegex.Match(trimmed); - if (mTable.Success) - { - if (manifestTables == 0) manifestTables++; - if (long.TryParse(mTable.Groups[2].Value, out long r) && manifestRows == 0) manifestRows += r; - continue; - } - var mSha = ManifestSha256Regex.Match(trimmed); - if (mSha.Success) - { - manifestSha = mSha.Groups[1].Value; - } - } - - // 註解行若在引號外，不計入語句實質內容 - if (!inSingleQuote && !inDoubleQuote && !inBlockComment) - { - continue; - } - } - - // 探測 SpatiaLite / GeoPackage 關鍵字 - if (!detectedSpatiaLite) - { - for (int k = 0; k < SpatialKeywords.Length; k++) - { - if (line.IndexOf(SpatialKeywords[k], StringComparison.OrdinalIgnoreCase) >= 0) - { - detectedSpatiaLite = true; - break; - } - } - } - - // 逐字元狀態機解析，以精準識別分號 - int len = line.Length; - for (int i = 0; i < len; i++) - { - char c = line[i]; - - if (inBlockComment) - { - if (c == '*' && i + 1 < len && line[i + 1] == '/') - { - inBlockComment = false; - i++; - } - continue; - } - - if (!inSingleQuote && !inDoubleQuote && !inBracket) - { - if (c == '-' && i + 1 < len && line[i + 1] == '-') - { - // 遇到單行註解，本行後續忽略 - break; - } - if (c == '/' && i + 1 < len && line[i + 1] == '*') - { - inBlockComment = true; - i++; - continue; - } - } - - if (c == '\'' && !inDoubleQuote && !inBracket) - { - inSingleQuote = !inSingleQuote; - hasStatementContent = true; - currentBatchLength++; - continue; - } - - if (c == '"' && !inSingleQuote && !inBracket) - { - inDoubleQuote = !inDoubleQuote; - hasStatementContent = true; - currentBatchLength++; - continue; - } - - if (c == '[' && !inSingleQuote && !inDoubleQuote) - { - inBracket = true; - hasStatementContent = true; - currentBatchLength++; - continue; - } - - if (c == ']' && inBracket) - { - inBracket = false; - hasStatementContent = true; - currentBatchLength++; - continue; - } - - if (!char.IsWhiteSpace(c)) - { - hasStatementContent = true; - } - currentBatchLength++; - - // 檢查語句終止分號（必須在字串引號與註解之外） - if (c == ';' && !inSingleQuote && !inDoubleQuote && !inBracket) - { - if (hasStatementContent) - { - statementCount++; - hasStatementContent = false; - currentBatchLength = 0; - } - } - } - - // 單批次大小檢查 (32 MiB) - if (currentBatchLength > MaxBatchBytes) - { - issues.Add($"Line {lineIndex}: Single SQL statement exceeds safe limit of 32 MiB."); - break; - } + if (caseDepth > 0) { caseDepth--; end = false; } + else end = true; } - - if (hasStatementContent) + else end = false; + } + }; + while (true) + { + cancellation.ThrowIfCancellationRequested(); + int next = reader.Read(); + if (next < 0) + { + flushWord(); + if (bytes > MaxStatementBytes) throw new InvalidDataException("Single SQL statement exceeds safe limit of 32 MiB."); + if (inline && comment != null) comment(lineComment.ToString()); + if (quote != 0 || block || (trigger && !end)) + throw new InvalidDataException("Unterminated SQLite quote, comment or trigger at line " + StatementLine + "."); + return meaningful ? sql.ToString() : null; + } + char c = (char)next; + if (first) { first = false; if (c == '\uFEFF') continue; } + if (c == '\0') throw new InvalidDataException("Line " + line + ": Contains Embedded NUL character; use CAST(X'...' AS TEXT)."); + // UTF-8 decoding is strict. Count surrogate pairs as four bytes. + bytes += c < 0x80 ? 1 : c < 0x800 ? 2 : char.IsHighSurrogate(c) ? 2 : char.IsLowSurrogate(c) ? 2 : 3; + if (bytes > MaxStatementBytes) throw new InvalidDataException("Line " + StatementLine + ": Single SQL statement exceeds safe limit of 32 MiB."); + sql.Append(c); + if (c == '\n') line++; + if (inline) + { + if (c == '\r' || c == '\n') + { + inline = false; + if (comment != null) comment(lineComment.ToString()); + lineComment.Clear(); + } + else if (lineComment.Length < 1024 * 1024) lineComment.Append(c); + else throw new InvalidDataException("Native comment metadata exceeds the safe 1 MiB limit."); + continue; + } + if (block) + { + if (c == '*' && reader.Peek() == '/') { sql.Append((char)reader.Read()); bytes++; block = false; } + continue; + } + if (quote != 0) + { + char closer = quote == '[' ? ']' : (char)quote; + if (c == closer) { - statementCount++; + if (quote != '[' && reader.Peek() == closer) { sql.Append((char)reader.Read()); bytes += closer < 128 ? 1 : 3; } + else quote = 0; } + continue; + } + if (c == '-' && reader.Peek() == '-') + { + flushWord(); sql.Append((char)reader.Read()); bytes++; inline = true; lineComment.Append("--"); continue; + } + if (c == '/' && reader.Peek() == '*') + { + flushWord(); sql.Append((char)reader.Read()); bytes++; block = true; continue; + } + if (c == '\'' || c == '"' || c == '`' || c == '[') + { + flushWord(); quote = c; meaningful = true; end = false; continue; } + if (char.IsLetterOrDigit(c) || c == '_') { if (word.Length < 64) word.Append(c); meaningful = true; continue; } + flushWord(); + if (c == ';') + { + if (meaningful && (!trigger || (body && end))) { if (bytes > MaxStatementBytes) throw new InvalidDataException("Single SQL statement exceeds safe limit of 32 MiB."); return sql.ToString(); } + } + else if (!char.IsWhiteSpace(c)) { meaningful = true; end = false; } } - catch (OperationCanceledException) + } + internal static string ExecutableSql(string sql) + { + int i = 0; + while (i < sql.Length) { - return new SqliteImportPreflight(false, fileBytes, statementCount, hasManifest, manifestTables, manifestRows, manifestSha, detectedSpatiaLite, new[] { "Scan cancelled by user." }); + while (i < sql.Length && char.IsWhiteSpace(sql[i])) i++; + if (i + 1 < sql.Length && sql[i] == '-' && sql[i + 1] == '-') + { + i += 2; while (i < sql.Length && sql[i] != '\r' && sql[i] != '\n') i++; continue; + } + if (i + 1 < sql.Length && sql[i] == '/' && sql[i + 1] == '*') + { + int end = sql.IndexOf("*/", i + 2, StringComparison.Ordinal); + if (end < 0) return ""; + i = end + 2; continue; + } + break; } - catch (Exception ex) + return sql.Substring(i).Trim(); + } + internal static string PolicySql(string sql) + { + var result = new StringBuilder(sql.Length); + int quote = 0; + for (int i = 0; i < sql.Length; i++) { - issues.Add("Error during preflight scan: " + ex.Message); + char c = sql[i]; + if (quote != 0) + { + result.Append(c); + char closer = quote == '[' ? ']' : (char)quote; + if (c == closer) + { + if (quote != '[' && i + 1 < sql.Length && sql[i + 1] == closer) result.Append(sql[++i]); + else quote = 0; + } + } + else if (c == '\'' || c == '"' || c == '`' || c == '[') { quote = c; result.Append(c); } + else if (c == '-' && i + 1 < sql.Length && sql[i + 1] == '-') + { + result.Append(' '); i += 2; + while (i < sql.Length && sql[i] != '\r' && sql[i] != '\n') i++; + if (i < sql.Length) result.Append(sql[i]); + } + else if (c == '/' && i + 1 < sql.Length && sql[i + 1] == '*') + { + result.Append(' '); int end = sql.IndexOf("*/", i + 2, StringComparison.Ordinal); + i = end < 0 ? sql.Length : end + 1; + } + else result.Append(c); } + return result.ToString().Trim(); + } + public void Dispose() { reader.Dispose(); } + } + + public static class SqliteImportPreflightScanner + { + internal static readonly Regex ManifestTableRegex = new Regex(@"^--\s*Manifest-Table(?64)?:\s*(?.*?)\s*\|\s*Rows:\s*(?\d+)\s*$", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ManifestShaRegex = new Regex(@"^--\s*Manifest-(?:Payload-)?SHA256:\s*([a-fA-F0-9]{64})\s*$", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] SpatialKeywords = { "InitSpatialMetaData", "AddGeometryColumn", "CreateSpatialIndex", "mod_spatialite", "spatial_ref_sys", "geometry_columns", "gpkg_contents", "GeomFromText", "ST_GeomFromText" }; - bool passed = issues.Count == 0; - return new SqliteImportPreflight( - passed, - fileBytes, - statementCount, - hasManifest, - manifestTables, - manifestRows, - manifestSha, - detectedSpatiaLite, - issues); + internal static Tuple ParseTableManifest(string comment) + { + var match = ManifestTableRegex.Match(comment); + if (!match.Success) return null; + string name = match.Groups["name"].Value; + if (match.Groups["base64"].Success) name = new UTF8Encoding(false, true).GetString(Convert.FromBase64String(name)); + return Tuple.Create(name, long.Parse(match.Groups["rows"].Value, System.Globalization.CultureInfo.InvariantCulture)); + } + public static SqliteImportPreflight Scan(string filePath, CancellationToken cancellation) + { + if (string.IsNullOrWhiteSpace(filePath)) throw new ArgumentException("File path is required", nameof(filePath)); + if (!File.Exists(filePath)) return new SqliteImportPreflight(false, 0, 0, false, 0, 0, null, false, new[] { "File not found: " + filePath }); + long bytes = new FileInfo(filePath).Length, rows = 0; + int count = 0, tables = 0; + bool manifest = false, spatial = false; + string checksum = null; + var issues = new List(); + try + { + Action onComment = text => + { + if (text.IndexOf("mySQLPunk Native Manifest", StringComparison.OrdinalIgnoreCase) >= 0) manifest = true; + if (!manifest) return; + var t = Regex.Match(text, @"^--\s*Tables:\s*(\d+)", RegexOptions.IgnoreCase); + var r = Regex.Match(text, @"^--\s*TotalRows:\s*(\d+)", RegexOptions.IgnoreCase); + if (t.Success) tables = int.Parse(t.Groups[1].Value); + if (r.Success) rows = long.Parse(r.Groups[1].Value); + ParseTableManifest(text); // Validate encoded names too. + var h = ManifestShaRegex.Match(text); + if (h.Success) { if (checksum != null) throw new InvalidDataException("Multiple checksum footers are not supported."); checksum = h.Groups[1].Value; } + }; + using (var reader = new SqliteStatementReader(filePath, cancellation, onComment)) + { + string sql; + while ((sql = reader.Read()) != null) + { + count++; + foreach (string keyword in SpatialKeywords) + if (sql.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) >= 0) { spatial = true; break; } + } + } + if (checksum != null) VerifyPayloadChecksum(filePath, checksum, cancellation); + } + catch (OperationCanceledException) { issues.Add("Scan cancelled by user."); } + catch (Exception ex) { issues.Add("Error during preflight scan: " + ex.Message); } + return new SqliteImportPreflight(issues.Count == 0, bytes, count, manifest, tables, rows, checksum, spatial, issues); + } + private static void VerifyPayloadChecksum(string path, string expected, CancellationToken cancellation) + { + using (var stream = File.OpenRead(path)) + { + int tailLength = (int)Math.Min(512, stream.Length); + var tail = new byte[tailLength]; + stream.Position = stream.Length - tailLength; + int read = stream.Read(tail, 0, tailLength); + string ending = Encoding.ASCII.GetString(tail, 0, read); + var match = Regex.Match(ending, @"(?m)^-- Manifest-(?:Payload-)?SHA256: [a-fA-F0-9]{64}\r?\n?\z"); + if (!match.Success) throw new InvalidDataException("Checksum must be the final native footer line."); + long payloadBytes = stream.Length - tailLength + match.Index; + stream.Position = 0; + using (var sha = SHA256.Create()) + { + var buffer = new byte[65536]; + long remaining = payloadBytes; + while (remaining > 0) + { + cancellation.ThrowIfCancellationRequested(); + int n = stream.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); + if (n == 0) throw new EndOfStreamException(); + sha.TransformBlock(buffer, 0, n, null, 0); remaining -= n; + } + sha.TransformFinalBlock(new byte[0], 0, 0); + string actual = BitConverter.ToString(sha.Hash).Replace("-", ""); + if (!string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase)) throw new InvalidDataException("Native payload SHA-256 mismatch; target was not modified."); + } + } } } } diff --git a/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs b/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs index c44a879..1ef5296 100644 --- a/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs +++ b/mySQLPunk/lib/SqliteImport/SqliteImportRunner.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Data.SQLite; using System.Diagnostics; using System.IO; using System.Text; @@ -12,40 +13,14 @@ namespace mySQLPunk.lib.SqliteImport { public sealed class SqliteImportRunner { - private const string IdentifierPattern = @"""(?:""""|[^""])+""|`(?:``|[^`])+`|\[[^\]]+\]|'[^']+'|[A-Za-z0-9_$]+"; - - private static readonly Regex CreateObjectRegex = new Regex( - @"^\s*CREATE\s+(?:(?:TEMP|TEMPORARY|UNIQUE|VIRTUAL)\s+)?(?TABLE|VIEW|INDEX|TRIGGER)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?" + IdentifierPattern + @")(?:\s+USING\s+|\s+ON\s+(?" + IdentifierPattern + @"))?", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex DropObjectRegex = new Regex( - @"^\s*DROP\s+(?TABLE|VIEW|INDEX|TRIGGER)\s+(?:IF\s+EXISTS\s+)?(?" + IdentifierPattern + @")", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex InsertTargetRegex = new Regex( - @"^\s*INSERT(?:\s+OR\s+(?:REPLACE|IGNORE|ROLLBACK|ABORT|FAIL))?\s+INTO\s+(?" + IdentifierPattern + @")", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex BeginTxRegex = new Regex( - @"^\s*BEGIN(?:\s+(?:IMMEDIATE|DEFERRED|EXCLUSIVE))?(?:\s+TRANSACTION)?\s*;?$", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex CommitTxRegex = new Regex( - @"^\s*(?:COMMIT|END)(?:\s+TRANSACTION)?\s*;?$", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex RollbackTxRegex = new Regex( - @"^\s*ROLLBACK(?:\s+TRANSACTION)?\s*;?$", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex ManifestTableRegex = new Regex( - @"^--\s*Manifest-Table:\s*([^\s|]+)\s*\|\s*Rows:\s*(\d+)", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private static readonly Regex ManifestSha256Regex = new Regex( - @"^--\s*Manifest-SHA256:\s*([a-fA-F0-9]{64})", - RegexOptions.IgnoreCase | RegexOptions.Compiled); - + private const string IdentifierPattern = @"""(?:""""|[^""])+""|`(?:``|[^`])+`|\[[^\]]+\]|'(?:''|[^'])+'|(?:[A-Za-z0-9_$]|[^\x00-\x7F])+"; + private static readonly Regex CreateObjectRegex = new Regex(@"^CREATE\s+(?:(?:TEMP|TEMPORARY|UNIQUE|VIRTUAL)\s+)?(?TABLE|VIEW|INDEX|TRIGGER)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex DropObjectRegex = new Regex(@"^DROP\s+(?TABLE|VIEW|INDEX|TRIGGER)\s+(?:IF\s+EXISTS\s+)?(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex DataTargetRegex = new Regex(@"^(?:INSERT(?:\s+OR\s+(?:REPLACE|IGNORE|ROLLBACK|ABORT|FAIL))?\s+INTO|REPLACE\s+INTO|UPDATE(?:\s+OR\s+(?:REPLACE|IGNORE|ROLLBACK|ABORT|FAIL))?|DELETE\s+FROM|ALTER\s+TABLE)\s+(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex OnTargetRegex = new Regex(@"\bON\s+(?:(?" + IdentifierPattern + @")\s*\.\s*)?(?" + IdentifierPattern + @")", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex BeginTxRegex = new Regex(@"^BEGIN(?:\s+(?:IMMEDIATE|DEFERRED|EXCLUSIVE))?(?:\s+TRANSACTION)?\s*;?$", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex CommitTxRegex = new Regex(@"^(?:COMMIT|END)(?:\s+TRANSACTION)?\s*;?$", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex RollbackTxRegex = new Regex(@"^ROLLBACK(?:\s+TRANSACTION)?\s*;?$", RegexOptions.IgnoreCase | RegexOptions.Compiled); private readonly IDatabase db; private readonly string databaseName; @@ -54,620 +29,306 @@ public SqliteImportRunner(IDatabase db, string databaseName) this.db = db ?? throw new ArgumentNullException(nameof(db)); this.databaseName = !string.IsNullOrWhiteSpace(databaseName) ? databaseName : throw new ArgumentException("Database name is required", nameof(databaseName)); } - - public Task RunAsync( - SqliteImportRequest request, - SqliteImportOptions options, - IProgress progress, - CancellationToken cancellation) + public Task RunAsync(SqliteImportRequest request, SqliteImportOptions options, IProgress progress, CancellationToken cancellation) { - return Task.Run(() => RunInternal(request, options, progress, cancellation), cancellation); + return Task.Run(() => RunInternal(request, options, progress, cancellation)); } - - private SqliteImportResult RunInternal( - SqliteImportRequest request, - SqliteImportOptions options, - IProgress progress, - CancellationToken cancellation) + private SqliteImportResult RunInternal(SqliteImportRequest request, SqliteImportOptions options, IProgress progress, CancellationToken cancellation) { var clock = Stopwatch.StartNew(); - var log = new SqliteImportLog(request.JobId); - long fileBytes = 0; - if (File.Exists(request.SourcePath)) fileBytes = new FileInfo(request.SourcePath).Length; - - log.LogInfo($"Target SQLite Database: {databaseName}"); - log.LogInfo($"Source SQL File: {request.SourcePath} ({fileBytes:N0} bytes)"); - log.LogInfo($"Options: ContinueOnError={options.ContinueOnError}, Strategy={options.ExistingObjectStrategy}, WrapInTransaction={options.WrapInTransaction}"); - - int executed = 0; - int failed = 0; - int skipped = 0; - long totalRowsAffected = 0; - var errors = new List(); - var manifestTables = new List>(); - bool readingManifest = false; - bool haltedOnError = false; - - var existingTables = new HashSet(StringComparer.OrdinalIgnoreCase); - var existingViews = new HashSet(StringComparer.OrdinalIgnoreCase); - var skippedTables = new HashSet(StringComparer.OrdinalIgnoreCase); - - bool runnerManagedTx = false; - - try + using (var log = new SqliteImportLog(request.JobId)) { - // 1. 若啟用 SpatiaLite 自動加載，嘗試載入模組 - if (options.AutoLoadSpatiaLite && db is my_sqlite sqliteDb) + int executed = 0, failed = 0, skipped = 0; + long rows = 0, fileBytes = File.Exists(request.SourcePath) ? new FileInfo(request.SourcePath).Length : 0; + bool txOpen = false, managed = false, cancelled = false, manifestVerified = false; + int? originalFk = null, originalSync = null; + string manifestReport = null; + var errors = new List(); + var manifests = new List>(); + var initial = new HashSet(StringComparer.OrdinalIgnoreCase); + var current = new HashSet(StringComparer.OrdinalIgnoreCase); + var initialTables = new HashSet(StringComparer.OrdinalIgnoreCase); + var owners = new Dictionary(StringComparer.OrdinalIgnoreCase); + log.LogInfo("Target SQLite Database: " + databaseName); + log.LogInfo("Source SQL File: " + request.SourcePath); + try { - try + // Retain a read-only file handle through preflight and execution so a + // writer cannot replace the verified bytes on Windows in between. + using (var sourceLock = new FileStream(request.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - sqliteDb.RetryLoadSpatiaLite(); - if (sqliteDb.SpatiaLiteEnabled) + cancellation.ThrowIfCancellationRequested(); + var preflight = SqliteImportPreflightScanner.Scan(request.SourcePath, cancellation); + cancellation.ThrowIfCancellationRequested(); + if (!preflight.Passed) throw new InvalidDataException(string.Join("; ", preflight.Issues)); + if (db is my_sqlite session && session.MCT != null && !session.MCT.AutoCommit) + throw new InvalidOperationException("Import requires an owned SQLite session with no existing transaction."); + DataTable catalog = SelectChecked("SELECT type, name, tbl_name FROM sqlite_master WHERE type IN ('table','view','index','trigger');"); + foreach (DataRow row in catalog.Rows) { - log.LogInfo("SpatiaLite runtime dynamic extension (mod_spatialite) loaded successfully."); + string type = Convert.ToString(row["type"]).ToUpperInvariant(), name = Convert.ToString(row["name"]); + initial.Add(type + ":" + name); current.Add(type + ":" + name); + owners[type + ":" + name] = Convert.ToString(row["tbl_name"]); + if (type == "TABLE" || type == "VIEW") initialTables.Add(name); } - else if (!string.IsNullOrWhiteSpace(sqliteDb.SpatiaLiteLoadError)) + bool explicitTx = false; + using (var scan = new SqliteStatementReader(request.SourcePath, cancellation, text => { - log.LogInfo("SpatiaLite runtime extension note: " + sqliteDb.SpatiaLiteLoadError); - } - } - catch (Exception ex) - { - log.LogInfo("SpatiaLite runtime extension attempt: " + ex.Message); - } - } - - // 2. 設定 PRAGMA 環境變數以加快寫入 - if (options.PragmaForeignKeysOff) - { - try { db.ExecSQL("PRAGMA foreign_keys = OFF;"); } catch { } - } - if (options.PragmaSynchronousOff) - { - try { db.ExecSQL("PRAGMA synchronous = OFF;"); } catch { } - } - - // 3. 快取目標資料庫中既有物件 - try - { - var tList = db.GetTables(databaseName); - if (tList != null) existingTables.UnionWith(tList); - } - catch { } - - try - { - var vList = db.GetViews(databaseName); - if (vList != null) existingViews.UnionWith(vList); - } - catch { } - - // 4. 預先檢查腳本是否自帶 BEGIN TRANSACTION - bool scriptHasExplicitTx = CheckIfScriptContainsExplicitTx(request.SourcePath); - if (options.WrapInTransaction && !scriptHasExplicitTx) - { - var beginRes = db.ExecSQL("BEGIN IMMEDIATE;"); - if (IsExecutionSuccess(beginRes)) - { - runnerManagedTx = true; - log.LogInfo("Transaction started by runner: BEGIN IMMEDIATE;"); - } - } - - using (var fs = new FileStream(request.SourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536)) - using (var reader = new StreamReader(fs, Encoding.UTF8, true, 65536)) - { - var currentBatch = new StringBuilder(); - bool inSingleQuote = false; - bool inDoubleQuote = false; - bool inBracket = false; - bool inBlockComment = false; - int batchStartLine = 1; - int currentLine = 0; - string line; - long lastReportMs = 0; - - while ((line = reader.ReadLine()) != null) - { - cancellation.ThrowIfCancellationRequested(); - currentLine++; - - string trimmed = line.Trim(); - - // 檢查 Native Manifest - if (trimmed.StartsWith("--")) + var pair = SqliteImportPreflightScanner.ParseTableManifest(text); + if (pair != null) manifests.Add(pair); + })) { - if (trimmed.Contains("mySQLPunk Native Manifest") && !readingManifest) - { - readingManifest = true; - log.LogInfo("Detected Native Manifest in SQL script footer."); - } - - var match = ManifestTableRegex.Match(trimmed); - if (match.Success && long.TryParse(match.Groups[2].Value, out long r)) + string statement; + while ((statement = scan.Read()) != null) { - manifestTables.Add(Tuple.Create(match.Groups[1].Value, r)); - } - - if (!inSingleQuote && !inDoubleQuote && !inBlockComment) - { - continue; + string sql = SqliteStatementReader.PolicySql(statement); + if (BeginTxRegex.IsMatch(sql)) explicitTx = true; + if (Regex.IsMatch(sql, @"^(ATTACH|DETACH|SAVEPOINT|RELEASE)\b|^ROLLBACK\s+TO\b", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Attached databases and savepoint scripts are not supported by this import wizard."); + if (options.ExistingObjectStrategy == SqliteExistingObjectStrategy.SkipExisting && initialTables.Count > 0 && Regex.IsMatch(sql, @"^WITH\b", RegexOptions.IgnoreCase)) + throw new NotSupportedException("CTE statements cannot be safely classified under SkipExisting with existing objects."); + var change = CreateObjectRegex.Match(sql); + if (!change.Success) change = DropObjectRegex.Match(sql); + if (Regex.IsMatch(sql, @"^CREATE\s+(TEMP|TEMPORARY)\b", RegexOptions.IgnoreCase)) + throw new NotSupportedException("Temporary schema import is not supported by this wizard."); + var target = change.Success ? change : DataTargetRegex.Match(sql); + if (target.Success && target.Groups["schema"].Success && !Unquote(target.Groups["schema"].Value).Equals("main", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException("Import may only modify the main SQLite schema."); + if (change.Success && options.ExistingObjectStrategy == SqliteExistingObjectStrategy.AbortIfExists && IsInitiallyProtected(change, sql, initial, initialTables, owners)) + throw new InvalidOperationException("Object " + Unquote(change.Groups["name"].Value) + " already exists in database (AbortIfExists policy)."); } } - - if (currentBatch.Length == 0) batchStartLine = currentLine; - - // 逐字元狀態機解析語句結束分號 - int len = line.Length; - int lastSegmentStart = 0; - - for (int i = 0; i < len; i++) + originalFk = ReadPragma("foreign_keys"); originalSync = ReadPragma("synchronous"); + if (options.AutoLoadSpatiaLite && db is my_sqlite sqlite) sqlite.RetryLoadSpatiaLite(); + if (options.PragmaForeignKeysOff) ExecChecked("PRAGMA foreign_keys = OFF;"); + if (options.PragmaSynchronousOff) ExecChecked("PRAGMA synchronous = OFF;"); + managed = options.WrapInTransaction && !explicitTx; + using (var reader = new SqliteStatementReader(request.SourcePath, cancellation)) { - char c = line[i]; - - if (inBlockComment) + string statement; + while ((statement = reader.Read()) != null) { - if (c == '*' && i + 1 < len && line[i + 1] == '/') + cancellation.ThrowIfCancellationRequested(); + string sql = SqliteStatementReader.PolicySql(statement); + if (txOpen && db is my_sqlite nativeState && nativeState.MCT != null && nativeState.MCT.AutoCommit) { - inBlockComment = false; - i++; + txOpen = false; + throw new InvalidOperationException("SQLite ended the import transaction unexpectedly; further statements were not executed."); } - continue; - } - - if (!inSingleQuote && !inDoubleQuote && !inBracket) - { - if (c == '-' && i + 1 < len && line[i + 1] == '-') + if (ShouldSkip(sql, options.ExistingObjectStrategy, initial, initialTables, owners)) { - // 遇到單行註解，本行此處後續為註解 - currentBatch.Append(line.Substring(lastSegmentStart, i - lastSegmentStart)); - lastSegmentStart = len; - break; + skipped++; log.LogStatement(reader.StatementLine, Preview(sql), true, null, "SKIP_PROTECTED_OBJECT"); continue; } - if (c == '/' && i + 1 < len && line[i + 1] == '*') + if (managed && !txOpen && !Regex.IsMatch(sql, @"^PRAGMA\b", RegexOptions.IgnoreCase)) { - inBlockComment = true; - i++; - continue; + ExecChecked("BEGIN IMMEDIATE;"); txOpen = true; } - } - - if (c == '\'' && !inDoubleQuote && !inBracket) - { - inSingleQuote = !inSingleQuote; - continue; - } - - if (c == '"' && !inSingleQuote && !inBracket) - { - inDoubleQuote = !inDoubleQuote; - continue; - } - - if (c == '[' && !inSingleQuote && !inDoubleQuote) - { - inBracket = true; - continue; - } - - if (c == ']' && inBracket) - { - inBracket = false; - continue; - } - - // 偵測到語句結束分號 - if (c == ';' && !inSingleQuote && !inDoubleQuote && !inBracket) - { - currentBatch.Append(line.Substring(lastSegmentStart, i - lastSegmentStart + 1)); - lastSegmentStart = i + 1; - - string statementSql = currentBatch.ToString().Trim(); - currentBatch.Clear(); - - if (!string.IsNullOrWhiteSpace(statementSql)) + var create = CreateObjectRegex.Match(sql); + if (create.Success && options.ExistingObjectStrategy == SqliteExistingObjectStrategy.DropAndRecreate) { - bool continueRunning = ExecuteStatement( - statementSql, - batchStartLine, - options, - log, - existingTables, - existingViews, - skippedTables, - ref executed, - ref failed, - ref skipped, - ref totalRowsAffected, - errors); - - if (!continueRunning) + string key = Key(create); + if (current.Contains(key) && !(key.Equals("TABLE:" + my_sqlite.ColumnCommentTableName, StringComparison.OrdinalIgnoreCase) && Regex.IsMatch(sql, @"^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b", RegexOptions.IgnoreCase))) { - haltedOnError = true; - break; + ExecChecked("DROP " + create.Groups["type"].Value + " IF EXISTS " + QuoteIdentifier(Unquote(create.Groups["name"].Value)) + ";"); + current.Remove(key); } - - // 每 150ms 節流回報進度至 UI - if (clock.ElapsedMilliseconds - lastReportMs > 150) + } + if (failed > 0 && CommitTxRegex.IsMatch(sql)) + throw new InvalidOperationException("Transaction contains failed SQL statements; COMMIT was withheld."); + var result = ExecuteSql(statement, cancellation); + cancellation.ThrowIfCancellationRequested(); + if (!IsExecutionSuccess(result)) + { + failed++; + string reason = "Line " + reader.StatementLine + ": " + DatabaseExecutionResultService.GetFailureReason(result); + errors.Add(reason); log.LogStatement(reader.StatementLine, Preview(sql), false, reason, "FAIL"); + if (txOpen && db is my_sqlite errorState && errorState.MCT != null && errorState.MCT.AutoCommit) { - lastReportMs = clock.ElapsedMilliseconds; - ReportProgress(progress, fs.Position, fileBytes, executed, failed, skipped, totalRowsAffected, clock, statementSql); + txOpen = false; + throw new InvalidOperationException("SQLite rolled back the import transaction after the statement error; further statements were not executed."); } + // A failed transaction command always stops: a failed + // COMMIT can leave the transaction and locks open. + if (!options.ContinueOnError || BeginTxRegex.IsMatch(sql) || CommitTxRegex.IsMatch(sql) || RollbackTxRegex.IsMatch(sql)) throw new InvalidOperationException(reason); + continue; } - - batchStartLine = currentLine; + executed++; + if (result.TryGetValue("rowsAffected", out string affected) && long.TryParse(affected, out long n)) rows += Math.Max(0, n); + if (BeginTxRegex.IsMatch(sql)) txOpen = true; + if (CommitTxRegex.IsMatch(sql) || RollbackTxRegex.IsMatch(sql)) txOpen = false; + if (create.Success) current.Add(Key(create)); + var drop = DropObjectRegex.Match(sql); + if (drop.Success) current.Remove(Key(drop)); + log.LogStatement(reader.StatementLine, Preview(sql), true, null, "OK"); log.Flush(); + ReportProgress(progress, fileBytes, executed, failed, skipped, rows, clock, Preview(sql)); } } - - if (haltedOnError) break; - - if (lastSegmentStart < len) - { - currentBatch.AppendLine(line.Substring(lastSegmentStart)); - } - } - - // 處理檔尾若有未以分號結尾的最後一條語句 - if (!haltedOnError && currentBatch.Length > 0) - { - string remainingSql = currentBatch.ToString().Trim(); - if (!string.IsNullOrWhiteSpace(remainingSql) && !remainingSql.StartsWith("--")) + cancellation.ThrowIfCancellationRequested(); + if (failed > 0) throw new InvalidOperationException("Import contains failed SQL statements; transaction was not committed."); + if (managed && txOpen) { ExecChecked("COMMIT;"); txOpen = false; } + if (txOpen || (db is my_sqlite completedSession && completedSession.MCT != null && !completedSession.MCT.AutoCommit)) throw new InvalidOperationException("Script ended with an active transaction; rolling back instead of reporting completion."); + if (manifests.Count > 0) { - ExecuteStatement( - remainingSql, - batchStartLine, - options, - log, - existingTables, - existingViews, - skippedTables, - ref executed, - ref failed, - ref skipped, - ref totalRowsAffected, - errors); + var report = new StringBuilder(); int matches = 0; + foreach (var pair in manifests) + { + long actual = db.CountRows(databaseName, pair.Item1); + if (actual < 0) throw new InvalidOperationException("Manifest count failed for " + pair.Item1); + bool equal = actual == pair.Item2; if (equal) matches++; + log.LogManifestVerification(pair.Item1, pair.Item2, actual, equal); + report.Append(pair.Item1 + ": " + actual + "/" + pair.Item2 + "; "); + } + manifestVerified = matches == manifests.Count; manifestReport = report.ToString(); + // Preserving existing data intentionally changes row counts. + // ManifestVerified is row-count evidence, not data equivalence. } } } - - // 5. 交易提交或回滾 - if (runnerManagedTx) + catch (OperationCanceledException) { cancelled = true; errors.Add("Import cancelled by user."); } + catch (Exception ex) { if (!errors.Contains(ex.Message)) errors.Add(ex.Message); failed = Math.Max(1, failed); } + finally { - if (haltedOnError && !options.ContinueOnError) + if (txOpen) { - try { db.ExecSQL("ROLLBACK;"); } catch { } - log.LogInfo("Runner managed transaction rolled back due to error: ROLLBACK;"); + try + { + if (!(db is my_sqlite cleanSession) || cleanSession.MCT == null || !cleanSession.MCT.AutoCommit) ExecChecked("ROLLBACK;"); + if (db is my_sqlite rolledBack && rolledBack.MCT != null && !rolledBack.MCT.AutoCommit) throw new InvalidOperationException("SQLite transaction is still active."); + txOpen = false; log.LogInfo("Active import transaction rolled back."); + } + catch (Exception ex) { errors.Add("ROLLBACK cleanup failed: " + ex.Message); failed++; cancelled = false; } } - else + foreach (var pragma in new[] { Tuple.Create("foreign_keys", originalFk), Tuple.Create("synchronous", originalSync) }) { - try { db.ExecSQL("COMMIT;"); } catch { } - log.LogInfo("Runner managed transaction committed successfully: COMMIT;"); + if (!pragma.Item2.HasValue) continue; + try + { + ExecChecked("PRAGMA " + pragma.Item1 + " = " + pragma.Item2.Value + ";"); + if (ReadPragma(pragma.Item1) != pragma.Item2.Value) throw new InvalidOperationException(pragma.Item1 + " was not restored."); + } + catch (Exception ex) { errors.Add("PRAGMA cleanup failed: " + ex.Message); failed++; cancelled = false; } } } - - // 恢復 foreign_keys - if (options.PragmaForeignKeysOff) - { - try { db.ExecSQL("PRAGMA foreign_keys = ON;"); } catch { } - } - clock.Stop(); - - // 6. Native Manifest 自動比對 - bool manifestVerified = false; - string manifestReport = null; - if (manifestTables.Count > 0) - { - int matchCount = 0; - var reportBuilder = new StringBuilder(); - foreach (var pair in manifestTables) - { - string tName = pair.Item1; - long expected = pair.Item2; - long actual = 0; - try { actual = db.CountRows(databaseName, tName); } catch { } - bool matched = (actual == expected); - if (matched) matchCount++; - log.LogManifestVerification(tName, expected, actual, matched); - reportBuilder.Append($"{tName}: {actual}/{expected} {(matched ? "OK" : "MISMATCH")}; "); - } - manifestVerified = (matchCount == manifestTables.Count); - manifestReport = reportBuilder.ToString().Trim(); - log.LogInfo($"Manifest Verification: {matchCount}/{manifestTables.Count} tables matched row counts exactly. Verified={manifestVerified}"); - } - - var status = haltedOnError - ? SqliteImportStatus.Failed - : (failed > 0 && !options.ContinueOnError ? SqliteImportStatus.Failed : SqliteImportStatus.Completed); - - var finalResult = new SqliteImportResult( - status, - executed, - failed, - skipped, - totalRowsAffected, - log.Path, - clock.Elapsed, - errors, - manifestVerified, - manifestReport); - - log.WriteEnd(finalResult); - log.Dispose(); - - ReportProgress(progress, fileBytes, fileBytes, executed, failed, skipped, totalRowsAffected, clock, null, true); - - return finalResult; - } - catch (OperationCanceledException) - { - clock.Stop(); - if (runnerManagedTx) - { - try { db.ExecSQL("ROLLBACK;"); } catch { } - } - log.LogInfo("Import cancelled by user. Rolled back any runner-managed transaction."); - var cancelResult = new SqliteImportResult( - SqliteImportStatus.Cancelled, - executed, - failed, - skipped, - totalRowsAffected, - log.Path, - clock.Elapsed, - new[] { "Import cancelled by user." }); - log.WriteEnd(cancelResult); - log.Dispose(); - return cancelResult; - } - catch (Exception ex) - { - clock.Stop(); - if (runnerManagedTx) - { - try { db.ExecSQL("ROLLBACK;"); } catch { } - } - log.LogInfo("Import terminated with unhandled exception: " + ex.Message); - errors.Add(ex.Message); - var errResult = new SqliteImportResult( - SqliteImportStatus.Failed, - executed, - failed, - skipped, - totalRowsAffected, - log.Path, - clock.Elapsed, - errors); - log.WriteEnd(errResult); - log.Dispose(); - return errResult; + var status = cancelled ? SqliteImportStatus.Cancelled : failed > 0 ? SqliteImportStatus.Failed : SqliteImportStatus.Completed; + var final = new SqliteImportResult(status, executed, failed, skipped, rows, log.Path, clock.Elapsed, errors, manifestVerified, manifestReport); + log.WriteEnd(final); + return final; } } - - private bool ExecuteStatement( - string sql, - int lineNumber, - SqliteImportOptions options, - SqliteImportLog log, - HashSet existingTables, - HashSet existingViews, - HashSet skippedTables, - ref int executed, - ref int failed, - ref int skipped, - ref long totalRowsAffected, - List errors) + private bool ShouldSkip(string sql, SqliteExistingObjectStrategy strategy, HashSet initial, HashSet tables, Dictionary owners) { - string trimmed = sql.Trim(); - string preview = MakeStatementPreview(trimmed); - - // 1. 處理 CREATE 物件策略 - var createMatch = CreateObjectRegex.Match(trimmed); - if (createMatch.Success) - { - string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); - string objName = Unquote(createMatch.Groups["name"].Value); - - bool isExisting = false; - if (objType == "TABLE") isExisting = existingTables.Contains(objName); - else if (objType == "VIEW") isExisting = existingViews.Contains(objName); - - if (isExisting) - { - switch (options.ExistingObjectStrategy) - { - case SqliteExistingObjectStrategy.AbortIfExists: - string abortMsg = $"Object {objType} \"{objName}\" already exists in database (AbortIfExists policy)."; - log.LogStatement(lineNumber, preview, false, abortMsg, "ABORT"); - failed++; - errors.Add(abortMsg); - return false; // 中斷匯入 - - case SqliteExistingObjectStrategy.SkipExisting: - log.LogStatement(lineNumber, preview, true, null, "SKIP_EXISTING"); - skipped++; - if (objType == "TABLE") skippedTables.Add(objName); - return true; // 略過本句，繼續 - - case SqliteExistingObjectStrategy.DropAndRecreate: - string dropSql = $"DROP {objType} IF EXISTS {QuoteIdentifier(objName)};"; - db.ExecSQL(dropSql); - log.LogStatement(lineNumber, $"DROP {objType} \"{objName}\" for recreation", true, null, "DROP_EXISTING"); - if (objType == "TABLE") existingTables.Remove(objName); - else if (objType == "VIEW") existingViews.Remove(objName); - break; - - case SqliteExistingObjectStrategy.OverwriteKeepExisting: - log.LogStatement(lineNumber, preview, true, null, "KEEP_STRUCTURE"); - skipped++; - return true; - } - } - } - - // 2. 處理 INSERT 資料策略 (若該表於前面被標記為 SkipExisting，則一併略過 INSERT) - var insertMatch = InsertTargetRegex.Match(trimmed); - if (insertMatch.Success) + if (strategy != SqliteExistingObjectStrategy.SkipExisting && strategy != SqliteExistingObjectStrategy.OverwriteKeepExisting) return false; + var change = CreateObjectRegex.Match(sql); + if (!change.Success) change = DropObjectRegex.Match(sql); + if (change.Success) return IsInitiallyProtected(change, sql, initial, tables, owners); + var data = DataTargetRegex.Match(sql); + if (!data.Success) return false; + string name = Unquote(data.Groups["name"].Value); + if (strategy == SqliteExistingObjectStrategy.SkipExisting && tables.Contains(name) && !name.Equals(my_sqlite.ColumnCommentTableName, StringComparison.OrdinalIgnoreCase) && !name.Equals("sqlite_sequence", StringComparison.OrdinalIgnoreCase)) return true; + if (Regex.IsMatch(sql, @"^ALTER\s+TABLE\b", RegexOptions.IgnoreCase) && tables.Contains(name)) return true; + // Comments and sequence records belong to the table named in their + // literal, rather than to the sidecar/system table itself. + if (name.Equals(my_sqlite.ColumnCommentTableName, StringComparison.OrdinalIgnoreCase) || name.Equals("sqlite_sequence", StringComparison.OrdinalIgnoreCase)) { - string targetTable = Unquote(insertMatch.Groups["name"].Value); - if (skippedTables.Contains(targetTable)) + if (tables.Count > 0) { - skipped++; - log.LogStatement(lineNumber, preview, true, null, "SKIP_DATA_FOR_SKIPPED_TABLE"); - return true; + string owner; + // A broad DELETE/UPDATE or multi-owner VALUES list cannot be + // applied safely while preserving existing owners. Only the + // native exporter's single-owner forms are permitted here. + if (!TryScopedMetadataOwner(sql, data, name, out owner)) return true; + if (tables.Contains(owner)) return true; } } - - // 3. 實際執行 SQL - var res = db.ExecSQL(trimmed); - bool success = IsExecutionSuccess(res); - - if (success) + return false; + } + private static bool TryScopedMetadataOwner(string sql, Match target, string name, out string owner) + { + owner = null; + const string text = @"'(?:''|[^'])*'"; + string suffix = sql.Substring(target.Length); + string pattern; + bool insert = Regex.IsMatch(sql, @"^(?:INSERT|REPLACE)\b", RegexOptions.IgnoreCase); + if (name.Equals("sqlite_sequence", StringComparison.OrdinalIgnoreCase)) { - executed++; - long affected = 0; - if (res != null && res.ContainsKey("rowsAffected") && long.TryParse(res["rowsAffected"], out long a)) - { - affected = a; - totalRowsAffected += a; - } - - // 若為 CREATE TABLE / VIEW 成功，更新快取 - if (createMatch.Success) - { - string objType = createMatch.Groups["type"].Value.ToUpperInvariant(); - string objName = Unquote(createMatch.Groups["name"].Value); - if (objType == "TABLE") existingTables.Add(objName); - else if (objType == "VIEW") existingViews.Add(objName); - } - - log.LogStatement(lineNumber, preview, true, null, affected > 0 ? $"OK (+{affected} rows)" : "OK"); - return true; + pattern = insert + ? @"^\s*\(\s*name\s*,\s*seq\s*\)\s*VALUES\s*\(\s*'(?(?:''|[^'])*)'\s*,\s*[+-]?\d+\s*\)\s*;?\s*$" + : Regex.IsMatch(sql, @"^DELETE\b", RegexOptions.IgnoreCase) + ? @"^\s+WHERE\s+name\s*=\s*'(?(?:''|[^'])*)'\s*;?\s*$" : null; } else { - failed++; - string reason = DatabaseExecutionResultService.GetFailureReason(res); - string errText = $"Line {lineNumber}: {reason}"; - errors.Add(errText); - log.LogStatement(lineNumber, preview, false, reason, "FAIL"); - - if (!options.ContinueOnError) - { - return false; // 停止匯入 - } - return true; // 繼續略過執行 + string comment = @"(?:" + text + @"|CAST\s*\(\s*X'[0-9a-f]*'\s+AS\s+TEXT\s*\))"; + pattern = insert ? @"^\s*\(\s*table_name\s*,\s*column_name\s*,\s*comment\s*\)\s*VALUES\s*\(\s*'(?(?:''|[^'])*)'\s*,\s*" + text + @"\s*,\s*" + comment + @"\s*\)\s*;?\s*$" : null; } + if (pattern == null) return false; + Match match = Regex.Match(suffix, pattern, RegexOptions.IgnoreCase); + if (!match.Success) return false; + owner = match.Groups["owner"].Value.Replace("''", "'"); + return true; } - - private static bool IsExecutionSuccess(Dictionary result) + private static bool IsInitiallyProtected(Match change, string sql, HashSet initial, HashSet tables, Dictionary owners) { - if (result == null) return false; - if (result.TryGetValue("status", out string status)) + string key = Key(change), type = change.Groups["type"].Value.ToUpperInvariant(); + if (key.Equals("TABLE:" + my_sqlite.ColumnCommentTableName, StringComparison.OrdinalIgnoreCase) && Regex.IsMatch(sql, @"^CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b", RegexOptions.IgnoreCase)) return false; + if (initial.Contains(key)) return true; + if (type == "INDEX" || type == "TRIGGER") { - return string.Equals(status, "OK", StringComparison.OrdinalIgnoreCase); + if (owners.TryGetValue(key, out string owner) && tables.Contains(owner)) return true; + var on = OnTargetRegex.Match(sql.Substring(change.Length)); + if (on.Success && tables.Contains(Unquote(on.Groups["name"].Value))) return true; } - return true; + return false; } - - private static string Unquote(string identifier) + private DataTable SelectChecked(string sql) { - if (string.IsNullOrWhiteSpace(identifier)) return string.Empty; - string s = identifier.Trim(); - if ((s.StartsWith("\"") && s.EndsWith("\"")) || - (s.StartsWith("`") && s.EndsWith("`")) || - (s.StartsWith("'") && s.EndsWith("'"))) - { - return s.Substring(1, s.Length - 2).Replace("\"\"", "\"").Replace("``", "`").Replace("''", "'"); - } - if (s.StartsWith("[") && s.EndsWith("]")) + DataTable result = db.SelectSQL(sql, new Dictionary()); + if (result == null || result.ExtendedProperties.ContainsKey("QueryError")) + throw new InvalidOperationException("SQLite query failed: " + (result == null ? "no result" : Convert.ToString(result.ExtendedProperties["QueryError"]))); + return result; + } + private int ReadPragma(string name) + { + DataTable result = SelectChecked("PRAGMA " + name + ";"); + if (result.Rows.Count != 1 || result.Columns.Count == 0) throw new InvalidOperationException("Cannot read PRAGMA " + name); + return Convert.ToInt32(result.Rows[0][0]); + } + private Dictionary ExecuteSql(string sql, CancellationToken cancellation) + { + if (!(db is my_sqlite native) || native.MCT == null) return db.ExecSQL(sql); + using (var command = new SQLiteCommand(sql, native.MCT)) + using (cancellation.Register(() => { try { native.MCT.Cancel(); } catch { } })) { - return s.Substring(1, s.Length - 2); + try { return new Dictionary { { "status", "OK" }, { "rowsAffected", command.ExecuteNonQuery().ToString(System.Globalization.CultureInfo.InvariantCulture) } }; } + catch (SQLiteException ex) { return new Dictionary { { "status", "NO" }, { "reason", ex.Message } }; } } - return s; } - - private static string QuoteIdentifier(string name) + private void ExecChecked(string sql) { - name = name ?? string.Empty; - return "\"" + name.Replace("\"", "\"\"") + "\""; + var result = db.ExecSQL(sql); + if (!IsExecutionSuccess(result)) throw new InvalidOperationException(sql + " " + DatabaseExecutionResultService.GetFailureReason(result)); } - - private static string MakeStatementPreview(string sql) + private static bool IsExecutionSuccess(Dictionary result) { - if (string.IsNullOrWhiteSpace(sql)) return string.Empty; - string singleLine = Regex.Replace(sql, @"\s+", " ").Trim(); - if (singleLine.Length > 120) return singleLine.Substring(0, 117) + "..."; - return singleLine; + return result != null && result.TryGetValue("status", out string status) && string.Equals(status, "OK", StringComparison.OrdinalIgnoreCase); } - - private static bool CheckIfScriptContainsExplicitTx(string filePath) + private static string Key(Match match) { return match.Groups["type"].Value.ToUpperInvariant() + ":" + Unquote(match.Groups["name"].Value); } + private static string Unquote(string name) { - if (!File.Exists(filePath)) return false; - try + if (name.Length > 1 && (name[0] == '"' || name[0] == '`' || name[0] == '\'' || name[0] == '[')) { - using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096)) - using (var reader = new StreamReader(fs, Encoding.UTF8, true, 4096)) - { - string line; - int count = 0; - while ((line = reader.ReadLine()) != null && count < 200) - { - count++; - string trimmed = line.Trim(); - if (BeginTxRegex.IsMatch(trimmed)) return true; - } - } + char quote = name[0]; name = name.Substring(1, name.Length - 2); + if (quote != '[') name = name.Replace(new string(quote, 2), quote.ToString()); } - catch { } - return false; + return name; } - - private static void ReportProgress( - IProgress progress, - long currentBytes, - long totalBytes, - int executed, - int failed, - int skipped, - long totalRowsAffected, - Stopwatch clock, - string currentPreview, - bool isComplete = false) + private static string QuoteIdentifier(string name) { return "\"" + name.Replace("\"", "\"\"") + "\""; } + private static string Preview(string sql) { return Regex.Replace(sql.Substring(0, Math.Min(120, sql.Length)), @"\s+", " ").Trim(); } + private static void ReportProgress(IProgress progress, long bytes, int executed, int failed, int skipped, long rows, Stopwatch clock, string preview) { - if (progress == null) return; - - double elapsedSec = clock.Elapsed.TotalSeconds; - double bytesPerSec = elapsedSec > 0.05 ? currentBytes / elapsedSec : 0; - TimeSpan? remaining = null; - if (bytesPerSec > 0 && totalBytes > currentBytes) - { - double remainingSec = (totalBytes - currentBytes) / bytesPerSec; - if (remainingSec < 86400) remaining = TimeSpan.FromSeconds(remainingSec); - } - - string msg; - if (isComplete) - { - msg = $"Import execution finished: {executed} executed, {failed} failed, {skipped} skipped, {totalRowsAffected:N0} rows affected."; - } - else - { - msg = $"Executing: {executed} executed, {failed} failed, {skipped} skipped..."; - } - - progress.Report(new SqliteImportProgress( - currentBytes, - totalBytes, - executed, - failed, - skipped, - totalRowsAffected, - bytesPerSec, - clock.Elapsed, - remaining, - msg, - currentPreview)); + progress?.Report(new SqliteImportProgress(0, bytes, executed, failed, skipped, rows, 0, clock.Elapsed, null, + "Executing: " + executed + " executed, " + failed + " failed, " + skipped + " skipped...", preview)); } } } diff --git a/mySQLPunk/lib/WizardOwnedSession.cs b/mySQLPunk/lib/WizardOwnedSession.cs new file mode 100644 index 0000000..9ae684d --- /dev/null +++ b/mySQLPunk/lib/WizardOwnedSession.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace mySQLPunk.lib +{ + internal sealed class WizardOwnedSession : IDisposable + { + private IDatabase owner; + internal IDatabase Database { get; private set; } + + private WizardOwnedSession(ConnectionOpenResult opened) + { + owner = opened.Database; + // Native streaming readers need the underlying provider; owner retains the SSH lease. + Database = opened.NativeDatabase ?? opened.Database; + } + + internal static Dictionary CopySettings(IDictionary source, string provider, string database) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (string.IsNullOrWhiteSpace(database)) throw new ArgumentException("An explicit target database is required."); + var settings = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in source) + { + if (string.Equals(pair.Key, "pdo", StringComparison.OrdinalIgnoreCase) + || string.Equals(pair.Key, "isConnect", StringComparison.OrdinalIgnoreCase) + || string.Equals(pair.Key, "connString", StringComparison.OrdinalIgnoreCase)) continue; + if (pair.Value == null || pair.Value is string || pair.Value.GetType().IsPrimitive || pair.Value is decimal) + settings[pair.Key] = pair.Value; + } + settings["db_kind"] = ConnectionConfigurationService.NormalizeProvider(provider); + settings["initial_database"] = database; + return settings; + } + + internal static WizardOwnedSession Open(IDictionary source, string provider, string database) + { + var settings = CopySettings(source, provider, database); + if (string.Equals(Convert.ToString(settings["db_kind"]), "sqlite", StringComparison.OrdinalIgnoreCase)) + { + string path = ConnectionConfigurationService.GetValue(settings, "path"); + if (!string.Equals(database, "main", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException("SQLite wizard currently requires the main database."); + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + throw new IOException("The selected SQLite file must already exist; the wizard will not create an empty source."); + } + return new WizardOwnedSession(ConnectionOpenService.OpenWizardConnection(settings)); + } + + public void Dispose() + { + var current = owner; owner = null; + if (current != null) current.Dispose(); + } + } +} diff --git a/mySQLPunk/mySQLPunk.csproj b/mySQLPunk/mySQLPunk.csproj index 9ce21b0..a6bc6d4 100644 --- a/mySQLPunk/mySQLPunk.csproj +++ b/mySQLPunk/mySQLPunk.csproj @@ -465,6 +465,9 @@ + + + diff --git a/tests/Run-SmokeTests.ps1 b/tests/Run-SmokeTests.ps1 index 63bf168..d68f1a4 100644 --- a/tests/Run-SmokeTests.ps1 +++ b/tests/Run-SmokeTests.ps1 @@ -147,6 +147,7 @@ if (!(Test-Path -LiteralPath $mysqlConnector)) { (Join-Path $PSScriptRoot "SqlServerRestoreCompatibilityTests.cs") ` (Join-Path $PSScriptRoot "SqlServerMigrationTests.cs") ` (Join-Path $PSScriptRoot "SqlServerMigrationCatalogTests.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.WizardRepairs.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MySqlExport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MySqlImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.SqliteExport.cs") ` diff --git a/tests/SmokeTests.MySqlExport.cs b/tests/SmokeTests.MySqlExport.cs index fbb607a..a8afda6 100644 --- a/tests/SmokeTests.MySqlExport.cs +++ b/tests/SmokeTests.MySqlExport.cs @@ -88,10 +88,71 @@ private static void MySqlExport_ServiceAndManifest_AtomicWrite() } } + private static void MySqlExport_Regression_GeneratedBoundsAndFailures() + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_export_repair_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "dump.sql"); + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, path) { SelectedTables = new List { "users" } }; + var db = new MockMySqlExportDatabase { Generated = true }; + var result = new MySqlSqlExportService(db, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(result.Success, "Generated column export succeeds by omitting writable generated value: " + result.ErrorMessage); + string dump = File.ReadAllText(path); + AssertContains(dump, "GENERATED ALWAYS", "Generated DDL retained"); + AssertContains(dump, "INSERT INTO `users` (`id`, `name`)", "Generated column absent from explicit INSERT column list"); + Assert(!dump.Contains("`id`, `name`, `computed`"), "Computed values are not inserted"); + Assert(db.ExecutedSql.Contains("ROLLBACK;") && !db.ExecutedSql.Contains("COMMIT;"), "Read snapshot ends with validated rollback"); + Assert(result.FileBytes == new FileInfo(path).Length, "Reported final bytes include checksum footer"); + Assert(mySQLPunk.lib.MySqlImport.MySqlImportPreflightScanner.Scan(path, CancellationToken.None).Passed, "Native generated dump passes its own parser/digest preflight"); + File.AppendAllText(path, "-- tampered"); + Assert(!mySQLPunk.lib.MySqlImport.MySqlImportPreflightScanner.Scan(path, CancellationToken.None).Passed, "Tampered native dump rejected"); + + foreach (string failure in new[] { "snapshot", "rollback", "engine", "large" }) + { + File.WriteAllText(path, "existing complete backup"); + db = new MockMySqlExportDatabase { Failure = failure, MaxRowBytes = failure == "large" ? 1048577 : 32 }; + result = new MySqlSqlExportService(db, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(!result.Success && result.Stage == MySqlExportStage.Failed, failure + " fails explicitly"); + AssertEquals("existing complete backup", File.ReadAllText(path), failure + " preserves existing backup"); + if (failure == "large") Assert(db.PageReads == 0, "Oversized row rejected before materializing source page"); + } + db = new MockMySqlExportDatabase(); + using (var locked = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + { + result = new MySqlSqlExportService(db, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(result.Stage == MySqlExportStage.Failed, "Replacement lock failure is reported"); + } + AssertEquals("existing complete backup", File.ReadAllText(path), "Replacement failure preserves previous complete backup"); + Assert(Directory.GetFiles(directory, "dump.sql.staging.*.tmp").Length == 1, "Completed staging survives publication failure"); + foreach (string retained in Directory.GetFiles(directory, "dump.sql.staging.*.tmp")) + Assert(mySQLPunk.lib.MySqlImport.MySqlImportPreflightScanner.Scan(retained, CancellationToken.None).Passed, "Retained staging is a complete verified dump"); + db = new MockMySqlExportDatabase { Failure = "page-race" }; + result = new MySqlSqlExportService(db, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(!result.Success, "Concurrent enlargement marker fails explicitly"); + AssertEquals("existing complete backup", File.ReadAllText(path), "Concurrent enlargement leaves prior backup intact"); + using (var cancel = new CancellationTokenSource()) + { + db = new MockMySqlExportDatabase { OnPageRead = () => cancel.Cancel() }; + result = new MySqlSqlExportService(db, "test_db").ExportAsync(options, null, cancel.Token).Result; + Assert(result.Stage == MySqlExportStage.Cancelled, "Cancellation at last source read cannot publish"); + AssertEquals("existing complete backup", File.ReadAllText(path), "Cancelled export preserves prior backup"); + } + } + finally { Directory.Delete(directory, true); } + } + private sealed class MockMySqlExportDatabase : IDatabase { public ConnectionState State => ConnectionState.Open; public string ProviderName => "MySQL"; + public bool Generated; + public string Failure; + public long MaxRowBytes = 32; + public int PageReads; + public Action OnPageRead; + public readonly List ExecutedSql = new List(); public void SetConn(string connectionString) { } public void Open() { } @@ -101,6 +162,17 @@ public void Dispose() { } public DataTable SelectSQL(string sql, Dictionary parameters = null) { var dt = new DataTable(); + if (sql.Contains("AS MaxRowBytes")) { dt.Columns.Add("MaxRowBytes", typeof(long)); dt.Rows.Add(MaxRowBytes); return dt; } + if (sql.Contains("AS __mysqlpunk_row_bytes")) + { + bool first = sql.Contains("LIMIT 0,"); + dt = SelectTablePage("test_db", "users", first ? 0 : 16, 16); + dt.Columns.Add("__mysqlpunk_row_bytes", typeof(long)); + foreach (DataRow row in dt.Rows) row["__mysqlpunk_row_bytes"] = Failure == "page-race" ? 1048577L : MaxRowBytes; + PageReads++; + OnPageRead?.Invoke(); + return dt; + } if (sql.Contains("ROUTINES")) { dt.Columns.Add("ROUTINE_NAME"); @@ -140,7 +212,9 @@ public DataTable SelectSQL(string sql, Dictionary parameters = n public Dictionary ExecSQL(string sql, Dictionary parameters = null) { - return new Dictionary(); + ExecutedSql.Add(sql); + bool fail = Failure == "snapshot" && sql.StartsWith("START TRANSACTION") || Failure == "rollback" && sql.StartsWith("ROLLBACK"); + return fail ? new Dictionary { { "status", "NO" }, { "reason", "injected " + Failure } } : new Dictionary { { "status", "OK" } }; } public Task SelectSQLAsync(string sql, Dictionary parameters = null) @@ -162,17 +236,24 @@ public DataTable GetColumns(string databaseName, string tableName) var dt = new DataTable(); dt.Columns.Add("COLUMN_NAME"); dt.Columns.Add("DATA_TYPE"); - dt.Rows.Add("id", "int"); - dt.Rows.Add("name", "varchar"); + dt.Columns.Add("Extra"); + dt.Rows.Add("id", "int", ""); + dt.Rows.Add("name", "varchar", ""); + if (Generated) dt.Rows.Add("computed", "int", "VIRTUAL GENERATED"); return dt; } public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); - public DataTable GetTableStatus(string databaseName) => new DataTable(); + public DataTable GetTableStatus(string databaseName) + { + var dt = new DataTable(); dt.Columns.Add("Name"); dt.Columns.Add("Engine"); + dt.Rows.Add("users", Failure == "engine" ? "MyISAM" : "InnoDB"); return dt; + } public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); public string GetTableCreateStatement(string databaseName, string tableName) { + if (Generated) return "CREATE TABLE `users` (`id` int, `name` varchar(50), `computed` int GENERATED ALWAYS AS (id+1) VIRTUAL) ENGINE=InnoDB;"; return "CREATE TABLE `users` (`id` int NOT NULL, `name` varchar(50) DEFAULT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; } @@ -192,11 +273,13 @@ public DataTable SelectTablePage(string databaseName, string tableName, long off var dt = new DataTable(); dt.Columns.Add("id", typeof(int)); dt.Columns.Add("name", typeof(string)); + if (Generated) dt.Columns.Add("computed", typeof(int)); if (offset == 0) { dt.Rows.Add(1, "Alice"); dt.Rows.Add(2, "Bob"); } + if (Generated) foreach (DataRow row in dt.Rows) row["computed"] = Convert.ToInt32(row["id"]) + 1; return dt; } diff --git a/tests/SmokeTests.MySqlImport.cs b/tests/SmokeTests.MySqlImport.cs index 63a1a41..3d51762 100644 --- a/tests/SmokeTests.MySqlImport.cs +++ b/tests/SmokeTests.MySqlImport.cs @@ -375,6 +375,144 @@ private static void MySqlImport_Runner_ManifestMismatchDetection() } } + private static void MySqlImport_Regression_TargetLiteralsAndStrategies() + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_repair_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "fixture.sql"); + string payload = "src.orders\r\n-- keep this literal line\r\n# and this one\r\n/* src.orders */\r\nlast"; + File.WriteAllText(path, "# restore src\r\nUSE /* choose */ `src`; INSERT INTO src.t VALUES ('" + payload + "'); " + + "/* src.t comment */ INSERT INTO `src`.`t` VALUES ('escaped \\' src.t', '{\"name\":\"src.orders\"}');", new UTF8Encoding(false)); + var db = new MockMySqlImportDatabase(); + var result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Completed, "Inline/comment USE and inserts complete: " + result.Summary); + Assert(result.ExecutedStatements == 3, "Inline statements split consistently"); + Assert(db.ExecutedSql.Exists(sql => sql.Contains("INSERT INTO `target`.t") && sql.Contains(payload)), "Database target changes while multiline payload remains exact"); + Assert(db.ExecutedSql.Exists(sql => sql.Contains("/* src.t comment */") && sql.Contains("'{\"name\":\"src.orders\"}'")), "Comment and JSON literal remain exact"); + string rewritten = MySqlImportRunner.RewriteDatabaseQualifier("INSERT INTO src.t VALUES ('src.t', 'xsrc.t') /* src.t */", "src", "target"); + AssertContains(rewritten, "`target`.t VALUES ('src.t', 'xsrc.t') /* src.t */", "Only identifier token rewritten"); + AssertEquals("SELECT xsrc.t", MySqlImportRunner.RewriteDatabaseQualifier("SELECT xsrc.t", "src", "target"), "Longer identifiers are not source qualifiers"); + + File.WriteAllText(path, "INSERT INTO other.t VALUES (1);", new UTF8Encoding(false)); + db = new MockMySqlImportDatabase(); + result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Completed && db.ExecutedSql.Exists(sql => sql.Contains("INSERT INTO `target`.t")), "Qualified INSERT without USE cannot write to other database"); + + File.WriteAllText(path, "/* preserve */ DROP TABLE IF EXISTS old; CREATE TABLE old (id int); INSERT INTO old VALUES (1); CREATE TABLE fresh (id int); INSERT INTO fresh VALUES (2); INSERT INTO fresh VALUES (3);", new UTF8Encoding(false)); + db = new MockMySqlImportDatabase { GetTablesHandler = () => new List { "old" }, TableExistsHandler = (database, table) => table == "old" }; + result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.SkipExisting }, null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Completed, "Mixed existing/new SkipExisting import completes"); + Assert(result.ExecutedStatements == 3 && result.SkippedStatements == 3, "Fresh CREATE and both INSERT batches execute; existing object protected"); + Assert(!db.ExecutedSql.Exists(sql => MySqlScriptReader.CodeOnly(sql).TrimStart().StartsWith("DROP", StringComparison.OrdinalIgnoreCase)), "Comment-prefixed DROP skipped"); + File.WriteAllText(path, "DROP DATABASE src;", new UTF8Encoding(false)); + db = new MockMySqlImportDatabase(); + result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Failed && !db.ExecutedSql.Exists(sql => sql.Contains("DROP DATABASE")), "Database-wide DROP rejected"); + } + finally { Directory.Delete(directory, true); } + } + + private static void MySqlImport_Regression_CancelTransactionsAndPreflight() + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_repair_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "fixture.sql"); + File.WriteAllText(path, "INSERT INTO t VALUES (1);", new UTF8Encoding(false)); + using (var cancel = new CancellationTokenSource()) + { + var db = new MockMySqlImportDatabase { ExecSqlHandler = sql => { if (sql.StartsWith("INSERT")) cancel.Cancel(); return new Dictionary { { "status", "OK" } }; } }; + var result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, cancel.Token).Result; + Assert(result.Status == MySqlImportStatus.Cancelled, "Cancellation inside last command cannot report Completed"); + Assert(db.ExecutedSql.Contains("ROLLBACK;"), "Cancelled job attempts owned transaction cleanup"); + } + File.WriteAllText(path, "BEGIN; INSERT INTO t VALUES (1);", new UTF8Encoding(false)); + var txDb = new MockMySqlImportDatabase(); + var txResult = new MySqlImportRunner(txDb, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(txResult.Status == MySqlImportStatus.Failed, "Uncommitted script cannot report restored rows as complete"); + File.WriteAllText(path, "BEGIN; INSERT INTO t VALUES (1); COMMIT;", new UTF8Encoding(false)); + txDb = new MockMySqlImportDatabase { ExecSqlHandler = sql => new Dictionary { { "status", sql.StartsWith("COMMIT") ? "NO" : "OK" }, { "reason", "injected commit failure" } } }; + txResult = new MySqlImportRunner(txDb, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(txResult.Status == MySqlImportStatus.Failed, "Real status=NO commit cannot complete"); + File.WriteAllText(path, "INSERT INTO t VALUES (1);", new UTF8Encoding(false)); + txDb = new MockMySqlImportDatabase { ExecSqlHandler = sql => new Dictionary { { "status", sql.StartsWith("ROLLBACK") ? "NO" : "OK" }, { "reason", "injected cleanup failure" } } }; + txResult = new MySqlImportRunner(txDb, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(txResult.Status == MySqlImportStatus.Failed, "Cleanup status=NO cannot report completion"); + File.WriteAllText(path, "/*!40101 USE src */;", new UTF8Encoding(false)); + Assert(!MySqlImportPreflightScanner.Scan(path, CancellationToken.None).Passed, "Executable version comments explicitly unsupported"); + File.WriteAllText(path, "INSERT INTO t VALUES ('unterminated);", new UTF8Encoding(false)); + Assert(!MySqlImportPreflightScanner.Scan(path, CancellationToken.None).Passed, "Unterminated literal fails before execution"); + File.WriteAllText(path, "-- mySQLPunk Native Manifest\r\n-- Manifest-SHA256: " + new string('0', 64) + "\r\n", new UTF8Encoding(false)); + Assert(!MySqlImportPreflightScanner.Scan(path, CancellationToken.None).Passed, "Native digest is verified instead of displayed only"); + } + finally { Directory.Delete(directory, true); } + } + + private static void MySqlImport_Regression_ModifiedInsertProtectionAndTriggerPseudoRows() + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_review_repair_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "fixture.sql"); + string[] forms = + { + "INSERT LOW_PRIORITY INTO `TABLE`", "INSERT HIGH_PRIORITY `TABLE`", + "INSERT DELAYED IGNORE INTO src.`TABLE`", "INSERT IGNORE `src`.`TABLE`", + "INSERT `TABLE`", "REPLACE LOW_PRIORITY INTO `TABLE`", + "REPLACE DELAYED src.TABLE", "REPLACE `TABLE`" + }; + string sourceLookingValue = "'src.old NEW.id OLD.id'"; + var script = new StringBuilder("CREATE TABLE fresh (value text);\r\n"); + foreach (string form in forms) + { + script.Append(form.Replace("TABLE", "old")).Append(" VALUES (").Append(sourceLookingValue).Append(");\r\n"); + script.Append(form.Replace("TABLE", "fresh")).Append(" VALUES (").Append(sourceLookingValue).Append(");\r\n"); + } + File.WriteAllText(path, script.ToString(), new UTF8Encoding(false)); + var db = new MockMySqlImportDatabase { GetTablesHandler = () => new List { "old" }, TableExistsHandler = (database, table) => table == "old" }; + var options = new MySqlImportOptions { ExistingObjectStrategy = MySqlExistingObjectStrategy.SkipExisting }; + var result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), options, null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Completed, "Priority/optional-INTO forms restore safely: " + result.Summary); + Assert(result.SkippedStatements == forms.Length && result.ExecutedStatements == forms.Length + 1, "All old-table INSERT/REPLACE variants skipped and fresh writes execute"); + foreach (string sql in db.ExecutedSql) + { + string code = MySqlScriptReader.CodeOnly(sql).TrimStart(); + if (!code.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase) && !code.StartsWith("REPLACE", StringComparison.OrdinalIgnoreCase)) continue; + Assert(code.IndexOf("old", StringComparison.OrdinalIgnoreCase) < 0, "Protected table never reaches provider for any accepted variant"); + AssertContains(sql, sourceLookingValue, "Source/pseudo-row-looking literal remains exact in fresh-table writes"); + Assert(!code.Contains("src." ) && !code.Contains("`src`."), "Qualified optional-INTO source targets redirected without a USE"); + } + script.Clear(); + foreach (string form in forms) script.Append(form.Replace("TABLE", "old")).Append(" VALUES (1);\r\n"); + File.WriteAllText(path, script.ToString(), new UTF8Encoding(false)); + db = new MockMySqlImportDatabase { GetTablesHandler = () => new List { "old" } }; + result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), options, null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Completed && result.ExecutedStatements == 0 && result.SkippedStatements == forms.Length, "Data-only modifier/optional-INTO forms all preserve initial existing table"); + + File.WriteAllText(path, "DELIMITER //\r\n" + + "CREATE TRIGGER `tr` BEFORE UPDATE ON `src`.`t` FOR EACH ROW\r\n" + + "BEGIN\r\nSET NEW.id = OLD.id;\r\n" + + "INSERT LOW_PRIORITY INTO src.audit VALUES (NEW.id, 'src.audit NEW.id OLD.id');\r\nEND//\r\nDELIMITER ;\r\n", new UTF8Encoding(false)); + db = new MockMySqlImportDatabase(); + result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Completed && result.ExecutedStatements == 1, "Native DELIMITER trigger with NEW/OLD imports as one definition: " + result.Summary); + string trigger = db.ExecutedSql.Find(sql => sql.Contains("CREATE TRIGGER")); + AssertContains(trigger, "ON `target`.`t`", "Trigger table qualified to selected target without USE"); + AssertContains(trigger, "INSERT LOW_PRIORITY INTO `target`.audit", "Trigger body real object qualified to selected target"); + AssertContains(trigger, "SET NEW.id = OLD.id;", "Trigger pseudo-row identifiers remain unchanged"); + AssertContains(trigger, "'src.audit NEW.id OLD.id'", "Trigger source-looking literal remains unchanged"); + File.WriteAllText(path, "INSERT INTO t VALUES (NEW.id);", new UTF8Encoding(false)); + db = new MockMySqlImportDatabase(); + result = new MySqlImportRunner(db, "target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(result.Status == MySqlImportStatus.Failed, "NEW/OLD exemption is confined to recognized trigger definitions"); + } + finally { Directory.Delete(directory, true); } + } + private sealed class MockMySqlImportDatabase : IDatabase { public ConnectionState State => ConnectionState.Open; @@ -382,6 +520,7 @@ private sealed class MockMySqlImportDatabase : IDatabase public List ExecutedSql = new List(); public Func> ExecSqlHandler = null; public Func TableExistsHandler = null; + public Func> GetTablesHandler = null; public Func CountRowsHandler = null; public void SetConn(string connectionString) { } @@ -408,7 +547,7 @@ public Dictionary ExecSQL(string sql, Dictionary public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); public List GetDatabases() => new List { "test_db" }; - public List GetTables(string databaseName) => new List { "test" }; + public List GetTables(string databaseName) => GetTablesHandler != null ? GetTablesHandler() : new List { "test" }; public List GetViews(string databaseName) => new List(); public DataTable GetColumns(string databaseName, string tableName) => new DataTable(); public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); diff --git a/tests/SmokeTests.OracleExport.cs b/tests/SmokeTests.OracleExport.cs index 78c78a6..1dffa36 100644 --- a/tests/SmokeTests.OracleExport.cs +++ b/tests/SmokeTests.OracleExport.cs @@ -60,6 +60,7 @@ private static void OracleExport_ServiceAndManifest_StreamingAndSdoGeometry() Assert(shadow.IsSpatialMetadata, "MDRT_* should be recognized as internal spatial shadow table"); Assert(!shadow.Selected, "MDRT_* should be unchecked by default"); + mockDb.Commands.Clear(); var options = new OracleExportWizardOptions(OracleExportMode.SchemaAndData, targetFile) { UseTransactionSnapshot = true, @@ -83,14 +84,15 @@ private static void OracleExport_ServiceAndManifest_StreamingAndSdoGeometry() string sql = File.ReadAllText(targetFile); // 2. 驗證 Oracle SET TRANSACTION READ ONLY 交易快照 - AssertContains(sql, "SET TRANSACTION READ ONLY;", "Header must initialize READ ONLY snapshot"); + Assert(!sql.Contains("SET TRANSACTION READ ONLY"), "Source read-only transaction must not appear in restore script"); + Assert(mockDb.Commands[0] == "SET TRANSACTION READ ONLY" && mockDb.Commands[mockDb.Commands.Count - 1] == "ROLLBACK", "Real source snapshot surrounds reads and is ended"); // 3. 驗證 USER_SDO_GEOM_METADATA 空間中繼註冊 AssertContains(sql, "INSERT INTO USER_SDO_GEOM_METADATA", "Must generate USER_SDO_GEOM_METADATA insert"); AssertContains(sql, "MDSYS.SDO_DIM_ARRAY", "Must generate dimension array"); // 4. 驗證 SDO_GEOMETRY 幾何資料轉換為無損 SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW('...')) - AssertContains(sql, "SDO_UTIL.FROM_WKBGEOMETRY(HEXTORAW(", "Geometry values must be formatted as WKB HEXTORAW"); + AssertContains(sql, "SDO_UTIL.FROM_WKBGEOMETRY(TO_BLOB(HEXTORAW(", "Geometry values must be formatted as WKB HEXTORAW"); AssertContains(sql, "INSERT INTO \"SCOTT\".\"PARCELS\"", "Insert header should target qualified table name"); // 5. 驗證 Oracle Spatial 空間索引 @@ -98,7 +100,7 @@ private static void OracleExport_ServiceAndManifest_StreamingAndSdoGeometry() // 6. 驗證 Native Manifest AssertContains(sql, "-- mySQLPunk Native Manifest", "Manifest header present"); - AssertContains(sql, "-- Manifest-Table: PARCELS | Rows: 2", "Manifest table row count accurate"); + AssertContains(sql, "-- Manifest-Table: \"SCOTT\".\"PARCELS\" | Rows: 2", "Manifest table row count accurate"); AssertContains(sql, "-- Manifest-Checksum-SHA256:", "Manifest SHA-256 checksum present"); Assert(!string.IsNullOrWhiteSpace(result.Sha256Checksum), "Manifest SHA256 string populated in result"); } @@ -108,18 +110,299 @@ private static void OracleExport_ServiceAndManifest_StreamingAndSdoGeometry() } } - private sealed class MockOracleExportDatabase : IDatabase + private static void OracleExport_SpatialMetadataAndOptionGates() + { + string directory = Path.Combine(Path.GetTempPath(), "ora_options_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + foreach (int flag in new[] { 0, 1, 2, 3 }) + { + var db = new MockOracleExportDatabase { VirtualColumn = true }; + string path = Path.Combine(directory, "flags" + flag + ".sql"); + var options = new OracleExportWizardOptions(OracleExportMode.SchemaAndData, path) { IncludeComments = flag == 1, IncludeIndexes = flag == 2, IncludeSpatialIndexes = flag == 3 }; + options.SelectedTables.Add("\"SCOTT\".\"PARCELS\""); + new OracleSqlExportService(db, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); + string sql = File.ReadAllText(path); + Assert(sql.Contains("actual comment") == (flag == 1), "IncludeComments gate"); + Assert(sql.Contains("IX_NAME") == (flag == 2), "IncludeIndexes gate"); + Assert(sql.Contains("REAL_SPATIAL_INDEX") == (flag == 3), "IncludeSpatialIndexes gate preserves actual name"); + Assert(!sql.Contains("SIDX_PARCELS_SHAPE") && !sql.Contains("4326"), "No fabricated spatial index or SRID"); + Assert(sql.Contains("'E', 0, 1000000, 0.01") && sql.Contains("'Z', -100, 9000, 0.1"), "Actual three-dimensional bounds and tolerance preserved"); + Assert(sql.Contains(", 3826)") && sql.Contains(", NULL)"), "Per-value projected and null SRIDs preserved"); + Assert(!db.Commands.Exists(command => command.StartsWith("SELECT ") && command.Contains("CALC")), "Virtual column omitted from reader projection"); + if (flag == 3) Assert(sql.IndexOf("INSERT INTO USER_SDO_GEOM_METADATA") < sql.IndexOf("REAL_SPATIAL_INDEX"), "Spatial metadata precedes actual spatial index"); + } + var nullDb = new MockOracleExportDatabase { NullMetadataDimensions = true, GeometrySrid = null }; + var nullOptions = new OracleExportWizardOptions(OracleExportMode.SchemaOnly, Path.Combine(directory, "null.sql")); nullOptions.SelectedTables.Add("PARCELS"); + new OracleSqlExportService(nullDb, "SCOTT").ExportAsync(nullOptions).GetAwaiter().GetResult(); + Assert(File.ReadAllText(nullOptions.TargetPath).Contains("'SHAPE', NULL, NULL)"), "NULL DIMINFO and metadata SRID retained"); + } + finally { Directory.Delete(directory, true); } + } + + private static void OracleExport_BoundedLobsSnapshotFailureAndCancel() + { + string directory = Path.Combine(Path.GetTempPath(), "ora_lobs_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + string path = Path.Combine(directory, "dump.sql"); + try + { + var db = new MockOracleExportDatabase { TextValue = new string('中', 5000), GeometryBytes = 3000 }; + var options = new OracleExportWizardOptions(OracleExportMode.DataOnly, path) { BatchSize = 1 }; options.SelectedTables.Add("PARCELS"); + bool outputBeforeExhaustion = false; + var observer = new OracleProgressObserver(p => + { + if (p.RowsExported == 1) + { + string[] staging = Directory.GetFiles(directory, "dump.sql.*.writing"); + if (staging.Length == 1) + using (var pending = new StreamReader(new FileStream(staging[0], FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) + outputBeforeExhaustion = pending.ReadToEnd().Contains("DBMS_LOB.WRITEAPPEND") && db.Reader != null && !db.Reader.IsClosed; + } + }); + new OracleSqlExportService(db, "SCOTT").ExportAsync(options, observer).GetAwaiter().GetResult(); + Assert(outputBeforeExhaustion, "First row is written and flushed while source reader remains active"); + string sql = File.ReadAllText(path); + Assert(sql.Contains("DBMS_LOB.WRITEAPPEND") && sql.Contains(" BLOB;") && sql.Contains(" CLOB;"), "Large WKB/CLOB uses bounded temporary LOB blocks"); + Assert(!sql.Contains("SET TRANSACTION READ ONLY"), "Data-only restore remains writable"); + foreach (System.Text.RegularExpressions.Match literal in System.Text.RegularExpressions.Regex.Matches(sql, @"HEXTORAW\('([0-9A-F]+)'\)")) Assert(literal.Groups[1].Length <= 4000, "Every binary hex literal stays within STANDARD limit"); + string parsePath = Path.Combine(directory, "parse.sql"); File.WriteAllText(parsePath, sql); + var preflight = new mySQLPunk.lib.OracleImport.OracleImportPreflightScanner().Scan(parsePath); + Assert(preflight.CanProceed && preflight.HasPlSqlBlocks, "Real exporter LOB blocks parse as complete executable statements"); + var restore = new MockOracleImportDatabase { ActualRows = 2 }; + var restored = new mySQLPunk.lib.OracleImport.OracleImportRunner(restore, new mySQLPunk.lib.OracleImport.OracleImportRequest(parsePath, "SCOTT")).Run(); + Assert(restored.Success && restore.ExecutedQueries.Exists(command => command.StartsWith("DECLARE")), "Actual generated LOB script passes owner validation and executes as whole blocks (recording provider only)"); + foreach (string failure in new[] { "SET TRANSACTION", "ROLLBACK" }) + { + File.WriteAllText(path, "KEEP ORIGINAL"); + var failing = new MockOracleExportDatabase { FailCommand = failure }; + bool threw = false; + try { new OracleSqlExportService(failing, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); } catch { threw = true; } + Assert(threw && File.ReadAllText(path) == "KEEP ORIGINAL", "Source command NO prevents publication: " + failure); + } + using (var cancel = new CancellationTokenSource()) + { + var cancelled = new MockOracleExportDatabase { BeforeReaderOpen = () => cancel.Cancel() }; + File.WriteAllText(path, "KEEP ORIGINAL"); bool threw = false; + try { new OracleSqlExportService(cancelled, "SCOTT").ExportAsync(options, null, cancel.Token).GetAwaiter().GetResult(); } catch (OperationCanceledException) { threw = true; } + Assert(threw && cancelled.Commands.Contains("ROLLBACK") && File.ReadAllText(path) == "KEEP ORIGINAL", "Cancellation ends source transaction and preserves existing dump"); + } + var empty = new MockOracleExportDatabase { EmptyLobs = true }; + new OracleSqlExportService(empty, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); + string emptySql = File.ReadAllText(path); + Assert(emptySql.Contains("EMPTY_BLOB()") && emptySql.Contains("EMPTY_CLOB()") && emptySql.Contains("TO_BLOB(HEXTORAW("), "Empty non-null LOBs remain distinct from NULL and small geometry uses explicit BLOB conversion"); + var tooLarge = new MockOracleExportDatabase { GeometryBytes = OracleSqlExportService.MaxRowValueBytes + 1 }; + File.WriteAllText(path, "KEEP ORIGINAL"); bool overBudget = false; + try { new OracleSqlExportService(tooLarge, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); } catch (NotSupportedException) { overBudget = true; } + Assert(overBudget && File.ReadAllText(path) == "KEEP ORIGINAL", "Row budget failure preserves previous output"); + var identity = new MockOracleExportDatabase { Identity = true }; bool rejected = false; + try { new OracleSqlExportService(identity, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); } catch (NotSupportedException) { rejected = true; } + Assert(rejected, "Identity data rejected explicitly until sequence-state restoration verified"); + foreach (bool cycle in new[] { false, true }) + { + var dependency = new MockOracleExportDatabase { ForeignKeyCycle = cycle, MissingForeignKeyParent = !cycle }; + File.WriteAllText(path, "KEEP ORIGINAL"); bool refused = false; + try { new OracleSqlExportService(dependency, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); } catch (NotSupportedException) { refused = true; } + Assert(refused && File.ReadAllText(path) == "KEEP ORIGINAL", "Selected FK cycle or absent parent is rejected before publication"); + } + bool oversized = false; + try { OracleSqlExportService.FormatOracleLiteral(new string('中', 2000), false); } catch (NotSupportedException) { oversized = true; } + Assert(oversized, "Scalar UTF-8 literal bound is measured in bytes"); + } + finally { Directory.Delete(directory, true); } + } + + private static void OracleExport_ScopedBinaryFloatRoundTrip() + { + foreach (double value in new[] { 1.2345678901234567d, 1.7e308d, double.Epsilon, 0d }) + { + string literal = OracleSqlExportService.FormatOracleLiteral(value, false); + Assert(literal == value.ToString("G17", System.Globalization.CultureInfo.InvariantCulture) + "d", "BINARY_DOUBLE uses exact G17 typed token"); + double parsed = double.Parse(literal.Substring(0, literal.Length - 1), System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture); + Assert(BitConverter.DoubleToInt64Bits(parsed) == BitConverter.DoubleToInt64Bits(value), "Double token roundtrips exactly in .NET"); + } + foreach (float value in new[] { 1.2345678f, 1.7e30f, float.Epsilon, 0f }) + { + string literal = OracleSqlExportService.FormatOracleLiteral(value, false); + Assert(literal == value.ToString("G9", System.Globalization.CultureInfo.InvariantCulture) + "f", "BINARY_FLOAT uses exact G9 typed token"); + float parsed = float.Parse(literal.Substring(0, literal.Length - 1), System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture); + Assert(BitConverter.ToInt32(BitConverter.GetBytes(parsed), 0) == BitConverter.ToInt32(BitConverter.GetBytes(value), 0), "Float token roundtrips exactly in .NET"); + } + foreach (object value in new object[] { BitConverter.Int64BitsToDouble(long.MinValue), BitConverter.ToSingle(BitConverter.GetBytes(int.MinValue), 0) }) + { + bool rejected = false; try { OracleSqlExportService.FormatOracleLiteral(value, false); } catch (NotSupportedException) { rejected = true; } + Assert(rejected, "Signed zero rejects until native storage fidelity is verified"); + } + string path = Path.Combine(Path.GetTempPath(), "ora_float_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + var source = new MockOracleExportDatabase { FloatingValues = true }; + var options = new OracleExportWizardOptions(OracleExportMode.DataOnly, path); options.SelectedTables.Add("PARCELS"); + new OracleSqlExportService(source, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); + string dump = File.ReadAllText(path); + Assert(!dump.Contains(((double)1.2345678f).ToString("G17", System.Globalization.CultureInfo.InvariantCulture) + "d"), "Float is emitted as float rather than widened double"); + Assert(dump.Contains((1.2345678f).ToString("G9", System.Globalization.CultureInfo.InvariantCulture) + "f") && dump.Contains("1.2345678901234567d") && dump.Contains((1.7e308d).ToString("G17", System.Globalization.CultureInfo.InvariantCulture) + "d"), "Recording reader retains finite high precision/exponent binary values"); + File.WriteAllText(path, "KEEP ORIGINAL"); source.NegativeZero = true; bool rejected = false; + try { new OracleSqlExportService(source, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); } catch (NotSupportedException) { rejected = true; } + Assert(rejected && File.ReadAllText(path) == "KEEP ORIGINAL", "Signed-zero source failure prevents publication"); + } + finally { File.Delete(path); } + } + + private static void OracleExport_ScopedNativeViewsAndDisabledForeignKeys() + { + string directory = Path.Combine(Path.GetTempPath(), "ora_view_fk_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + string path = Path.Combine(directory, "schema.sql"); + try + { + var source = new MockOracleExportDatabase { NativeView = true }; + var options = new OracleExportWizardOptions(OracleExportMode.SchemaOnly, path); options.SelectedTables.Add("PARCELS"); options.SelectedTables.Add("V"); + new OracleSqlExportService(source, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); + string dump = File.ReadAllText(path); + Assert(dump.Contains("CREATE OR REPLACE FORCE EDITIONABLE VIEW \"SCOTT\".\"V\""), "Representative native view DDL exported"); + var destination = new MockOracleImportDatabase(); + Assert(new mySQLPunk.lib.OracleImport.OracleImportRunner(destination, new mySQLPunk.lib.OracleImport.OracleImportRequest(path, "SCOTT")).Run().Success, "Exported native view grammar passes real import prepass"); + foreach (string changed in new[] { dump.Replace("VIEW \"SCOTT\".\"V\"", "VIEW \"A\".\"V\""), dump.Replace("SELECT \"ID\" FROM \"SCOTT\".\"PARCELS\"", "SELECT \"ID\" FROM \"A\".\"PARCELS\"") }) + { + File.WriteAllText(path, changed); var rejected = new MockOracleImportDatabase(); + Assert(!new mySQLPunk.lib.OracleImport.OracleImportRunner(rejected, new mySQLPunk.lib.OracleImport.OracleImportRequest(path, "SCOTT")).Run().Success && rejected.ExecutedQueries.Count == 0, "Native view owner/reference mismatch fails before session/mutations"); + } + source.JoinedNativeView = true; + new OracleSqlExportService(source, "SCOTT").ExportAsync(options).GetAwaiter().GetResult(); + var joined = new MockOracleImportDatabase(); + Assert(new mySQLPunk.lib.OracleImport.OracleImportRunner(joined, new mySQLPunk.lib.OracleImport.OracleImportRequest(path, "SCOTT")).Run().Success, "Native FORCE/EDITIONABLE joined view does not mistake ON aliases for owners"); + string joinedDump = File.ReadAllText(path).Replace("JOIN \"SCOTT\".\"PARCELS\" b", "JOIN \"A\".\"PARCELS\" b"); + File.WriteAllText(path, joinedDump); var joinedWrongOwner = new MockOracleImportDatabase(); + Assert(!new mySQLPunk.lib.OracleImport.OracleImportRunner(joinedWrongOwner, new mySQLPunk.lib.OracleImport.OracleImportRequest(path, "SCOTT")).Run().Success && joinedWrongOwner.ExecutedQueries.Count == 0, "Joined native view still validates actual referenced owner"); + var dependency = new MockOracleExportDatabase { DisabledForeignKey = true, AddParent = true }; + var schema = new OracleExportWizardOptions(OracleExportMode.SchemaOnly, path); schema.SelectedTables.Add("PARCELS"); schema.SelectedTables.Add("PARENT"); + new OracleSqlExportService(dependency, "SCOTT").ExportAsync(schema).GetAwaiter().GetResult(); + string ordered = File.ReadAllText(path); + Assert(ordered.IndexOf("CREATE TABLE \"SCOTT\".\"PARENT\"", StringComparison.Ordinal) < ordered.IndexOf("CREATE TABLE \"SCOTT\".\"PARCELS\"", StringComparison.Ordinal), "Disabled FK parent DDL precedes child DDL"); + Assert(ordered.Contains("DISABLE") && dependency.Commands.Exists(command => command.Contains("FROM ALL_CONSTRAINTS c JOIN") && !command.Contains("c.STATUS='ENABLED'")), "All schema FK definitions participate, preserving disabled DDL"); + foreach (int failure in new[] { 0, 1, 2 }) + { + var refused = new MockOracleExportDatabase { DisabledForeignKey = true, AddParent = failure != 0, CrossOwnerForeignKey = failure == 1, ForeignKeyCycle = failure == 2 }; + File.WriteAllText(path, "KEEP ORIGINAL"); bool rejected = false; + try { new OracleSqlExportService(refused, "SCOTT").ExportAsync(schema).GetAwaiter().GetResult(); } catch (NotSupportedException) { rejected = true; } + Assert(rejected && File.ReadAllText(path) == "KEEP ORIGINAL", "Disabled missing/cross-owner/cyclic FK rejects before publication"); + } + } + finally { Directory.Delete(directory, true); } + } + + // DataTable's DoubleStorage normalizes signed zero. This recording reader returns the provider bit pattern explicitly. + private sealed class OracleSignedZeroReader : IDataReader + { + private readonly IDataReader inner; + public OracleSignedZeroReader(IDataReader inner) { this.inner = inner; } + public object GetValue(int i) { object value = inner.GetValue(i); return i == 1 && value is double && (double)value == 0d ? BitConverter.Int64BitsToDouble(long.MinValue) : value; } + public object this[int i] => GetValue(i); + public object this[string name] => inner[name]; + public int Depth => inner.Depth; + public bool IsClosed => inner.IsClosed; + public int RecordsAffected => inner.RecordsAffected; + public int FieldCount => inner.FieldCount; + public void Close() => inner.Close(); + public void Dispose() => inner.Dispose(); + public bool Read() => inner.Read(); + public bool NextResult() => inner.NextResult(); + public DataTable GetSchemaTable() => inner.GetSchemaTable(); + public bool GetBoolean(int i) => inner.GetBoolean(i); + public byte GetByte(int i) => inner.GetByte(i); + public long GetBytes(int i, long offset, byte[] buffer, int start, int length) => inner.GetBytes(i, offset, buffer, start, length); + public char GetChar(int i) => inner.GetChar(i); + public long GetChars(int i, long offset, char[] buffer, int start, int length) => inner.GetChars(i, offset, buffer, start, length); + public IDataReader GetData(int i) => inner.GetData(i); + public string GetDataTypeName(int i) => inner.GetDataTypeName(i); + public DateTime GetDateTime(int i) => inner.GetDateTime(i); + public decimal GetDecimal(int i) => inner.GetDecimal(i); + public double GetDouble(int i) => (double)GetValue(i); + public Type GetFieldType(int i) => inner.GetFieldType(i); + public float GetFloat(int i) => inner.GetFloat(i); + public Guid GetGuid(int i) => inner.GetGuid(i); + public short GetInt16(int i) => inner.GetInt16(i); + public int GetInt32(int i) => inner.GetInt32(i); + public long GetInt64(int i) => inner.GetInt64(i); + public string GetName(int i) => inner.GetName(i); + public int GetOrdinal(string name) => inner.GetOrdinal(name); + public string GetString(int i) => inner.GetString(i); + public int GetValues(object[] values) { int count = inner.GetValues(values); for (int i = 0; i < count; i++) values[i] = GetValue(i); return count; } + public bool IsDBNull(int i) => inner.IsDBNull(i); + } + + private sealed class OracleProgressObserver : IProgress + { + private readonly Action callback; + public OracleProgressObserver(Action callback) { this.callback = callback; } + public void Report(OracleExportProgress progress) { callback(progress); } + } + + private sealed class MockOracleExportDatabase : IDatabase, IOracleExportReaderSource { public ConnectionState State => ConnectionState.Open; public string ProviderName => "oracle"; + public List Commands = new List(); + public string FailCommand, TextValue; + public bool Identity, VirtualColumn, EmptyLobs, ForeignKeyCycle, MissingForeignKeyParent; + public bool AllLargeLobRows, FloatingValues, NegativeZero, NativeView, JoinedNativeView, DisabledForeignKey, AddParent, CrossOwnerForeignKey; + public int GeometryBytes = 21; + public object GeometrySrid = 3826; + public bool NullMetadataDimensions; + public Action BeforeReaderOpen; + public IDataReader Reader; + public IDataReader OpenExportReader(string sql, CancellationToken token) + { + Commands.Add(sql); + BeforeReaderOpen?.Invoke(); + var data = new DataTable(); + if (FloatingValues) + { + data.Columns.Add("F", typeof(float)); data.Columns.Add("D", typeof(double)); + data.Rows.Add(1.2345678f, NegativeZero ? BitConverter.Int64BitsToDouble(long.MinValue) : 1.2345678901234567d); + data.Rows.Add(1.7e30f, 1.7e308d); + Reader = NegativeZero ? (IDataReader)new OracleSignedZeroReader(data.CreateDataReader()) : data.CreateDataReader(); return Reader; + } + data.Columns.Add("ID", typeof(int)); data.Columns.Add("NAME", typeof(char[])); data.Columns.Add("SHAPE", typeof(byte[])); + data.Columns.Add("SRID", typeof(int)); data.Columns.Add("GTYPE", typeof(int)); + if (EmptyLobs) + { + data.Columns.Add("BINARY", typeof(byte[])); + data.Rows.Add(1, new char[0], new byte[GeometryBytes], GeometrySrid ?? DBNull.Value, 3001, new byte[0]); + data.Rows.Add(2, DBNull.Value, new byte[21], DBNull.Value, 2001, DBNull.Value); + } + else + { + data.Rows.Add(1, (TextValue ?? "宗地一").ToCharArray(), new byte[GeometryBytes], GeometrySrid ?? DBNull.Value, 3001); + data.Rows.Add(2, (AllLargeLobRows ? TextValue : "宗地二").ToCharArray(), new byte[AllLargeLobRows ? GeometryBytes : 21], DBNull.Value, 2001); + } + Reader = data.CreateDataReader(); + return Reader; + } public void Open() { } public void Close() { } public void Dispose() { } public DataTable SelectSQL(string sql, Dictionary parameters = null) { + Commands.Add(sql); var dt = new DataTable(); - + if (sql.Contains("FROM ALL_CONSTRAINTS c JOIN")) + { + dt.Columns.Add("CHILD_TABLE"); dt.Columns.Add("PARENT_OWNER"); dt.Columns.Add("PARENT_TABLE"); + if (DisabledForeignKey && sql.Contains("c.STATUS='ENABLED'")) return dt; + if (ForeignKeyCycle || MissingForeignKeyParent || DisabledForeignKey) dt.Rows.Add("PARCELS", CrossOwnerForeignKey ? "A" : "SCOTT", ForeignKeyCycle ? "PARCELS" : MissingForeignKeyParent ? "UNSELECTED_PARENT" : "PARENT"); + return dt; + } + if (sql.Contains("TABLE(m.DIMINFO)")) + { + dt.Columns.Add("DIM_ORD", typeof(int)); dt.Columns.Add("DIM_NAME"); dt.Columns.Add("LOWER_BOUND", typeof(decimal)); dt.Columns.Add("UPPER_BOUND", typeof(decimal)); dt.Columns.Add("TOLERANCE", typeof(decimal)); + dt.Rows.Add(1, "E", 0m, 1000000m, 0.01m); dt.Rows.Add(2, "N", 0m, 4000000m, 0.02m); dt.Rows.Add(3, "Z", -100m, 9000m, 0.1m); return dt; + } + if (sql.Contains("FROM ALL_SDO_GEOM_METADATA")) + { + dt.Columns.Add("SRID", typeof(int)); dt.Columns.Add("DIMINFO_NULL", typeof(int)); dt.Columns.Add("DIM_COUNT", typeof(int)); + dt.Rows.Add(GeometrySrid ?? DBNull.Value, NullMetadataDimensions ? 1 : 0, NullMetadataDimensions ? (object)DBNull.Value : 3); return dt; + } if (sql.Contains("FROM ALL_TABLES")) { dt.Columns.Add("OWNER", typeof(string)); @@ -129,6 +412,7 @@ public DataTable SelectSQL(string sql, Dictionary parameters = n dt.Rows.Add("SCOTT", "PARCELS", 2L, 65536L); dt.Rows.Add("SCOTT", "MDRT_12345$", 50L, 16384L); + if (AddParent) dt.Rows.Add("SCOTT", "PARENT", 0L, 0L); return dt; } @@ -136,6 +420,7 @@ public DataTable SelectSQL(string sql, Dictionary parameters = n { dt.Columns.Add("OWNER", typeof(string)); dt.Columns.Add("VIEW_NAME", typeof(string)); + if (NativeView) dt.Rows.Add("SCOTT", "V"); return dt; } @@ -152,14 +437,8 @@ public DataTable SelectSQL(string sql, Dictionary parameters = n if (sql.Contains("FROM \"SCOTT\".\"PARCELS\"")) { - dt.Columns.Add("ID", typeof(int)); - dt.Columns.Add("NAME", typeof(string)); - dt.Columns.Add("SHAPE", typeof(byte[])); + throw new InvalidOperationException("Full-table DataTable must never be used for rows."); - byte[] wkbSample = new byte[] { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F }; - dt.Rows.Add(1, "宗地一", wkbSample); - dt.Rows.Add(2, "宗地二", wkbSample); - return dt; } return dt; @@ -167,6 +446,8 @@ public DataTable SelectSQL(string sql, Dictionary parameters = n public Dictionary ExecSQL(string sql, Dictionary parameters = null) { + Commands.Add(sql); + if (FailCommand != null && sql.Contains(FailCommand)) return new Dictionary { { "status", "NO" }, { "reason", "Simulated source command failure" } }; return new Dictionary { { "status", "OK" }, { "rowsAffected", "1" } }; } @@ -178,11 +459,16 @@ public Dictionary ExecSQL(string sql, Dictionary public DataTable GetColumns(string databaseName, string tableName) { var dt = new DataTable(); + if (FloatingValues) { dt.Columns.Add("COLUMN_NAME"); dt.Columns.Add("DATA_TYPE"); dt.Rows.Add("F", "BINARY_FLOAT"); dt.Rows.Add("D", "BINARY_DOUBLE"); return dt; } dt.Columns.Add("COLUMN_NAME", typeof(string)); dt.Columns.Add("DATA_TYPE", typeof(string)); + dt.Columns.Add("VIRTUAL_COLUMN"); dt.Columns.Add("IDENTITY_COLUMN"); dt.Columns.Add("IDENTITY_GENERATION"); dt.Rows.Add("ID", "NUMBER"); - dt.Rows.Add("NAME", "VARCHAR2"); + dt.Rows.Add("NAME", TextValue == null && !EmptyLobs ? "VARCHAR2" : "CLOB"); dt.Rows.Add("SHAPE", "SDO_GEOMETRY"); + if (Identity) { dt.Rows[0]["IDENTITY_COLUMN"] = "YES"; dt.Rows[0]["IDENTITY_GENERATION"] = "ALWAYS"; } + if (VirtualColumn) dt.Rows.Add("CALC", "NUMBER", "YES"); + if (EmptyLobs) dt.Rows.Add("BINARY", "BLOB"); return dt; } public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); @@ -190,7 +476,9 @@ public DataTable GetColumns(string databaseName, string tableName) public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); public string GetTableCreateStatement(string databaseName, string tableName) { - return "CREATE TABLE \"SCOTT\".\"PARCELS\" (\"ID\" NUMBER, \"NAME\" VARCHAR2(50), \"SHAPE\" MDSYS.SDO_GEOMETRY);"; + if (tableName == "PARENT") return "CREATE TABLE \"SCOTT\".\"PARENT\" (\"ID\" NUMBER PRIMARY KEY);"; + if (DisabledForeignKey) return "CREATE TABLE \"SCOTT\".\"PARCELS\" (\"ID\" NUMBER, CONSTRAINT \"FK_DISABLED\" FOREIGN KEY (\"ID\") REFERENCES \"SCOTT\".\"PARENT\"(\"ID\") DISABLE);"; + return "CREATE TABLE \"SCOTT\".\"PARCELS\" (\"ID\" NUMBER, \"NAME\" VARCHAR2(50), \"SHAPE\" MDSYS.SDO_GEOMETRY); COMMENT ON TABLE \"SCOTT\".\"PARCELS\" IS 'actual comment'; CREATE INDEX \"SCOTT\".\"IX_NAME\" ON \"SCOTT\".\"PARCELS\" (\"NAME\"); CREATE INDEX \"SCOTT\".\"REAL_SPATIAL_INDEX\" ON \"SCOTT\".\"PARCELS\" (\"SHAPE\") INDEXTYPE IS MDSYS.SPATIAL_INDEX;"; } public bool TableExists(string databaseName, string tableName) => true; public bool ViewExists(string databaseName, string viewName) => false; @@ -205,7 +493,7 @@ public void DropTableForCopy(string databaseName, string tableName) { } public void CreateIndexesForCopy(string databaseName, string tableName, DataTable sourceIndexes, string sourceProvider) { } public DataTable SelectTablePage(string databaseName, string tableName, long offset, int limit) => new DataTable(); public void InsertTableBatch(string databaseName, string tableName, DataTable rows) { } - public string GetViewCreateStatement(string databaseName, string viewName) => ""; + public string GetViewCreateStatement(string databaseName, string viewName) => NativeView ? (JoinedNativeView ? "CREATE OR REPLACE FORCE EDITIONABLE VIEW \"SCOTT\".\"V\" AS SELECT a.\"ID\" FROM \"SCOTT\".\"PARCELS\" a JOIN \"SCOTT\".\"PARCELS\" b ON a.\"ID\"=b.\"ID\";" : "CREATE OR REPLACE FORCE EDITIONABLE VIEW \"SCOTT\".\"V\" AS SELECT \"ID\" FROM \"SCOTT\".\"PARCELS\";") : ""; public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } } } diff --git a/tests/SmokeTests.OracleImport.cs b/tests/SmokeTests.OracleImport.cs index f91e8c3..0300172 100644 --- a/tests/SmokeTests.OracleImport.cs +++ b/tests/SmokeTests.OracleImport.cs @@ -117,7 +117,7 @@ private static void OracleImport_Runner_ExecutionAndManifestVerification() Assert(File.Exists(result.LogFilePath), "Independent log file must exist: " + result.LogFilePath); // 驗證交易包覆有正確 COMMIT - Assert(mockDb.ExecutedQueries.Contains("COMMIT;"), "Transaction COMMIT must be executed"); + Assert(mockDb.ExecutedQueries.Contains("COMMIT"), "Transaction COMMIT must be executed"); } finally { @@ -159,7 +159,7 @@ private static void OracleImport_Runner_ExistingObjectStrategies() var runner3 = new OracleImportRunner(mockDb3, new OracleImportRequest(tempFile, "SCOTT", opt3)); runner3.Run(); - bool hasTruncate = mockDb3.ExecutedQueries.Exists(q => q.Contains("TRUNCATE TABLE \"SCOTT\".\"ROADS\";")); + bool hasTruncate = mockDb3.ExecutedQueries.Exists(q => q.Contains("TRUNCATE TABLE \"SCOTT\".\"ROADS\"")); Assert(hasTruncate, "TruncateAndInsert should execute TRUNCATE TABLE"); } finally @@ -175,12 +175,12 @@ private static void OracleImport_Runner_TransactionRollbackOnFailure() { string sql = @" CREATE TABLE ""SCOTT"".""VALID"" (""ID"" NUMBER); -INVALID SQL SYNTAX HERE; +INSERT INTO ""SCOTT"".""VALID"" VALUES (1); "; File.WriteAllText(tempFile, sql, System.Text.Encoding.UTF8); var mockDb = new MockOracleImportDatabase(); - mockDb.FailOnSql = "INVALID SQL"; + mockDb.FailOnSql = "INSERT INTO"; var opt = new OracleImportOptions { @@ -193,7 +193,7 @@ private static void OracleImport_Runner_TransactionRollbackOnFailure() var result = runner.Run(); Assert(!result.Success, "Runner should report failure"); - Assert(mockDb.ExecutedQueries.Contains("ROLLBACK;"), "Transaction ROLLBACK must be executed upon statement failure"); + Assert(mockDb.ExecutedQueries.Contains("ROLLBACK"), "Transaction ROLLBACK must be executed upon statement failure"); } finally { @@ -201,6 +201,153 @@ private static void OracleImport_Runner_TransactionRollbackOnFailure() } } + private static void OracleImport_ReturnedNoAndTransactionControls() + { + string path = Path.Combine(Path.GetTempPath(), "ora_no_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + foreach (string failure in new[] { "INSERT", "COMMIT", "TRUNCATE", "DROP TABLE" }) + { + File.WriteAllText(path, failure == "DROP TABLE" ? "CREATE TABLE T (ID NUMBER);" : "INSERT INTO T VALUES (1);"); + var db = new MockOracleImportDatabase { FailOnSql = failure }; + var options = new OracleImportOptions { ConflictStrategy = failure == "TRUNCATE" ? OracleConflictStrategy.TruncateAndInsert : failure == "DROP TABLE" ? OracleConflictStrategy.DropAndRecreate : OracleConflictStrategy.KeepExisting }; + var result = new OracleImportRunner(db, new OracleImportRequest(path, "SCOTT", options)).Run(); + Assert(!result.Success && result.StatementsFailed > 0, "Returned NO must fail " + failure); + Assert(result.ErrorMessage.Contains("Simulated"), "Provider failure reason retained: " + failure); + Assert(db.ExecutedQueries.Contains("SET TRANSACTION READ WRITE"), "Transaction option requests a transaction"); + Assert(db.ExecutedQueries.Contains("ROLLBACK"), "Failed pending transaction rolls back"); + } + File.WriteAllText(path, "INSERT INTO T VALUES (1);"); + var noTransaction = new MockOracleImportDatabase(); + Assert(new OracleImportRunner(noTransaction, new OracleImportRequest(path, "SCOTT", new OracleImportOptions { UseTransaction = false })).Run().Success, "Nontransaction import succeeds"); + Assert(!noTransaction.ExecutedQueries.Contains("SET TRANSACTION READ WRITE") && !noTransaction.ExecutedQueries.Contains("COMMIT"), "Unchecked transaction option does not force a transaction"); + File.WriteAllText(path, "SET TRANSACTION READ WRITE; INSERT INTO T VALUES (1);"); + using (var scriptCancel = new CancellationTokenSource()) + { + var scriptOwned = new MockOracleImportDatabase(); + scriptOwned.AfterExecute = sql => { if (sql.StartsWith("INSERT")) scriptCancel.Cancel(); }; + var scriptResult = new OracleImportRunner(scriptOwned, new OracleImportRequest(path, "SCOTT", new OracleImportOptions { UseTransaction = false })).Run(null, scriptCancel.Token); + Assert(!scriptResult.Success && scriptOwned.ExecutedQueries.Contains("ROLLBACK"), "Script-owned transaction is cleaned after cancellation with option unchecked"); + } + File.WriteAllText(path, "INSERT INTO T VALUES (1);"); + var rollbackNo = new MockOracleImportDatabase { FailOnSql = "ROLLBACK" }; + using (var cancel = new CancellationTokenSource()) + { + rollbackNo.AfterExecute = sql => { if (sql.StartsWith("INSERT")) cancel.Cancel(); }; + var result = new OracleImportRunner(rollbackNo, new OracleImportRequest(path, "SCOTT")).Run(null, cancel.Token); + Assert(!result.Success && result.ErrorMessage.Contains("rollback failed"), "Cancellation rollback NO is retained"); + } + } + finally { File.Delete(path); } + } + + private static void OracleImport_TargetOwnerPreflightAndCancellation() + { + string path = Path.Combine(Path.GetTempPath(), "ora_owner_" + Guid.NewGuid().ToString("N") + ".sql"); + try + { + foreach (string sql in new[] { "INSERT INTO A.T VALUES (1);", "CREATE INDEX B.I ON A.T(ID);", "CREATE TABLE B.T(ID NUMBER REFERENCES A.T(ID));", "COMMENT ON COLUMN A.T.ID IS 'x';", "ALTER SESSION SET CURRENT_SCHEMA=A;", "BEGIN A.P(); END;\n/" }) + { + File.WriteAllText(path, sql); + var db = new MockOracleImportDatabase(); + var result = new OracleImportRunner(db, new OracleImportRequest(path, "B")).Run(); + Assert(!result.Success && db.ExecutedQueries.Count == 0, "Cross-owner/ambiguous SQL rejected before execution: " + sql); + } + File.WriteAllText(path, "DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME='T'; INSERT INTO T VALUES (1);"); + var noMetadata = new MockOracleImportDatabase(); + var noMetadataResult = new OracleImportRunner(noMetadata, new OracleImportRequest(path, "B", new OracleImportOptions { AutoRegisterSpatialMetadata = false })).Run(); + Assert(noMetadataResult.Success && !noMetadata.ExecutedQueries.Exists(command => command.StartsWith("DELETE FROM USER_SDO_GEOM_METADATA")), "Unchecked spatial registration skips script metadata mutations"); + File.WriteAllText(path, "-- A.T stays comment\nINSERT INTO T VALUES ('A.T; literal');\nINSERT INTO T VALUES (2);"); + using (var cancel = new CancellationTokenSource()) + { + var db = new MockOracleImportDatabase(); + db.AfterExecute = sql => { if (sql.StartsWith("INSERT")) cancel.Cancel(); }; + var result = new OracleImportRunner(db, new OracleImportRequest(path, "B")).Run(null, cancel.Token); + Assert(!result.Success, "Cancelled import fails"); + Assert(db.ExecutedQueries[0] == "ALTER SESSION SET CURRENT_SCHEMA = \"B\"", "Chosen owner pinned before writes"); + Assert(db.ExecutedQueries.Exists(sql => sql.Contains("'A.T; literal'")), "Owner-like literal remains unchanged"); + Assert(db.ExecutedQueries.Contains("ROLLBACK") && !db.ExecutedQueries.Contains("COMMIT"), "Cancellation cleans pending transaction without commit"); + } + } + finally { File.Delete(path); } + } + + private static void OracleImport_ParserBytesCommentsAndAlternativeQuotes() + { + string sql = "/*x*/ SELECT '中文' FROM dual;\r\nINSERT INTO T VALUES (q'[I'm;here]');\nBEGIN\nNULL;\nEND;\n/\n"; + var parse = typeof(OracleImportRunner).GetMethod("EnumerateStatements", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic); + using (var reader = new StreamReader(new MemoryStream(System.Text.Encoding.UTF8.GetBytes(sql)), System.Text.Encoding.UTF8)) + { + var statements = (System.Collections.IEnumerable)parse.Invoke(null, new object[] { reader, CancellationToken.None, null }); + int count = 0; long bytes = 0; + foreach (object item in statements) + { + string statement = (string)item.GetType().GetField("Sql").GetValue(item); + int length = (int)item.GetType().GetField("ByteLength").GetValue(item); + int line = (int)item.GetType().GetField("LineNumber").GetValue(item); + Assert(length > 0, "Terminated statement has nonzero byte count"); bytes += length; + if (count == 0) Assert(statement == "SELECT '中文' FROM dual" && line == 1, "Comment discarded and SQL terminator removed"); + if (count == 1) Assert(statement.Contains("q'[I'm;here]'") && line == 2, "Alternative quoted semicolon stays in one statement"); + if (count == 2) Assert(statement == "BEGIN\nNULL;\nEND;" && line == 3, "PL/SQL keeps terminal semicolon and original start line"); + count++; + } + Assert(count == 3 && bytes == System.Text.Encoding.UTF8.GetByteCount(sql), "Actual UTF-8/CRLF source bytes accounted"); + } + } + + private static void OracleImport_ScopedTargetVariantsMetadataAndLobTruncate() + { + string directory = Path.Combine(Path.GetTempPath(), "ora_scoped_targets_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + string path = Path.Combine(directory, "import.sql"); + try + { + foreach (string sql in new[] { "DELETE A.T WHERE ID=1;", "CREATE UNIQUE INDEX A.I ON B.T(ID);", "CREATE BITMAP INDEX A.I ON B.T(ID);", "INSERT INTO B.T@REMOTE VALUES (1);", "DELETE B.T@REMOTE WHERE ID=1;", "DELETE FROM ONLY(A.V) WHERE ID=1;", "UPDATE ONLY(A.V) SET ID=1;" }) + { + File.WriteAllText(path, sql); var database = new MockOracleImportDatabase(); + var result = new OracleImportRunner(database, new OracleImportRequest(path, "B")).Run(); + Assert(!result.Success && database.ExecutedQueries.Count == 0, "Direct cross-owner/database-link variant fails before session/mutations: " + sql); + } + File.WriteAllText(path, "/*A.T @REMOTE*/ DELETE B.T WHERE NAME='A.T @REMOTE'; CREATE UNIQUE INDEX B.I ON B.T(ID); CREATE BITMAP INDEX B.J ON B.T(ID);"); + var sameOwner = new MockOracleImportDatabase(); + Assert(new OracleImportRunner(sameOwner, new OracleImportRequest(path, "B")).Run().Success, "Same-owner optional FROM and modified indexes accepted"); + Assert(sameOwner.ExecutedQueries.Exists(sql => sql.Contains("'A.T @REMOTE'")), "Literal/comment owner and at-sign text does not trigger remapping/rejection"); + foreach (string sql in new[] { "INSERT INTO \"USER_SDO_GEOM_METADATA\" (TABLE_NAME) VALUES ('T');", "DELETE \"USER_SDO_GEOM_METADATA\" WHERE TABLE_NAME='T';", "DELETE USER_SDO_GEOM_METADATA WHERE TABLE_NAME='T';", "UPDATE \"USER_SDO_GEOM_METADATA\" SET SRID=NULL;" }) + { + File.WriteAllText(path, sql); + var wrongUser = new MockOracleImportDatabase { SessionUser = "SCOTT" }; + Assert(!new OracleImportRunner(wrongUser, new OracleImportRequest(path, "B")).Run().Success, "Every metadata spelling enforces SESSION_USER"); + Assert(wrongUser.ExecutedQueries.Count == 1 && wrongUser.ExecutedQueries[0].Contains("SESSION_USER"), "Metadata mismatch fails before ALTER SESSION/mutation"); + var disabled = new MockOracleImportDatabase { SessionUser = "SCOTT" }; + var options = new OracleImportOptions { AutoRegisterSpatialMetadata = false }; + Assert(new OracleImportRunner(disabled, new OracleImportRequest(path, "B", options)).Run().Success, "Every metadata spelling is skipped when registration disabled"); + Assert(!disabled.ExecutedQueries.Exists(command => command.Contains("USER_SDO_GEOM_METADATA") || command.Contains("SESSION_USER")), "Disabled metadata spelling executes no registration or USER gate query"); + var ownerUser = new MockOracleImportDatabase { SessionUser = "B" }; + Assert(new OracleImportRunner(ownerUser, new OracleImportRequest(path, "B")).Run().Success, "Matching user can execute quoted/optional FROM metadata target"); + } + foreach (bool allLarge in new[] { false, true }) + { + var source = new MockOracleExportDatabase { TextValue = new string('中', 5000), GeometryBytes = 3000, AllLargeLobRows = allLarge }; + var export = new mySQLPunk.lib.OracleExport.OracleExportWizardOptions(mySQLPunk.lib.OracleExport.OracleExportMode.DataOnly, path) { IncludeManifest = false }; + export.SelectedTables.Add("PARCELS"); + new mySQLPunk.lib.OracleExport.OracleSqlExportService(source, "SCOTT").ExportAsync(export).GetAwaiter().GetResult(); + if (!allLarge) + { + string dump = File.ReadAllText(path); + int small = dump.LastIndexOf("INSERT INTO \"SCOTT\".\"PARCELS\"", StringComparison.Ordinal); + dump = dump.Substring(0, small) + dump.Substring(small).Replace("INSERT INTO \"SCOTT\".\"PARCELS\"", "INSERT INTO PARCELS"); + File.WriteAllText(path, dump); + } + var destination = new MockOracleImportDatabase { TrackRows = true, StoredRows = 9 }; + var restore = new OracleImportRunner(destination, new OracleImportRequest(path, "SCOTT", new OracleImportOptions { ConflictStrategy = OracleConflictStrategy.TruncateAndInsert })).Run(); + Assert(restore.Success && destination.StoredRows == 2, "No-manifest all-large/mixed dump keeps both restored rows"); + int firstTruncate = destination.ExecutedQueries.FindIndex(command => command.StartsWith("TRUNCATE TABLE")); + int firstLob = destination.ExecutedQueries.FindIndex(command => command.StartsWith("DECLARE")); + Assert(firstTruncate >= 0 && firstTruncate < firstLob && destination.ExecutedQueries.FindAll(command => command.StartsWith("TRUNCATE TABLE")).Count == 1, "Normalized truncate runs exactly once before first LOB row"); + } + } + finally { Directory.Delete(directory, true); } + } + private sealed class MockOracleImportDatabase : IDatabase { public ConnectionState State => ConnectionState.Open; @@ -208,6 +355,11 @@ private sealed class MockOracleImportDatabase : IDatabase public List ExecutedQueries { get; } = new List(); public string FailOnSql { get; set; } = null; public bool SimulateExistingObjectError { get; set; } = false; + public Action AfterExecute { get; set; } + public long ActualRows = 1; + public string SessionUser = "SCOTT"; + public bool TrackRows; + public long StoredRows; public void Open() { } public void Close() { } @@ -217,11 +369,12 @@ public DataTable SelectSQL(string sql, Dictionary parameters = n { ExecutedQueries.Add(sql); var dt = new DataTable(); + if (sql.Contains("SESSION_USER")) { dt.Columns.Add("OWNER"); dt.Rows.Add(SessionUser); return dt; } if (sql.Contains("SELECT COUNT(*) FROM \"SCOTT\".\"PARCELS\"")) { dt.Columns.Add("COUNT", typeof(long)); - dt.Rows.Add(1L); + dt.Rows.Add(ActualRows); return dt; } @@ -234,14 +387,20 @@ public Dictionary ExecSQL(string sql, Dictionary if (!string.IsNullOrWhiteSpace(FailOnSql) && sql.Contains(FailOnSql)) { - throw new InvalidOperationException("Simulated syntax error: " + sql); + return new Dictionary { { "status", "NO" }, { "reason", "Simulated syntax error: " + sql } }; } if (SimulateExistingObjectError && sql.StartsWith("CREATE TABLE", StringComparison.OrdinalIgnoreCase)) { - throw new InvalidOperationException("ORA-00955: name is already used by an existing object"); + return new Dictionary { { "status", "NO" }, { "reason", "ORA-00955: name is already used by an existing object" } }; } + if (TrackRows) + { + if (sql.StartsWith("TRUNCATE TABLE")) StoredRows = 0; + if (sql.StartsWith("INSERT") || sql.StartsWith("DECLARE") && sql.Contains("INSERT INTO")) StoredRows++; + } + AfterExecute?.Invoke(sql); long rows = sql.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase) ? 1 : 0; return new Dictionary { diff --git a/tests/SmokeTests.SqliteExport.cs b/tests/SmokeTests.SqliteExport.cs index 7aaca84..eb5f302 100644 --- a/tests/SmokeTests.SqliteExport.cs +++ b/tests/SmokeTests.SqliteExport.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using mySQLPunk.lib; using mySQLPunk.lib.SqliteExport; +using mySQLPunk.lib.SqliteImport; public static partial class SmokeTests { @@ -33,81 +34,146 @@ private static void SqliteExport_OptionsAndModels_Defaults() AssertContains(item.SpatialInfo, "GIS", "SpatialInfo match"); } + private static mySQLPunk.lib.SqliteExport.SqliteExportResult SqliteExportFixture(SqliteFixtureDatabase database,string directory, params string[] tables) + { + var options=new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData,Path.Combine(directory,Guid.NewGuid().ToString("N")+".sql")) + { + SelectedTables=new List(tables), IncludeViews=false, IncludeTriggers=false, IncludeComments=true, + IncludeDropStatements=true, PragmaForeignKeysOff=false, PragmaSynchronousOff=false + }; + return new SqliteSqlExportService(database,"main").ExportAsync(options,null,CancellationToken.None).GetAwaiter().GetResult(); + } private static void SqliteExport_ServiceAndManifest_StreamingAndHexBlob() { - string tempDir = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_export_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tempDir); - string targetFile = Path.Combine(tempDir, "sqlite_export_test.sql"); - + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_export_native_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); try { - var mockDb = new MockSqliteExportDatabase(); - var service = new SqliteSqlExportService(mockDb, "gis_demo.sqlite"); - - // 1. 驗證 LoadTableCatalog 正確辨識空間圖層並過濾影子表 - var catalog = service.LoadTableCatalog(); - Assert(catalog.Count == 1, "Catalog should filter out shadow tables like idx_roads_geom_node, expected 1 table but got " + catalog.Count); - Assert(catalog[0].Name == "spatial_roads", "Export item table name match"); - Assert(catalog[0].IsSpatial, "spatial_roads should be recognized as spatial layer"); - AssertContains(catalog[0].SpatialInfo, "SpatiaLite", "Spatial info contains provider note"); - - var options = new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData, targetFile) + using(var source=new SqliteFixtureDatabase()) using(var target=new SqliteFixtureDatabase()) { - SingleTransaction = true, - PragmaForeignKeysOff = true, - PragmaSynchronousOff = true, - IncludeDropStatements = true, - IncludeIndexes = true, - IncludeViews = true, - IncludeTriggers = true - }; - options.SelectedTables.Add("spatial_roads"); - options.SelectedViews.Add("v_roads"); - options.SelectedTriggers.Add("trg_roads_audit"); - - SqliteExportProgress lastProgress = null; - var progress = new Progress(p => lastProgress = p); - - var task = service.ExportAsync(options, progress, CancellationToken.None); - task.Wait(); - var result = task.Result; - - Assert(result.Success, "Export should succeed"); - Assert(File.Exists(targetFile), "Target file must exist after atomic move"); - Assert(result.TableCount == 1, "Exported table count should be 1"); - Assert(result.ViewCount == 1, "Exported view count should be 1"); - Assert(result.TriggerCount == 1, "Exported trigger count should be 1"); - Assert(result.IndexCount == 1, "Exported index count should be 1"); - Assert(result.TotalRows == 2, "Exported row count should be 2"); - - string content = File.ReadAllText(targetFile); - AssertContains(content, "-- mySQLPunk SQLite Logical Dump", "Header comment present"); - AssertContains(content, "PRAGMA foreign_keys = OFF;", "Pragma foreign_keys OFF present"); - AssertContains(content, "PRAGMA synchronous = OFF;", "Pragma synchronous OFF present"); - AssertContains(content, "BEGIN TRANSACTION;", "BEGIN TRANSACTION present"); - AssertContains(content, "DROP TABLE IF EXISTS \"spatial_roads\";", "DROP TABLE statement present"); - AssertContains(content, "CREATE TABLE \"spatial_roads\"", "CREATE TABLE DDL present"); - AssertNotContains(content, "CREATE TABLE \"idx_roads_geom_node\"", "Shadow table should NEVER be exported as CREATE TABLE"); - - // 驗證 BLOB 幾何二進位輸出為 X'...' 格式，無損精度 - AssertContains(content, "X'00010000007C0101000000'", "GIS binary geometry must be exported as X'...' hex literal"); - AssertContains(content, "COMMIT;", "COMMIT transaction present"); - AssertContains(content, "PRAGMA foreign_keys = ON;", "Restore foreign_keys ON present"); - - // 驗證 Native Manifest - AssertContains(content, "-- mySQLPunk Native Manifest", "Native Manifest header present"); - AssertContains(content, "-- Tables: 1", "Manifest total tables count present"); - AssertContains(content, "-- TotalRows: 2", "Manifest total rows count present"); - AssertContains(content, "-- Manifest-Table: spatial_roads | Rows: 2", "Manifest table row entry present"); - AssertContains(content, "-- Manifest-SHA256:", "Manifest SHA-256 present"); + source.Must("CREATE TABLE roads(id INTEGER PRIMARY KEY,name TEXT,geom BLOB); INSERT INTO roads VALUES(1,'Main Street',X'00010000007C0101000000'); INSERT INTO roads VALUES(2,'Broadway',X''); CREATE VIEW v_roads AS SELECT id,name FROM roads; CREATE TRIGGER tr_roads AFTER INSERT ON roads BEGIN SELECT CASE WHEN new.id>0 THEN 'END;' ELSE 'x' END; END; CREATE INDEX idx_roads_name ON roads(name);"); + var service=new SqliteSqlExportService(source,"main"); + Assert(service.LoadTableCatalog().Count==1,"Catalog has ordinary table only"); + string file=Path.Combine(directory,"roads.sql"); + var options=new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData,file) { SelectedTables=new List{"roads"}, SelectedViews=service.LoadViews(),SelectedTriggers=service.LoadTriggers() }; + var result=service.ExportAsync(options,null,CancellationToken.None).GetAwaiter().GetResult(); + Assert(result.Success,"Actual native SQLite export succeeds: "+result.ErrorMessage); + Assert(result.TableCount==1 && result.TotalRows==2 && result.ViewCount==1 && result.TriggerCount==1 && result.IndexCount==1,"Native object and row counts match"); + string content=File.ReadAllText(file); + AssertContains(content,"X'00010000007C0101000000'","Binary geometry bytes use hex without conversion"); + AssertContains(content,"-- Manifest-Payload-SHA256:","Payload checksum scope is explicit"); + Assert(result.FileBytes==new FileInfo(file).Length,"FileBytes describe delivered bytes including footer"); + using(var sha=System.Security.Cryptography.SHA256.Create()) + AssertEquals(BitConverter.ToString(sha.ComputeHash(File.ReadAllBytes(file))).Replace("-","").ToLowerInvariant(),result.Sha256Hash,"Result hash matches external full-file checksum"); + Assert(SqliteImportPreflightScanner.Scan(file,CancellationToken.None).Passed,"Own native dump checksum validates before writes"); + var restored=SqliteRunFixture(target,directory,content); + Assert(restored.Status==SqliteImportStatus.Completed && restored.ManifestVerified,"Native export imports and row manifest verifies: "+string.Join(";",restored.Errors)); + AssertEquals("00010000007C0101000000",Convert.ToString(target.Scalar("SELECT hex(geom) FROM roads WHERE id=1;")),"Blob bytes restored"); + Assert(target.Exists("trigger","tr_roads") && target.Exists("view","v_roads") && target.Exists("index","idx_roads_name"),"Native DDL restored"); + } } - finally + finally { Directory.Delete(directory,true); } + } + private static void SqliteExport_NativeRoundtrip_GeneratedRowidSequenceTextComments() + { + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_roundtrip_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + using(var source=new SqliteFixtureDatabase()) using(var target=new SqliteFixtureDatabase()) + { + source.Must("CREATE TABLE g(a INT,b INT GENERATED ALWAYS AS(a*2),c TEXT,d INT GENERATED ALWAYS AS(a+3) STORED); INSERT INTO g(a,c) VALUES(5,'c'); CREATE TABLE t(x TEXT,y BLOB); INSERT INTO t(rowid,x,y) VALUES(-7,'a'||char(10)||'b'||char(13)||'c'||char(13)||char(10)||'中'||char(0)||char(39),X'00FF01'); INSERT INTO t(rowid,x,y) VALUES(9223372036854775806,'large rowid',X''); CREATE TABLE ai(id INTEGER PRIMARY KEY AUTOINCREMENT,x); INSERT INTO ai VALUES(1,'one'); INSERT INTO ai VALUES(100,'removed'); DELETE FROM ai WHERE id=100; CREATE TABLE empty_ai(id INTEGER PRIMARY KEY AUTOINCREMENT); INSERT INTO empty_ai VALUES(200); DELETE FROM empty_ai; CREATE TABLE wr(k TEXT PRIMARY KEY,v) WITHOUT ROWID /* trailing clause comment */; INSERT INTO wr VALUES('k','v'); CREATE TABLE \"space | name\"(x); INSERT INTO \"space | name\" VALUES(1); CREATE TABLE other(x); CREATE TRIGGER other_tr AFTER INSERT ON other BEGIN SELECT 1; END; CREATE TABLE __mysqlpunk_column_comments(table_name TEXT NOT NULL,column_name TEXT NOT NULL,comment TEXT NOT NULL,PRIMARY KEY(table_name,column_name)); INSERT INTO __mysqlpunk_column_comments VALUES('t','x','line'||char(0)||'note'); INSERT INTO __mysqlpunk_column_comments VALUES('other','x','excluded');"); + var service=new SqliteSqlExportService(source,"main"); + Assert(!service.LoadTableCatalog().Exists(i=>i.Name=="__mysqlpunk_column_comments"),"Sidecar excluded from ordinary table selection"); + string file=Path.Combine(directory,"roundtrip.sql"); + var options=new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData,file) { SelectedTables=new List{"g","t","ai","empty_ai","wr","space | name","__mysqlpunk_column_comments"}, IncludeViews=false,SelectedTriggers=service.LoadTriggers(),PragmaForeignKeysOff=false,PragmaSynchronousOff=false }; + var exported=service.ExportAsync(options,null,CancellationToken.None).GetAwaiter().GetResult(); + Assert(exported.Success,"Generated/rowid/sequence/text/comments export succeeds: "+exported.ErrorMessage); + string content=File.ReadAllText(file); + AssertNotContains(content,"CREATE TRIGGER other_tr","Unselected table trigger is excluded"); + Assert(content.IndexOf('\0')<0,"Text NUL uses lossless SQL representation"); + var restored=SqliteRunFixture(target,directory,content); + Assert(restored.Status==SqliteImportStatus.Completed && restored.ManifestVerified,"Generated/rowid/sequence/comments dump imports: "+string.Join(";",restored.Errors)); + Assert(Convert.ToInt64(target.Scalar("SELECT b+d FROM g;"))==18,"Virtual/stored generated columns recompute"); + AssertEquals(Convert.ToString(source.Scalar("SELECT hex(x)||':'||hex(y) FROM t WHERE rowid=-7;")),Convert.ToString(target.Scalar("SELECT hex(x)||':'||hex(y) FROM t WHERE rowid=-7;")),"Mixed newline/Unicode/quote/NUL/text and BLOB bytes exact"); + Assert(Convert.ToInt64(target.Scalar("SELECT max(rowid) FROM t;"))==9223372036854775806L,"Sparse negative/large implicit rowids survive"); + target.Must("INSERT INTO ai(x) VALUES('next'); INSERT INTO empty_ai DEFAULT VALUES;"); + Assert(Convert.ToInt64(target.Scalar("SELECT max(id) FROM ai;"))==101 && Convert.ToInt64(target.Scalar("SELECT id FROM empty_ai;"))==201,"Deleted and empty AUTOINCREMENT history survives"); + AssertEquals("v",Convert.ToString(target.Scalar("SELECT v FROM wr WHERE k='k';")),"WITHOUT ROWID keys survive"); + Assert(Convert.ToInt64(target.Scalar("SELECT count(*) FROM __mysqlpunk_column_comments;"))==1,"Selected comments exported exactly once"); + AssertEquals(Convert.ToString(source.Scalar("SELECT hex(comment) FROM __mysqlpunk_column_comments WHERE table_name='t';")),Convert.ToString(target.Scalar("SELECT hex(comment) FROM __mysqlpunk_column_comments WHERE table_name='t';")),"Comment NUL bytes survive"); + } + } + finally { Directory.Delete(directory,true); } + } + private static void SqliteExport_Failures_PreserveOutputAndRejectTamper() + { + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_export_fail_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + using(var source=new SqliteFixtureDatabase()) + { + source.Must("CREATE TABLE t(x); INSERT INTO t VALUES('old');"); + foreach(string table in new[]{"missing","t"}) + { + string file=Path.Combine(directory,"old.sql"); File.WriteAllText(file,"previous good dump"); + var options=new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData,file) { SelectedTables=new List{table},IncludeViews=false,IncludeTriggers=false }; + source.FailQueryContains=table=="t" ? "PRAGMA table_xinfo" : null; + var result=new SqliteSqlExportService(source,"main").ExportAsync(options,null,CancellationToken.None).GetAwaiter().GetResult(); + Assert(result.Stage==SqliteExportStage.Failed,"Missing object and QueryError cannot report Completed"); + AssertEquals("previous good dump",File.ReadAllText(file),"Failed source preserves previous dump"); + Assert(source.MCT.AutoCommit,"Failed export releases source snapshot"); + } + source.FailQueryContains=null; + var good=SqliteExportFixture(source,directory,"t"); Assert(good.Success,"Checksum fixture export succeeds"); + byte[] data=File.ReadAllBytes(good.TargetPath); data[5]^=1; File.WriteAllBytes(good.TargetPath,data); + Assert(!SqliteImportPreflightScanner.Scan(good.TargetPath,CancellationToken.None).Passed,"One-byte alteration is detected before writes"); + using(var target=new SqliteFixtureDatabase()) + { + var result=SqliteRunFixture(target,directory,File.ReadAllText(good.TargetPath)); + Assert(result.Status==SqliteImportStatus.Failed && !target.TableExists("main","t"),"Tampered dump cannot mutate target"); + } + source.Must("CREATE TABLE oversized(x BLOB); INSERT INTO oversized VALUES(zeroblob(8388609));"); + var large=SqliteExportFixture(source,directory,"oversized"); + Assert(large.Stage==SqliteExportStage.Failed && !File.Exists(large.TargetPath),"Oversize native row explicitly rejected before publication"); + source.Must("CREATE TABLE shadow(rowid,_rowid_,oid); INSERT INTO shadow VALUES(1,2,3);"); + var shadow=SqliteExportFixture(source,directory,"shadow"); + Assert(shadow.Stage==SqliteExportStage.Failed,"Inaccessible implicit rowid explicitly rejected"); + source.Must("CREATE VIRTUAL TABLE virtual_fixture USING rtree(id,minx,maxx,miny,maxy);"); + var virtualTable=SqliteExportFixture(source,directory,"virtual_fixture"); + Assert(virtualTable.Stage==SqliteExportStage.Failed,"Untested virtual module backup explicitly rejected"); + } + } + finally { Directory.Delete(directory,true); } + } + + private static void SqliteExport_ReviewRegression_RepeatedCloseAndInitialOptions() + { + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_review_options_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try { - if (Directory.Exists(tempDir)) + using(var source=new SqliteFixtureDatabase()) { - try { Directory.Delete(tempDir, true); } catch { } + source.Must("CREATE TABLE t(x); INSERT INTO t VALUES(1); CREATE VIEW v_t AS SELECT x FROM t; CREATE TRIGGER t_external AFTER INSERT ON t BEGIN INSERT INTO missing_dependency VALUES(new.x); END;"); + var initial=new SqliteExportWizardOptions(SqliteExportMode.SchemaAndData,Path.Combine(directory,"one.sql")) { SelectedTables=new List{"t"},IncludeViews=false,IncludeTriggers=false }; + using(var form=SqliteCreateWizard("SqliteExportWizardForm",source,"main",initial)) + { + Assert(!((System.Windows.Forms.CheckBox)SqliteWizardField(form,"includeViews").GetValue(form)).Checked && !((System.Windows.Forms.CheckBox)SqliteWizardField(form,"includeTriggers").GetValue(form)).Checked,"Single-table flags seed the actual wizard controls"); + var load=(Task)form.GetType().GetMethod("LoadCatalogAsync",System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.NonPublic).Invoke(form,null); SqliteWaitWizardTask(load); + var options=(SqliteExportWizardOptions)form.GetType().GetMethod("BuildExportOptions",System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.NonPublic).Invoke(form,null); + Assert(!options.IncludeViews && !options.IncludeTriggers && options.SelectedTables.Count==1 && options.SelectedTables[0]=="t","Options rebuilt from the actual controls retain the single-table scope"); + var result=new SqliteSqlExportService(source,"main").ExportAsync(options,null,CancellationToken.None).GetAwaiter().GetResult(); + Assert(result.Success && result.ViewCount==0 && result.TriggerCount==0,"Actual wizard options omit views and dependency-bearing triggers even in a one-table database"); + string output=File.ReadAllText(result.TargetPath); AssertNotContains(output,"CREATE VIEW","No implicit view DDL"); AssertNotContains(output,"CREATE TRIGGER","No implicit trigger DDL"); + SqliteAssertRepeatedCloseBlocked(form,source); + } + using(var form=SqliteCreateWizard("SqliteExportWizardForm",source,"main",initial)) + { + SqliteWizardField(form,"closing").SetValue(form,true); + var task=(Task)form.GetType().GetMethod("LoadCatalogAsync",System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.NonPublic).Invoke(form,null); SqliteWaitWizardTask(task); + Assert(form.IsDisposed,"Pending close is honored after export catalog worker cleanup"); + } } } + finally { Directory.Delete(directory,true); } } private sealed class MockSqliteExportDatabase : IDatabase diff --git a/tests/SmokeTests.SqliteImport.cs b/tests/SmokeTests.SqliteImport.cs index e5d7c94..f76cb3e 100644 --- a/tests/SmokeTests.SqliteImport.cs +++ b/tests/SmokeTests.SqliteImport.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Data; +using System.Data.SQLite; +using System.Linq; using System.IO; using System.Text; using System.Threading; @@ -35,7 +37,13 @@ private static void SqliteImport_PreflightScanner_SpatiaLiteAndManifest() "-- Manifest-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\r\n" + "-- ========================================================\r\n"; + sqlContent = sqlContent.Replace("-- Manifest-SHA256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\r\n", ""); File.WriteAllText(sqlFile, sqlContent, Encoding.UTF8); + using (var hash=System.Security.Cryptography.SHA256.Create()) + { + string checksum=BitConverter.ToString(hash.ComputeHash(File.ReadAllBytes(sqlFile))).Replace("-", "").ToLowerInvariant(); + File.AppendAllText(sqlFile,"-- Manifest-Payload-SHA256: "+checksum+"\r\n",new UTF8Encoding(false)); + } var preflight = SqliteImportPreflightScanner.Scan(sqlFile, CancellationToken.None); Assert(preflight.Passed, "Preflight scan should pass"); @@ -44,7 +52,7 @@ private static void SqliteImport_PreflightScanner_SpatiaLiteAndManifest() Assert(preflight.HasManifest, "Preflight must detect Native Manifest"); Assert(preflight.ManifestTables == 1, "ManifestTables count match 1"); Assert(preflight.ManifestRows == 1, "ManifestRows count match 1"); - AssertEquals("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", preflight.ManifestSha256, "Manifest SHA-256 match"); + Assert(preflight.ManifestSha256?.Length == 64, "Manifest payload SHA-256 validated"); // 測試 Embedded NUL 偵測 string badFile = Path.Combine(tempDir, "corrupted_nul.sql"); @@ -208,6 +216,274 @@ private static void SqliteImport_Runner_TransactionRollbackOnFailure() } } + private static SqliteImportOptions SqliteSafeImportOptions(SqliteExistingObjectStrategy strategy = SqliteExistingObjectStrategy.AbortIfExists) + { + return new SqliteImportOptions { ExistingObjectStrategy = strategy, WrapInTransaction = true, PragmaForeignKeysOff = false, PragmaSynchronousOff = false, AutoLoadSpatiaLite = false }; + } + private static SqliteImportResult SqliteRunFixture(SqliteFixtureDatabase database, string directory, string sql, SqliteImportOptions options = null, CancellationToken token = default(CancellationToken)) + { + string path = Path.Combine(directory, Guid.NewGuid().ToString("N") + ".sql"); + File.WriteAllText(path, sql, new UTF8Encoding(false, true)); + return new SqliteImportRunner(database, "main").RunAsync(new SqliteImportRequest(Guid.NewGuid(), path, "main"), options ?? SqliteSafeImportOptions(), null, token).GetAwaiter().GetResult(); + } + private static void SqliteImport_NativeConflictPolicies_ProtectDropAndDependencies() + { + var expected = new[] { SqliteExistingObjectStrategy.SkipExisting, SqliteExistingObjectStrategy.DropAndRecreate, SqliteExistingObjectStrategy.OverwriteKeepExisting, SqliteExistingObjectStrategy.AbortIfExists }; + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_policy_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + for (int index = 0; index < expected.Length; index++) + { + Assert(SqliteExistingObjectSelection.FromIndex(index) == expected[index], "Visible conflict label maps to its policy"); + using (var database = new SqliteFixtureDatabase()) + { + database.Must("CREATE TABLE t(x PRIMARY KEY); INSERT INTO t VALUES(99); CREATE TABLE audit(x); CREATE INDEX t_old ON t(x); CREATE TRIGGER t_ai AFTER INSERT ON t BEGIN INSERT INTO audit VALUES(new.x); END;"); + string sql = "BEGIN; DROP TRIGGER IF EXISTS main.t_ai; DROP TABLE IF EXISTS main.t; CREATE TABLE main.t(x PRIMARY KEY); INSERT INTO main.t VALUES(1); CREATE INDEX t_new ON t(x); CREATE TRIGGER t_new_ai AFTER INSERT ON t BEGIN INSERT INTO audit VALUES(new.x+100); END; COMMIT;"; + var result = SqliteRunFixture(database, directory, sql, SqliteSafeImportOptions(expected[index])); + Assert(database.MCT.AutoCommit, "Conflict policy leaves no transaction active"); + Assert(database.TableExists("main", "t"), "Every conflict policy retains a table"); + if (expected[index] == SqliteExistingObjectStrategy.AbortIfExists) + { + Assert(result.Status == SqliteImportStatus.Failed, "Abort policy fails before DROP"); + Assert(Convert.ToInt64(database.Scalar("SELECT x FROM t;")) == 99, "Abort preserves original rows"); + Assert(database.Exists("trigger", "t_ai"), "Abort preserves dependent trigger"); + } + else if (expected[index] == SqliteExistingObjectStrategy.DropAndRecreate) + { + Assert(result.Status == SqliteImportStatus.Completed, "Recreate completes"); + Assert(Convert.ToInt64(database.Scalar("SELECT x FROM t;")) == 1, "Recreate restores dump row"); + Assert(!database.Exists("index", "t_old") && database.Exists("index", "t_new"), "Recreate replaces dependent indexes"); + } + else + { + Assert(result.Status == SqliteImportStatus.Completed, "Preserving policy completes"); + Assert(Convert.ToInt64(database.Scalar("SELECT count(*) FROM t WHERE x=99;")) == 1, "Preserving policy keeps original rows"); + Assert(database.Exists("index", "t_old") && !database.Exists("index", "t_new"), "Preserving policy protects dependent structure"); + Assert(database.Exists("trigger", "t_ai") && !database.Exists("trigger", "t_new_ai"), "Preserving policy protects dependent triggers"); + long count = Convert.ToInt64(database.Scalar("SELECT count(*) FROM t;")); + Assert(count == (expected[index] == SqliteExistingObjectStrategy.SkipExisting ? 1 : 2), "Skip protects data and Keep inserts data"); + } + } + } + using(var database=new SqliteFixtureDatabase()) + { + database.Must("CREATE TABLE 中文(x); INSERT INTO 中文 VALUES(99);"); + var result=SqliteRunFixture(database,directory,"DROP TABLE 中文; CREATE TABLE 中文(x); INSERT INTO 中文 VALUES(1);",SqliteSafeImportOptions(SqliteExistingObjectStrategy.SkipExisting)); + Assert(result.Status==SqliteImportStatus.Completed && Convert.ToInt64(database.Scalar("SELECT x FROM 中文;"))==99,"Unicode unquoted names cannot bypass SkipExisting"); + } + foreach(var policy in expected) + using(var database=new SqliteFixtureDatabase()) + { + database.Must("CREATE TABLE __mysqlpunk_column_comments(table_name TEXT NOT NULL,column_name TEXT NOT NULL,comment TEXT NOT NULL,PRIMARY KEY(table_name,column_name)); INSERT INTO __mysqlpunk_column_comments VALUES('other','x','preserve');"); + var result=SqliteRunFixture(database,directory,"CREATE TABLE t(x); CREATE TABLE IF NOT EXISTS __mysqlpunk_column_comments(table_name TEXT NOT NULL,column_name TEXT NOT NULL,comment TEXT NOT NULL,PRIMARY KEY(table_name,column_name)); INSERT OR REPLACE INTO __mysqlpunk_column_comments(table_name,column_name,comment) VALUES('t','x','new');",SqliteSafeImportOptions(policy)); + Assert(result.Status==SqliteImportStatus.Completed && Convert.ToInt64(database.Scalar("SELECT count(*) FROM __mysqlpunk_column_comments;"))==2,"Idempotent sidecar preserves other tables' comments under "+policy); + } + } + finally { Directory.Delete(directory, true); } + } + private static void SqliteImport_NativeLexer_TriggerAndExactText() + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_sqlite_lexer_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + using (var database = new SqliteFixtureDatabase()) + { + string text = "a\nb\rc\r\nd'中; END"; + string sql = "-- leading\nBEGIN;\nCREATE TABLE `t; name`(x TEXT);\nCREATE TABLE audit(x TEXT);\n" + + "CREATE TRIGGER `t; trigger` AFTER INSERT ON `t; name` BEGIN INSERT INTO audit VALUES(CASE WHEN new.x IS NULL THEN 'END; --' ELSE new.x END); /* END; */ UPDATE audit SET x=x||';' WHERE rowid=last_insert_rowid(); END;\n" + + "INSERT INTO `t; name` VALUES('" + text.Replace("'", "''") + "'); -- inline keeps next token separate\nCOMMIT;"; + var result = SqliteRunFixture(database, directory, sql); + Assert(result.Status == SqliteImportStatus.Completed, "Trigger/CASE/quotes/comments import succeeds: " + string.Join(";",result.Errors)); + AssertEquals(text, Convert.ToString(database.Scalar("SELECT x FROM `t; name`;")), "Literal CR/LF bytes remain exact"); + AssertEquals(text + ";", Convert.ToString(database.Scalar("SELECT x FROM audit;")), "Multi-statement trigger executes"); + Assert(database.MCT.AutoCommit, "Trigger import committed"); + } + foreach (string sql in new[] { "SELECT 'unterminated", "SELECT 1; /* unterminated", "CREATE TRIGGER tr AFTER INSERT ON t BEGIN SELECT 1;" }) + { + string path=Path.Combine(directory, Guid.NewGuid().ToString("N")+".sql"); File.WriteAllText(path, sql,new UTF8Encoding(false)); + Assert(!SqliteImportPreflightScanner.Scan(path,CancellationToken.None).Passed, "Unterminated lexical input rejected"); + } + string malformed=Path.Combine(directory,"malformed.sql"); File.WriteAllBytes(malformed,new byte[]{0x53,0x45,0x4C,0x45,0x43,0x54,0x20,0xC3,0x28}); + Assert(!SqliteImportPreflightScanner.Scan(malformed,CancellationToken.None).Passed,"Malformed UTF-8 rejected without replacement"); + } + finally { Directory.Delete(directory,true); } + } + private static void SqliteImport_NativeTransactions_FailureCancelAndPragmas() + { + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_cleanup_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + foreach(string sql in new[] { "BEGIN; CREATE TABLE t(x); INSERT INTO t VALUES(1); FAILING_SQL; COMMIT;", "CREATE TABLE t(x); INSERT INTO t VALUES(1); FAILING_SQL" }) + using(var database=new SqliteFixtureDatabase()) + { + database.Must("PRAGMA foreign_keys=ON; PRAGMA synchronous=NORMAL;"); + var options=SqliteSafeImportOptions(); options.PragmaForeignKeysOff=true; options.PragmaSynchronousOff=true; + var result=SqliteRunFixture(database,directory,sql,options); + Assert(result.Status==SqliteImportStatus.Failed,"SQL failure including unterminated tail fails"); + Assert(database.MCT.AutoCommit && !database.TableExists("main","t"),"Script and managed transactions roll back schema/rows"); + Assert(Convert.ToInt64(database.Scalar("PRAGMA foreign_keys;"))==1 && Convert.ToInt64(database.Scalar("PRAGMA synchronous;"))==1,"Original PRAGMAs restored on failure"); + } + using(var database=new SqliteFixtureDatabase()) + { + database.Must("PRAGMA foreign_keys=ON;"); + var result=SqliteRunFixture(database,directory,"CREATE TABLE parent(id INTEGER PRIMARY KEY); CREATE TABLE child(id REFERENCES parent(id) DEFERRABLE INITIALLY DEFERRED); INSERT INTO child VALUES(7);"); + Assert(result.Status==SqliteImportStatus.Failed,"Deferred foreign-key COMMIT failure cannot report Completed"); + Assert(database.MCT.AutoCommit && !database.TableExists("main","child"),"Rejected COMMIT rolls back pending schema/rows"); + } + foreach(string command in new[]{"BEGIN","COMMIT","ROLLBACK"}) + using(var database=new SqliteFixtureDatabase()) + { + database.FailExecPrefix=command; + string sql=command=="ROLLBACK" ? "CREATE TABLE t(x); FAILING_SQL;" : "CREATE TABLE t(x);"; + var result=SqliteRunFixture(database,directory,sql); + Assert(result.Status==SqliteImportStatus.Failed,"Provider NO for "+command+" is failure"); + if(command=="ROLLBACK") { Assert(result.Errors.Any(e=>e.Contains("ROLLBACK cleanup failed")),"Rollback NO is visible"); database.FailExecPrefix=null; database.Must("ROLLBACK;"); } + else Assert(database.MCT.AutoCommit && !database.TableExists("main","t"),command+" failure cannot leak pending work"); + } + string cancellationDatabase = Path.Combine(directory,"cancel-fixture.sqlite"); + using(var database=new SqliteFixtureDatabase(cancellationDatabase)) + using(var cancellation=new CancellationTokenSource()) + { + database.Must("PRAGMA foreign_keys=OFF; PRAGMA synchronous=FULL;"); + cancellation.CancelAfter(100); + var latency=System.Diagnostics.Stopwatch.StartNew(); + var result=SqliteRunFixture(database,directory,"BEGIN; CREATE TABLE t(x); WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000000) INSERT INTO t SELECT x FROM n; COMMIT;",SqliteSafeImportOptions(),cancellation.Token); + Assert(latency.Elapsede.Contains("32 MiB")),"Completed single-line multibyte statement is bounded in bytes before semicolon"); + } + finally { Directory.Delete(directory,true); } + } + + private static void SqliteImport_ReviewRegression_ManagedDropAndImplicitRollback() + { + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_review_tx_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + foreach(string sql in new[]{"CREATE TABLE t(x); INSERT INTO t VALUES(1); FAILING_SQL;", "CREATE TABLE t(x,);"}) + using(var database=new SqliteFixtureDatabase()) + { + database.Must("CREATE TABLE t(x); INSERT INTO t VALUES(99);"); + var result=SqliteRunFixture(database,directory,sql,SqliteSafeImportOptions(SqliteExistingObjectStrategy.DropAndRecreate)); + Assert(result.Status==SqliteImportStatus.Failed && database.MCT.AutoCommit,"Replacement failure rolls back managed transaction"); + Assert(database.TableExists("main","t") && Convert.ToInt64(database.Scalar("SELECT x FROM t;"))==99,"Policy-generated DROP is rolled back with replacement failure"); + } + foreach(string sql in new[]{ + "CREATE TABLE t(x UNIQUE); INSERT INTO t VALUES(1); INSERT OR ROLLBACK INTO t VALUES(1); CREATE TABLE leaked(x); INSERT INTO leaked VALUES(2);", + "BEGIN; CREATE TABLE t(x UNIQUE); INSERT INTO t VALUES(1); INSERT OR ROLLBACK INTO t VALUES(1); CREATE TABLE leaked(x); INSERT INTO leaked VALUES(2); COMMIT;", + "CREATE TABLE t(x); CREATE TRIGGER rollback_trigger BEFORE INSERT ON t WHEN new.x=2 BEGIN SELECT RAISE(ROLLBACK,'stop'); END; INSERT INTO t VALUES(1); INSERT INTO t VALUES(2); CREATE TABLE leaked(x);"}) + using(var database=new SqliteFixtureDatabase()) + { + database.Must("CREATE TABLE original(x); INSERT INTO original VALUES(99);"); + var options=SqliteSafeImportOptions(); options.ContinueOnError=true; + var result=SqliteRunFixture(database,directory,sql,options); + Assert(result.Status==SqliteImportStatus.Failed && database.MCT.AutoCommit,"Statement-level full rollback stops continued execution"); + Assert(!database.TableExists("main","t") && !database.TableExists("main","leaked"),"No later statement leaks into autocommit after OR/RAISE ROLLBACK"); + Assert(Convert.ToInt64(database.Scalar("SELECT x FROM original;"))==99,"Initial rows survive implicit rollback"); + } + } + finally { Directory.Delete(directory,true); } + } + private static void SqliteImport_ReviewRegression_MetadataOwnerPreservation() + { + string directory=Path.Combine(Path.GetTempPath(),"mysqlpunk_sqlite_review_metadata_"+Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + foreach(var policy in new[]{SqliteExistingObjectStrategy.SkipExisting,SqliteExistingObjectStrategy.OverwriteKeepExisting}) + using(var database=new SqliteFixtureDatabase()) + { + database.Must("CREATE TABLE ai(id INTEGER PRIMARY KEY AUTOINCREMENT); INSERT INTO ai VALUES(1); INSERT INTO ai VALUES(100); DELETE FROM ai WHERE id=100; CREATE TABLE __mysqlpunk_column_comments(table_name TEXT NOT NULL,column_name TEXT NOT NULL,comment TEXT NOT NULL,PRIMARY KEY(table_name,column_name)); INSERT INTO __mysqlpunk_column_comments VALUES('ai','id','preserve'); INSERT INTO __mysqlpunk_column_comments VALUES('other','x','unrelated');"); + string sql="DELETE FROM sqlite_sequence; INSERT INTO sqlite_sequence(name,seq) VALUES('ai',1); DELETE FROM __mysqlpunk_column_comments; INSERT OR REPLACE INTO __mysqlpunk_column_comments VALUES('ai','id','overwrite'); CREATE TABLE new_ai(id INTEGER PRIMARY KEY AUTOINCREMENT); INSERT INTO new_ai VALUES(1); DELETE FROM sqlite_sequence WHERE name='new_ai'; INSERT INTO sqlite_sequence(name,seq) VALUES('new_ai',77); DELETE FROM sqlite_sequence WHERE name='new_ai' OR 1=1; CREATE TABLE new_t(x); INSERT OR REPLACE INTO __mysqlpunk_column_comments(table_name,column_name,comment) VALUES('new_t','x',CAST(X'6E6577' AS TEXT));"; + var result=SqliteRunFixture(database,directory,sql,SqliteSafeImportOptions(policy)); + Assert(result.Status==SqliteImportStatus.Completed,"Preserving policy handles mixed protected/new owner metadata"); + database.Must("INSERT INTO ai DEFAULT VALUES; INSERT INTO new_ai DEFAULT VALUES;"); + Assert(Convert.ToInt64(database.Scalar("SELECT max(id) FROM ai;"))==101,"Unscoped sequence DELETE cannot erase protected high-water mark under "+policy); + Assert(Convert.ToInt64(database.Scalar("SELECT max(id) FROM new_ai;"))==78,"Exact single-owner sequence operations remain supported"); + AssertEquals("preserve",Convert.ToString(database.Scalar("SELECT comment FROM __mysqlpunk_column_comments WHERE table_name='ai';")),"Protected table comment remains"); + AssertEquals("unrelated",Convert.ToString(database.Scalar("SELECT comment FROM __mysqlpunk_column_comments WHERE table_name='other';")),"Unscoped sidecar DELETE cannot remove unrelated owners"); + AssertEquals("new",Convert.ToString(database.Scalar("SELECT comment FROM __mysqlpunk_column_comments WHERE table_name='new_t';")),"Exact single-owner native comment operation remains supported"); + } + } + finally { Directory.Delete(directory,true); } + } + private static System.Windows.Forms.Form SqliteCreateWizard(string typeName,params object[] arguments) + { + Type type=typeof(IDatabase).Assembly.GetType("mySQLPunk."+typeName,true); + return (System.Windows.Forms.Form)Activator.CreateInstance(type,System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.NonPublic,null,arguments,null); + } + private static System.Reflection.FieldInfo SqliteWizardField(System.Windows.Forms.Form form,string name) + { + return form.GetType().GetField(name,System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.NonPublic); + } + private static void SqliteAssertRepeatedCloseBlocked(System.Windows.Forms.Form form,SqliteFixtureDatabase database) + { + using(var cancellation=new CancellationTokenSource()) + { + SqliteWizardField(form,"busy").SetValue(form,true); SqliteWizardField(form,"cancellation").SetValue(form,cancellation); + var method=typeof(System.Windows.Forms.Form).GetMethod("OnFormClosing",System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.NonPublic); + for(int attempt=0;attempt<2;attempt++) + { + var closing=new System.Windows.Forms.FormClosingEventArgs(System.Windows.Forms.CloseReason.UserClosing,false); + method.Invoke(form,new object[]{closing}); + Assert(closing.Cancel && !form.IsDisposed && database.State==ConnectionState.Open,"Every close attempt remains cancelled until worker cleanup completes"); + } + Assert(cancellation.IsCancellationRequested,"First close requests cancellation"); + SqliteWizardField(form,"busy").SetValue(form,false); + var completed=new System.Windows.Forms.FormClosingEventArgs(System.Windows.Forms.CloseReason.UserClosing,false); method.Invoke(form,new object[]{completed}); + Assert(!completed.Cancel,"Close is allowed once worker is no longer busy"); + } + } + private static void SqliteWaitWizardTask(Task task) + { + var timeout=System.Diagnostics.Stopwatch.StartNew(); + while(!task.IsCompleted) + { + Assert(timeout.Elapsed ConnectionState.Open; @@ -223,7 +499,10 @@ public void Dispose() { } public DataTable SelectSQL(string sql, Dictionary parameters = null) { - return new DataTable(); + var result = new DataTable(); + if (sql.StartsWith("PRAGMA ", StringComparison.OrdinalIgnoreCase)) { result.Columns.Add("value",typeof(int)); result.Rows.Add(0); } + else if (sql.Contains("SELECT type, name, tbl_name")) { result.Columns.Add("type"); result.Columns.Add("name"); result.Columns.Add("tbl_name"); foreach(string name in ExistingTables) result.Rows.Add("table",name,name); } + return result; } public Dictionary ExecSQL(string sql, Dictionary parameters = null) @@ -239,7 +518,7 @@ public Dictionary ExecSQL(string sql, Dictionary } var res = new Dictionary { { "status", "OK" } }; - if (sql.StartsWith("INSERT", StringComparison.OrdinalIgnoreCase)) + if (sql.TrimStart().StartsWith("INSERT", StringComparison.OrdinalIgnoreCase)) { res["rowsAffected"] = "1"; } @@ -272,4 +551,72 @@ public void InsertTableBatch(string databaseName, string tableName, DataTable ro public string GetViewCreateStatement(string databaseName, string viewName) => ""; public void CreateViewFromStatement(string databaseName, string viewName, string sourceViewSql) { } } +#pragma warning disable 0108 + private sealed class SqliteFixtureDatabase : my_sqlite, IDatabase + { + private readonly SQLiteConnection connection; + public string FailExecPrefix; + public string FailQueryContains; + public SqliteFixtureDatabase(string path = null) { connection = new SQLiteConnection(new SQLiteConnectionStringBuilder { DataSource=path ?? ":memory:", Version=3, Pooling=false }.ToString()); connection.Open(); MCT = connection; } + public ConnectionState State { get { return connection.State; } } + public string ProviderName { get { return "sqlite"; } } + public void Dispose() { connection.Dispose(); } + public void SetConn(string value) { throw new NotSupportedException(); } + public void Open() { throw new NotSupportedException(); } + public void Close() { throw new NotSupportedException(); } + public object Scalar(string sql) { using(var cmd = new SQLiteCommand(sql, connection)) return cmd.ExecuteScalar(); } + public void Must(string sql) { var r = ExecSQL(sql); if (r["status"] != "OK") throw new Exception(r["reason"]); } + public DataTable SelectSQL(string sql, Dictionary parameters=null) + { + var t = new DataTable(); + if (FailQueryContains != null && sql.Contains(FailQueryContains)) { t.ExtendedProperties["QueryError"] = "injected source read failure"; return t; } + try { using(var cmd = new SQLiteCommand(sql, connection)) { Bind(cmd, parameters); using(var reader=cmd.ExecuteReader()) t.Load(reader); } } + catch(Exception ex) { t.ExtendedProperties["QueryError"] = ex.Message; } + return t; + } + public Dictionary ExecSQL(string sql, Dictionary parameters=null) + { + if (FailExecPrefix != null && sql.TrimStart().StartsWith(FailExecPrefix, StringComparison.OrdinalIgnoreCase)) + return new Dictionary{{"status","NO"},{"reason","injected execution failure"}}; + try { using(var cmd = new SQLiteCommand(sql, connection)) { Bind(cmd, parameters); int n=cmd.ExecuteNonQuery(); return new Dictionary{{"status","OK"},{"rowsAffected",n.ToString()}}; } } + catch(Exception ex) { return new Dictionary{{"status","NO"},{"reason",ex.Message}}; } + } + private static void Bind(SQLiteCommand cmd, Dictionary p) { if(p!=null) foreach(var pair in p) cmd.Parameters.AddWithValue("@"+pair.Key,pair.Value); } + private static string Q(string s) { return "\""+s.Replace("\"","\"\"")+"\""; } + public bool Exists(string type,string name) { return Convert.ToInt64(Scalar("SELECT COUNT(*) FROM sqlite_master WHERE type='"+type+"' AND name='"+name.Replace("'","''")+"';"))>0; } + public List GetDatabases() { return new List{"main"}; } + private List Names(string type) { return SelectSQL("SELECT name FROM sqlite_master WHERE type='"+type+"' AND name NOT LIKE 'sqlite_%';").Rows.Cast().Select(r=>r[0].ToString()).ToList(); } + public List GetTables(string name) { return Names("table").Where(x=>x!="__mysqlpunk_column_comments" && x!="__mysqlpunk_table_comments").ToList(); } + public List GetViews(string name) { return Names("view"); } + public DataTable GetColumns(string db,string table) + { + var t=SelectSQL("PRAGMA table_info("+Q(table)+");"); t.Columns.Add("Comment",typeof(string)); + if(Exists("table","__mysqlpunk_column_comments")) foreach(DataRow r in t.Rows) + r["Comment"]=Scalar("SELECT comment FROM __mysqlpunk_column_comments WHERE table_name='"+table.Replace("'","''")+"' AND column_name='"+r["name"].ToString().Replace("'","''")+"';")??DBNull.Value; + return t; + } + public long CountRows(string db,string table) { var t=SelectSQL("SELECT COUNT(*) FROM "+Q(table)+";"); return t.ExtendedProperties.ContainsKey("QueryError")?-1:Convert.ToInt64(t.Rows[0][0]); } + public DataTable SelectTablePage(string db,string table,long offset,int limit) { return SelectSQL("SELECT * FROM "+Q(table)+" LIMIT "+limit+" OFFSET "+offset+";"); } + public bool TableExists(string db,string table) { return Exists("table",table); } + public bool ViewExists(string db,string view) { return Exists("view",view); } + public Task SelectSQLAsync(string s,Dictionary p=null) { return Task.FromResult(SelectSQL(s,p)); } + public Task> ExecSQLAsync(string s,Dictionary p=null) { return Task.FromResult(ExecSQL(s,p)); } + public DataTable GetIndexes(string d,string t) { return new DataTable(); } + public DataTable GetTableStatus(string d) { return new DataTable(); } + public Dictionary GetDatabaseInfo(string d) { return new Dictionary(); } + public string GetTableCreateStatement(string d,string t) { return ""; } + public void RenameTable(string d,string a,string b) { throw new NotSupportedException(); } + public void RenameView(string d,string a,string b) { throw new NotSupportedException(); } + public DataTable GetCopyColumns(string d,string t) { return new DataTable(); } + public DataTable GetCopyIndexes(string d,string t) { return new DataTable(); } + public void CreateTableForCopy(string d,string t,DataTable c,string p) { throw new NotSupportedException(); } + public void DropTableForCopy(string d,string t) { throw new NotSupportedException(); } + public void CreateIndexesForCopy(string d,string t,DataTable i,string p) { throw new NotSupportedException(); } + public void InsertTableBatch(string d,string t,DataTable r) { throw new NotSupportedException(); } + public string GetViewCreateStatement(string d,string v) { return ""; } + public void CreateViewFromStatement(string d,string v,string s) { throw new NotSupportedException(); } + } + +#pragma warning restore 0108 + } diff --git a/tests/SmokeTests.WizardRepairs.cs b/tests/SmokeTests.WizardRepairs.cs new file mode 100644 index 0000000..2a075ec --- /dev/null +++ b/tests/SmokeTests.WizardRepairs.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Reflection; +using mySQLPunk.lib; + +public static partial class SmokeTests +{ + private static void WizardRepairs_MySqlTableRouting() + { + foreach (string handler in new[] { "DumpSelectedTableSql", "DumpCurrentSelectionSqlWithDialog" }) + foreach (bool dataOnly in new[] { true, false }) + using (var fixture = new RoutingFixture("mysql", "Tables", "sales.orders")) + { + mySQLPunk.lib.MySqlExport.MySqlExportWizardOptions captured = null; + typeof(mySQLPunk.Form1).GetField("MySqlExportWizardRunner", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(fixture.Form, + new Func((db, name, options) => { captured = options; return null; })); + fixture.Invoke(handler, dataOnly); + Assert(captured != null, "Actual table route must open MySQL wizard"); + Assert(captured.Mode == (dataOnly ? mySQLPunk.lib.MySqlExport.MySqlExportMode.DataOnly : mySQLPunk.lib.MySqlExport.MySqlExportMode.SchemaAndData), "Actual route must retain data-only mode"); + Assert(captured.SelectedTables.Count == 1 && captured.SelectedTables[0] == "sales.orders", "Actual route must retain raw selected table identity"); + Assert(!captured.IncludeViews && !captured.IncludeRoutines && !captured.IncludeTriggers, "A table route must not implicitly include unrelated database objects"); + } + } + + private static void WizardRepairs_AtomicPublication() + { + Type type = typeof(IDatabase).Assembly.GetType("mySQLPunk.lib.SqlExportPublication", true); + MethodInfo publish = type.GetMethod("Publish", BindingFlags.Static | BindingFlags.NonPublic); + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk-publish-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string target = Path.Combine(directory, "backup.sql"); + string candidate = Path.Combine(directory, "candidate.tmp"); + File.WriteAllText(target, "original"); File.WriteAllText(candidate, "new backup"); + using (var locked = new FileStream(target, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + bool rejected = false; + try { publish.Invoke(null, new object[] { candidate, target }); } + catch (TargetInvocationException ex) { rejected = ex.InnerException is IOException; } + Assert(rejected, "A locked destination must fail publication"); + AssertEquals("original", File.ReadAllText(target), "Publication failure must preserve original bytes"); + AssertEquals("new backup", File.ReadAllText(candidate), "Publication failure must retain candidate"); + } + publish.Invoke(null, new object[] { candidate, target }); + AssertEquals("new backup", File.ReadAllText(target), "Replacement must publish candidate"); + Assert(!File.Exists(candidate), "Successful replacement consumes candidate"); + string fresh = Path.Combine(directory, "fresh.sql"); + File.WriteAllText(candidate, "fresh"); publish.Invoke(null, new object[] { candidate, fresh }); + AssertEquals("fresh", File.ReadAllText(fresh), "First publication must work"); + } + finally { Directory.Delete(directory, true); } + } + + private static void WizardRepairs_NativeTableScope() + { + foreach (string provider in new[] { "sqlite", "oracle" }) + foreach (string handler in new[] { "DumpSelectedTableSql", "DumpCurrentSelectionSqlWithDialog" }) + using (var fixture = new RoutingFixture(provider, "Tables", "sales.orders")) + { + bool called = false; + if (provider == "sqlite") + typeof(mySQLPunk.Form1).GetField("SqliteExportWizardRunner", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(fixture.Form, + new Func((db, name, options) => + { + called = true; + Assert(options.SelectedTables.Count == 1 && options.SelectedTables[0] == "sales.orders", "SQLite raw seed retained"); + Assert(!options.IncludeViews && !options.IncludeTriggers, "SQLite table route must not include unrelated objects"); + return null; + })); + else + typeof(mySQLPunk.Form1).GetField("OracleExportWizardRunner", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(fixture.Form, + new Func((db, name, options) => + { + called = true; + Assert(options.SelectedTables.Count == 1 && options.SelectedTables[0] == "sales.orders", "Oracle raw seed retained"); + Assert(!options.IncludeViews, "Oracle table route must not include unrelated views"); + return null; + })); + fixture.Invoke(handler, false); + Assert(called, "Actual provider table route must open its wizard"); + } + } + + private static void WizardRepairs_OwnedSession() + { + Type type = typeof(IDatabase).Assembly.GetType("mySQLPunk.lib.WizardOwnedSession", true); + MethodInfo copy = type.GetMethod("CopySettings", BindingFlags.Static | BindingFlags.NonPublic); + var original = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "pdo", new object() }, { "runtime", new object() }, { "CONNSTRING", "stale" }, + { "isConnect", "T" }, { "host", "fixture.invalid" }, { "initial_database", "source" } + }; + var settings = (Dictionary)copy.Invoke(null, new object[] { original, "mysql", "destination" }); + Assert(!settings.ContainsKey("pdo") && !settings.ContainsKey("runtime") && !settings.ContainsKey("connString") && !settings.ContainsKey("isConnect"), "No cached session or runtime values may cross boundary"); + AssertEquals("destination", Convert.ToString(settings["initial_database"]), "Explicit target must win"); + AssertEquals("source", Convert.ToString(original["initial_database"]), "Source settings must not change"); + + string file = Path.Combine(Path.GetTempPath(), "mysqlpunk-owned-" + Guid.NewGuid().ToString("N") + ".sqlite"); + try + { + using (var editor = new my_sqlite()) + { + editor.SetConn("Data Source=" + file + ";Version=3;Pooling=False"); editor.MCT.Open(); + editor.ExecSQL("CREATE TABLE fixture (id INTEGER);"); + var config = new Dictionary { { "path", file }, { "pdo", editor } }; + MethodInfo open = type.GetMethod("Open", BindingFlags.Static | BindingFlags.NonPublic); + var owned = (IDisposable)open.Invoke(null, new object[] { config, "sqlite", "main" }); + var job = (IDatabase)type.GetProperty("Database", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(owned); + Assert(!object.ReferenceEquals(job, editor), "Wizard must have an independent provider"); + job.ExecSQL("BEGIN; INSERT INTO fixture VALUES (42);"); + Assert(editor.CountRows("main", "fixture") == 0, "Uncommitted job writes must not affect editor session"); + owned.Dispose(); + Assert(editor.State == ConnectionState.Open, "Disposing job must leave editor open"); + Assert(editor.CountRows("main", "fixture") == 0, "Closing job must rollback its transaction"); + } + } + finally { if (File.Exists(file)) File.Delete(file); } + } +} diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index 507b22d..b79b516 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -32,6 +32,14 @@ public static int Main(string[] args) if (Environment.GetEnvironmentVariable("MYSQLPUNK_MSSQL_IMPORT_WIZARD_PREVIEW") == "1") return ImportWizardPreview(); if (Environment.GetEnvironmentVariable("MYSQLPUNK_MSSQL_WIZARD_PREVIEW") == "1") return WizardPreview(); int passed = 0; + Run("Wizard repairs atomic publication", WizardRepairs_AtomicPublication, ref passed); + Run("Wizard repairs owned session", WizardRepairs_OwnedSession, ref passed); + Run("Wizard repairs MySQL table routing", WizardRepairs_MySqlTableRouting, ref passed); + Run("Wizard repairs native table scope", WizardRepairs_NativeTableScope, ref passed); + Run("MySQL import modified INSERT and trigger pseudo rows", MySqlImport_Regression_ModifiedInsertProtectionAndTriggerPseudoRows, ref passed); + Run("MySQL import regression target literals and strategies", MySqlImport_Regression_TargetLiteralsAndStrategies, ref passed); + Run("MySQL import regression cancel transactions and preflight", MySqlImport_Regression_CancelTransactionsAndPreflight, ref passed); + Run("MySQL export regression generated bounds and failures", MySqlExport_Regression_GeneratedBoundsAndFailures, ref passed); Run("MySQL export options and models defaults", MySqlExport_OptionsAndModels_Defaults, ref passed); Run("MySQL export service and manifest atomic write", MySqlExport_ServiceAndManifest_AtomicWrite, ref passed); Run("MySQL import preflight scanner delimiters and manifest", MySqlImport_PreflightScanner_DelimitersAndManifest, ref passed); @@ -40,6 +48,16 @@ public static int Main(string[] args) Run("MySQL import runner database redirection", MySqlImport_Runner_DatabaseRedirection, ref passed); Run("MySQL import runner existing object strategies", MySqlImport_Runner_ExistingObjectStrategies, ref passed); Run("MySQL import runner manifest mismatch detection", MySqlImport_Runner_ManifestMismatchDetection, ref passed); + Run("SQLite import managed drop and auto rollback", SqliteImport_ReviewRegression_ManagedDropAndImplicitRollback, ref passed); + Run("SQLite import protected metadata owners", SqliteImport_ReviewRegression_MetadataOwnerPreservation, ref passed); + Run("SQLite import repeated close and preflight", SqliteImport_ReviewRegression_RepeatedCloseAndPreflight, ref passed); + Run("SQLite export repeated close and initial options", SqliteExport_ReviewRegression_RepeatedCloseAndInitialOptions, ref passed); + Run("SQLite import native conflict policies", SqliteImport_NativeConflictPolicies_ProtectDropAndDependencies, ref passed); + Run("SQLite import native trigger and exact text", SqliteImport_NativeLexer_TriggerAndExactText, ref passed); + Run("SQLite import native transactions and pragmas", SqliteImport_NativeTransactions_FailureCancelAndPragmas, ref passed); + Run("SQLite import multibyte statement budget", SqliteImport_StatementLimit_MultibyteAndTerminated, ref passed); + Run("SQLite export native fidelity roundtrip", SqliteExport_NativeRoundtrip_GeneratedRowidSequenceTextComments, ref passed); + Run("SQLite export protected failure and tamper", SqliteExport_Failures_PreserveOutputAndRejectTamper, ref passed); Run("SQLite export options and models defaults", SqliteExport_OptionsAndModels_Defaults, ref passed); Run("SQLite export service and manifest streaming and hex blob", SqliteExport_ServiceAndManifest_StreamingAndHexBlob, ref passed); Run("SQLite import preflight scanner SpatiaLite and manifest", SqliteImport_PreflightScanner_SpatiaLiteAndManifest, ref passed); @@ -53,6 +71,14 @@ public static int Main(string[] args) Run("PostgreSQL import runner execution and manifest verification", PostgreSqlImport_Runner_ExecutionAndManifestVerification, ref passed); Run("PostgreSQL import runner existing object strategies", PostgreSqlImport_Runner_ExistingObjectStrategies, ref passed); Run("PostgreSQL import runner transaction rollback on failure", PostgreSqlImport_Runner_TransactionRollbackOnFailure, ref passed); + Run("Oracle import scoped owner metadata and LOB conflict policy", OracleImport_ScopedTargetVariantsMetadataAndLobTruncate, ref passed); + Run("Oracle export exact binary float tokens", OracleExport_ScopedBinaryFloatRoundTrip, ref passed); + Run("Oracle export native view and disabled FK gates", OracleExport_ScopedNativeViewsAndDisabledForeignKeys, ref passed); + Run("Oracle import returned NO and transaction controls", OracleImport_ReturnedNoAndTransactionControls, ref passed); + Run("Oracle import owner confinement and cancel", OracleImport_TargetOwnerPreflightAndCancellation, ref passed); + Run("Oracle import parser bytes comments and quotes", OracleImport_ParserBytesCommentsAndAlternativeQuotes, ref passed); + Run("Oracle export spatial metadata and option gates", OracleExport_SpatialMetadataAndOptionGates, ref passed); + Run("Oracle export bounded LOB snapshot and cancellation", OracleExport_BoundedLobsSnapshotFailureAndCancel, ref passed); Run("Oracle export options and models defaults", OracleExport_OptionsAndModels_Defaults, ref passed); Run("Oracle export service and manifest streaming and SDO_GEOMETRY", OracleExport_ServiceAndManifest_StreamingAndSdoGeometry, ref passed); Run("Oracle import preflight scanner SDO_GEOMETRY and PL/SQL", OracleImport_PreflightScanner_SdoGeometryAndPlSql, ref passed); From 9f5f3ed82f15f18c277ba094205e9db982df7481 Mon Sep 17 00:00:00 2001 From: John Date: Fri, 16 Oct 2026 14:06:59 +0800 Subject: [PATCH 10/12] fix(mysql): check snapshot compatibility before export and allow retry --- docs/design/db-wizard-repairs/plan.md | 4 + docs/design/db-wizard-repairs/verification.md | 15 ++ mySQLPunk/MySqlExportWizardForm.cs | 92 ++++++++-- .../lib/MySqlExport/MySqlExportModels.cs | 2 + .../lib/MySqlExport/MySqlSqlExportService.cs | 33 +++- tests/SmokeTests.MySqlExport.cs | 158 +++++++++++++++++- tests/SmokeTests.cs | 3 + 7 files changed, 287 insertions(+), 20 deletions(-) diff --git a/docs/design/db-wizard-repairs/plan.md b/docs/design/db-wizard-repairs/plan.md index 9bd5cba..6a91492 100644 --- a/docs/design/db-wizard-repairs/plan.md +++ b/docs/design/db-wizard-repairs/plan.md @@ -49,3 +49,7 @@ The accepted review already supplies the design and root causes; no additional d - Ruling: keep native MySQL source snapshots limited to verified InnoDB, keep Oracle SYS read-only and cross-session-user spatial metadata restrictions explicit, and reject ambiguous SQL scope. Conservative failures can require the native client but preserve data rather than pretending general SQL confinement. - Final root gate: all 294 smoke groups plus AutoReleasePolicy passed; explicit Debug and Release application builds passed. Optional live tests were skipped. All three independent scoped reviews/re-reviews approved. Existing-file BOM/newlines and diff whitespace were audited; no code edits after final gate. + +## MySQL snapshot failure follow-up (2026-10-16) + +The user screenshot shows an engine compatibility failure at execution; it does not establish the real engine. Localized correction: load/show engine metadata before confirmation, disable unsupported/unknown snapshots with a visible ordinary-export warning, remember the user's snapshot preference, exempt schema-only mode, retain the fresh service guard, and permit failed/cancelled attempts to return to settings. The user confirms the displayed ordinary mode before execution; the service never silently falls back. Add catalog, ordinary/guard, and form recovery regressions, then run smoke/build and one independent scoped review. No real database or saved connection access. diff --git a/docs/design/db-wizard-repairs/verification.md b/docs/design/db-wizard-repairs/verification.md index b6d9b61..091beb8 100644 --- a/docs/design/db-wizard-repairs/verification.md +++ b/docs/design/db-wizard-repairs/verification.md @@ -40,3 +40,18 @@ Local baseline, red reproductions, focused test logs and final build logs are re Real MySQL/MariaDB and Oracle/Spatial roundtrips, live SSH/permissions/source-editor concurrency, multi-GiB memory/performance, native UI/DPI/audio and production are not verified. MySQL stops after the current command returns/times out; immediate server-command cancellation is not promised. A failed import can retain earlier committed DDL/DML; no automatic cleanup/retry or all-or-nothing restore guarantee is introduced. Oracle reference semantics used during review: [ODP.NET transactions](https://docs.oracle.com/en/database/oracle/oracle-database/19/odpnt/featOraCommand.html), [SET TRANSACTION](https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/SET-TRANSACTION.html), [SDO_UTIL WKB and SRID](https://docs.oracle.com/en/database/oracle/oracle-database/19/spatl/SDO_UTIL-reference.html). These references are design evidence, not a live acceptance result. + +## MySQL snapshot availability follow-up (2026-10-16) + +The supplied screenshot fails at the fresh verified-InnoDB guard. It does not establish whether the real table is non-InnoDB or its engine metadata is unavailable. The catalog previously had no engine information and the wizard defaulted to snapshot-on, so this compatibility limitation appeared only after starting. + +- Catalog reads engine status once and displays the actual engine (or Unknown); matching uses exact table identity so case-sensitive names cannot borrow another table's engine. +- Data export with a selected non-InnoDB/unknown engine disables and unchecks snapshot before confirmation. Settings and confirmation explain ordinary unlocked reads, cross-table inconsistency and foreign-key restore risk. Compatible selections restore the user's previous choice; explicitly turning snapshot off remains off. Schema-only never claims a data snapshot. +- Runtime snapshot checks remain fresh and mandatory; metadata/engine changes fail with actionable settings guidance, without reading data pages, publishing or silently downgrading. Ordinary mode already existed and remains available; no engine conversion or database configuration change occurs. +- After failure/cancellation and cleanup, Adjust settings returns to step 2 with the existing choices. Another attempt still requires confirmation, resets prior result/progress/output buttons, and ignores queued progress from previous attempts. + +Regression evidence: new wizard test against the previous actual Debug assembly fails with `Mixed engines must offer ordinary export before execution` (exit 1); repaired actual-product MySQL20 groups and AutoReleasePolicy pass. Added cases cover mixed/search-hidden selections, unknown/missing/status-query failure, explicit preference, schema-only, exact-case identity, fresh engine drift, old-file preservation, failure/cancel recovery and confirmed ordinary retry. Independent read-only scoped review found no material correctness issue; its three suggested edge cases were added. Logs are `snapshot-options-red.log` and `snapshot-options-mysql.log` in the local evidence directory. Full-suite and build results are appended after verification below. + +Reference: [MySQL single-transaction semantics](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html). InnoDB snapshots do not protect nontransactional tables or concurrent DDL. This implementation uses its own exporter; it does not invoke mysqldump or add table locks for ordinary mode. Real MySQL/MariaDB engines, privileges, live concurrent writes and rendered UI/DPI remain unverified; the screenshot is the production symptom, not a live database acceptance result. + +Final follow-up gate: all **297 smoke groups and AutoReleasePolicy PASS** (`snapshot-options-full.log`); explicit Debug and Release builds PASS. The initially running Debug application locked its executable, so an independent `bin/SnapshotFixDebug` build succeeded first; after the user requested retry, normal `bin/Debug/mySQLPunk.exe` rebuilt successfully (`snapshot-options-debug-retry.log`). No application process was terminated. Existing source/test CRLF and document LF, UTF-8/BOM conventions and diff whitespace are preserved. Real database and rendered UI acceptance remain pending. diff --git a/mySQLPunk/MySqlExportWizardForm.cs b/mySQLPunk/MySqlExportWizardForm.cs index a85d68b..534b022 100644 --- a/mySQLPunk/MySqlExportWizardForm.cs +++ b/mySQLPunk/MySqlExportWizardForm.cs @@ -28,6 +28,7 @@ internal sealed class MySqlExportWizardForm : Form private readonly CheckBox disableUnique = new CheckBox(), noAutoValueOnZero = new CheckBox(); private readonly CheckBox includeViews = new CheckBox(), includeRoutines = new CheckBox(), includeTriggers = new CheckBox(); private readonly Label stepsLabel = new Label(), statusLabel = new Label(), selectionSummary = new Label(), confirmSummary = new Label(); + private readonly Label snapshotHint = new Label(); private readonly Label currentObjectLabel = new Label(), percentLabel = new Label(), writtenLabel = new Label(); private readonly Label elapsedLabel = new Label(), speedLabel = new Label(), etaLabel = new Label(), resultText = new Label(), mascotHint = new Label(); private readonly Button nextButton = new Button(), backButton = new Button(), cancelButton = new Button(), browseButton = new Button(), soundButton = new Button(); @@ -45,6 +46,7 @@ internal sealed class MySqlExportWizardForm : Form private CancellationTokenSource cancellation; private MySqlExportProgress lastProgress; private bool busy, closing, exporting; + private bool snapshotPreferred, updatingSnapshot; private int step; internal MySqlExportResult Result { get; private set; } @@ -116,6 +118,17 @@ internal MySqlExportWizardForm(IDatabase db, string databaseName, MySqlExportWiz includeTriggers.Checked = initialOptions.IncludeTriggers; } + snapshotPreferred = singleTransaction.Checked; + singleTransaction.CheckedChanged += (s, e) => + { + if (updatingSnapshot) return; + snapshotPreferred = singleTransaction.Checked; + UpdateSnapshotAvailability(); + }; + schemaData.CheckedChanged += (s, e) => UpdateSnapshotAvailability(); + schemaOnly.CheckedChanged += (s, e) => UpdateSnapshotAvailability(); + dataOnly.CheckedChanged += (s, e) => UpdateSnapshotAvailability(); + statusLabel.AutoSize = true; statusLabel.Margin = new Padding(0, 8, 0, 6); root.Controls.Add(statusLabel, 0, 2); @@ -129,7 +142,14 @@ internal MySqlExportWizardForm(IDatabase db, string databaseName, MySqlExportWiz }; ConfigureButton(cancelButton, "exportCancel", T("取消", "Cancel"), (s, e) => { if (busy) RequestCancel(); else Close(); }); ConfigureButton(nextButton, "exportNext", T("下一步 →", "Next →"), async (s, e) => await OnNextAsync()); - ConfigureButton(backButton, "exportBack", T("← 上一步", "← Back"), (s, e) => { if (!busy && step > 0 && step < 3) { step--; ShowPage(); } }); + ConfigureButton(backButton, "exportBack", T("← 上一步", "← Back"), (s, e) => + { + if (!busy && step > 0 && (step < 3 || Result == null || !Result.Success)) + { + step = step == 3 ? 1 : step - 1; + ShowPage(); + } + }); footer.Controls.AddRange(new Control[] { cancelButton, nextButton, backButton }); root.Controls.Add(footer, 0, 3); @@ -241,6 +261,7 @@ private void BuildPage0() tableGrid.Columns.Add(new DataGridViewCheckBoxColumn { Name = "ColSelect", HeaderText = T("選取", "Select"), FillWeight = 20 }); tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColName", HeaderText = T("資料表名稱", "Table Name"), FillWeight = 55, ReadOnly = true }); tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColRows", HeaderText = T("預估列數", "Estimated Rows"), FillWeight = 25, ReadOnly = true }); + tableGrid.Columns.Add(new DataGridViewTextBoxColumn { Name = "ColEngine", HeaderText = T("引擎", "Engine"), FillWeight = 20, ReadOnly = true }); tableGrid.CellValueChanged += (s, e) => { @@ -308,12 +329,12 @@ private void BuildPage1(MySqlExportWizardOptions initialOptions) { Text = T("交易與結構相容性保障 (Transaction & Portability)", "Transaction & Portability"), Location = new Point(12, 134), - Size = new Size(740, 150), + Size = new Size(740, 260), Padding = new Padding(12) }; var transFlow = new FlowLayoutPanel { Dock = DockStyle.Fill, FlowDirection = FlowDirection.TopDown, WrapContents = false }; - singleTransaction.Text = T("啟用一致性快照 (--single-transaction，以 REPEATABLE READ 交易無鎖讀取，保障關聯一致)", "Consistent snapshot (--single-transaction, non-locking repeatable read)"); + singleTransaction.Text = T("啟用一致性快照（僅支援 InnoDB，--single-transaction）", "Consistent snapshot (InnoDB only, --single-transaction)"); singleTransaction.Checked = true; singleTransaction.AutoSize = true; singleTransaction.Margin = new Padding(0, 4, 0, 6); @@ -332,7 +353,10 @@ private void BuildPage1(MySqlExportWizardOptions initialOptions) includeCreateDb.Checked = false; includeCreateDb.AutoSize = true; - transFlow.Controls.AddRange(new Control[] { singleTransaction, removeDefiner, includeDrop, includeCreateDb }); + snapshotHint.AutoSize = true; + snapshotHint.MaximumSize = new Size(700, 0); + snapshotHint.Margin = new Padding(0, 0, 0, 10); + transFlow.Controls.AddRange(new Control[] { singleTransaction, snapshotHint, removeDefiner, includeDrop, includeCreateDb }); transGroup.Controls.Add(transFlow); content.Controls.Add(transGroup); @@ -340,7 +364,7 @@ private void BuildPage1(MySqlExportWizardOptions initialOptions) var optGroup = new GroupBox { Text = T("MySQL 專屬高速還原與約束優化 (Engine Optimizations & Safety)", "Engine Optimizations & Safety"), - Location = new Point(12, 294), + Location = new Point(12, 404), Size = new Size(740, 130), Padding = new Padding(12) }; @@ -373,6 +397,7 @@ private void BuildPage2() content.Controls.Add(HintLabel(T("請確認本次匯出參數，點擊「開始匯出」即會串流寫入本機檔案。", "Review export parameters. Click 'Start Export' to begin streaming to local file."))); confirmSummary.AutoSize = true; + confirmSummary.MaximumSize = new Size(950, 0); confirmSummary.Margin = new Padding(0, 12, 0, 16); content.Controls.Add(confirmSummary); } @@ -493,7 +518,7 @@ private async Task LoadCatalogAsync() foreach (var item in catalog) item.IsSelected = selected.Contains(item.Name); } tableGrid.Rows.Clear(); - foreach (var item in catalog) tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0")); + foreach (var item in catalog) tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0"), item.Engine ?? T("無法確認", "Unknown")); statusLabel.Text = ""; UpdateSelectionSummary(); } @@ -516,7 +541,7 @@ private void ApplySearchFilter() { if (string.IsNullOrWhiteSpace(term) || item.Name.IndexOf(term, StringComparison.OrdinalIgnoreCase) >= 0) { - tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0")); + tableGrid.Rows.Add(item.IsSelected, item.Name, item.EstimatedRows.ToString("N0"), item.Engine ?? T("無法確認", "Unknown")); } } UpdateSelectionSummary(); @@ -536,6 +561,40 @@ private void UpdateSelectionSummary() selectionSummary.Text = string.Format( T("已選取 {0} / {1} 張資料表（預估約 {2:N0} 筆資料）", "Selected {0} / {1} tables (approx. {2:N0} rows)"), selectedCount, catalog.Count, totalRows); + UpdateSnapshotAvailability(); + } + + private void UpdateSnapshotAvailability() + { + var selected = catalog.Where(t => t.IsSelected).ToList(); + var incompatible = selected.Where(t => !t.SupportsConsistentSnapshot).ToList(); + bool available = !schemaOnly.Checked && selected.Count > 0 && incompatible.Count == 0; + updatingSnapshot = true; + try + { + singleTransaction.Enabled = available; + singleTransaction.Checked = available && snapshotPreferred; + } + finally { updatingSnapshot = false; } + + string ordinaryWarning = T( + "一般匯出不鎖定資料表；持續寫入時，跨表資料可能不一致，還原時也可能發生外鍵衝突。", + "Ordinary export does not lock tables; concurrent writes can cause cross-table inconsistencies and foreign key conflicts on restore."); + if (schemaOnly.Checked) + snapshotHint.Text = T("僅匯出結構，不需要一致性快照。", "Structure only: no consistency snapshot is needed."); + else if (selected.Count == 0) + snapshotHint.Text = T("未選取資料表，不使用資料快照。", "No tables selected; no data snapshot is used."); + else if (incompatible.Count > 0) + { + string examples = string.Join("、", incompatible.Take(3).Select(t => + (t.Name.Length > 60 ? t.Name.Substring(0, 60) + "…" : t.Name) + " (" + (t.Engine ?? T("無法確認", "unknown")) + ")")); + if (incompatible.Count > 3) examples += string.Format(T(" 等 {0} 張", " and {0} tables total"), incompatible.Count); + snapshotHint.Text = T("部分引擎不支援或無法確認為 InnoDB，已停用快照並使用一般匯出：", "Some engines are unsupported or unverified; snapshot is disabled and ordinary export will be used: ") + examples + "\n" + ordinaryWarning; + } + else if (!singleTransaction.Checked) snapshotHint.Text = ordinaryWarning; + else snapshotHint.Text = T( + "所有選取資料表均為 InnoDB。快照期間仍請避免 ALTER / DROP / TRUNCATE 等結構變更。", + "All selected tables are InnoDB. Avoid ALTER / DROP / TRUNCATE and other DDL during the snapshot."); } private void ShowPage() @@ -550,7 +609,9 @@ private void ShowPage() }; stepsLabel.Text = string.Join(" → ", stepTitles.Select((t, i) => i == step ? $"【{t}】" : t)); - backButton.Enabled = (step > 0 && step < 3 && !busy); + UpdateSnapshotAvailability(); + backButton.Enabled = step > 0 && !busy && (step < 3 || Result == null || !Result.Success); + backButton.Text = step == 3 ? T("← 調整設定", "← Adjust settings") : T("← 上一步", "← Back"); if (step == 0) { nextButton.Text = T("下一步 →", "Next →"); @@ -576,6 +637,7 @@ private void ShowPage() private void UpdateConfirmSummary() { + UpdateSnapshotAvailability(); var selectedTables = catalog.Where(t => t.IsSelected).ToList(); confirmSummary.Text = $"● {T("目標資料庫：", "Target Database: ")} {databaseName}\n" + @@ -583,7 +645,7 @@ private void UpdateConfirmSummary() $"● {T("選取資料表：", "Selected Tables: ")} {selectedTables.Count} 張（預估 {selectedTables.Sum(t => t.EstimatedRows):N0} 列）\n" + $"● {T("選取檢視：", "Selected Views: ")} {(includeViews.Checked ? views.Count : 0)} 個\n" + $"● {T("選取常式/觸發器：", "Selected Routines/Triggers: ")} {(includeRoutines.Checked ? routines.Count : 0)} / {(includeTriggers.Checked ? triggers.Count : 0)} 個\n" + - $"● {T("一致性快照：", "Consistency Snapshot: ")} {(singleTransaction.Checked ? T("啟用 (--single-transaction，不鎖表)", "Enabled (--single-transaction)") : T("停用", "Disabled"))}\n" + + $"● {T("一致性快照：", "Consistency Snapshot: ")} {(singleTransaction.Checked ? T("啟用 (--single-transaction)", "Enabled (--single-transaction)") : T("停用", "Disabled"))}\n{snapshotHint.Text}\n\n" + $"● {T("MySQL 引擎優化：", "MySQL Engine Optimizations: ")} {(disableUnique.Checked ? T("停用唯一檢查 (加速 2~5x)", "Unique Checks Off") : "")} {(disableFk.Checked ? T("・停用外鍵", "・FK Checks Off") : "")} {(noAutoValueOnZero.Checked ? "・NO_AUTO_VALUE_ON_ZERO" : "")}\n" + $"● {T("輸出檔案路徑：", "Target Path: ")} {targetPathBox.Text}\n\n" + $"{T("提示：匯出過程將全串流寫入暫存檔案，完成驗證後原子發布，不污染既有檔案。", "Notice: Streaming to staging file first, published atomically upon completion.")}"; @@ -618,6 +680,14 @@ private async Task StartExportAsync() backButton.Enabled = false; cancelButton.Text = T("停止匯出", "Stop Export"); logList.Items.Clear(); + Result = null; + lastProgress = null; + resultText.Text = ""; + currentObjectLabel.Text = T("正在準備匯出...", "Preparing export..."); + overallBar.Value = 0; + percentLabel.Text = "0%"; + writtenLabel.Text = speedLabel.Text = elapsedLabel.Text = etaLabel.Text = ""; + openFileButton.Visible = openFolderButton.Visible = false; var mode = schemaData.Checked ? MySqlExportMode.SchemaAndData : schemaOnly.Checked ? MySqlExportMode.SchemaOnly : MySqlExportMode.DataOnly; var opt = new MySqlExportWizardOptions(mode, targetPathBox.Text.Trim()) @@ -640,8 +710,10 @@ private async Task StartExportAsync() SelectedTriggers = triggers.Select(tr => tr.Name).ToList() }; + var attemptCancellation = cancellation; var progress = new Progress(p => { + if (!exporting || !ReferenceEquals(cancellation, attemptCancellation)) return; lastProgress = p; RenderProgress(p); }); @@ -692,7 +764,7 @@ private async Task StartExportAsync() cancelButton.Enabled = true; cancellation.Dispose(); cancellation = null; - nextButton.Enabled = true; + ShowPage(); if (closing) Close(); } } diff --git a/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs b/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs index 1a89dc5..a54a5a2 100644 --- a/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs +++ b/mySQLPunk/lib/MySqlExport/MySqlExportModels.cs @@ -28,6 +28,8 @@ public sealed class MySqlTableExportItem { public string Name { get; set; } public long EstimatedRows { get; set; } + public string Engine { get; set; } + public bool SupportsConsistentSnapshot => string.Equals(Engine, "InnoDB", StringComparison.OrdinalIgnoreCase); public bool IsSelected { get; set; } public MySqlTableExportItem(string name, long estimatedRows, bool isSelected = true) diff --git a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs index 784a8ff..6276bc6 100644 --- a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs +++ b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs @@ -28,12 +28,14 @@ public List LoadTableCatalog() { var list = new List(); var tables = db.GetTables(databaseName); + DataTable status = null; + try { status = db.GetTableStatus(databaseName); } catch { } foreach (var table in tables) { if (string.IsNullOrWhiteSpace(table)) continue; long count = 0; try { count = db.CountRows(databaseName, table); } catch { } - list.Add(new MySqlTableExportItem(table, count, true)); + list.Add(new MySqlTableExportItem(table, count, true) { Engine = ReadTableEngine(status, table) }); } return list; } @@ -460,16 +462,33 @@ private static string NormalizeCreateSql(string sql, bool removeDefiner) return sql; } + private static string ReadTableEngine(DataTable status, string table) + { + if (status == null || !status.Columns.Contains("Name") || !status.Columns.Contains("Engine")) return null; + // Match the catalog identity exactly; case-sensitive servers may contain both Foo and foo. + DataRow entry = status.Rows.Cast().FirstOrDefault(r => string.Equals(Convert.ToString(r["Name"]), table, StringComparison.Ordinal)); + string engine = entry == null ? null : Convert.ToString(entry["Engine"]); + return string.IsNullOrWhiteSpace(engine) ? null : engine.Trim(); + } + private void RequireTransactionalTables(List selectedTables) { - DataTable status = db.GetTableStatus(databaseName); - if (status == null || !status.Columns.Contains("Name") || !status.Columns.Contains("Engine")) - throw new InvalidOperationException("Cannot verify table engines for a consistent snapshot."); + DataTable status; + try { status = db.GetTableStatus(databaseName); } + catch (Exception ex) + { + throw new InvalidOperationException(ReportText.Get( + "無法確認資料表引擎，不能使用一致性快照。請返回輸出設定，取消一致性快照後使用一般匯出。", + "Cannot verify table engines for a consistent snapshot. Return to output settings and turn off the snapshot for an ordinary export."), ex); + } foreach (string table in selectedTables) { - DataRow entry = status.Rows.Cast().FirstOrDefault(r => string.Equals(Convert.ToString(r["Name"]), table, StringComparison.OrdinalIgnoreCase)); - if (entry == null || !string.Equals(Convert.ToString(entry["Engine"]), "InnoDB", StringComparison.OrdinalIgnoreCase)) - throw new NotSupportedException("Consistent snapshot requires verified InnoDB tables: " + table); + var item = new MySqlTableExportItem(table, 0) { Engine = ReadTableEngine(status, table) }; + if (!item.SupportsConsistentSnapshot) + throw new NotSupportedException(string.Format(ReportText.Get( + "一致性快照只支援已確認的 InnoDB 資料表：{0}（引擎：{1}）。請返回輸出設定，取消一致性快照後使用一般匯出；跨表資料可能不一致。", + "Consistent snapshot requires verified InnoDB tables: {0} (engine: {1}). Return to output settings and turn off the snapshot for an ordinary export; cross-table data may be inconsistent."), + table, item.Engine ?? ReportText.Get("無法確認", "unknown"))); } } diff --git a/tests/SmokeTests.MySqlExport.cs b/tests/SmokeTests.MySqlExport.cs index a8afda6..67e0b9d 100644 --- a/tests/SmokeTests.MySqlExport.cs +++ b/tests/SmokeTests.MySqlExport.cs @@ -2,8 +2,10 @@ using System.Collections.Generic; using System.Data; using System.IO; +using System.Reflection; using System.Threading; using System.Threading.Tasks; +using mySQLPunk; using mySQLPunk.lib; using mySQLPunk.lib.MySqlExport; @@ -143,12 +145,155 @@ private static void MySqlExport_Regression_GeneratedBoundsAndFailures() finally { Directory.Delete(directory, true); } } + private static void MySqlExport_SnapshotCatalogMetadata() + { + var engineProperty = typeof(MySqlTableExportItem).GetProperty("Engine"); + Assert(engineProperty != null, "Catalog must expose verified engine metadata before export"); + foreach (string engine in new[] { "InnoDB", "MyISAM", "MEMORY", null }) + { + var db = new MockMySqlExportDatabase { Engine = engine }; + var items = new MySqlSqlExportService(db, "test_db").LoadTableCatalog(); + AssertEquals(engine ?? "", Convert.ToString(engineProperty.GetValue(items[0])), "Catalog retains engine or unknown"); + Assert(db.StatusReads == 1, "Catalog loads status once rather than per table"); + } + foreach (string failure in new[] { "throw", "null", "columns", "missing" }) + { + var items = new MySqlSqlExportService(new MockMySqlExportDatabase { StatusFailure = failure }, "test_db").LoadTableCatalog(); + Assert(items.Count == 1 && string.IsNullOrEmpty(Convert.ToString(engineProperty.GetValue(items[0]))), "Unavailable engine stays unknown; ordinary catalog remains usable: " + failure); + } + } + + private static void MySqlExport_OrdinaryModeAndSnapshotGuard() + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk-snapshot-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + string path = Path.Combine(directory, "dump.sql"); + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, path) { SingleTransaction = false, SelectedTables = new List { "users" } }; + foreach (string engine in new[] { "MyISAM", "MEMORY", null }) + { + var db = new MockMySqlExportDatabase { Engine = engine }; + var result = new MySqlSqlExportService(db, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(result.Success && result.TotalRows == 2, "Ordinary export works without an InnoDB guarantee: " + result.ErrorMessage); + AssertContains(File.ReadAllText(path), "-- SingleTransaction: False", "Dump must not claim snapshot consistency"); + Assert(!db.ExecutedSql.Exists(sql => sql.Contains("START TRANSACTION") || sql.Contains("LOCK TABLES")), "Ordinary export must not acquire snapshot/table locks"); + } + options.SingleTransaction = true; + var drift = new MockMySqlExportDatabase(); + new MySqlSqlExportService(drift, "test_db").LoadTableCatalog(); + drift.Engine = "MyISAM"; + File.WriteAllText(path, "previous backup"); + var failed = new MySqlSqlExportService(drift, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(!failed.Success && drift.PageReads == 0, "Fresh runtime guard rejects engine changes before reading pages"); + AssertContains(failed.ErrorMessage, "MyISAM", "Failure identifies actual engine"); + AssertEquals("previous backup", File.ReadAllText(path), "Guard preserves previous backup"); + Assert(Directory.GetFiles(directory, "*.tmp").Length == 0, "Guard leaves no incomplete artifact"); + foreach (var unavailable in new[] { new MockMySqlExportDatabase { StatusFailure = "throw" }, new MockMySqlExportDatabase { StatusName = "Users" } }) + { + failed = new MySqlSqlExportService(unavailable, "test_db").ExportAsync(options, null, CancellationToken.None).Result; + Assert(!failed.Success && unavailable.PageReads == 0, "Missing metadata or case-mismatched identity must not borrow another table's engine"); + Assert(failed.ErrorMessage.IndexOf("snapshot", StringComparison.OrdinalIgnoreCase) >= 0 || failed.ErrorMessage.Contains("一致性快照"), "Guard explains snapshot requirement in the selected language"); + Assert(failed.ErrorMessage.IndexOf("settings", StringComparison.OrdinalIgnoreCase) >= 0 || failed.ErrorMessage.Contains("輸出設定"), "Guard provides actionable settings guidance"); + AssertEquals("previous backup", File.ReadAllText(path), "Metadata uncertainty preserves old output"); + } + var caseItems = new MySqlSqlExportService(new MockMySqlExportDatabase { StatusName = "Users" }, "test_db").LoadTableCatalog(); + Assert(!caseItems[0].SupportsConsistentSnapshot, "Catalog must not assume case-insensitive table identity"); + options.Mode = MySqlExportMode.SchemaOnly; + Assert(new MySqlSqlExportService(drift, "test_db").ExportAsync(options, null, CancellationToken.None).Result.Success, "Schema-only export requires no transactional engine"); + } + finally { Directory.Delete(directory, true); } + } + + private static void MySqlExport_SnapshotWizardRecovery() + { + string previousLanguage = Localization.CurrentLanguage; + Localization.SetLanguage(Localization.English, false); + var wizardType = typeof(IDatabase).Assembly.GetType("mySQLPunk.MySqlExportWizardForm", true); + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk-snapshot-ui-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + var db = new MockMySqlExportDatabase { OtherEngine = "MyISAM" }; + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, Path.Combine(directory, "dump.sql")); + using (var form = (System.Windows.Forms.Form)Activator.CreateInstance(wizardType, BindingFlags.Instance | BindingFlags.NonPublic, null, new object[] { db, "test_db", options }, null)) + { + MySqlSnapshotPump((Task)wizardType.GetMethod("LoadCatalogAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + var snapshot = GetPrivateField(form, "singleTransaction"); + var items = GetPrivateField>(form, "catalog"); + Action refresh = () => wizardType.GetMethod("UpdateSelectionSummary", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null); + Assert(!snapshot.Enabled && !snapshot.Checked, "Mixed engines must offer ordinary export before execution"); + AssertContains(GetPrivateField(form, "snapshotHint").Text, "legacy (MyISAM)", "Hint identifies incompatible selection"); + GetPrivateField(form, "search").Text = "users"; + Assert(!snapshot.Enabled, "Search-hidden selected table still controls consistency"); + items[1].IsSelected = false; refresh(); + Assert(snapshot.Enabled && snapshot.Checked, "All selected InnoDB tables restore preferred snapshot"); + snapshot.Checked = false; + items[1].IsSelected = true; refresh(); items[1].IsSelected = false; refresh(); + Assert(snapshot.Enabled && !snapshot.Checked, "Explicit snapshot-off survives selection changes"); + var schema = GetPrivateField(form, "schemaOnly"); + snapshot.Checked = true; schema.Checked = true; + Assert(!snapshot.Enabled && !snapshot.Checked, "Schema-only must not claim a snapshot"); + wizardType.GetMethod("UpdateConfirmSummary", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null); + AssertContains(GetPrivateField(form, "confirmSummary").Text, "snapshot", "English schema-only summary remains explicit about consistency"); + GetPrivateField(form, "schemaData").Checked = true; + Assert(snapshot.Enabled && snapshot.Checked, "Returning to data mode restores preferred snapshot"); + db.Engine = "MyISAM"; + wizardType.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(form, 3); + MySqlSnapshotPump((Task)wizardType.GetMethod("StartExportAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + Assert(GetPrivateField(form, "backButton").Enabled, "Failure permits settings recovery after cleanup"); + typeof(System.Windows.Forms.Button).GetMethod("OnClick", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(GetPrivateField(form, "backButton"), new object[] { EventArgs.Empty }); + Assert((int)wizardType.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form) == 1, "Failure returns to output settings"); + snapshot.Checked = false; + wizardType.GetMethod("UpdateConfirmSummary", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null); + AssertContains(GetPrivateField(form, "confirmSummary").Text, "cross-table", "Ordinary confirmation warns about cross-table inconsistency"); + MySqlSnapshotPump((Task)wizardType.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + Assert(!File.Exists(options.TargetPath), "Returning to settings requires confirmation before retry"); + MySqlSnapshotPump((Task)wizardType.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + var result = (mySQLPunk.lib.MySqlExport.MySqlExportResult)wizardType.GetProperty("Result", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form); + Assert(result.Success && File.Exists(options.TargetPath), "User can confirm ordinary mode and retry successfully"); + Assert(!GetPrivateField(form, "backButton").Enabled, "Success only offers closing/output actions"); + db.OnPageRead = () => GetPrivateField(form, "cancellation").Cancel(); + MySqlSnapshotPump((Task)wizardType.GetMethod("StartExportAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + result = (mySQLPunk.lib.MySqlExport.MySqlExportResult)wizardType.GetProperty("Result", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form); + Assert(result.Stage == MySqlExportStage.Cancelled && GetPrivateField(form, "backButton").Enabled, "Cancellation also permits settings recovery"); + Assert(!GetPrivateField(form, "openFileButton").Visible, "Retry/cancel must remove previous success affordances"); + Assert(GetPrivateField(form, "overallBar").Value < 100, "Cancelled retry cannot retain completed progress"); + Assert(mySQLPunk.lib.MySqlImport.MySqlImportPreflightScanner.Scan(options.TargetPath, CancellationToken.None).Passed, "Cancelled retry preserves the prior verified backup"); + } + foreach (string failure in new[] { "throw", "missing", "columns" }) + using (var form = (System.Windows.Forms.Form)Activator.CreateInstance(wizardType, BindingFlags.Instance | BindingFlags.NonPublic, null, new object[] { new MockMySqlExportDatabase { StatusFailure = failure }, "test_db", options }, null)) + { + MySqlSnapshotPump((Task)wizardType.GetMethod("LoadCatalogAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + var snapshot = GetPrivateField(form, "singleTransaction"); + Assert(!snapshot.Enabled && !snapshot.Checked, "Unknown engines permit only ordinary export: " + failure); + } + } + finally { Directory.Delete(directory, true); Localization.SetLanguage(previousLanguage, false); } + } + + private static void MySqlSnapshotPump(Task task) + { + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!task.IsCompleted && watch.ElapsedMilliseconds < 15000) + { + System.Windows.Forms.Application.DoEvents(); + Thread.Sleep(1); + } + Assert(task.IsCompleted, "Synthetic wizard task must complete within 15 seconds"); + task.GetAwaiter().GetResult(); + System.Windows.Forms.Application.DoEvents(); + } + private sealed class MockMySqlExportDatabase : IDatabase { public ConnectionState State => ConnectionState.Open; public string ProviderName => "MySQL"; public bool Generated; public string Failure; + public string Engine = "InnoDB", OtherEngine, StatusFailure; + public string StatusName = "users"; + public int StatusReads; public long MaxRowBytes = 32; public int PageReads; public Action OnPageRead; @@ -228,7 +373,7 @@ public Task> ExecSQLAsync(string sql, Dictionary GetDatabases() => new List { "test_db" }; - public List GetTables(string databaseName) => new List { "users" }; + public List GetTables(string databaseName) => OtherEngine == null ? new List { "users" } : new List { "users", "legacy" }; public List GetViews(string databaseName) => new List { "active_users" }; public DataTable GetColumns(string databaseName, string tableName) @@ -246,8 +391,15 @@ public DataTable GetColumns(string databaseName, string tableName) public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); public DataTable GetTableStatus(string databaseName) { - var dt = new DataTable(); dt.Columns.Add("Name"); dt.Columns.Add("Engine"); - dt.Rows.Add("users", Failure == "engine" ? "MyISAM" : "InnoDB"); return dt; + StatusReads++; + if (StatusFailure == "throw") throw new InvalidOperationException("metadata unavailable"); + if (StatusFailure == "null") return null; + var dt = new DataTable(); + if (StatusFailure == "columns") return dt; + dt.Columns.Add("Name"); dt.Columns.Add("Engine"); + if (StatusFailure != "missing") dt.Rows.Add(StatusName, Failure == "engine" ? "MyISAM" : Engine); + if (OtherEngine != null) dt.Rows.Add("legacy", OtherEngine); + return dt; } public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index b79b516..23c4d6d 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -40,6 +40,9 @@ public static int Main(string[] args) Run("MySQL import regression target literals and strategies", MySqlImport_Regression_TargetLiteralsAndStrategies, ref passed); Run("MySQL import regression cancel transactions and preflight", MySqlImport_Regression_CancelTransactionsAndPreflight, ref passed); Run("MySQL export regression generated bounds and failures", MySqlExport_Regression_GeneratedBoundsAndFailures, ref passed); + Run("MySQL export snapshot catalog metadata", MySqlExport_SnapshotCatalogMetadata, ref passed); + Run("MySQL export ordinary mode and snapshot guard", MySqlExport_OrdinaryModeAndSnapshotGuard, ref passed); + Run("MySQL export snapshot wizard recovery", MySqlExport_SnapshotWizardRecovery, ref passed); Run("MySQL export options and models defaults", MySqlExport_OptionsAndModels_Defaults, ref passed); Run("MySQL export service and manifest atomic write", MySqlExport_ServiceAndManifest_AtomicWrite, ref passed); Run("MySQL import preflight scanner delimiters and manifest", MySqlImport_PreflightScanner_DelimitersAndManifest, ref passed); From 165ef3cdd8458ac20e6a054f20e7ed905b25a24f Mon Sep 17 00:00:00 2001 From: John Date: Fri, 16 Oct 2026 14:57:42 +0800 Subject: [PATCH 11/12] fix(mysql): export bounded geometry values and validate before writing --- CHANGELOG.md | 1 + TODO.md | 2 +- .../mysql-geometry-report.md | 58 ++++ .../db-wizard-repairs/mysql-repair-report.md | 4 +- .../mysql-spatial-fix-plan.md | 14 + docs/design/db-wizard-repairs/verification.md | 10 +- mySQLPunk/MySqlExportWizardForm.cs | 71 +++- .../lib/MySqlExport/MySqlSqlExportService.cs | 169 ++++++++-- tests/Run-SmokeTests.ps1 | 1 + tests/SmokeTests.MySqlExport.cs | 99 +++++- tests/SmokeTests.MySqlGeometry.cs | 316 ++++++++++++++++++ tests/SmokeTests.cs | 7 + 12 files changed, 713 insertions(+), 39 deletions(-) create mode 100644 docs/design/db-wizard-repairs/mysql-geometry-report.md create mode 100644 docs/design/db-wizard-repairs/mysql-spatial-fix-plan.md create mode 100644 tests/SmokeTests.MySqlGeometry.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7111950..57ef454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - **獨立日誌與即時審計**：所有匯入執行步驟與錯誤皆獨立記錄至 `%LOCALAPPDATA%\mySQLPunk\ImportLogs`，並於匯入完成後自動比對 Native Manifest 目標列數；筆數符合不代表全部資料值與結構已驗收。 ### 🛠️ 問題修正與優化 +- **MySQL 空間欄位匯出與提前檢查**：普通 2D Geometry 改以 WKB 與每筆 SRID 匯出，保留 NULL、排除 generated 欄位；全部選表先檢查欄位與大小，再開始寫檔。失敗後清除速率／ETA，顯示未發布狀態；100% 只代表成功產檔。來源單列仍限 1 MiB，實機還原與跨引擎 SRS 相容性尚待驗收。 - **MySQL／SQLite／Oracle 匯出匯入修整**：修正衝突策略誤刪／漏匯、SQL 字串誤改、provider 失敗被當成功、交易與取消清理；改用獨立工作連線與保留舊檔的原子發布。SQLite 補上 trigger、generated 欄位、rowid／序號及文字 bytes 往返；Oracle 補來源快照、逐列 reader、SRID／metadata 與 bounded LOB 分段。單表入口預設不包含不相關物件。支援上限與尚未驗收項目見 [驗證文件](docs/design/db-wizard-repairs/verification.md)。 - **MySQL 匯出與匯入精靈版面與體驗優化**： - **修復版面跑版坍塌**：修正步驟 2 與步驟 3 之設定 GroupBox 在 FlowLayoutPanel 內因 AutoSize 導致寬度坍塌至邊框寬度 (~15px) 造成文字嚴重擠壓折疊之跑版問題，改採標準固定寬度 (740px) 與適應性內部流式排版。 diff --git a/TODO.md b/TODO.md index 6458e51..d169395 100644 --- a/TODO.md +++ b/TODO.md @@ -32,7 +32,7 @@ | 資料庫 Provider | SQL 傾印 (匯出) 現況 | SQL 檔案匯入現況 | 主要缺口與風險 | | :--- | :--- | :--- | :--- | -| **MySQL / MariaDB** | 四步精靈、選表／模式、獨立連線、InnoDB 快照、generated 欄位投影與原子發布已補強。 | 共用預檢／執行 lexer、識別字限定的目標處理、衝突策略、交易／清理結果與錯誤日誌已補強。 | 本機回歸後仍需實機往返；來源單列目前限 1 MiB、INSERT 限 8 MiB。更大 LOB 分段與主動中斷執行中命令尚待擴充。 | +| **MySQL / MariaDB** | 四步精靈、選表／模式、獨立連線、InnoDB 快照、generated 欄位投影、WKB／SRID 空間欄位、提前檢查與原子發布已補強。 | 共用預檢／執行 lexer、識別字限定的目標處理、衝突策略、交易／清理結果與錯誤日誌已補強。 | 本機回歸後仍需實機往返；來源單列目前限 1 MiB、INSERT 限 8 MiB。更大 LOB 分段與主動中斷執行中命令尚待擴充。 | | **SQLite** | 原生逐列 reader、來源快照、generated 欄位投影、rowid／序號、文字／BLOB bytes 與原子發布已補強。 | 策略值綁定、DROP 保護、trigger 解析、嚴格 UTF-8、交易／PRAGMA 清理與日誌已補強。 | 私有 SQLite 往返驗收；單列目前限 8 MiB。SpatiaLite／GeoPackage 擴充模組與多 GiB、實體 UI 仍需另外驗收，不能只靠筆數宣稱無損。 | | **PostgreSQL** | ✅ 已具備完整四步精靈 (`PostgreSqlExportWizardForm`)，支援選表/View、PostGIS 空間幾何無損 Hex EWKB 傾印、排除 spatial_ref_sys、REPEATABLE READ 快照與 Native Manifest。 | ✅ 已具備完整四步精靈 (`PostgreSqlImportWizardForm`)，支援狀態機 Lexer (精準解析 Dollar Quotes 與巢狀註解)、PostGIS 自動安裝開關、4種衝突策略、2,000行滑動視窗日誌與列數核對。 | 已達標（完全對齊 MSSQL 標竿規格，並深度整合崩琦 GIS PostGIS 空間特性）。 | | **Oracle** | 獨立連線、來源 READ ONLY、原生逐列 reader、Spatial 原始 metadata／SRID、bounded LOB 與原子發布修整。 | 檢查 provider 結果、目標 owner、交易、共用 lexer、日誌與進度 bytes 修整。 | 仍待 Oracle／Spatial 實機往返；大型值、identity、LRS／曲線及跨 owner 腳本有明確支援邊界，詳見修整驗證文件。尚未宣稱完全對齊 MSSQL。 | diff --git a/docs/design/db-wizard-repairs/mysql-geometry-report.md b/docs/design/db-wizard-repairs/mysql-geometry-report.md new file mode 100644 index 0000000..03400e5 --- /dev/null +++ b/docs/design/db-wizard-repairs/mysql-geometry-report.md @@ -0,0 +1,58 @@ +# MySQL spatial export repair + +Date: 2026-10-16. Baseline: `9f5f3ed82f15f18c277ba094205e9db982df7481` on `codex/db-wizard-repairs`. + +## Problem and behavior + +The exporter rejected spatial column metadata with `Spatial column values require a native MySQL export`. Because metadata validation occurred immediately before exporting each table, an unsupported table late in the selection could stop a long-running export after earlier tables had been processed. This explains the reported `city_geometry.SHAPE` failure; no source database inspection was performed. + +`MySqlSqlExportService.ValidateTableDataCompatibility` now provides a metadata-only check for the wizard before export confirmation. It does not count or scan rows, start transactions, create output, or retain metadata across attempts. Unsupported or absent writable-column type metadata fails explicitly with the table/type context and native-export guidance. + +Each actual export independently reloads every selected table's shape. After the optional verified-InnoDB consistent snapshot has started, it first validates all selected table metadata, then establishes all row bounds and counts, and only then creates a staging file or reads data pages. `ReadingCatalog` progress identifies the table being checked. Schema-only export does not require a data compatibility check. + +## Spatial representation and bounded operation + +The eight supported ordinary spatial types are `GEOMETRY`, `POINT`, `LINESTRING`, `POLYGON`, `MULTIPOINT`, `MULTILINESTRING`, `MULTIPOLYGON`, and `GEOMETRYCOLLECTION`. Writable spatial columns are projected as binary `ST_AsWKB(column)` plus their per-row `ST_SRID(column)`. Output uses `ST_GeomFromWKB(0x...,srid)` with exact hexadecimal WKB and invariant numeric SRID. Zero and nonzero SRIDs are preserved; no SRID is synthesized or transformed. + +Both-null WKB/SRID pairs emit SQL `NULL`. Non-null geometry requires nonempty binary WKB and an integral unsigned 32-bit SRID. One-sided nulls, missing projected helpers, empty/nonbinary WKB, fractional/negative/out-of-range SRIDs and unsupported value types fail explicitly. + +Generated columns retain their DDL and are omitted before source projection, row-size bounding and INSERT column selection. Helper aliases are allocated against every original column name using case-insensitive comparison. INSERT targets come exclusively from original writable metadata; helper values never become target columns. Existing identifier quoting handles backticks, Unicode and embedded dots. + +The existing 1 MiB source-row materialization bound, 16-row page cap, 8 MiB INSERT limit, atomic publication and prior-output preservation remain. Spatial bounds include WKB length and ten conservative SRID bytes. Server-side CASE guards cover both WKB and SRID so enlargement after the initial bound cannot send an oversized geometry page to the client. Hex expansion and constructor overhead count toward the final INSERT limit. + +## Compatibility boundary + +The common two-argument constructor deliberately pairs the source's default WKB axis convention with the compatible target's default constructor convention. MySQL 8 uses its SRS-defined geographic axis order by default; MariaDB documents `ST_GeomFromWKB(wkb[,srid])`. No unconditional MySQL-only third axis-order argument is emitted. + +Supported intent is a compatible-engine, compatible-SRS round trip. Cross-vendor or cross-version geographic migration, coordinate transformations, custom SRS-definition transfer, and unknown target SRS availability are outside this repair. WKB preserves binary coordinates without a WKT decimal conversion, but actual server constructor behavior and spatial fidelity remain to be verified on the user's selected source/target engines. The SQL INSERT cap is independent of a target server's potentially smaller `max_allowed_packet`. + +Official references: + +- [MySQL spatial formats and internal SRID prefix](https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html) +- [MySQL WKB conversion and default axis handling](https://dev.mysql.com/doc/refman/8.0/en/gis-format-conversion-functions.html) +- [MySQL WKB construction, axis order and undefined SRS errors](https://dev.mysql.com/doc/refman/8.0/en/gis-wkb-functions.html) +- [MariaDB two-argument WKB constructor](https://mariadb.com/docs/server/reference/sql-statements/geometry-constructors/wkb/st_geomfromwkb) +- [MySQL spatial upgrade incompatibilities](https://dev.mysql.com/blog-archive/upgrading-to-mysql-8-0-with-spatial-data/) +- [MySQL packet-size limits](https://dev.mysql.com/doc/refman/8.0/en/packet-too-large.html) + +## Local evidence + +The root integration run `C:/Users/johnho/Documents/mySQLPunk-reviews/2026-10-16/spatial-mysql.log` passed all 27 MySQL smoke groups plus AutoReleasePolicy. The five new recording-provider groups are: + +1. `MySqlExport_Geometry_MetadataPreflightAndFreshAttempt`: all eight metadata types, scalar enum containing spatial words, metadata-only work, fresh attempt metadata and cancellation. +2. `MySqlExport_Geometry_ValuesAndImportExpressions`: exact binary WKB, NULL, zero/nonzero/maximum SRIDs, ordinary BLOB values, actual digest/parser preflight, and the actual import runner accepting intact constructor expressions through a recording provider. +3. `MySqlExport_Geometry_GeneratedColumnsAndIdentifiers`: generated geometry DDL, exclusion from scans/INSERTs, multiple geometries, escaped identifiers and case-insensitive helper collisions. +4. `MySqlExport_Geometry_BoundsRaceAndMalformedValues`: malformed WKB/SRID pairs, initial row bound, guarded enlargement race, explicit failure and preservation of the previous complete backup. +5. `MySqlExport_Geometry_BatchCapsAndEarlyFailure`: unsupported last-table rejection before any bounds/counts/pages/staging, whole-selection preflight order, six synthetic near-limit binary projection payloads, 16-row pages, and SQL statements below 8 MiB after hex expansion. + +`spatial-values-red.log` verifies that the previous actual Debug binary fails the new spatial-values test with its original spatial-column error. `spatial-ui-red.log` separately records the old wizard advancing unsupported metadata to confirmation. The successful integrated run includes the root-owned UI regressions. + +Read-only diff whitespace verification passed for the service. The service and new test file were audited as UTF-8 without BOM, with CRLF and no bare CR/LF. This report uses LF. No other file was edited by this implementation subagent. + +## Deferred acceptance + +No actual MySQL/MariaDB connection, server startup, saved profile/credential access, user-source scan, real import/restore or production write was performed. The user explicitly deferred real database restoration. Recording-provider completion proves generated-expression/parser/runner behavior, not live geometry fidelity or data landing. + +Later authorized private live acceptance should compare restored `ST_SRID`, `HEX(ST_AsWKB(...))`, subtype and asymmetric point coordinates, including geographic SRID 4326, zero/custom SRIDs where supported, empty geometry collections, spatial indexes and generated spatial expressions. Row-count/hash manifests alone do not prove spatial equality. Full-suite/build/review results are maintained by the root in the integration verification document. + +Root final gate: complete304 smoke groups and AutoReleasePolicy PASS; explicit default Debug and Release builds PASS. Independent integrated source/test review found no material findings. All twelve modified/new files passed final UTF-8/BOM/newline and whitespace audits. Actual database acceptance remains deferred. diff --git a/docs/design/db-wizard-repairs/mysql-repair-report.md b/docs/design/db-wizard-repairs/mysql-repair-report.md index b1369cd..179f246 100644 --- a/docs/design/db-wizard-repairs/mysql-repair-report.md +++ b/docs/design/db-wizard-repairs/mysql-repair-report.md @@ -8,7 +8,7 @@ Baseline: d41825cead810bc67f263753bb5f986a7f69c65c. Review input: C:/Users/johnh - `MySqlImportRunner.cs` redirects comment-prefixed USE and recognized schema-qualified object positions even without prior USE. Identifier token replacement preserves string/JSON/comment content and longer identifiers. Ambiguous qualifiers and dynamic execution/CALL/client commands fail explicitly; database-wide destructive commands fail rather than writing outside the chosen target. This is a conservative logical-dump restore boundary, not an arbitrary SQL sandbox. - Initial target-table catalog is separate from the live object cache. SkipExisting protects initial objects while fresh CREATE plus repeated INSERT batches execute. Comment-prefixed DROP cannot bypass protected strategies; synthetic DROP checks actual status=NO results. Session initialization and cleanup validate execution results. Uncommitted explicit transactions, failed COMMIT, and failed cleanup cannot produce Completed. Logs auto-flush each statement. Cancellation is checked after returned SQL, including the final statement. - `MySqlImportPreflightScanner.cs` uses the same parser and verifies the native SHA-256 footer against the exact byte prefix. The footer lookup operates on raw bytes, including non-ASCII input near the final buffer boundary. -- `MySqlSqlExportService.cs` validates snapshot setup, requires verified InnoDB selected tables for SingleTransaction, uses UTC session/output time zone, validates snapshot rollback before publication, and never commits the source session. Missing table DDL/metadata/bounds/pages or source row-count discrepancies fail explicitly. Generated columns retain DDL but are omitted from INSERT columns and values. Unsupported spatial values and unsupported runtime value types fail before publication. TIME microseconds are preserved. +- `MySqlSqlExportService.cs` validates snapshot setup, requires verified InnoDB selected tables for SingleTransaction, uses UTC session/output time zone, validates snapshot rollback before publication, and never commits the source session. Missing table DDL/metadata/bounds/pages or source row-count discrepancies fail explicitly. Generated columns retain DDL but are omitted from INSERT columns and values. Ordinary 2D spatial values now use bounded WKB/SRID export (see [spatial follow-up](mysql-geometry-report.md)); unsupported runtime value types fail before publication. TIME microseconds are preserved. - Export has a 1 MiB source-row limit, at most 16 rows per source page, and an 8 MiB UTF-8 INSERT limit. Server-side OCTET_LENGTH preflight and CASE-projected pages prevent oversized LOB materialization, including an enlargement after the initial bound query when SingleTransaction is disabled. Oversized values fail explicitly; no all-size/streaming-LOB claim is made. - Export uses the shared `SqlExportPublication.Publish` helper. Replacement failure keeps the completed staging file and reports its path; failure/cancellation before publication preserves the previous destination. Hashing is cancellable and final byte count includes the footer. - `MySqlExportWizardForm.cs` applies supplied initial mode, table selection and option flags. Catalog loading remains busy until awaited work finishes; close requests wait for cleanup. Both forms restore the cancel button and dispose per-operation cancellation sources after awaited completion. Shared Form1 owned-session routing is integrated by the root implementer. @@ -41,7 +41,7 @@ Durable coverage includes comment-prefixed/inline USE, same-line splitting, expl - IDatabase exposes no cancellable active command. Stop is observed when the current command/read returns or times out; it does not promise immediate server command cancellation. The forms keep the owned job alive while awaiting cleanup. - Imports may have applied earlier autocommitted DDL/DML before a later rejected statement; failure logs and counters remain authoritative and no all-or-nothing restore claim is made. Explicit uncommitted transactions are rolled back and reported failed. - Concurrent source DDL and source pagination without a snapshot remain outside a consistency guarantee. SingleTransaction is restricted to verified InnoDB tables; disabling it permits ordinary non-snapshot reads, with bounded pages and row-count discrepancy detection. -- Executable version comments, dynamic SQL/CALL, ambiguous dotted identifiers, noncanonical SQL_MODE changes, AUTOCOMMIT changes, complex transaction-control modifiers, spatial data values, oversized source rows, and unknown runtime value types require the native-client path. Logical import is intentionally conservative about unsupported constructs. +- Executable version comments, dynamic SQL/CALL, ambiguous dotted identifiers, noncanonical SQL_MODE changes, AUTOCOMMIT changes, complex transaction-control modifiers, cross-engine geographic SRS migration, oversized source rows, and unknown runtime value types require the native-client path. Logical import is intentionally conservative about unsupported constructs. - Full supported-type fidelity remains pending real-server fixtures; in particular timestamp/zero-date/provider type mapping and MariaDB variants are not certified by these recording tests. Shared owned-session and routing/publication test results belong to the root report. ## Scoped review correction wave diff --git a/docs/design/db-wizard-repairs/mysql-spatial-fix-plan.md b/docs/design/db-wizard-repairs/mysql-spatial-fix-plan.md new file mode 100644 index 0000000..1e09733 --- /dev/null +++ b/docs/design/db-wizard-repairs/mysql-spatial-fix-plan.md @@ -0,0 +1,14 @@ +# MySQL spatial export and early validation repair + +2026-10-16. User screenshot: city_geometry.SHAPE is rejected at 77% after 12:36. This is an explicit unsupported-type failure, not evidence of a stalled connection. User authorizes code and simulated testing now; native restore is deferred. Keep the current repair worktree and architecture. + +1. Reproduce late spatial failure with the actual exporter and recording source; preserve old output and retain regression evidence. Independently trace the native importer/parser and official MySQL/MariaDB format contracts. +2. Support the eight ordinary 2D MySQL geometry types using bounded ST_AsWKB + per-row ST_SRID and ST_GeomFromWKB(binary,srid). Skip generated values; retain exact quoted identifiers and collision-free helper aliases. Require correct null pairing, binary payload and integral uint32 SRID. Keep 1 MiB source-row, 16-row page, 8 MiB INSERT budgets and server CASE guards. +3. Metadata-only checks before confirmation; a failed check retains settings and blocks advance. The worker independently inspects all selected metadata first, then all byte bounds/counts before staging/data pages, within its requested snapshot. Never trust confirmation metadata as runtime cache. Schema-only bypasses data-only exclusions. +4. Terminal UI must show failure rather than an apparent stalled job: no live ETA/rate after stop; processed counts remain labelled unpublished. Reserve100% for successfully published output. Retain settings recovery and retry confirmation. +5. Root owns Form/integration/shared smoke registration; provider worker owns service + new geometry tests; independent reviewer inspects the finished patch. Test malformed/null/type/SRID/alias/generated/bounds/races, late unsupported selection, schema-only, parser/import expressions and terminal/confirmation UI. Run focused tests, full smoke, Debug/Release and encoding audits before local commit. +6. Preserve UTF-8 no-BOM and existing CRLF source/test / LF document conventions. Do not read saved connections, connect to/start any database, convert engines, change DB settings, push or merge. + +Compatibility boundary: default two-argument functions preserve semantics for compatible engine/SRS round trips. MySQL8 uses SRS-defined geographic axis order; migration between MySQL8 and older MySQL/MariaDB geographic semantics or custom SRS definitions is not certified. WKB tests and parser/manifest success are not database value-fidelity proof. Native restore and production acceptance remain pending. + +References: [MySQL conversion](https://dev.mysql.com/doc/refman/8.0/en/gis-format-conversion-functions.html), [MySQL constructor](https://dev.mysql.com/doc/refman/8.0/en/gis-wkb-functions.html), [MariaDB constructor](https://mariadb.com/docs/server/reference/sql-statements/geometry-constructors/wkb/st_geomfromwkb). diff --git a/docs/design/db-wizard-repairs/verification.md b/docs/design/db-wizard-repairs/verification.md index 091beb8..b26a62d 100644 --- a/docs/design/db-wizard-repairs/verification.md +++ b/docs/design/db-wizard-repairs/verification.md @@ -13,7 +13,7 @@ Baseline: `d41825cead810bc67f263753bb5f986a7f69c65c`. Local repair branch: `code | Provider | Limit / unsupported cases | | --- | --- | -| MySQL | Source row <=1 MiB; 16-row source pages; INSERT <=8 MiB. Snapshot requires verified InnoDB. Executable version comments, dynamic SQL/CALL, ambiguous scope and native spatial values are explicitly unsupported. Bigger LOB chunking remains future work. | +| MySQL | Source row <=1 MiB; 16-row source pages; INSERT <=8 MiB. Snapshot requires verified InnoDB. Ordinary 2D geometry uses bounded WKB plus per-row SRID for compatible engine/SRS round trips; native restore is pending. Executable version comments, dynamic SQL/CALL and ambiguous scope remain unsupported. Bigger LOB chunking remains future work. | | SQLite | Source row <=8 MiB; strict UTF-8 SQL and <=32 MiB decoded statement. Virtual modules and inaccessible row identities fail explicitly. SpatiaLite/GeoPackage extension-dependent acceptance is pending. | | Oracle | Row values <=8 MiB, generated statement <=28 MiB, input statement <=32 MiB. BLOB chunks <=2000 bytes. Native export currently requires AL32UTF8; identity data, signed zero, LRS/curved/compound spatial cases and unverified types fail explicitly. Cross-owner targets and general stored/dynamic PL/SQL are rejected; generated bounded LOB and DROP wrappers are supported. | @@ -55,3 +55,11 @@ Regression evidence: new wizard test against the previous actual Debug assembly Reference: [MySQL single-transaction semantics](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html). InnoDB snapshots do not protect nontransactional tables or concurrent DDL. This implementation uses its own exporter; it does not invoke mysqldump or add table locks for ordinary mode. Real MySQL/MariaDB engines, privileges, live concurrent writes and rendered UI/DPI remain unverified; the screenshot is the production symptom, not a live database acceptance result. Final follow-up gate: all **297 smoke groups and AutoReleasePolicy PASS** (`snapshot-options-full.log`); explicit Debug and Release builds PASS. The initially running Debug application locked its executable, so an independent `bin/SnapshotFixDebug` build succeeded first; after the user requested retry, normal `bin/Debug/mySQLPunk.exe` rebuilt successfully (`snapshot-options-debug-retry.log`). No application process was terminated. Existing source/test CRLF and document LF, UTF-8/BOM conventions and diff whitespace are preserved. Real database and rendered UI acceptance remain pending. + +## MySQL spatial export follow-up (2026-10-16) + +The city_geometry.SHAPE screenshot shows an explicit unsupported spatial-type failure, with stale progress/ETA remaining on screen. Implemented bounded WKB + per-row SRID export for the eight ordinary 2D geometry types, writable-column/source helper isolation, metadata-only confirmation checks and fresh all-table metadata/bounds checks before staging/data. Terminal UI clears rate/ETA, labels processed rows unpublished on failure/cancellation and reserves100% for successful publication. Repeated close/cancel during asynchronous compatibility checks is covered. + +Final evidence: previous actual Debug binary fails geometry-values regression with the same original Spatial column values error (`spatial-values-red.log`); previous wizard fails early-check regression (`spatial-ui-red.log`). Repaired actual-product MySQL27 groups, complete304 groups plus AutoReleasePolicy PASS (`spatial-mysql.log`, `spatial-full.log`). Explicit default Debug and Release builds PASS (`spatial-debug.log`, `spatial-release.log`). A separate read-only integrated reviewer found no material issues; the asynchronous-lifetime and actual-late-progress test gaps were addressed before the final gate. All twelve source/test/document encoding/BOM/newline conventions and diff whitespace pass. + +[Implementation and scoped evidence](mysql-geometry-report.md), [repair plan](mysql-spatial-fix-plan.md). Source and test files were frozen before the final gate. User explicitly deferred actual database restoration: no MySQL/MariaDB session, saved profile/credential, live source scan, actual restore or production write was used. Real WKB/SRID/SRS fidelity remains pending. The1MiB source-row/16-rowpage/8MiBINSERT budgets remain; compatible engine/SRS round trips are the intended scope, with cross-version/vendor geographic migration and custom SRS transfer unverified. Recording-provider import and manifest success do not prove native data landing. diff --git a/mySQLPunk/MySqlExportWizardForm.cs b/mySQLPunk/MySqlExportWizardForm.cs index 534b022..e433ee4 100644 --- a/mySQLPunk/MySqlExportWizardForm.cs +++ b/mySQLPunk/MySqlExportWizardForm.cs @@ -130,6 +130,7 @@ internal MySqlExportWizardForm(IDatabase db, string databaseName, MySqlExportWiz dataOnly.CheckedChanged += (s, e) => UpdateSnapshotAvailability(); statusLabel.AutoSize = true; + statusLabel.MaximumSize = new Size(1000, 0); statusLabel.Margin = new Padding(0, 8, 0, 6); root.Controls.Add(statusLabel, 0, 2); @@ -414,6 +415,7 @@ private void BuildPage3() var statsFlow = Stack(); currentObjectLabel.Font = UiKit.Subtitle; currentObjectLabel.AutoSize = true; + currentObjectLabel.MaximumSize = new Size(680, 0); statsFlow.Controls.Add(currentObjectLabel); overallBar.Width = 520; @@ -438,6 +440,7 @@ private void BuildPage3() statsFlow.Controls.Add(timeRow); resultText.AutoSize = true; + resultText.MaximumSize = new Size(680, 0); resultText.Margin = new Padding(0, 6, 0, 8); statsFlow.Controls.Add(resultText); @@ -630,9 +633,10 @@ private void ShowPage() } else if (step == 3) { - nextButton.Text = T("完成", "Finished"); + nextButton.Text = Result != null && Result.Success ? T("完成", "Finished") : T("關閉", "Close"); nextButton.Enabled = !busy; } + if (busy) nextButton.Enabled = false; } private void UpdateConfirmSummary() @@ -654,11 +658,50 @@ private void UpdateConfirmSummary() private async Task OnNextAsync() { if (busy) return; - if (step < 2) + if (step == 0) { + statusLabel.Text = ""; step++; ShowPage(); } + else if (step == 1) + { + busy = true; + cancellation = new CancellationTokenSource(); + ShowPage(); + statusLabel.Text = T("正在檢查所有選取資料表的欄位相容性...", "Checking column compatibility for all selected tables..."); + try + { + if (!schemaOnly.Checked) + { + var selected = catalog.Where(t => t.IsSelected).Select(t => t.Name).ToList(); + await Task.Run(() => service.ValidateTableDataCompatibility(selected, cancellation.Token)); + } + cancellation.Token.ThrowIfCancellationRequested(); + if (!closing) + { + statusLabel.Text = ""; + step = 2; + } + } + catch (OperationCanceledException) + { + statusLabel.Text = T("已取消相容性檢查。", "Compatibility check cancelled."); + } + catch (Exception ex) + { + statusLabel.Text = T("尚無法匯出資料：", "Data export is unavailable: ") + ex.Message; + } + finally + { + busy = false; + cancellation.Dispose(); + cancellation = null; + cancelButton.Enabled = true; + ShowPage(); + if (closing) Close(); + } + } else if (step == 2) { step = 3; @@ -677,6 +720,7 @@ private async Task StartExportAsync() exporting = true; cancellation = new CancellationTokenSource(); nextButton.Enabled = false; + nextButton.Text = T("匯出中...", "Exporting..."); backButton.Enabled = false; cancelButton.Text = T("停止匯出", "Stop Export"); logList.Items.Clear(); @@ -741,7 +785,7 @@ private async Task StartExportAsync() else if (Result.Stage == MySqlExportStage.Cancelled) { currentObjectLabel.Text = T("⚠️ 匯出已被使用者取消。", "⚠️ Export cancelled by user."); - resultText.Text = T("暫存檔案已自動清理，未更動目標檔案。", "Staging file was deleted, destination untouched."); + resultText.Text = T("本次匯出未完成，沒有發布新的目標檔案；可調整設定後重試。", "Export did not complete; no new target file was published. Adjust settings to retry."); } else { @@ -754,12 +798,29 @@ private async Task StartExportAsync() timer.Stop(); clock.Stop(); currentObjectLabel.Text = T("❌ 發生非預期錯誤：", "❌ Unexpected error: ") + ex.Message; + resultText.Text = ex.Message; } finally { + exporting = false; + timer.Stop(); + clock.Stop(); + speedLabel.Text = etaLabel.Text = ""; + elapsedLabel.Text = string.Format(T("耗時：{0:mm\\:ss}", "Elapsed: {0:mm\\:ss}"), Result?.Elapsed ?? clock.Elapsed); + if (Result != null && Result.Success) + { + overallBar.Value = 100; + percentLabel.Text = "100%"; + } + else + { + overallBar.Value = Math.Min(99, overallBar.Value); + percentLabel.Text = overallBar.Value + T("%（未完成）", "% (incomplete)"); + writtenLabel.Text = string.Format(T("已處理：{0:N0} 列（未發布）", "Processed: {0:N0} rows (unpublished)"), Result?.TotalRows ?? lastProgress?.ProcessedRows ?? 0); + openFileButton.Visible = openFolderButton.Visible = false; + } punky.EndExport(); busy = false; - exporting = false; cancelButton.Text = T("關閉", "Close"); cancelButton.Enabled = true; cancellation.Dispose(); @@ -773,7 +834,7 @@ private void RenderProgress(MySqlExportProgress p) { if (p.TotalTables > 0) { - int val = Math.Min(100, Math.Max(0, (int)(p.CompletedTables * 100.0 / p.TotalTables))); + int val = p.Stage == MySqlExportStage.Completed ? 100 : Math.Min(99, Math.Max(0, (int)(p.CompletedTables * 100.0 / p.TotalTables))); overallBar.Value = val; percentLabel.Text = $"{val}%"; } diff --git a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs index 6276bc6..35302a7 100644 --- a/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs +++ b/mySQLPunk/lib/MySqlExport/MySqlSqlExportService.cs @@ -46,6 +46,19 @@ public List LoadViews() catch { return new List(); } } + // The wizard may call this before confirmation. It must not scan data, + // count rows, start a transaction, or cache metadata across attempts. + public void ValidateTableDataCompatibility(IEnumerable selectedTables, CancellationToken cancellation) + { + if (selectedTables == null) throw new ArgumentNullException(nameof(selectedTables)); + foreach (string table in selectedTables) + { + cancellation.ThrowIfCancellationRequested(); + LoadTableShape(table, cancellation); + } + cancellation.ThrowIfCancellationRequested(); + } + public List LoadRoutines() { try { return MySqlExportService.GetRoutines(db, databaseName) ?? new List(); } @@ -93,6 +106,8 @@ private MySqlExportResult ExportInternal( var selectedViews = options.IncludeViews ? (options.SelectedViews?.ToList() ?? new List()) : new List(); var selectedRoutines = options.IncludeRoutines ? (options.SelectedRoutines?.ToList() ?? new List()) : new List(); var selectedTriggers = options.IncludeTriggers ? (options.SelectedTriggers?.ToList() ?? new List()) : new List(); + var tableShapes = new Dictionary(StringComparer.Ordinal); + var tableRowCounts = new Dictionary(StringComparer.Ordinal); bool inTransaction = false; bool publicationAttempted = false; @@ -108,7 +123,28 @@ private MySqlExportResult ExportInternal( inTransaction = true; } if (options.Mode != MySqlExportMode.SchemaOnly) - foreach (var table in selectedTables) estimatedTotalRows += db.CountRows(databaseName, table); + { + // Validate the whole selection before any potentially expensive + // row scan. All metadata is fresh and belongs to this attempt. + foreach (string table in selectedTables) + { + ReportProgress(progress, MySqlExportStage.ReadingCatalog, table, 0, selectedTables.Count, 0, 0, 0, clock); + tableShapes[table] = LoadTableShape(table, cancellation); + } + foreach (string table in selectedTables) + { + cancellation.ThrowIfCancellationRequested(); + ReportProgress(progress, MySqlExportStage.ReadingCatalog, table, 0, selectedTables.Count, 0, estimatedTotalRows, 0, clock); + CheckRowBound(table, tableShapes[table], cancellation); + long count = db.CountRows(databaseName, table); + if (count < 0) throw new InvalidOperationException("Invalid source row count: " + table); + tableRowCounts[table] = count; + estimatedTotalRows += count; + cancellation.ThrowIfCancellationRequested(); + } + } + + cancellation.ThrowIfCancellationRequested(); using (var fileStream = new FileStream(stagingPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 65536)) using (var writer = new StreamWriter(fileStream, new UTF8Encoding(false))) @@ -183,8 +219,8 @@ private MySqlExportResult ExportInternal( cancellation.ThrowIfCancellationRequested(); string table = selectedTables[i]; long tableRows = 0; - long totalTableRows = db.CountRows(databaseName, table); - var shape = LoadGeneratedColumnsAndCheckRowBound(table, cancellation); + long totalTableRows = tableRowCounts[table]; + var shape = tableShapes[table]; ReportProgress(progress, MySqlExportStage.WritingData, table, i, selectedTables.Count, totalProcessedRows, estimatedTotalRows, fileStream.Position, clock); @@ -196,10 +232,9 @@ private MySqlExportResult ExportInternal( cancellation.ThrowIfCancellationRequested(); if (page.Rows.Count == 0) break; - var writableColumns = page.Columns.Cast().Where(c => !shape.Generated.Contains(c.ColumnName)).ToList(); - if (writableColumns.Count == 0) throw new NotSupportedException("Tables containing only generated columns require a native client: " + table); + var writableColumns = shape.Writable; string prefix = "INSERT INTO " + MySqlImportRunner.QuoteIdentifier(table) + " (" + - string.Join(", ", writableColumns.Select(c => MySqlImportRunner.QuoteIdentifier(c.ColumnName))) + ") VALUES\r\n"; + string.Join(", ", writableColumns.Select(c => MySqlImportRunner.QuoteIdentifier(c.Name))) + ") VALUES\r\n"; var insert = new StringBuilder(prefix); foreach (DataRow row in page.Rows) { @@ -208,7 +243,8 @@ private MySqlExportResult ExportInternal( for (int c = 0; c < writableColumns.Count; c++) { if (c > 0) rowText.Append(", "); - rowText.Append(ToBoundedLiteral(row[writableColumns[c]])); + var column = writableColumns[c]; + rowText.Append(column.Spatial ? ToSpatialLiteral(row[column.Name], row[column.SridAlias], table + "." + column.Name) : ToBoundedLiteral(row[column.Name])); if (rowText.Length > MaxInsertBytes) throw new NotSupportedException("Single row exceeds safe logical-export limit: " + table); } rowText.Append(")"); @@ -494,6 +530,17 @@ private void RequireTransactionalTables(List selectedTables) private const int MaxSourceRowBytes = 1024 * 1024; private const int MaxInsertBytes = 8 * 1024 * 1024; + private static readonly HashSet SpatialTypes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "geometry", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon", "geometrycollection" + }; + private static readonly HashSet ScalarTypes = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "tinyint", "smallint", "mediumint", "int", "integer", "bigint", "decimal", "dec", "numeric", "fixed", + "float", "double", "real", "bit", "bool", "boolean", "char", "varchar", "binary", "varbinary", + "tinyblob", "blob", "mediumblob", "longblob", "tinytext", "text", "mediumtext", "longtext", + "enum", "set", "json", "date", "datetime", "timestamp", "time", "year" + }; private void RequireExecution(string sql) { @@ -502,72 +549,135 @@ private void RequireExecution(string sql) throw new InvalidOperationException(DatabaseExecutionResultService.GetFailureReason(result)); } - private TableShape LoadGeneratedColumnsAndCheckRowBound(string table, CancellationToken cancellation) + private TableShape LoadTableShape(string table, CancellationToken cancellation) { + cancellation.ThrowIfCancellationRequested(); DataTable columns = db.GetColumns(databaseName, table); + cancellation.ThrowIfCancellationRequested(); if (columns == null || columns.Rows.Count == 0) throw new InvalidOperationException("Missing column metadata: " + table); string nameColumn = columns.Columns.Contains("Field") ? "Field" : "COLUMN_NAME"; string extraColumn = columns.Columns.Contains("Extra") ? "Extra" : "EXTRA"; if (!columns.Columns.Contains(nameColumn) || !columns.Columns.Contains(extraColumn)) throw new NotSupportedException("Generated-column metadata unavailable: " + table); + string typeColumn = columns.Columns.Contains("Type") ? "Type" : "DATA_TYPE"; + if (!columns.Columns.Contains(typeColumn)) + throw new NotSupportedException("Column type metadata unavailable; use a native MySQL export: " + table); var shape = new TableShape(); - var generated = shape.Generated; var sizes = new List(); foreach (DataRow column in columns.Rows) { + cancellation.ThrowIfCancellationRequested(); string name = Convert.ToString(column[nameColumn]); + if (string.IsNullOrWhiteSpace(name) || !shape.Names.Add(name)) + throw new InvalidOperationException("Missing or duplicate column identity: " + table); string extra = Convert.ToString(column[extraColumn]); string expression = columns.Columns.Contains("GenerationExpression") ? Convert.ToString(column["GenerationExpression"]) : ""; if (extra.IndexOf("VIRTUAL", StringComparison.OrdinalIgnoreCase) >= 0 || - extra.IndexOf("STORED", StringComparison.OrdinalIgnoreCase) >= 0 || !string.IsNullOrWhiteSpace(expression)) generated.Add(name); - string type = columns.Columns.Contains("Type") ? Convert.ToString(column["Type"]) : - columns.Columns.Contains("DATA_TYPE") ? Convert.ToString(column["DATA_TYPE"]) : ""; - if (type.IndexOf("geometry", StringComparison.OrdinalIgnoreCase) >= 0 || type.IndexOf("point", StringComparison.OrdinalIgnoreCase) >= 0 || - type.IndexOf("polygon", StringComparison.OrdinalIgnoreCase) >= 0 || type.IndexOf("linestring", StringComparison.OrdinalIgnoreCase) >= 0) - throw new NotSupportedException("Spatial column values require a native MySQL export: " + table + "." + name); - shape.Names.Add(name); - sizes.Add("COALESCE(OCTET_LENGTH(" + MySqlImportRunner.QuoteIdentifier(name) + "),0)"); + extra.IndexOf("STORED", StringComparison.OrdinalIgnoreCase) >= 0 || !string.IsNullOrWhiteSpace(expression)) continue; + string declaredType = Convert.ToString(column[typeColumn]).Trim(); + int suffix = declaredType.IndexOfAny(new[] { '(', ' ', '\t', '\r', '\n' }); + string type = suffix < 0 ? declaredType : declaredType.Substring(0, suffix); + bool spatial = SpatialTypes.Contains(type); + if (!spatial && !ScalarTypes.Contains(type)) + throw new NotSupportedException("Unsupported MySQL column type '" + (declaredType.Length == 0 ? "unknown" : declaredType) + "'; use a native MySQL export: " + table + "." + name); + string quoted = MySqlImportRunner.QuoteIdentifier(name); + var writable = new ColumnShape { Name = name, Spatial = spatial, SourceExpression = spatial ? "ST_AsWKB(" + quoted + ")" : quoted }; + shape.Writable.Add(writable); + sizes.Add(spatial ? "(COALESCE(OCTET_LENGTH(" + writable.SourceExpression + "),0) + CASE WHEN " + quoted + " IS NULL THEN 0 ELSE 10 END)" : + "COALESCE(OCTET_LENGTH(" + quoted + "),0)"); } + if (shape.Writable.Count == 0) + throw new NotSupportedException("Tables containing only generated columns require a native MySQL export: " + table); + shape.RowBytesAlias = AllocateHelperAlias(shape.Names, "__mysqlpunk_row_bytes"); + int spatialIndex = 0; + foreach (ColumnShape column in shape.Writable.Where(c => c.Spatial)) + column.SridAlias = AllocateHelperAlias(shape.Names, "__mysqlpunk_srid_" + (spatialIndex++).ToString(CultureInfo.InvariantCulture)); + shape.ByteExpression = string.Join(" + ", sizes); + return shape; + } + + private void CheckRowBound(string table, TableShape shape, CancellationToken cancellation) + { cancellation.ThrowIfCancellationRequested(); - string sql = "SELECT COALESCE(MAX(" + string.Join(" + ", sizes) + "),0) AS MaxRowBytes FROM " + + string sql = "SELECT COALESCE(MAX(" + shape.ByteExpression + "),0) AS MaxRowBytes FROM " + MySqlImportRunner.QuoteIdentifier(databaseName) + "." + MySqlImportRunner.QuoteIdentifier(table) + ";"; DataTable bounds = db.SelectSQL(sql); + cancellation.ThrowIfCancellationRequested(); if (bounds == null || bounds.Rows.Count != 1 || !bounds.Columns.Contains("MaxRowBytes") || bounds.Rows[0]["MaxRowBytes"] == DBNull.Value) throw new InvalidOperationException("Cannot establish a safe source row-size bound: " + table); long maxBytes = Convert.ToInt64(bounds.Rows[0]["MaxRowBytes"], CultureInfo.InvariantCulture); if (maxBytes < 0 || maxBytes > MaxSourceRowBytes) throw new NotSupportedException("Source row exceeds 1 MiB materialization limit; use a native MySQL export: " + table); - shape.ByteExpression = string.Join(" + ", sizes); - return shape; } private sealed class TableShape { - public readonly List Names = new List(); - public readonly HashSet Generated = new HashSet(StringComparer.OrdinalIgnoreCase); - public string ByteExpression; + public readonly HashSet Names = new HashSet(StringComparer.OrdinalIgnoreCase); + public readonly List Writable = new List(); + public string ByteExpression, RowBytesAlias; + } + + private sealed class ColumnShape + { + public string Name, SourceExpression, SridAlias; + public bool Spatial; + } + + private static string AllocateHelperAlias(HashSet names, string preferred) + { + string alias = preferred; + while (!names.Add(alias)) alias += "_"; + return alias; } private DataTable ReadBoundedPage(string table, TableShape shape, long offset, int limit) { - string marker = "__mysqlpunk_row_bytes"; - while (shape.Names.Contains(marker)) marker += "_"; + string marker = shape.RowBytesAlias; // Server-side CASE prevents a concurrently enlarged LOB from being // materialized even when the user disabled the snapshot option. string predicate = "(" + shape.ByteExpression + ") <= " + MaxSourceRowBytes; - string projection = string.Join(", ", shape.Names.Select(name => "CASE WHEN " + predicate + " THEN " + - MySqlImportRunner.QuoteIdentifier(name) + " ELSE NULL END AS " + MySqlImportRunner.QuoteIdentifier(name))); + var expressions = new List(); + foreach (ColumnShape column in shape.Writable) + { + expressions.Add("CASE WHEN " + predicate + " THEN " + column.SourceExpression + " ELSE NULL END AS " + MySqlImportRunner.QuoteIdentifier(column.Name)); + if (column.Spatial) + expressions.Add("CASE WHEN " + predicate + " THEN ST_SRID(" + MySqlImportRunner.QuoteIdentifier(column.Name) + ") ELSE NULL END AS " + MySqlImportRunner.QuoteIdentifier(column.SridAlias)); + } + string projection = string.Join(", ", expressions); DataTable page = db.SelectSQL("SELECT " + projection + ", (" + shape.ByteExpression + ") AS " + marker + " FROM " + MySqlImportRunner.QuoteIdentifier(databaseName) + "." + MySqlImportRunner.QuoteIdentifier(table) + " LIMIT " + offset.ToString(CultureInfo.InvariantCulture) + ", " + limit.ToString(CultureInfo.InvariantCulture) + ";"); if (page == null || !page.Columns.Contains(marker)) throw new InvalidOperationException("Bounded source page query failed: " + table); + foreach (ColumnShape column in shape.Writable) + if (!page.Columns.Contains(column.Name) || (column.Spatial && !page.Columns.Contains(column.SridAlias))) + throw new InvalidOperationException("Missing bounded source column: " + table + "." + column.Name); foreach (DataRow row in page.Rows) - if (row[marker] == DBNull.Value || Convert.ToInt64(row[marker], CultureInfo.InvariantCulture) > MaxSourceRowBytes) + if (row[marker] == DBNull.Value || Convert.ToInt64(row[marker], CultureInfo.InvariantCulture) < 0 || Convert.ToInt64(row[marker], CultureInfo.InvariantCulture) > MaxSourceRowBytes) throw new NotSupportedException("Source row exceeds 1 MiB materialization limit: " + table); page.Columns.Remove(marker); return page; } + private static string ToSpatialLiteral(object wkb, object srid, string column) + { + bool noWkb = wkb == null || wkb == DBNull.Value; + bool noSrid = srid == null || srid == DBNull.Value; + if (noWkb && noSrid) return "NULL"; + byte[] bytes = wkb as byte[]; + if (noWkb || noSrid || bytes == null || bytes.Length == 0) + throw new InvalidOperationException("Invalid spatial WKB/SRID value: " + column); + if (!(srid is byte) && !(srid is sbyte) && !(srid is short) && !(srid is ushort) && !(srid is int) && !(srid is uint) && + !(srid is long) && !(srid is ulong) && !(srid is decimal)) + throw new InvalidOperationException("Spatial SRID must be an unsigned 32-bit integer: " + column); + decimal value = Convert.ToDecimal(srid, CultureInfo.InvariantCulture); + if (value < 0 || value > uint.MaxValue || decimal.Truncate(value) != value) + throw new InvalidOperationException("Spatial SRID must be an unsigned 32-bit integer: " + column); + // Use the matching source/target default SRS axis convention. The + // two-argument form is shared by MySQL and MariaDB; this is not a + // cross-vendor geographic SRS migration or coordinate transform. + return "ST_GeomFromWKB(" + ToBoundedLiteral(bytes) + "," + ((uint)value).ToString(CultureInfo.InvariantCulture) + ")"; + } + private static string ToBoundedLiteral(object value) { if (value is byte[] && ((byte[])value).Length > MaxSourceRowBytes || @@ -624,6 +734,9 @@ private static void ReportProgress( string message; switch (stage) { + case MySqlExportStage.ReadingCatalog: + message = string.Format(ReportText.Get("正在檢查資料表型別、列大小與筆數：{0}", "Checking table types, row size and row count: {0}"), currentObject); + break; case MySqlExportStage.WritingSchema: message = Localization.Format("MySqlExport.ProgressTableStructure", currentObject); break; diff --git a/tests/Run-SmokeTests.ps1 b/tests/Run-SmokeTests.ps1 index d68f1a4..5f74bfc 100644 --- a/tests/Run-SmokeTests.ps1 +++ b/tests/Run-SmokeTests.ps1 @@ -149,6 +149,7 @@ if (!(Test-Path -LiteralPath $mysqlConnector)) { (Join-Path $PSScriptRoot "SqlServerMigrationCatalogTests.cs") ` (Join-Path $PSScriptRoot "SmokeTests.WizardRepairs.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MySqlExport.cs") ` + (Join-Path $PSScriptRoot "SmokeTests.MySqlGeometry.cs") ` (Join-Path $PSScriptRoot "SmokeTests.MySqlImport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.SqliteExport.cs") ` (Join-Path $PSScriptRoot "SmokeTests.SqliteImport.cs") ` diff --git a/tests/SmokeTests.MySqlExport.cs b/tests/SmokeTests.MySqlExport.cs index 67e0b9d..bdda040 100644 --- a/tests/SmokeTests.MySqlExport.cs +++ b/tests/SmokeTests.MySqlExport.cs @@ -272,6 +272,97 @@ private static void MySqlExport_SnapshotWizardRecovery() finally { Directory.Delete(directory, true); Localization.SetLanguage(previousLanguage, false); } } + private static void MySqlExport_EarlyCompatibilityAndTerminalUi() + { + string previousLanguage = Localization.CurrentLanguage; + Localization.SetLanguage(Localization.English, false); + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk-preflight-ui-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + Type wizard = typeof(IDatabase).Assembly.GetType("mySQLPunk.MySqlExportWizardForm", true); + var db = new MockMySqlExportDatabase { MetadataType = "vector" }; + string path = Path.Combine(directory, "dump.sql"); + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, path) { SingleTransaction = false }; + using (var form = (System.Windows.Forms.Form)Activator.CreateInstance(wizard, BindingFlags.Instance | BindingFlags.NonPublic, null, new object[] { db, "test_db", options }, null)) + { + MySqlSnapshotPump((Task)wizard.GetMethod("LoadCatalogAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(form, 1); + wizard.GetMethod("ShowPage", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null); + MySqlSnapshotPump((Task)wizard.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + Assert((int)wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form) == 1, "Unsupported data type must block advance to export confirmation"); + AssertContains(GetPrivateField(form, "statusLabel").Text, "vector", "Preflight identifies unsupported type before starting"); + Assert(db.PageReads == 0 && !File.Exists(path), "Early compatibility checks must not read data pages or write the target"); + db.MetadataType = "varchar"; + MySqlSnapshotPump((Task)wizard.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + Assert((int)wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form) == 2, "Corrected selection/type permits a fresh preflight and confirmation"); + db.MetadataType = "vector"; + wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(form, 1); + GetPrivateField(form, "schemaOnly").Checked = true; + MySqlSnapshotPump((Task)wizard.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + Assert((int)wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form) == 2, "Schema-only bypasses data-value type restriction"); + GetPrivateField(form, "schemaData").Checked = true; + var progress = new MySqlExportProgress(MySqlExportStage.WritingRoutines, "procedure", 1, 1, 2, 2, 128, 10, TimeSpan.FromSeconds(12), TimeSpan.FromSeconds(4), "Writing routine"); + wizard.GetMethod("RenderProgress", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, new object[] { progress }); + Assert(GetPrivateField(form, "overallBar").Value == 99, "All data tables done cannot imply publication success"); + db.MetadataType = "varchar"; + db.RowCount = 4; // Two rows are read; the next empty page forces a late count failure. + db.OnPageRead = () => Thread.Sleep(150); // Give actual progress a measurable speed/ETA. + wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(form, 3); + MySqlSnapshotPump((Task)wizard.GetMethod("StartExportAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + var failed = (mySQLPunk.lib.MySqlExport.MySqlExportResult)wizard.GetProperty("Result", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form); + Assert(!failed.Success && failed.TotalRows == 2 && db.PageReads >= 2, "Terminal assertions exercise failure after real page progress, not an empty initial screen"); + AssertEquals("", GetPrivateField(form, "etaLabel").Text, "Failed export must remove stale ETA"); + AssertEquals("", GetPrivateField(form, "speedLabel").Text, "Failed export must remove stale speed"); + AssertContains(GetPrivateField(form, "writtenLabel").Text, "unpublished", "Failure makes unpublished state explicit"); + Assert(GetPrivateField(form, "backButton").Enabled, "Early runtime failure still permits returning to settings"); + } + } + finally { Directory.Delete(directory, true); Localization.SetLanguage(previousLanguage, false); } + } + + private static void MySqlExport_CompatibilityCancellationLifetime() + { + Type wizard = typeof(IDatabase).Assembly.GetType("mySQLPunk.MySqlExportWizardForm", true); + foreach (bool closeWhileChecking in new[] { false, true }) + using (var entered = new ManualResetEventSlim()) + using (var release = new ManualResetEventSlim()) + { + var db = new MockMySqlExportDatabase { OnMetadataRead = () => { entered.Set(); Assert(release.Wait(10000), "Synthetic metadata gate must be released"); } }; + var options = new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, Path.Combine(Path.GetTempPath(), "unused-preflight-" + Guid.NewGuid().ToString("N") + ".sql")); + using (var form = (System.Windows.Forms.Form)Activator.CreateInstance(wizard, BindingFlags.Instance | BindingFlags.NonPublic, null, new object[] { db, "test_db", options }, null)) + { + MySqlSnapshotPump((Task)wizard.GetMethod("LoadCatalogAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + IntPtr handle = form.Handle; + wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).SetValue(form, 1); + Task check = (Task)wizard.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null); + try + { + Assert(entered.Wait(5000), "Metadata work must execute asynchronously"); + Assert(!GetPrivateField(form, "nextButton").Enabled && !GetPrivateField(form, "backButton").Enabled, "Precheck blocks navigation while the worker owns the session"); + if (closeWhileChecking) + { + form.Close(); form.Close(); + Assert(!form.IsDisposed, "Repeated close must wait for metadata worker cleanup"); + } + else wizard.GetMethod("RequestCancel", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null); + } + finally { release.Set(); } + MySqlSnapshotPump(check); + Assert(!(bool)wizard.GetField("busy", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form), "Precheck must release busy state"); + Assert(GetPrivateField(form, "cancellation") == null, "Precheck owns and releases exactly one CTS"); + Assert((int)wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form) == 1, "Cancelled/closing precheck cannot advance to confirmation"); + if (closeWhileChecking) Assert(form.IsDisposed, "Requested close completes after worker cleanup"); + else + { + db.OnMetadataRead = null; + MySqlSnapshotPump((Task)wizard.GetMethod("OnNextAsync", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(form, null)); + Assert((int)wizard.GetField("step", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(form) == 2, "Cancelled check can be retried with a fresh CTS"); + } + } + } + } + private static void MySqlSnapshotPump(Task task) { var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -293,8 +384,11 @@ private sealed class MockMySqlExportDatabase : IDatabase public string Failure; public string Engine = "InnoDB", OtherEngine, StatusFailure; public string StatusName = "users"; + public string MetadataType = "varchar"; + public Action OnMetadataRead; public int StatusReads; public long MaxRowBytes = 32; + public long RowCount = 2; public int PageReads; public Action OnPageRead; public readonly List ExecutedSql = new List(); @@ -378,12 +472,13 @@ public Task> ExecSQLAsync(string sql, Dictionary true; public void RenameTable(string databaseName, string oldTableName, string newTableName) { } public void RenameView(string databaseName, string oldViewName, string newViewName) { } - public long CountRows(string databaseName, string tableName) => 2; + public long CountRows(string databaseName, string tableName) => RowCount; public DataTable GetCopyColumns(string databaseName, string tableName) => new DataTable(); public DataTable GetCopyIndexes(string databaseName, string tableName) => new DataTable(); public void CreateTableForCopy(string databaseName, string tableName, DataTable sourceColumns, string sourceProvider) { } diff --git a/tests/SmokeTests.MySqlGeometry.cs b/tests/SmokeTests.MySqlGeometry.cs new file mode 100644 index 0000000..c81c5f3 --- /dev/null +++ b/tests/SmokeTests.MySqlGeometry.cs @@ -0,0 +1,316 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using mySQLPunk.lib; +using mySQLPunk.lib.MySqlExport; +using mySQLPunk.lib.MySqlImport; + +public static partial class SmokeTests +{ + private static readonly byte[] GeometryPointWkb = new byte[] { 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 63, 0, 0, 0, 0, 0, 0, 240, 191 }; + + private static void MySqlExport_Geometry_MetadataPreflightAndFreshAttempt() + { + GeometryWithDirectory((directory, path) => + { + var db = new GeometryRecordingDatabase(); + var table = db.AddTable("shapes"); + foreach (string type in new[] { "geometry", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon", "geometrycollection" }) + table.Columns.Add(new GeometryTestColumn(type, type.ToUpperInvariant())); + table.Columns.Add(new GeometryTestColumn("ordinary_enum", "enum('point','geometry')")); + var service = new MySqlSqlExportService(db, "geo_source"); + GeometryValidateMetadata(service, new[] { "shapes" }, CancellationToken.None); + Assert(db.MetadataReads == 1 && db.Queries.Count == 0 && db.Executed.Count == 0 && db.CountReads == 0, "Metadata preflight must not query rows, counts, status or open a session transaction"); + Assert(Directory.GetFiles(directory).Length == 0, "Metadata preflight creates no output"); + table.Columns[7].Type = "vector(3)"; + File.WriteAllText(path, "previous complete backup"); + var result = service.ExportAsync(GeometryOptions(path, "shapes"), null, CancellationToken.None).Result; + Assert(!result.Success && result.ErrorMessage.Contains("shapes.geometrycollection") && result.ErrorMessage.Contains("vector"), "Each attempt must re-read metadata and name unsupported type/column"); + Assert(db.MetadataReads == 2 && db.PageReads == 0 && db.BoundReads == 0 && db.CountReads == 0, "Fresh unsupported metadata fails before row scans and pages"); + AssertEquals("previous complete backup", File.ReadAllText(path), "Fresh incompatibility preserves backup"); + Assert(Directory.GetFiles(directory, "*.staging.*.tmp").Length == 0, "Fresh incompatibility creates no staging file"); + using (var cancel = new CancellationTokenSource()) + { + cancel.Cancel(); + int before = db.MetadataReads; + bool cancelled = false; + try { GeometryValidateMetadata(service, new[] { "shapes" }, cancel.Token); } + catch (OperationCanceledException) { cancelled = true; } + Assert(cancelled && db.MetadataReads == before, "Cancelled metadata preflight performs no provider call"); + } + }); + } + + private static void MySqlExport_Geometry_ValuesAndImportExpressions() + { + GeometryWithDirectory((directory, path) => + { + var db = new GeometryRecordingDatabase(); + var table = db.AddTable("shapes"); + table.Columns.Add(new GeometryTestColumn("id", "int")); + foreach (string type in new[] { "geometry", "point", "linestring", "polygon", "multipoint", "multilinestring", "multipolygon", "geometrycollection" }) + table.Columns.Add(new GeometryTestColumn(type, type)); + table.Columns.Add(new GeometryTestColumn("binary_payload", "blob")); + var row = table.AddRow(); row.Values["id"] = 1; row.Values["binary_payload"] = new byte[] { 0, 255, 128, 39 }; + int i = 0; + foreach (var column in table.Columns.Where(c => c.Name != "id" && c.Name != "binary_payload")) + { + row.Values[column.Name] = i == 3 ? DBNull.Value : GeometryPointWkb; + row.Srids[column.Name] = i == 3 ? (object)DBNull.Value : i == 0 ? 0U : i == 1 ? 4326U : i == 2 ? uint.MaxValue : 3857U; + i++; + } + var result = new MySqlSqlExportService(db, "geo_source").ExportAsync(GeometryOptions(path, "shapes"), null, CancellationToken.None).Result; + Assert(result.Success, "All eight native spatial types export through WKB: " + result.ErrorMessage); + string dump = File.ReadAllText(path); + string hex = "0x" + BitConverter.ToString(GeometryPointWkb).Replace("-", ""); + AssertContains(dump, "ST_GeomFromWKB(" + hex + ",0)", "Zero SRID preserved"); + AssertContains(dump, "ST_GeomFromWKB(" + hex + ",4326)", "Nonzero SRID preserved"); + AssertContains(dump, "ST_GeomFromWKB(" + hex + ",4294967295)", "Unsigned maximum SRID preserved"); + AssertContains(dump, ", NULL, ST_GeomFromWKB", "Nullable geometry emits SQL NULL"); + AssertContains(dump, "0x00FF8027", "Ordinary BLOB stays a binary literal"); + Assert(!dump.Contains("axis-order") && !dump.Contains("ST_GeomFromText"), "Use common two-argument constructor with matching default axis convention"); + Assert(MySqlImportPreflightScanner.Scan(path, CancellationToken.None).Passed, "Spatial dump passes actual parser and digest preflight"); + var target = new GeometryRecordingDatabase { ImportMode = true }; + target.Tables.Add("shapes", table); + var imported = new MySqlImportRunner(target, "geo_target").RunAsync(new MySqlImportRequest(Guid.NewGuid(), path, "geo_target"), new MySqlImportOptions(), null, CancellationToken.None).Result; + Assert(imported.Status == MySqlImportStatus.Completed, "Actual recording import accepts WKB constructor expressions: " + imported.Summary); + string inserted = target.Executed.Single(sql => sql.TrimStart().StartsWith("INSERT INTO", StringComparison.OrdinalIgnoreCase)); + AssertContains(inserted, "ST_GeomFromWKB(" + hex + ",4326)", "Import preserves exact WKB/SRID expression"); + Assert(target.Executed.Any(sql => sql.Contains("USE `geo_target`")), "Import selects requested target"); + string page = db.Queries.Single(sql => sql.Contains(" LIMIT ")); + AssertContains(page, "THEN ST_AsWKB(`point`) ELSE NULL END AS `point`", "WKB remains behind row-size CASE guard"); + AssertContains(page, "THEN ST_SRID(`point`) ELSE NULL END AS", "SRID remains behind row-size CASE guard"); + }); + } + + private static void MySqlExport_Geometry_GeneratedColumnsAndIdentifiers() + { + GeometryWithDirectory((directory, path) => + { + var db = new GeometryRecordingDatabase(); + var table = db.AddTable("city`geometry.測試"); + table.Columns.Add(new GeometryTestColumn("SHAPE`一", "point")); + table.Columns.Add(new GeometryTestColumn("other.geometry", "geometry")); + table.Columns.Add(new GeometryTestColumn("__MYSQLPUNK_SRID_0", "varchar(30)")); + table.Columns.Add(new GeometryTestColumn("__MYSQLPUNK_ROW_BYTES", "varchar(30)")); + table.Columns.Add(new GeometryTestColumn("generated_shape", "geometry", true)); + var row = table.AddRow(); row.Values["SHAPE`一"] = GeometryPointWkb; row.Srids["SHAPE`一"] = 4326; + row.Values["other.geometry"] = GeometryPointWkb; row.Srids["other.geometry"] = 3857; + row.Values["__MYSQLPUNK_SRID_0"] = "ordinary"; row.Values["__MYSQLPUNK_ROW_BYTES"] = "ordinary marker"; + var result = new MySqlSqlExportService(db, "geo_source").ExportAsync(GeometryOptions(path, table.Name), null, CancellationToken.None).Result; + Assert(result.Success, "Generated spatial values and quoted helper collisions export: " + result.ErrorMessage); + string dump = File.ReadAllText(path); + AssertContains(dump, "GENERATED ALWAYS", "Generated spatial DDL retained"); + string insert = MySqlScriptReader.Read(new StringReader(dump), CancellationToken.None).Single(s => s.Sql.StartsWith("INSERT INTO", StringComparison.OrdinalIgnoreCase)).Sql; + string columnList = insert.Substring(0, insert.IndexOf(" VALUES", StringComparison.Ordinal)); + AssertContains(columnList, "`SHAPE``一`, `other.geometry`, `__MYSQLPUNK_SRID_0`, `__MYSQLPUNK_ROW_BYTES`", "Only original writable identifiers enter INSERT"); + Assert(!columnList.Contains("generated_shape") && !columnList.Contains("__mysqlpunk_srid_"), "Generated and allocated helper columns omitted from INSERT"); + foreach (string sql in db.Queries) + Assert(!sql.Contains("generated_shape"), "Generated geometry is not fetched or counted in source bound"); + string page = db.Queries.Single(sql => sql.Contains(" LIMIT ")); + var aliases = GeometryRecordingDatabase.ReadAliases(page); + Assert(aliases.Count == aliases.Distinct(StringComparer.OrdinalIgnoreCase).Count(), "Source projection helper aliases are unique ignoring case"); + AssertContains(page, "`geo_source`.`city``geometry.測試`", "Database and table identifiers safely quoted"); + }); + } + + private static void MySqlExport_Geometry_BoundsRaceAndMalformedValues() + { + GeometryWithDirectory((directory, path) => + { + var malformed = new[] { + new object[] { GeometryPointWkb, DBNull.Value }, new object[] { DBNull.Value, 4326 }, + new object[] { new byte[0], 0 }, new object[] { "not binary", 0 }, + new object[] { GeometryPointWkb, -1 }, new object[] { GeometryPointWkb, 1.5m }, + new object[] { GeometryPointWkb, 4294967296UL }, new object[] { GeometryPointWkb, "4326" } + }; + foreach (object[] values in malformed) + { + File.WriteAllText(path, "previous complete backup"); + var db = GeometryOneRow(values[0], values[1]); + var result = new MySqlSqlExportService(db, "geo_source").ExportAsync(GeometryOptions(path, "shapes"), null, CancellationToken.None).Result; + Assert(!result.Success, "Malformed WKB/SRID provider result fails explicitly"); + AssertEquals("previous complete backup", File.ReadAllText(path), "Malformed value preserves previous complete output"); + Assert(Directory.GetFiles(directory, "*.staging.*.tmp").Length == 0, "Failed value leaves no incomplete staging file"); + } + foreach (bool race in new[] { false, true }) + { + File.WriteAllText(path, "previous complete backup"); + var db = GeometryOneRow(GeometryPointWkb, 4326); + if (race) db.PageRowBytes = 1048577; else db.Tables["shapes"].MaxRowBytes = 1048577; + var result = new MySqlSqlExportService(db, "geo_source").ExportAsync(GeometryOptions(path, "shapes"), null, CancellationToken.None).Result; + Assert(!result.Success, "Oversized geometry source or enlargement race fails"); + Assert(db.PageReads == (race ? 1 : 0), "Initial oversize fails before materialization; race fails at guarded page"); + AssertEquals("previous complete backup", File.ReadAllText(path), "Bound failure preserves previous backup"); + } + var nullable = GeometryOneRow(DBNull.Value, DBNull.Value); + Assert(new MySqlSqlExportService(nullable, "geo_source").ExportAsync(GeometryOptions(path, "shapes"), null, CancellationToken.None).Result.Success, "Both-null geometry pair succeeds"); + }); + } + + private static void MySqlExport_Geometry_BatchCapsAndEarlyFailure() + { + GeometryWithDirectory((directory, path) => + { + var db = GeometryOneRow(GeometryPointWkb, 0); + var last = db.AddTable("last_table"); last.Columns.Add(new GeometryTestColumn("unsupported", "vector(3)")); + File.WriteAllText(path, "previous complete backup"); + var failed = new MySqlSqlExportService(db, "geo_source").ExportAsync(GeometryOptions(path, "shapes", "last_table"), null, CancellationToken.None).Result; + Assert(!failed.Success && db.MetadataReads == 2 && db.BoundReads == 0 && db.CountReads == 0 && db.PageReads == 0, "Unsupported last table must fail before ANY bounds, counts or pages"); + Assert(Directory.GetFiles(directory, "*.staging.*.tmp").Length == 0, "Unsupported last table creates no staging"); + db = GeometryOneRow(new byte[1048566], 0); + db.Tables["shapes"].MaxRowBytes = 1048576; + for (int i = 0; i < 5; i++) { var row = db.Tables["shapes"].AddRow(); row.Values["shape"] = new byte[1048566]; row.Srids["shape"] = 0; } + var second = db.AddTable("later"); second.Columns.Add(new GeometryTestColumn("shape", "point")); + var secondRow = second.AddRow(); secondRow.Values["shape"] = GeometryPointWkb; secondRow.Srids["shape"] = 0; + db.BeforeSourceScan = () => Assert(Directory.GetFiles(directory, "*.staging.*.tmp").Length == 0, "All bounds and counts precede staging creation"); + var result = new MySqlSqlExportService(db, "geo_source").ExportAsync(GeometryOptions(path, "shapes", "later"), null, CancellationToken.None).Result; + Assert(result.Success && result.TotalRows == 7, "Bounded geometries split into safe INSERT statements: " + result.ErrorMessage); + Assert(db.Events.IndexOf("page:shapes") > db.Events.IndexOf("bound:later") && db.Events.IndexOf("page:shapes") > db.Events.IndexOf("count:later"), "All selected tables bounded and counted before first page"); + Assert(db.Events.IndexOf("bound:shapes") > db.Events.IndexOf("metadata:later"), "All shapes validated before first row bound"); + using (var reader = File.OpenText(path)) + { + var inserts = MySqlScriptReader.Read(reader, CancellationToken.None).Where(s => s.Sql.StartsWith("INSERT INTO", StringComparison.OrdinalIgnoreCase)).ToList(); + Assert(inserts.Count > 2 && inserts.All(s => Encoding.UTF8.GetByteCount(s.Sql) + 3 <= 8 * 1024 * 1024), "Hex expansion and constructor overhead stay within 8 MiB INSERT cap"); + } + Assert(db.Queries.Where(sql => sql.Contains(" LIMIT ")).All(sql => sql.Contains(", 16;")), "Materialized source pages remain capped at 16 rows"); + }); + } + + private static void GeometryValidateMetadata(MySqlSqlExportService service, IEnumerable tables, CancellationToken cancellation) + { + MethodInfo method = typeof(MySqlSqlExportService).GetMethod("ValidateTableDataCompatibility", BindingFlags.Instance | BindingFlags.Public); + Assert(method != null, "Exporter must expose metadata-only pre-confirmation validation"); + try { method.Invoke(service, new object[] { tables, cancellation }); } + catch (TargetInvocationException ex) { throw ex.InnerException; } + } + + private static MySqlExportWizardOptions GeometryOptions(string path, params string[] tables) + { + return new MySqlExportWizardOptions(MySqlExportMode.SchemaAndData, path) { SingleTransaction = false, IncludeViews = false, IncludeRoutines = false, IncludeTriggers = false, SelectedTables = tables.ToList() }; + } + + private static GeometryRecordingDatabase GeometryOneRow(object wkb, object srid) + { + var db = new GeometryRecordingDatabase(); var table = db.AddTable("shapes"); + table.Columns.Add(new GeometryTestColumn("shape", "geometry")); + var row = table.AddRow(); row.Values["shape"] = wkb; row.Srids["shape"] = srid; + return db; + } + + private static void GeometryWithDirectory(Action action) + { + string directory = Path.Combine(Path.GetTempPath(), "mysqlpunk_geometry_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try { action(directory, Path.Combine(directory, "dump.sql")); } + finally { Directory.Delete(directory, true); } + } + + private sealed class GeometryTestColumn + { + public string Name, Type; + public bool Generated; + public GeometryTestColumn(string name, string type, bool generated = false) { Name = name; Type = type; Generated = generated; } + } + + private sealed class GeometryTestRow + { + public readonly Dictionary Values = new Dictionary(StringComparer.OrdinalIgnoreCase); + public readonly Dictionary Srids = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + private sealed class GeometryTestTable + { + public string Name; + public long MaxRowBytes = 64; + public readonly List Columns = new List(); + public readonly List Rows = new List(); + public GeometryTestRow AddRow() { var row = new GeometryTestRow(); Rows.Add(row); return row; } + } + + private sealed class GeometryRecordingDatabase : IDatabase + { + public readonly Dictionary Tables = new Dictionary(StringComparer.Ordinal); + public readonly List Queries = new List(), Executed = new List(), Events = new List(); + public int MetadataReads, CountReads, BoundReads, PageReads; + public long PageRowBytes; + public bool ImportMode; + public Action BeforeSourceScan; + public ConnectionState State => ConnectionState.Open; + public string ProviderName => "MySQL"; + public GeometryTestTable AddTable(string name) { var table = new GeometryTestTable { Name = name }; Tables.Add(name, table); return table; } + public void SetConn(string value) { } public void Open() { } public void Close() { } public void Dispose() { } + public static List ReadAliases(string sql) + { + return Regex.Matches(sql, @"\bAS\s+(?:`(?(?:``|[^`])+)`|(?[A-Za-z0-9_]+))", RegexOptions.IgnoreCase).Cast().Select(m => m.Groups["quoted"].Success ? m.Groups["quoted"].Value.Replace("``", "`") : m.Groups["bare"].Value).ToList(); + } + public DataTable SelectSQL(string sql, Dictionary parameters = null) + { + Queries.Add(sql); + var table = Tables.Values.FirstOrDefault(t => sql.Contains("." + MySqlImportRunner.QuoteIdentifier(t.Name))); + if (table == null) throw new InvalidOperationException("Unexpected recording source query"); + var output = new DataTable(); + if (sql.Contains("AS MaxRowBytes")) + { + BeforeSourceScan?.Invoke(); BoundReads++; Events.Add("bound:" + table.Name); + output.Columns.Add("MaxRowBytes", typeof(long)); output.Rows.Add(table.MaxRowBytes); return output; + } + Assert(sql.Contains(" LIMIT "), "Exporter must use bounded source pages"); + PageReads++; Events.Add("page:" + table.Name); + var aliases = ReadAliases(sql); foreach (string alias in aliases) output.Columns.Add(alias, typeof(object)); + var sridAliases = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (Match match in Regex.Matches(sql, @"THEN ST_SRID\(`(?(?:``|[^`])+)`\) ELSE NULL END AS `(?(?:``|[^`])+)`")) + sridAliases.Add(match.Groups["alias"].Value.Replace("``", "`"), match.Groups["column"].Value.Replace("``", "`")); + Match limit = Regex.Match(sql, @"LIMIT\s+(\d+),\s*(\d+)"); + int offset = int.Parse(limit.Groups[1].Value, CultureInfo.InvariantCulture), count = int.Parse(limit.Groups[2].Value, CultureInfo.InvariantCulture); + foreach (var source in table.Rows.Skip(offset).Take(count)) + { + DataRow row = output.NewRow(); + foreach (string alias in aliases.Take(aliases.Count - 1)) + { + object value; + if (sridAliases.ContainsKey(alias)) source.Srids.TryGetValue(sridAliases[alias], out value); + else source.Values.TryGetValue(alias, out value); + row[alias] = PageRowBytes > 1048576 ? DBNull.Value : value ?? DBNull.Value; + } + row[aliases.Last()] = PageRowBytes == 0 ? table.MaxRowBytes : PageRowBytes; output.Rows.Add(row); + } + return output; + } + public Dictionary ExecSQL(string sql, Dictionary parameters = null) { Executed.Add(sql); return new Dictionary { { "status", "OK" }, { "rowsAffected", sql.TrimStart().StartsWith("INSERT", StringComparison.OrdinalIgnoreCase) ? "1" : "0" } }; } + public Task SelectSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(SelectSQL(sql, parameters)); + public Task> ExecSQLAsync(string sql, Dictionary parameters = null) => Task.FromResult(ExecSQL(sql, parameters)); + public List GetDatabases() => new List { "geo_source" }; + public List GetTables(string databaseName) => ImportMode ? new List() : Tables.Keys.ToList(); + public List GetViews(string databaseName) => new List(); + public DataTable GetColumns(string databaseName, string tableName) + { + MetadataReads++; Events.Add("metadata:" + tableName); + var output = new DataTable(); foreach (string name in new[] { "Field", "Type", "Extra", "GenerationExpression" }) output.Columns.Add(name); + foreach (var column in Tables[tableName].Columns) output.Rows.Add(column.Name, column.Type, column.Generated ? "VIRTUAL GENERATED" : "", column.Generated ? "ST_PointFromText('POINT(1 1)')" : ""); + return output; + } + public long CountRows(string databaseName, string tableName) { BeforeSourceScan?.Invoke(); CountReads++; Events.Add("count:" + tableName); return Tables[tableName].Rows.Count; } + public DataTable GetTableStatus(string databaseName) { var output = new DataTable(); output.Columns.Add("Name"); output.Columns.Add("Engine"); foreach (string name in Tables.Keys) output.Rows.Add(name, "InnoDB"); return output; } + public Dictionary GetDatabaseInfo(string databaseName) => new Dictionary(); + public string GetTableCreateStatement(string databaseName, string tableName) => "CREATE TABLE " + MySqlImportRunner.QuoteIdentifier(tableName) + " (" + string.Join(", ", Tables[tableName].Columns.Select(c => MySqlImportRunner.QuoteIdentifier(c.Name) + " " + c.Type + (c.Generated ? " GENERATED ALWAYS AS (ST_PointFromText('POINT(1 1)')) VIRTUAL" : ""))) + ") ENGINE=InnoDB;"; + public DataTable GetIndexes(string databaseName, string tableName) => new DataTable(); + public bool TableExists(string databaseName, string tableName) => !ImportMode && Tables.ContainsKey(tableName); + public bool ViewExists(string databaseName, string viewName) => false; + public void RenameTable(string a, string b, string c) { } public void RenameView(string a, string b, string c) { } + public DataTable GetCopyColumns(string a, string b) => new DataTable(); public DataTable GetCopyIndexes(string a, string b) => new DataTable(); + public void CreateTableForCopy(string a, string b, DataTable c, string d) { } public void DropTableForCopy(string a, string b) { } + public void CreateIndexesForCopy(string a, string b, DataTable c, string d) { } + public DataTable SelectTablePage(string a, string b, long c, int d) => throw new NotSupportedException("Unbounded table-page helper must not be used"); + public void InsertTableBatch(string a, string b, DataTable c) { } public string GetViewCreateStatement(string a, string b) => ""; + public void CreateViewFromStatement(string a, string b, string c) { } + } +} diff --git a/tests/SmokeTests.cs b/tests/SmokeTests.cs index 23c4d6d..0c7d07d 100644 --- a/tests/SmokeTests.cs +++ b/tests/SmokeTests.cs @@ -40,6 +40,13 @@ public static int Main(string[] args) Run("MySQL import regression target literals and strategies", MySqlImport_Regression_TargetLiteralsAndStrategies, ref passed); Run("MySQL import regression cancel transactions and preflight", MySqlImport_Regression_CancelTransactionsAndPreflight, ref passed); Run("MySQL export regression generated bounds and failures", MySqlExport_Regression_GeneratedBoundsAndFailures, ref passed); + Run("MySQL export geometry metadata preflight and fresh attempt", MySqlExport_Geometry_MetadataPreflightAndFreshAttempt, ref passed); + Run("MySQL export geometry values and import expressions", MySqlExport_Geometry_ValuesAndImportExpressions, ref passed); + Run("MySQL export geometry generated columns and identifiers", MySqlExport_Geometry_GeneratedColumnsAndIdentifiers, ref passed); + Run("MySQL export geometry bounds race and malformed values", MySqlExport_Geometry_BoundsRaceAndMalformedValues, ref passed); + Run("MySQL export geometry batch caps and early failure", MySqlExport_Geometry_BatchCapsAndEarlyFailure, ref passed); + Run("MySQL export compatibility cancellation lifetime", MySqlExport_CompatibilityCancellationLifetime, ref passed); + Run("MySQL export early compatibility and terminal UI", MySqlExport_EarlyCompatibilityAndTerminalUi, ref passed); Run("MySQL export snapshot catalog metadata", MySqlExport_SnapshotCatalogMetadata, ref passed); Run("MySQL export ordinary mode and snapshot guard", MySqlExport_OrdinaryModeAndSnapshotGuard, ref passed); Run("MySQL export snapshot wizard recovery", MySqlExport_SnapshotWizardRecovery, ref passed); From b1dfbf68c5c7307ab6f802ad2e10d5f65e47af1e Mon Sep 17 00:00:00 2001 From: John Date: Fri, 16 Oct 2026 16:16:08 +0800 Subject: [PATCH 12/12] docs: record user MySQL export and import acceptance --- docs/design/db-wizard-repairs/verification.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/design/db-wizard-repairs/verification.md b/docs/design/db-wizard-repairs/verification.md index b26a62d..12d9259 100644 --- a/docs/design/db-wizard-repairs/verification.md +++ b/docs/design/db-wizard-repairs/verification.md @@ -63,3 +63,7 @@ The city_geometry.SHAPE screenshot shows an explicit unsupported spatial-type fa Final evidence: previous actual Debug binary fails geometry-values regression with the same original Spatial column values error (`spatial-values-red.log`); previous wizard fails early-check regression (`spatial-ui-red.log`). Repaired actual-product MySQL27 groups, complete304 groups plus AutoReleasePolicy PASS (`spatial-mysql.log`, `spatial-full.log`). Explicit default Debug and Release builds PASS (`spatial-debug.log`, `spatial-release.log`). A separate read-only integrated reviewer found no material issues; the asynchronous-lifetime and actual-late-progress test gaps were addressed before the final gate. All twelve source/test/document encoding/BOM/newline conventions and diff whitespace pass. [Implementation and scoped evidence](mysql-geometry-report.md), [repair plan](mysql-spatial-fix-plan.md). Source and test files were frozen before the final gate. User explicitly deferred actual database restoration: no MySQL/MariaDB session, saved profile/credential, live source scan, actual restore or production write was used. Real WKB/SRID/SRS fidelity remains pending. The1MiB source-row/16-rowpage/8MiBINSERT budgets remain; compatible engine/SRS round trips are the intended scope, with cross-version/vendor geographic migration and custom SRS transfer unverified. Recording-provider import and manifest success do not prove native data landing. + +## User acceptance follow-up (2026-10-16) + +Following delivery of165ef3c, the user reported that MySQL export and import both completed successfully in their real environment. This records user-observed workflow acceptance and supersedes the earlier deferral for that tested MySQL workflow. Database/version/row-by-row geometry/SRID comparisons were not supplied or independently inspected. It does not extend native acceptance to SQLite, Oracle, PostgreSQL, cross-engine SRS migration or all large-value cases. The user then authorized push and a pull request against the mainline.