Skip to content

Atomic File Handling #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DimonByte merged 1 commit into from
Aug 30, 2026
Merged

Atomic File Handling #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/codeql.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches: [ "main", "develop" ]
pull_request:
branches: [ "main", "develop" ]
branches: [ "main" ]
schedule:
- cron: '34 20 * * 6'

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dotnet-desktop.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches: [ "main", "develop" ]
pull_request:
branches: [ "main", "develop" ]
branches: [ "main" ]

jobs:
build:
Expand Down
75 changes: 62 additions & 13 deletions HostDownloader/Modules/HostListDownloaderInternals/HostListManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
using System.Diagnostics;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Transactions;

namespace HostlistDownloader.Modules.HostListDownloaderInternals
{
Expand Down Expand Up @@ -76,7 +77,7 @@ public static void StartListProcessing(bool forceMode, CancellationToken cancell

ProcessDownloadLists(blockListIni,
IOManager.BlockListFolderLocation,
IOManager.CombinedBlockListFileLocation, forceMode, cancellationToken).GetAwaiter().GetResult();
IOManager.CombinedBlockListFileLocationTemp, forceMode, cancellationToken).GetAwaiter().GetResult();
}
else
{
Expand All @@ -86,7 +87,7 @@ public static void StartListProcessing(bool forceMode, CancellationToken cancell
if (userblockListIni.Length != 0)
{
TraceLogger.Log("User blocklist is configured. Merging user config...");
MergeUserDefinedDomains(IOManager.CombinedBlockListFileLocation, isBlocklist: true);
MergeUserDefinedDomains(IOManager.CombinedBlockListFileLocationTemp, isBlocklist: true);
}
else
{
Expand Down Expand Up @@ -115,7 +116,7 @@ public static void StartListProcessing(bool forceMode, CancellationToken cancell
TraceLogger.Log("Whitelist is configured. Updating whitelists...");
ProcessDownloadLists(whiteListIni,
IOManager.WhiteListFolderLocation,
IOManager.CombinedWhiteListFileLocation, forceMode, cancellationToken).GetAwaiter().GetResult();
IOManager.CombinedWhiteListFileLocationTemp, forceMode, cancellationToken).GetAwaiter().GetResult();
}
else
{
Expand All @@ -125,7 +126,7 @@ public static void StartListProcessing(bool forceMode, CancellationToken cancell
if (userwhiteListIni.Length != 0)
{
TraceLogger.Log("User Whitelist is configured. Merging user config...");
MergeUserDefinedDomains(IOManager.CombinedWhiteListFileLocation, isBlocklist: false);
MergeUserDefinedDomains(IOManager.CombinedWhiteListFileLocationTemp, isBlocklist: false);
}
else
{
Expand All @@ -136,9 +137,56 @@ public static void StartListProcessing(bool forceMode, CancellationToken cancell
{
GenerateCombinedList();
}
CommitToMasterLists();

TraceLogger.Log("Host lists update completed!");
}
/// <summary>
/// Commits the temporary generated hostfile files to the final master version.
/// This is done to allow a fallback in the event that something goes wrong, so if the generation fails spectacularly, it wont overwrite the existing master lists with a broken version.
/// </summary>
private static void CommitToMasterLists()
{
//Move the temp combined lists to their final locations
//e.g. HLDcombined-blocklist-TEMP.txt and HLDcombined-whitelist-TEMP.txt will be moved to HLDcombined-blocklist.txt and HLDcombined-whitelist.txt respectively.
//This is done to ensure that the combined lists are only updated if the entire process completes successfully. And also to prevent OS file locks from preventing the combined lists from being updated.
TraceLogger.Log("Committing temporary combined lists to final locations...");
try
{
if (File.Exists(IOManager.CombinedBlockListFileLocationTemp))
{
File.Move(IOManager.CombinedBlockListFileLocationTemp, IOManager.CombinedBlockListFileLocation, overwrite: true);
TraceLogger.Log($"Committed {IOManager.CombinedBlockListFileLocationTemp} to {IOManager.CombinedBlockListFileLocation}", Enums.StatusSeverityType.Debug);
}
if (File.Exists(IOManager.CombinedWhiteListFileLocationTemp))
{
File.Move(IOManager.CombinedWhiteListFileLocationTemp, IOManager.CombinedWhiteListFileLocation, overwrite: true);
TraceLogger.Log($"Committed {IOManager.CombinedWhiteListFileLocationTemp} to {IOManager.CombinedWhiteListFileLocation}", Enums.StatusSeverityType.Debug);
}
if (File.Exists(IOManager.CombinedListFileLocationTemp))
{
File.Move(IOManager.CombinedListFileLocationTemp, IOManager.CombinedListFileLocation, overwrite: true);
TraceLogger.Log($"Committed {IOManager.CombinedListFileLocationTemp} to {IOManager.CombinedListFileLocation}", Enums.StatusSeverityType.Debug);
}
string[] FilesToCreate = new string[]
{
IOManager.CombinedBlockListFileLocationTemp,
IOManager.CombinedWhiteListFileLocationTemp,
IOManager.CombinedListFileLocationTemp
};
foreach (string file in FilesToCreate)
{
File.Create(file).Dispose();
}
TraceLogger.Log("Temporary combined lists cleared after committing to final locations.",Enums.StatusSeverityType.Debug);
}
catch (Exception ex)
{
ProblemDuringUpdate = true;
TraceLogger.Log($"Failed to commit combined lists to final locations: {ex}", Enums.StatusSeverityType.Error);
TraceLogger.Log("[!] No changes were made to the final combined lists. Please check the logs for details.", Enums.StatusSeverityType.Error);
}
}

public static void StartOfflineListProcessing()
{
Expand All @@ -156,7 +204,7 @@ public static void StartOfflineListProcessing()
if (blockListIni.Length != 0)
{
TraceLogger.Log("Blocklist is configured. Merging...");
CompileList(IOManager.BlockListFolderLocation, IOManager.CombinedBlockListFileLocation, blockListIni.Length, 0, DateTime.Now);
CompileList(IOManager.BlockListFolderLocation, IOManager.CombinedBlockListFileLocationTemp, blockListIni.Length, 0, DateTime.Now);
}
else
{
Expand All @@ -165,7 +213,7 @@ public static void StartOfflineListProcessing()
if (whiteListIni.Length != 0)
{
TraceLogger.Log("Whitelist is configured. Merging user config...");
CompileList(IOManager.WhiteListFolderLocation, IOManager.CombinedWhiteListFileLocation, whiteListIni.Length, 0, DateTime.Now);
CompileList(IOManager.WhiteListFolderLocation, IOManager.CombinedWhiteListFileLocationTemp, whiteListIni.Length, 0, DateTime.Now);
}
else
{
Expand All @@ -175,7 +223,7 @@ public static void StartOfflineListProcessing()
if (userwhiteListIni.Length != 0)
{
TraceLogger.Log("User Whitelist is configured. Merging user config...");
MergeUserDefinedDomains(IOManager.CombinedWhiteListFileLocation, isBlocklist: false);
MergeUserDefinedDomains(IOManager.CombinedWhiteListFileLocationTemp, isBlocklist: false);
}
else
{
Expand All @@ -184,14 +232,15 @@ public static void StartOfflineListProcessing()
if (userblockListIni.Length != 0)
{
TraceLogger.Log("User blocklist is configured. Merging user config...");
MergeUserDefinedDomains(IOManager.CombinedBlockListFileLocation, isBlocklist: true);
MergeUserDefinedDomains(IOManager.CombinedBlockListFileLocationTemp, isBlocklist: true);
}
else
{
TraceLogger.Log("User Blocklist not configured. Ignoring", Enums.StatusSeverityType.Debug);
}

GenerateCombinedList();
CommitToMasterLists();
}

/// <summary>
Expand Down Expand Up @@ -625,11 +674,11 @@ private static bool CheckIntegrity(string ListFolderLocation, int urlCount, int

public static void GenerateCombinedList()
{
TraceLogger.Log($"Generating {Path.GetFileName(IOManager.CombinedListFileLocation)} list...");
TraceLogger.Log($"Generating {Path.GetFileName(IOManager.CombinedListFileLocationTemp)} list...");
try
{
var whiteList = ReadLinesFromFileCached(IOManager.CombinedWhiteListFileLocation);
var blockListLines = ReadLinesFromFile(IOManager.CombinedBlockListFileLocation);
var whiteList = ReadLinesFromFileCached(IOManager.CombinedWhiteListFileLocationTemp);
var blockListLines = ReadLinesFromFile(IOManager.CombinedBlockListFileLocationTemp);
var filteredLines = blockListLines.Where(line =>
!whiteList.Any(whiteItem =>
{
Expand All @@ -641,8 +690,8 @@ public static void GenerateCombinedList()
return line.Equals(whiteItem, StringComparison.OrdinalIgnoreCase);
})
).ToList();
File.WriteAllLines(IOManager.CombinedListFileLocation, filteredLines);
TraceLogger.Log($"Generated combined list to: {IOManager.CombinedListFileLocation} | Line count: {filteredLines.Count:N0}");
File.WriteAllLines(IOManager.CombinedListFileLocationTemp, filteredLines);
TraceLogger.Log($"Generated combined list to: {IOManager.CombinedListFileLocationTemp} | Line count: {filteredLines.Count:N0}");
}
catch (Exception ex)
{
Expand Down
14 changes: 12 additions & 2 deletions HostDownloader/Modules/WindowsSystem/IOManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,19 @@ internal class IOManager
//public static readonly string IniWhiteListFileLocation = "hostfiles/whitelist.ini";
//public static readonly string IniUserWebsiteBlockListFileLocation = "hostfiles/userwebsiteblocklist.ini";
//public static readonly string IniUserWebsiteWhiteListFileLocation = "hostfiles/userwebsitewhitelist.ini";
//Note these combined directories are the final saving location, when HostlistManager merges the files, it will save temp files and once the merge is complete, it will move the final merged file to this location. The temp files will be deleted after the merge is complete.
//Temp file names will be HLDcombined-*whitelist/blocklist*-TEMP.txt
//public static readonly string CombinedBlockListFileLocationTemp = "hostfiles/combined/HLDcombined-blocklist-TEMP.txt";
//public static readonly string CombinedWhiteListFileLocationTemp = "hostfiles/combined/HLDcombined-whitelist-TEMP.txt";
//public static readonly string CombinedListFileLocationTemp = "hostfiles/combined/HLDcombined-list-TEMP.txt";

public static readonly string CombinedBlockListFileLocation = "hostfiles/combined/HLDcombined-blocklist.txt";
public static readonly string CombinedWhiteListFileLocation = "hostfiles/combined/HLDcombined-whitelist.txt";
public static readonly string CombinedListFileLocation = "hostfiles/combined/HLDcombined-list.txt";

public static readonly string CombinedBlockListFileLocationTemp = "hostfiles/combined/HLDcombined-blocklist-TEMP.txt";
public static readonly string CombinedWhiteListFileLocationTemp = "hostfiles/combined/HLDcombined-whitelist-TEMP.txt";
public static readonly string CombinedListFileLocationTemp = "hostfiles/combined/HLDcombined-list-TEMP.txt";
//public static readonly string IniFormatTypeLocation = "hostfiles/formattype.ini";
public static readonly string LogsLocation = "logs";
public static readonly string SettingJsonFileLocation = "settings.json";
Expand All @@ -65,7 +75,7 @@ public static void CreateNecessaryDirectoriesAndFiles()
}
}
}
string[] files = [CombinedListFileLocation, CombinedBlockListFileLocation, CombinedWhiteListFileLocation];
string[] files = [CombinedListFileLocationTemp, CombinedBlockListFileLocationTemp, CombinedWhiteListFileLocationTemp];
foreach (string file in files)
{
if (!File.Exists(file))
Expand All @@ -77,7 +87,7 @@ public static void CreateNecessaryDirectoriesAndFiles()
if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory))
Directory.CreateDirectory(directory);
File.Create(file).Dispose();
TraceLogger.Log($"Created file: {file}");
TraceLogger.Log($"Created file: {file}", Enums.StatusSeverityType.Debug);
}
catch (Exception ex)
{
Expand Down
Loading