diff --git a/src/SIL.Machine/Corpora/DblBundleTextCorpus.cs b/src/SIL.Machine/Corpora/DblBundleTextCorpus.cs index 70dd23d2f..6a6f2d730 100644 --- a/src/SIL.Machine/Corpora/DblBundleTextCorpus.cs +++ b/src/SIL.Machine/Corpora/DblBundleTextCorpus.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Xml.Linq; using SIL.IO; +using SIL.Machine.Utils; namespace SIL.Machine.Corpora { @@ -17,7 +18,7 @@ public DblBundleTextCorpus(string fileName) using (ZipArchive archive = ZipFile.OpenRead(fileName)) { ZipArchiveEntry metadataEntry = archive.GetEntry("metadata.xml"); - using (Stream stream = metadataEntry.Open()) + using (Stream stream = metadataEntry.OpenBoundedStream()) { var doc = XDocument.Load(stream); var version = (string)doc.Root.Attribute("version"); diff --git a/src/SIL.Machine/Corpora/ZipEntryStreamContainer.cs b/src/SIL.Machine/Corpora/ZipEntryStreamContainer.cs index 680cef687..c3dea5575 100644 --- a/src/SIL.Machine/Corpora/ZipEntryStreamContainer.cs +++ b/src/SIL.Machine/Corpora/ZipEntryStreamContainer.cs @@ -1,5 +1,6 @@ using System.IO; using System.IO.Compression; +using SIL.Machine.Utils; using SIL.ObjectModel; namespace SIL.Machine.Corpora @@ -17,7 +18,7 @@ public ZipEntryStreamContainer(string archiveFileName, string entryPath) public Stream OpenStream() { - return _entry.Open(); + return _entry.OpenBoundedStream(); } protected override void DisposeManagedResources() diff --git a/src/SIL.Machine/Corpora/ZipParatextProjectFileHandler.cs b/src/SIL.Machine/Corpora/ZipParatextProjectFileHandler.cs index 2b614bbcd..2d177de14 100644 --- a/src/SIL.Machine/Corpora/ZipParatextProjectFileHandler.cs +++ b/src/SIL.Machine/Corpora/ZipParatextProjectFileHandler.cs @@ -2,6 +2,7 @@ using System.IO.Compression; using System.Linq; using SIL.IO; +using SIL.Machine.Utils; namespace SIL.Machine.Corpora { @@ -26,9 +27,7 @@ public Stream Open(string fileName) ZipArchiveEntry entry = _archive.Entries.FirstOrDefault(e => e.FullName.Equals(fileName, System.StringComparison.InvariantCultureIgnoreCase) ); - if (entry == null) - return null; - return entry.Open(); + return entry?.OpenBoundedStream(); } public string Find(string extension) @@ -36,9 +35,7 @@ public string Find(string extension) ZipArchiveEntry entry = _archive.Entries.FirstOrDefault(e => e.FullName.EndsWith(extension, System.StringComparison.InvariantCultureIgnoreCase) ); - if (entry == null) - return null; - return entry.FullName; + return entry?.FullName; } public UsfmStylesheet CreateStylesheet(string fileName) diff --git a/src/SIL.Machine/Utils/BoundedStream.cs b/src/SIL.Machine/Utils/BoundedStream.cs new file mode 100644 index 000000000..e45cb1139 --- /dev/null +++ b/src/SIL.Machine/Utils/BoundedStream.cs @@ -0,0 +1,96 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace SIL.Machine.Utils +{ + public class BoundedStream : Stream + { + private readonly Stream _innerStream; + private readonly long _maxSize; + private long _totalBytesProcessed; + + public BoundedStream(Stream innerStream, long maxSize) + { + _innerStream = innerStream ?? throw new ArgumentNullException(nameof(innerStream)); + _maxSize = maxSize < 0 ? throw new ArgumentOutOfRangeException(nameof(maxSize)) : maxSize; + } + + public override bool CanRead => _innerStream.CanRead; + public override bool CanSeek => _innerStream.CanSeek; + public override bool CanWrite => _innerStream.CanWrite; + public override long Length => _innerStream.Length; + + public override long Position + { + get => _innerStream.Position; + set => _innerStream.Position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int bytesRead = _innerStream.Read(buffer, offset, count); + TrackAndValidate(bytesRead); + return bytesRead; + } + + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + int bytesRead = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + TrackAndValidate(bytesRead); + return bytesRead; + } + + public override void Write(byte[] buffer, int offset, int count) + { + TrackAndValidate(count); + _innerStream.Write(buffer, offset, count); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + TrackAndValidate(count); + await _innerStream.WriteAsync(buffer, offset, count, cancellationToken); + } + + public override void Flush() => _innerStream.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + _innerStream.FlushAsync(cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); + + public override void SetLength(long value) + { + if (value > _maxSize) + { + throw new IOException( + $"SetLength value of {value} bytes exceeds the maximum limit of {_maxSize} bytes." + ); + } + + _innerStream.SetLength(value); + } + + private void TrackAndValidate(int bytesProcessed) + { + _totalBytesProcessed += bytesProcessed; + if (_totalBytesProcessed > _maxSize) + throw new IOException($"Stream operation aborted. Exceeded maximum limit of {_maxSize} bytes."); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + _innerStream.Dispose(); + + base.Dispose(disposing); + } + } +} diff --git a/src/SIL.Machine/Utils/ZipArchiveEntryExtensions.cs b/src/SIL.Machine/Utils/ZipArchiveEntryExtensions.cs new file mode 100644 index 000000000..34ab148c8 --- /dev/null +++ b/src/SIL.Machine/Utils/ZipArchiveEntryExtensions.cs @@ -0,0 +1,43 @@ +using System.IO; +using System.IO.Compression; + +namespace SIL.Machine.Utils +{ + public static class ZipArchiveEntryExtensions + { + private const long DefaultMaxUncompressedSize = 100 * 1024 * 1024; // 100 MB + private const double DefaultMaxCompressionRatio = 100.0; // 100:1 ratio limit + + /// + /// Opens a bounded stream that will not be larger than the specified maximum uncompressed size. + /// + /// The zip archive entry. + /// The maximum uncompressed size allowed in bytes. + /// The maximum compression ratio allowed. + /// A bounded stream. + /// + /// The entry's uncompressed size or ratio exceeds the maximum allowed limit. + /// + public static BoundedStream OpenBoundedStream( + this ZipArchiveEntry entry, + long maxUncompressedSize = DefaultMaxUncompressedSize, + double maxCompressionRatio = DefaultMaxCompressionRatio + ) + { + if (entry == null) + return null; + + if (entry.Length > maxUncompressedSize) + throw new InvalidDataException("Entry uncompressed size exceeds maximum allowed limit."); + + if (entry.CompressedLength > 0) + { + double ratio = (double)entry.Length / entry.CompressedLength; + if (ratio > maxCompressionRatio) + throw new InvalidDataException("Compression ratio exceeds safe threshold."); + } + + return new BoundedStream(entry.Open(), maxUncompressedSize); + } + } +} diff --git a/tests/SIL.Machine.Tests/Utils/ZipArchiveEntryExtensionTests.cs b/tests/SIL.Machine.Tests/Utils/ZipArchiveEntryExtensionTests.cs new file mode 100644 index 000000000..02227267d --- /dev/null +++ b/tests/SIL.Machine.Tests/Utils/ZipArchiveEntryExtensionTests.cs @@ -0,0 +1,88 @@ +using System.IO.Compression; +using NUnit.Framework; + +namespace SIL.Machine.Utils; + +[TestFixture] +public class ZipArchiveEntryExtensionTests +{ + [Test] + public async Task OpenBoundedStream_ReturnsReadableStream() + { + byte[] contentBytes = [.. "Hello World"u8]; + byte[] zipBytes = await CreateInMemoryZipFileAsync("test.txt", contentBytes); + using var memoryStream = new MemoryStream(zipBytes); + await using var archive = new ZipArchive(memoryStream); + ZipArchiveEntry? entry = archive.GetEntry("test.txt"); + + // SUT + await using BoundedStream stream = entry.OpenBoundedStream(maxUncompressedSize: 100); + using var reader = new StreamReader(stream); + string content = await reader.ReadToEndAsync(); + + Assert.That(content, Is.EqualTo("Hello World")); + } + + [Test] + public async Task OpenBoundedStream_ThrowsInvalidDataExceptionWhenHeaderSizeExceeded() + { + byte[] payload = new byte[200]; + byte[] zipBytes = await CreateInMemoryZipFileAsync("large.txt", payload); + using var memoryStream = new MemoryStream(zipBytes); + await using var archive = new ZipArchive(memoryStream); + ZipArchiveEntry? entry = archive.GetEntry("large.txt"); + + // SUT + Assert.Throws(() => entry.OpenBoundedStream(maxUncompressedSize: 100)); + } + + [Test] + public async Task OpenBoundedStream_ThrowsInvalidDataExceptionWhenCompressionRatioExceeded() + { + byte[] highlyCompressibleData = new byte[10_000]; + byte[] zipBytes = await CreateInMemoryZipFileAsync( + "bomb.txt", + highlyCompressibleData, + CompressionLevel.SmallestSize + ); + using var memoryStream = new MemoryStream(zipBytes); + await using var archive = new ZipArchive(memoryStream); + ZipArchiveEntry? entry = archive.GetEntry("bomb.txt"); + + // SUT + Assert.Throws(() => + entry.OpenBoundedStream(maxUncompressedSize: 20_000, maxCompressionRatio: 2.0) + ); + } + + [Test] + public void BoundedStream_ThrowsIOExceptionWhenRuntimeExpansionExceedsLimit() + { + byte[] rawData = [.. "1234567890"u8]; + using var memoryStream = new MemoryStream(rawData); + using var boundedStream = new BoundedStream(memoryStream, maxSize: 5); + + byte[] buffer = new byte[10]; + + // SUT + boundedStream.ReadExactly(buffer, 0, 4); + Assert.Throws(() => boundedStream.ReadExactly(buffer, 0, 4)); + } + + private static async Task CreateInMemoryZipFileAsync( + string fileName, + ReadOnlyMemory content, + CompressionLevel level = CompressionLevel.Fastest, + CancellationToken cancellationToken = default + ) + { + using var ms = new MemoryStream(); + await using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true)) + { + ZipArchiveEntry entry = archive.CreateEntry(fileName, level); + await using Stream entryStream = await entry.OpenAsync(cancellationToken); + await entryStream.WriteAsync(content, cancellationToken); + } + return ms.ToArray(); + } +}