forked from dotnet/runtime
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Set version in ZIP local header to ZIP64 when file offset is >4GB (do…
…tnet#102053) * ZipArchiveEntry didn't set ZIP64 in local headers for small files if their offset are > 4GB. * Added System.IO.Compression and System.IO.Packaging tests. --------- Co-authored-by: Gan Keyu <gankeyu@hotmail.com>
- Loading branch information
1 parent
e867965
commit 0cc0791
Showing
7 changed files
with
306 additions
and
101 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
129 changes: 105 additions & 24 deletions
129
src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,48 +1,129 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Reflection; | ||
| using Xunit; | ||
|
|
||
| namespace System.IO.Compression.Tests | ||
| namespace System.IO.Compression.Tests; | ||
|
|
||
| [Collection(nameof(DisableParallelization))] | ||
| public class zip_LargeFiles : ZipFileTestBase | ||
| { | ||
| [Collection(nameof(DisableParallelization))] | ||
| public class zip_LargeFiles : ZipFileTestBase | ||
| [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes | ||
| [OuterLoop("It requires almost 12 GB of free disk space")] | ||
| public static void UnzipOver4GBZipFile() | ||
| { | ||
| [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes | ||
| [OuterLoop("It requires almost 12 GB of free disk space")] | ||
| public static void UnzipOver4GBZipFile() | ||
| byte[] buffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // 1 GB | ||
|
|
||
| string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); | ||
| DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB")); | ||
|
|
||
| try | ||
| { | ||
| for (byte i = 0; i < 6; i++) | ||
| { | ||
| File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer); | ||
| } | ||
|
|
||
| ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false); | ||
|
|
||
| using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath); | ||
| foreach (ZipArchiveEntry entry in zipArchive.Entries) | ||
| { | ||
| using Stream entryStream = entry.Open(); | ||
|
|
||
| Assert.True(entryStream.CanRead); | ||
| Assert.Equal(buffer.Length, entryStream.Length); | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| byte[] buffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // 1 GB | ||
| File.Delete(zipArchivePath); | ||
|
|
||
| tempDir.Delete(recursive: true); | ||
| } | ||
| } | ||
|
|
||
| private static void FillWithHardToCompressData(byte[] buffer) | ||
| { | ||
| Random.Shared.NextBytes(buffer); | ||
| } | ||
|
|
||
| string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); | ||
| DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB")); | ||
| [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes | ||
| [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")] | ||
| [InlineData(false)] | ||
| [InlineData(true)] | ||
| public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed) | ||
| { | ||
| // issue #94899 | ||
|
|
||
| CompressionLevel compressLevel = isCompressed ? CompressionLevel.Optimal : CompressionLevel.NoCompression; | ||
| byte[] smallBuffer = GC.AllocateUninitializedArray<byte>(1000); | ||
| byte[] largeBuffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // ~1 GB | ||
| string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); | ||
| string LargeFileName = "largefile"; | ||
| string SmallFileName = "smallfile"; | ||
| uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4; | ||
| ushort Zip64Version = 45; | ||
|
|
||
| try | ||
| try | ||
| { | ||
| using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite); | ||
|
|
||
| // Create | ||
| using (ZipArchive archive = new(fs, ZipArchiveMode.Create, true)) | ||
| { | ||
| for (byte i = 0; i < 6; i++) | ||
| ZipArchiveEntry file = archive.CreateEntry(LargeFileName, compressLevel); | ||
|
|
||
| using (Stream stream = file.Open()) | ||
| { | ||
| File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer); | ||
| // Write 5GB of data | ||
| for (var i = 0; i < 5; i++) | ||
| { | ||
| if (isCompressed) | ||
| { | ||
| FillWithHardToCompressData(largeBuffer); | ||
| } | ||
|
|
||
| stream.Write(largeBuffer); | ||
| } | ||
| } | ||
|
|
||
| ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false); | ||
| file = archive.CreateEntry(SmallFileName, compressLevel); | ||
|
|
||
| using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath); | ||
| foreach (ZipArchiveEntry entry in zipArchive.Entries) | ||
| using (Stream stream = file.Open()) | ||
| { | ||
| using Stream entryStream = entry.Open(); | ||
|
|
||
| Assert.True(entryStream.CanRead); | ||
| Assert.Equal(buffer.Length, entryStream.Length); | ||
| stream.Write(smallBuffer); | ||
| } | ||
| } | ||
| finally | ||
|
|
||
| fs.Position = 0; | ||
|
|
||
| // Validate | ||
| using (ZipArchive archive = new(fs, ZipArchiveMode.Read)) | ||
| { | ||
| File.Delete(zipArchivePath); | ||
| using var reader = new BinaryReader(fs); | ||
|
|
||
| tempDir.Delete(recursive: true); | ||
| FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance); | ||
|
|
||
| if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long)) | ||
| { | ||
| Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written."); | ||
| } | ||
|
|
||
| foreach (ZipArchiveEntry entry in archive.Entries) | ||
| { | ||
| fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart; | ||
| ushort versionNeeded = reader.ReadUInt16(); | ||
|
|
||
| // Version is not ZIP64 for files with Local Header at >4GB offset. | ||
| Assert.Equal(Zip64Version, versionNeeded); | ||
| } | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| File.Delete(zipArchivePath); | ||
| } | ||
| } | ||
| } |
97 changes: 97 additions & 0 deletions
97
src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.IO.Compression; | ||
| using System.Net.Mime; | ||
| using System.Reflection; | ||
| using Xunit; | ||
|
|
||
| namespace System.IO.Packaging.Tests; | ||
|
|
||
| public partial class LargeFilesTests | ||
| { | ||
| private static void FillWithHardToCompressData(byte[] buffer) | ||
| { | ||
| Random.Shared.NextBytes(buffer); | ||
| } | ||
|
|
||
| [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes | ||
| [InlineData(false)] | ||
| [InlineData(true)] | ||
| [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")] | ||
| public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed) | ||
| { | ||
| // issue #94899 | ||
|
|
||
| CompressionOption compressionOption = isCompressed ? CompressionOption.Normal : CompressionOption.NotCompressed; | ||
| byte[] smallBuffer = GC.AllocateUninitializedArray<byte>(1000); | ||
| byte[] largeBuffer = GC.AllocateUninitializedArray<byte>(1_000_000_000); // ~1 GB | ||
| string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); | ||
| Uri largePartUri = PackUriHelper.CreatePartUri(new Uri("large.bin", UriKind.Relative)); | ||
| Uri smallPartUri = PackUriHelper.CreatePartUri(new Uri("small.bin", UriKind.Relative)); | ||
| uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4; | ||
| ushort Zip64Version = 45; | ||
|
|
||
| try | ||
| { | ||
| using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite); | ||
|
|
||
| // Create | ||
| using (Package package = Package.Open(fs, FileMode.Create, FileAccess.Write)) | ||
| { | ||
| PackagePart partLarge = package.CreatePart(largePartUri, MediaTypeNames.Application.Octet, compressionOption); | ||
|
|
||
| using (Stream streamLarge = partLarge.GetStream()) | ||
| { | ||
| // Write 5GB of data | ||
|
|
||
| for (var i = 0; i < 5; i++) | ||
| { | ||
| if (isCompressed) | ||
| { | ||
| FillWithHardToCompressData(largeBuffer); | ||
| } | ||
|
|
||
| streamLarge.Write(largeBuffer); | ||
| } | ||
| } | ||
|
|
||
| PackagePart partSmall = package.CreatePart(smallPartUri, MediaTypeNames.Application.Octet, compressionOption); | ||
|
|
||
| using (Stream streamSmall = partSmall.GetStream()) | ||
| { | ||
| streamSmall.Write(smallBuffer); | ||
| } | ||
| } | ||
|
|
||
|
|
||
| fs.Position = 0; | ||
|
|
||
| // Validate | ||
| using (ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Read)) | ||
| { | ||
| using var reader = new BinaryReader(fs); | ||
|
|
||
| FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance); | ||
|
|
||
| if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long)) | ||
| { | ||
| Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written."); | ||
| } | ||
|
|
||
| foreach (ZipArchiveEntry entry in archive.Entries) | ||
| { | ||
| fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart; | ||
| ushort versionNeeded = reader.ReadUInt16(); | ||
|
|
||
| // Version is not ZIP64 for files with Local Header at >4GB offset. | ||
| Assert.Equal(Zip64Version, versionNeeded); | ||
| } | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| File.Delete(zipArchivePath); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.