Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions src/libraries/System.Memory/tests/AllocationHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ static class AllocationHelper
private static readonly Mutex s_memoryLock = new Mutex();
private static readonly TimeSpan s_waitTimeout = TimeSpan.FromSeconds(120);

public static bool TryAllocNative(IntPtr size, out IntPtr memory)
public static unsafe bool TryAllocNative(IntPtr size, out IntPtr memory)
{
memory = IntPtr.Zero;

Expand All @@ -26,22 +26,29 @@ public static bool TryAllocNative(IntPtr size, out IntPtr memory)

try
{
memory = Marshal.AllocHGlobal(size);
memory = (IntPtr)NativeMemory.Alloc((nuint)size);
}
catch (OutOfMemoryException)
{
memory = IntPtr.Zero;
s_memoryLock.ReleaseMutex();
}
finally
{
// Only a successful allocation keeps the mutex; the matching ReleaseNative frees it.
// Any failure (OOM, a null result, or an unexpected throw) must release it here so a
// later large allocation doesn't hang waiting on a mutex that will never be freed.
if (memory == IntPtr.Zero)
s_memoryLock.ReleaseMutex();
}

return memory != IntPtr.Zero;
}

public static void ReleaseNative(ref IntPtr memory)
public static unsafe void ReleaseNative(ref IntPtr memory)
{
try
{
Marshal.FreeHGlobal(memory);
NativeMemory.Free((void*)memory);
memory = IntPtr.Zero;
}
finally
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using System.Runtime.CompilerServices;

using AllocationHelper = System.SpanTests.AllocationHelper;

Expand All @@ -23,69 +22,61 @@ public static partial class ParserTests
[Fact]
[OuterLoop]
[PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
public static void TestParser2GiBOverflow()
public static unsafe void TestParser2GiBOverflow()
{
if (IntPtr.Size < 8)
return;

IntPtr pMemory;
try
{
if (!AllocationHelper.TryAllocNative(size: new IntPtr(int.MaxValue), out pMemory))
return;
}
catch (OutOfMemoryException)
{
if (!AllocationHelper.TryAllocNative((IntPtr)TwoGiB, out IntPtr pMemory))
return;
}

try
{
TwoGiBOverflowHelper<int>(TwoGiBOverflowInt32TestData, pMemory);
Span<byte> span = new Span<byte>((void*)pMemory, TwoGiB);
Comment thread
tannergooding marked this conversation as resolved.
span.Fill((byte)'0'); // Fill the full 2GB once; each case restores only the bytes it touches.
TwoGiBOverflowHelper<int>(TwoGiBOverflowInt32TestData, span);
}
finally
{
AllocationHelper.ReleaseNative(ref pMemory);
}
}

private static void TwoGiBOverflowHelper<T>(IEnumerable<ParserTestData<T>> testDataCollection, IntPtr pMemory)
private static void TwoGiBOverflowHelper<T>(IEnumerable<ParserTestData<T>> testDataCollection, Span<byte> span)
{
Assert.All<ParserTestData<T>>(testDataCollection,
(testData) =>
foreach (ParserTestData<T> testData in testDataCollection)
{
ReadOnlySpan<byte> utf8Span = testData.Text.ToUtf8Span();
byte sign = utf8Span[0];
bool hasSign = sign == '-' || sign == '+';
if (hasSign)
{
unsafe
{
Span<byte> buffer = new Span<byte>((void*)pMemory, int.MaxValue);
ref byte memory = ref Unsafe.AsRef<byte>(pMemory.ToPointer());
Span<byte> span = new Span<byte>(pMemory.ToPointer(), TwoGiB);
span.Fill((byte)'0');
span[0] = sign;
utf8Span = utf8Span.Slice(1);
}

ReadOnlySpan<byte> utf8Span = testData.Text.ToUtf8Span();
byte sign = utf8Span[0];
if (sign == '-' || sign == '+')
{
span[0] = sign;
utf8Span = utf8Span.Slice(1);
}
utf8Span.CopyTo(span.Slice(TwoGiB - utf8Span.Length));
Span<byte> tail = span.Slice(TwoGiB - utf8Span.Length);
utf8Span.CopyTo(tail);

bool success = TryParseUtf8<T>(span, out T value, out int bytesConsumed, testData.FormatSymbol);
if (testData.ExpectedSuccess)
{
Assert.True(success);
Assert.Equal(testData.ExpectedValue, value);
Assert.Equal(testData.ExpectedBytesConsumed, bytesConsumed);
}
else
{
Assert.False(success);
Assert.Equal<T>(default, value);
Assert.Equal(0, bytesConsumed);
}
}
bool success = TryParseUtf8<T>(span, out T value, out int bytesConsumed, testData.FormatSymbol);
if (testData.ExpectedSuccess)
{
Assert.True(success);
Assert.Equal(testData.ExpectedValue, value);
Assert.Equal(testData.ExpectedBytesConsumed, bytesConsumed);
}
else
{
Assert.False(success);
Assert.Equal<T>(default, value);
Assert.Equal(0, bytesConsumed);
}

});
// Restore only the bytes this case wrote so the buffer is all '0' again for the next one.
tail.Fill((byte)'0');
if (hasSign)
span[0] = (byte)'0';
}
}

private static IEnumerable<ParserTestData<int>> TwoGiBOverflowInt32TestData
Expand Down
85 changes: 34 additions & 51 deletions src/libraries/System.Memory/tests/ReadOnlySpan/CopyTo.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Runtime.CompilerServices;
using Xunit;

namespace System.SpanTests
Expand Down Expand Up @@ -122,69 +121,53 @@ public static void Overlapping2()
Assert.Equal<int>(expected, a);
}

// This test case tests the Span.CopyTo method for large buffers of size 4GB or more. In the fast path,
// the CopyTo method performs copy in chunks of size 4GB (uint.MaxValue) with final iteration copying
// the residual chunk of size (bufferSize % 4GB). The inputs sizes to this method, 4GB and 4GB+256B,
// test the two size selection paths in CoptyTo method - memory size that is multiple of 4GB or,
// a multiple of 4GB + some more size.
[ActiveIssue("https://github.com/dotnet/runtime/issues/24139")]
// Verifies that ReadOnlySpan.CopyTo does not truncate the byte count when it exceeds
// uint.MaxValue. For a >4GB span, the element count times sizeof(T) must stay a 64-bit value
// all the way down to the native memmove; a 32-bit truncation would copy far fewer bytes. A
// single overlapping buffer is used so only one >4GB block is needed instead of two.
[Theory]
[OuterLoop]
[PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
[InlineData(4L * 1024L * 1024L * 1024L)]
[InlineData((4L * 1024L * 1024L * 1024L) + 256)]
public static void CopyToLargeSizeTest(long bufferSize)
public static unsafe void CopyToLargeSizeTest(long bufferSize)
{
// If this test is run in a 32-bit process, the large allocation will fail.
if (sizeof(IntPtr) != sizeof(long))
{
return;
}

int GuidCount = (int)(bufferSize / sizeof(Guid));
bool allocatedFirst = false;
bool allocatedSecond = false;
IntPtr memBlockFirst = IntPtr.Zero;
IntPtr memBlockSecond = IntPtr.Zero;
int guidCount = (int)(bufferSize / sizeof(Guid));
const int Gap = 1; // Offset source and destination so the copy is a real move, not a self-copy.

unsafe
if (!AllocationHelper.TryAllocNative((IntPtr)(bufferSize + (Gap * sizeof(Guid))), out IntPtr memBlock))
{
try
{
allocatedFirst = AllocationHelper.TryAllocNative((IntPtr)bufferSize, out memBlockFirst);
allocatedSecond = AllocationHelper.TryAllocNative((IntPtr)bufferSize, out memBlockSecond);

if (allocatedFirst && allocatedSecond)
{
ref Guid memoryFirst = ref Unsafe.AsRef<Guid>(memBlockFirst.ToPointer());
var spanFirst = new ReadOnlySpan<Guid>(memBlockFirst.ToPointer(), GuidCount);

ref Guid memorySecond = ref Unsafe.AsRef<Guid>(memBlockSecond.ToPointer());
var spanSecond = new Span<Guid>(memBlockSecond.ToPointer(), GuidCount);

Guid theGuid = Guid.Parse("900DBAD9-00DB-AD90-00DB-AD900DBADBAD");
for (int count = 0; count < GuidCount; ++count)
{
Unsafe.Add(ref memoryFirst, count) = theGuid;
}

spanFirst.CopyTo(spanSecond);

for (int count = 0; count < GuidCount; ++count)
{
Guid guidfirst = Unsafe.Add(ref memoryFirst, count);
Guid guidSecond = Unsafe.Add(ref memorySecond, count);
Assert.Equal(guidfirst, guidSecond);
}
}
}
finally
{
if (allocatedFirst)
AllocationHelper.ReleaseNative(ref memBlockFirst);
if (allocatedSecond)
AllocationHelper.ReleaseNative(ref memBlockSecond);
}
return;
}

try
{
var buffer = new Span<Guid>((void*)memBlock, guidCount + Gap);
Span<Guid> source = buffer.Slice(0, guidCount);
Span<Guid> destination = buffer.Slice(Gap, guidCount);

Guid fill = Guid.Parse("900DBAD9-00DB-AD90-00DB-AD900DBADBAD");
Guid tail = Guid.Parse("2B2B2B2B-2B2B-2B2B-2B2B-2B2B2B2B2B2B");

// Only the boundaries are initialized and checked. Filling all 4GB would add a full
// extra memory pass on top of the copy for no additional coverage.
source[0] = fill;
source[guidCount - 1] = tail;
destination[guidCount - 1] = fill; // A truncated copy leaves this as 'fill' rather than 'tail'.

((ReadOnlySpan<Guid>)source).CopyTo(destination);

Assert.Equal(fill, destination[0]);
Assert.Equal(tail, destination[guidCount - 1]);
}
finally
{
AllocationHelper.ReleaseNative(ref memBlock);
}
}

Expand Down
85 changes: 34 additions & 51 deletions src/libraries/System.Memory/tests/Span/CopyTo.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Runtime.CompilerServices;
using Xunit;

namespace System.SpanTests
Expand Down Expand Up @@ -191,69 +190,53 @@ public static void CopyToCovariantArray()
Assert.Equal("Hello", dst[0]);
}

// This test case tests the Span.CopyTo method for large buffers of size 4GB or more. In the fast path,
// the CopyTo method performs copy in chunks of size 4GB (uint.MaxValue) with final iteration copying
// the residual chunk of size (bufferSize % 4GB). The inputs sizes to this method, 4GB and 4GB+256B,
// test the two size selection paths in CoptyTo method - memory size that is multiple of 4GB or,
// a multiple of 4GB + some more size.
[ActiveIssue("https://github.com/dotnet/runtime/issues/24139")]
// Verifies that Span.CopyTo does not truncate the byte count when it exceeds uint.MaxValue.
// For a >4GB span, the element count times sizeof(T) must stay a 64-bit value all the way
// down to the native memmove; a 32-bit truncation would copy far fewer bytes. A single
// overlapping buffer is used so only one >4GB block is needed instead of two.
[Theory]
[OuterLoop]
[PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
[InlineData(4L * 1024L * 1024L * 1024L)]
[InlineData((4L * 1024L * 1024L * 1024L) + 256)]
public static void CopyToLargeSizeTest(long bufferSize)
public static unsafe void CopyToLargeSizeTest(long bufferSize)
{
// If this test is run in a 32-bit process, the large allocation will fail.
if (sizeof(IntPtr) != sizeof(long))
{
return;
}

int GuidCount = (int)(bufferSize / sizeof(Guid));
bool allocatedFirst = false;
bool allocatedSecond = false;
IntPtr memBlockFirst = IntPtr.Zero;
IntPtr memBlockSecond = IntPtr.Zero;
int guidCount = (int)(bufferSize / sizeof(Guid));
const int Gap = 1; // Offset source and destination so the copy is a real move, not a self-copy.

unsafe
if (!AllocationHelper.TryAllocNative((IntPtr)(bufferSize + (Gap * sizeof(Guid))), out IntPtr memBlock))
{
try
{
allocatedFirst = AllocationHelper.TryAllocNative((IntPtr)bufferSize, out memBlockFirst);
allocatedSecond = AllocationHelper.TryAllocNative((IntPtr)bufferSize, out memBlockSecond);

if (allocatedFirst && allocatedSecond)
{
ref Guid memoryFirst = ref Unsafe.AsRef<Guid>(memBlockFirst.ToPointer());
var spanFirst = new Span<Guid>(memBlockFirst.ToPointer(), GuidCount);

ref Guid memorySecond = ref Unsafe.AsRef<Guid>(memBlockSecond.ToPointer());
var spanSecond = new Span<Guid>(memBlockSecond.ToPointer(), GuidCount);

Guid theGuid = Guid.Parse("900DBAD9-00DB-AD90-00DB-AD900DBADBAD");
for (int count = 0; count < GuidCount; ++count)
{
Unsafe.Add(ref memoryFirst, count) = theGuid;
}

spanFirst.CopyTo(spanSecond);

for (int count = 0; count < GuidCount; ++count)
{
Guid guidfirst = Unsafe.Add(ref memoryFirst, count);
Guid guidSecond = Unsafe.Add(ref memorySecond, count);
Assert.Equal(guidfirst, guidSecond);
}
}
}
finally
{
if (allocatedFirst)
AllocationHelper.ReleaseNative(ref memBlockFirst);
if (allocatedSecond)
AllocationHelper.ReleaseNative(ref memBlockSecond);
}
return;
}

try
{
var buffer = new Span<Guid>((void*)memBlock, guidCount + Gap);
Span<Guid> source = buffer.Slice(0, guidCount);
Span<Guid> destination = buffer.Slice(Gap, guidCount);

Guid fill = Guid.Parse("900DBAD9-00DB-AD90-00DB-AD900DBADBAD");
Guid tail = Guid.Parse("2B2B2B2B-2B2B-2B2B-2B2B-2B2B2B2B2B2B");

// Only the boundaries are initialized and checked. Filling all 4GB would add a full
// extra memory pass on top of the copy for no additional coverage.
source[0] = fill;
source[guidCount - 1] = tail;
destination[guidCount - 1] = fill; // A truncated copy leaves this as 'fill' rather than 'tail'.

source.CopyTo(destination);

Assert.Equal(fill, destination[0]);
Assert.Equal(tail, destination[guidCount - 1]);
}
finally
{
AllocationHelper.ReleaseNative(ref memBlock);
}
}

Expand Down
Loading