Skip to content

Vectorize {Last}IndexOf{Any} and {Last}IndexOfAnyExcept without code duplication #73768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adamsitnik merged 37 commits into from
Aug 17, 2022
Merged

Vectorize {Last}IndexOf{Any} and {Last}IndexOfAnyExcept without code duplication #73768

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
9d01665
Vectorize LastIndexOf
adamsitnik Aug 11, 2022
14224a3
use structs to get performant code without code duplication
adamsitnik Aug 11, 2022
d05cf56
use it in all possible places
adamsitnik Aug 11, 2022
22f71ff
vectorize LastIndexOfAny(value0, value1)
adamsitnik Aug 11, 2022
bbc64e0
vectorize LastIndexOfAnyExcept(value0, value1)
adamsitnik Aug 11, 2022
f265d28
simplify it to make it easier to add 3 and 4 values overloads
adamsitnik Aug 11, 2022
b0f259c
vectorize LastIndexOfAny and LastIndexOfAnyExcept for 3 values
adamsitnik Aug 11, 2022
7dc03c3
rename (I am not convinced it's the best name yet)
adamsitnik Aug 11, 2022
628d429
vectorize Contains
adamsitnik Aug 12, 2022
114ca88
hide the implementation details
adamsitnik Aug 12, 2022
95a1b62
address review from Jan, don't cast Span to ROS. Introduce new helper…
adamsitnik Aug 12, 2022
ab8df3d
vectorize IndexOf(value)
adamsitnik Aug 12, 2022
bb13957
rename IndexOf used only by strlen to IndexOfNullByte and optimize it…
adamsitnik Aug 12, 2022
bbb7e4c
vectorize IndexOfAny<T>(value0, value1) and IndexOfAnyExcept<T>(value…
adamsitnik Aug 12, 2022
9186b8f
vectorize IndexOfAny<T>(value0, value1, value2) and IndexOfAnyExcept<…
adamsitnik Aug 12, 2022
5a39d21
vectorize IndexOfAny<T>(value0, value1, value2, value3) and IndexOfAn…
adamsitnik Aug 12, 2022
9099ef9
vectorize IndexOfAny<T>(value0, value1, value2, value3, value4)
adamsitnik Aug 12, 2022
5661793
remove dead code
adamsitnik Aug 12, 2022
8cb8208
use built-in helpers
adamsitnik Aug 12, 2022
ec447c7
Revert "address review from Jan, don't cast Span to ROS. Introduce ne…
adamsitnik Aug 12, 2022
62f16c1
add/remove special handling of some types
adamsitnik Aug 12, 2022
26c188d
fix NativeAOT build
adamsitnik Aug 12, 2022
2af01e3
address code review feedback
adamsitnik Aug 13, 2022
b305c54
perform manual loop unrolling in order to avoid perf regression for s…
adamsitnik Aug 13, 2022
f954ac5
Merge remote-tracking branch 'upstream/main' into vectorizeLastIndexOf
adamsitnik Aug 14, 2022
846e4fc
restore Vector<T> code path as Vector128 is not accelerated by Mono x…
adamsitnik Aug 14, 2022
8a74f28
remove redundant for loop that was left here by mistake
adamsitnik Aug 14, 2022
fab3143
apply manual loop unrolling to LastIndexOfAny methods
adamsitnik Aug 15, 2022
4274f54
vectorize LastIndexOf(4) and LastIndexOfAnyExcept(4)
adamsitnik Aug 15, 2022
59c867f
Apply suggestions from code review
adamsitnik Aug 15, 2022
3049470
Merge remote-tracking branch 'upstream/main' into vectorizeLastIndexOf
adamsitnik Aug 15, 2022
afc09bd
fix the build
adamsitnik Aug 15, 2022
efdff0e
fix the NativeAOT build
adamsitnik Aug 15, 2022
5831e09
Merge remote-tracking branch 'upstream/main' into vectorizeLastIndexOf
adamsitnik Aug 16, 2022
7f003da
optimize Except methods for small inputs
adamsitnik Aug 16, 2022
0dc257f
address code review feedback:
adamsitnik Aug 17, 2022
0c6b01d
reduce the regression for IndexOfAnyExcept(3) and LastIndexOfAnyExcep…
adamsitnik Aug 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/libraries/System.Private.CoreLib/src/System/Array.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1586,27 +1586,27 @@ public static int LastIndexOf<T>(T[] array, T value, int startIndex, int count)
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
int result = SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DontNegate<byte>>(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), endIndex),
Unsafe.As<T, byte>(ref value),
count);

return (result >= 0 ? endIndex : 0) + result;
}
else if (Unsafe.SizeOf<T>() == sizeof(char))
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<char[]>(array)), endIndex),
Unsafe.As<T, char>(ref value),
int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), endIndex),
Unsafe.As<T, short>(ref value),
count);

return (result >= 0 ? endIndex : 0) + result;
}
else if (Unsafe.SizeOf<T>() == sizeof(int))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
int result = SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DontNegate<int>>(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)), endIndex),
Unsafe.As<T, int>(ref value),
count);
Expand All @@ -1616,7 +1616,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)),
else if (Unsafe.SizeOf<T>() == sizeof(long))
{
int endIndex = startIndex - count + 1;
int result = SpanHelpers.LastIndexOf(
int result = SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DontNegate<long>>(
ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<long[]>(array)), endIndex),
Unsafe.As<T, long>(ref value),
count);
Expand Down
232 changes: 174 additions & 58 deletions src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,24 +408,7 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
/// <param name="value">The value to search for.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
{
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
return SpanHelpers.LastIndexOf(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value),
span.Length);

if (Unsafe.SizeOf<T>() == sizeof(char))
return SpanHelpers.LastIndexOf(
ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, char>(ref value),
span.Length);
}

return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
}
=> LastIndexOf((ReadOnlySpan<T>)span, value);

@jkotas jkotas Aug 11, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not inlineable when T is generic variable due to current generic inlining limitations. It means that this change to just forward Span to ReadOnlySpan will come with some perf regression in some situations.

(Just pointing it out. I will leave it up to you whether to take this regression for simplicity. Either way is fine with me.)


/// <summary>
/// Searches for the specified sequence and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
Expand Down Expand Up @@ -689,6 +672,38 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> val
/// </returns>
public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
{
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.Negate<byte>>(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.Negate<short>>(
ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, short>(ref value),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(int))
{
return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.Negate<int>>(
ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, int>(ref value),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(long))
{
return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.Negate<long>>(
ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, long>(ref value),
span.Length);
}
}

for (int i = span.Length - 1; i >= 0; i--)
{
if (!EqualityComparer<T>.Default.Equals(span[i], value))
Expand All @@ -711,6 +726,42 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) wh
/// </returns>
public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
{
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.Negate<byte>>(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.Negate<short>>(
ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, short>(ref value0),
Unsafe.As<T, short>(ref value1),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(int))
{
return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.Negate<int>>(
ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, int>(ref value0),
Unsafe.As<T, int>(ref value1),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(long))
{
return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.Negate<long>>(
ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, long>(ref value0),
Unsafe.As<T, long>(ref value1),
span.Length);
}
}

for (int i = span.Length - 1; i >= 0; i--)
{
if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
Expand All @@ -735,6 +786,29 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T
/// </returns>
public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
{
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.Negate<byte>>(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
Unsafe.As<T, byte>(ref value2),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.Negate<short>>(
ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, short>(ref value0),
Unsafe.As<T, short>(ref value1),
Unsafe.As<T, short>(ref value2),
span.Length);
}
// we can easily add int and long here, but there must be an evidence that shows that it's actually needed
}

for (int i = span.Length - 1; i >= 0; i--)
{
if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
Expand Down Expand Up @@ -902,16 +976,33 @@ public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : I
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
return SpanHelpers.LastIndexOf(
{
return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DontNegate<byte>>(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value),
span.Length);

if (Unsafe.SizeOf<T>() == sizeof(char))
return SpanHelpers.LastIndexOf(
ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, char>(ref value),
}
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, short>(ref value),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(int))
{
return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DontNegate<int>>(
ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, int>(ref value),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(long))
{
return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DontNegate<long>>(
ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, long>(ref value),
span.Length);
}
}

return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
Expand Down Expand Up @@ -1207,16 +1298,7 @@ private static unsafe int IndexOfAnyProbabilistic(ref char searchSpace, int sear
/// <param name="value1">One of the values to search for.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
{
if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
return SpanHelpers.LastIndexOfAny(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
span.Length);

return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
}
=> LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1);

/// <summary>
/// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
Expand All @@ -1227,17 +1309,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
/// <param name="value2">One of the values to search for.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
{
if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
return SpanHelpers.LastIndexOfAny(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
Unsafe.As<T, byte>(ref value2),
span.Length);

return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
}
=> LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1, value2);

/// <summary>
/// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
Expand Down Expand Up @@ -1266,12 +1338,41 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values)),
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
{
if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
return SpanHelpers.LastIndexOfAny(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
span.Length);
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DontNegate<byte>>(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DontNegate<short>>(
ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, short>(ref value0),
Unsafe.As<T, short>(ref value1),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(int))
{
return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.DontNegate<int>>(
ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, int>(ref value0),
Unsafe.As<T, int>(ref value1),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(long))
{
return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.DontNegate<long>>(
ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, long>(ref value0),
Unsafe.As<T, long>(ref value1),
span.Length);
}
}

return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
}
Expand All @@ -1286,13 +1387,28 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
{
if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
return SpanHelpers.LastIndexOfAny(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
Unsafe.As<T, byte>(ref value2),
span.Length);
if (RuntimeHelpers.IsBitwiseEquatable<T>())
{
if (Unsafe.SizeOf<T>() == sizeof(byte))
{
return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DontNegate<byte>>(
ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, byte>(ref value0),
Unsafe.As<T, byte>(ref value1),
Unsafe.As<T, byte>(ref value2),
span.Length);
}
else if (Unsafe.SizeOf<T>() == sizeof(short))
{
return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DontNegate<short>>(
ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
Unsafe.As<T, short>(ref value0),
Unsafe.As<T, short>(ref value1),
Unsafe.As<T, short>(ref value2),
span.Length);
}
// we can easily add int and long here, but there must be an evidence that shows that it's actually needed
Comment thread
adamsitnik marked this conversation as resolved.
Outdated
}

return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
}
Expand Down
Loading