From 15ffcff82fe8a3eea529ae5d67a9fc52be990d9f Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 3 Oct 2023 12:14:17 -0700 Subject: [PATCH] Expose an internal ISimdVector interface and being using it to deduplicate some SIMD code (#90764) * Adding an internal ISimdVector`2 interface * Move LastIndexOfValueType to use ISimdVector`2 * Fix a couple minor whitespace nits and remove an unnecessary local --- src/coreclr/jit/hwintrinsicarm64.cpp | 14 +- src/coreclr/jit/hwintrinsicxarch.cpp | 14 +- .../System.Private.CoreLib.Shared.projitems | 2 + .../Runtime/Intrinsics/ISimdVector_2.cs | 558 ++++++++++++++++++ .../Intrinsics/SimdVectorExtensions.cs | 171 ++++++ .../System/Runtime/Intrinsics/Vector128.cs | 262 +++----- .../System/Runtime/Intrinsics/Vector128_1.cs | 212 ++++++- .../System/Runtime/Intrinsics/Vector256.cs | 262 +++----- .../System/Runtime/Intrinsics/Vector256_1.cs | 213 ++++++- .../System/Runtime/Intrinsics/Vector512.cs | 262 +++----- .../System/Runtime/Intrinsics/Vector512_1.cs | 213 ++++++- .../src/System/Runtime/Intrinsics/Vector64.cs | 128 ++-- .../System/Runtime/Intrinsics/Vector64_1.cs | 212 ++++++- .../src/System/SpanHelpers.T.cs | 128 ++-- .../src/System/ThrowHelper.cs | 12 + 15 files changed, 1963 insertions(+), 700 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/ISimdVector_2.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/SimdVectorExtensions.cs diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 5c03636d71049..0234f4ed23cfa 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -547,7 +547,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_Ceiling: { assert(sig->numArgs == 1); - assert(varTypeIsFloating(simdBaseType)); + + if (!varTypeIsFloating(simdBaseType)) + { + retNode = impSIMDPopStack(); + break; + } op1 = impSIMDPopStack(); retNode = gtNewSimdCeilNode(retType, op1, simdBaseJitType, simdSize); @@ -1098,7 +1103,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector128_Floor: { assert(sig->numArgs == 1); - assert(varTypeIsFloating(simdBaseType)); + + if (!varTypeIsFloating(simdBaseType)) + { + retNode = impSIMDPopStack(); + break; + } op1 = impSIMDPopStack(); retNode = gtNewSimdFloorNode(retType, op1, simdBaseJitType, simdSize); diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 065999982a87a..89dc60031d531 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -1371,7 +1371,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector512_Ceiling: { assert(sig->numArgs == 1); - assert(varTypeIsFloating(simdBaseType)); + + if (!varTypeIsFloating(simdBaseType)) + { + retNode = impSIMDPopStack(); + break; + } if ((simdSize < 32) && !compOpportunisticallyDependsOn(InstructionSet_SSE41)) { @@ -1986,7 +1991,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, case NI_Vector512_Floor: { assert(sig->numArgs == 1); - assert(varTypeIsFloating(simdBaseType)); + + if (!varTypeIsFloating(simdBaseType)) + { + retNode = impSIMDPopStack(); + break; + } if ((simdSize < 32) && !compOpportunisticallyDependsOn(InstructionSet_SSE41)) { diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index db2751834350a..2b774a2f4fd55 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1014,7 +1014,9 @@ + + diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/ISimdVector_2.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/ISimdVector_2.cs new file mode 100644 index 0000000000000..a1d49045bb2ac --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/ISimdVector_2.cs @@ -0,0 +1,558 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Runtime.Intrinsics +{ + /// Defines a single instruction, multiple data (SIMD) vector type. + /// The type that implements the interface. + /// The type of the elements in the vector. + internal unsafe interface ISimdVector + : IAdditionOperators, + // IAdditiveIdentity, + IBitwiseOperators, + // IComparisonOperators, + // IDecrementOperators, + IDivisionOperators, + IEqualityOperators, + IEquatable, + // IIncrementOperators, + // IMinMaxValue, + // IModulusOperators, + // IMultiplicativeIdentity, + IMultiplyOperators, + IShiftOperators, + // ISpanFormattable, + ISubtractionOperators, + IUnaryNegationOperators, + IUnaryPlusOperators + // IUtf8SpanFormattable + where TSelf : ISimdVector? + { + /// Gets the natural alignment of the vector. + /// The type of the elements in the vector () is not supported. + static abstract int Alignment { get; } + + /// Gets an instance of the vector type in which all bits are set. + /// The type of the elements in the vector () is not supported. + static abstract TSelf AllBitsSet { get; } + + /// Gets the number of that are in the vector. + /// The type of the elements in the vector () is not supported. + static abstract int Count { get; } + + /// Gets a value that indicates whether the vector operations are subject to hardware acceleration through JIT intrinsic support. + /// if the vector operations are subject to hardware acceleration; otherwise, . + static abstract bool IsHardwareAccelerated { get; } + + /// Gets if is supported; otherwise, . + /// if is supported; otherwise, . + static abstract bool IsSupported { get; } + + /// Gets an instance of the vector type in which each element is the value one. + /// The type of the elements in the vector () is not supported. + static abstract TSelf One { get; } + + /// Gets an instance of the vector type in which each element is the value zero. + /// The type of the elements in the vector () is not supported. + static abstract TSelf Zero { get; } + + /// Gets the element at the specified index. + /// The index of the element to get. + /// The value of the element at . + /// was less than zero or greater than the number of elements. + /// The type of the elements in the vector () is not supported. + abstract T this[int index] { get; } + + /// Divides a vector by a scalar to compute the per-element quotient. + /// The vector that will be divided by . + /// The scalar that will divide . + /// The quotient of divided by . + static abstract TSelf operator /(TSelf left, T right); + + /// Multiplies a vector by a scalar to compute their product. + /// The vector to multiply with . + /// The scalar to multiply with . + /// The product of and . + /// The type of the vector () is not supported. + static abstract TSelf operator *(TSelf left, T right); + + /// Computes the absolute of a vector. + /// The vector for which to get its absolute. + /// A absolute of . + /// The type of the elements in the vector () is not supported. + static abstract TSelf Abs(TSelf vector); + + /// Adds two vectors to compute their sum. + /// The vector to add with . + /// The vector to add with . + /// The sum of and . + /// The type of and () is not supported. + static virtual TSelf Add(TSelf left, TSelf right) => left + right; + + /// Computes the bitwise-and of a given vector and the ones complement of another vector. + /// The vector to bitwise-and with . + /// The vector to that is ones-complemented before being bitwise-and with . + /// The bitwise-and of and the ones-complement of . + static virtual TSelf AndNot(TSelf left, TSelf right) => left & ~right; + + /// Computes the bitwise-and of two vectors. + /// The vector to bitwise-and with . + /// The vector to bitwise-and with . + /// The bitwise-and of and . + /// The type of and () is not supported. + static virtual TSelf BitwiseAnd(TSelf left, TSelf right) => left & right; + + /// Computes the bitwise-or of two vectors. + /// The vector to bitwise-or with . + /// The vector to bitwise-or with . + /// The bitwise-or of and . + /// The type of and () is not supported. + static virtual TSelf BitwiseOr(TSelf left, TSelf right) => left | right; + + /// Computes the ceiling of each element in a vector. + /// The vector that will have its ceiling computed. + /// A vector whose elements are the ceiling of the elements in . + static abstract TSelf Ceiling(TSelf vector); + + /// Conditionally selects bits from two vectors based on a given condition. + /// The mask that is used to select a value from or . + /// The vector that is selected when the corresponding bit in is one. + /// The vector that is selected when the corresponding bit in is zero. + /// A vector whose bits come from or based on the value of . + /// The type of the elements in the vector () is not supported. + static virtual TSelf ConditionalSelect(TSelf condition, TSelf left, TSelf right) => (left & condition) | (right & ~condition); + + /// Copies a vector to a given array. + /// The vector to be copied. + /// The array to which is copied. + /// The length of is less than . + /// The type of the elements in the vector () is not supported. + /// is null. + static virtual void CopyTo(TSelf vector, T[] destination) => TSelf.CopyTo(vector, destination.AsSpan()); + + /// Copies a vector to a given array starting at the specified index. + /// The vector to be copied. + /// The array to which is copied. + /// The starting index of which will be copied to. + /// The length of is less than . + /// is negative or greater than the length of . + /// The type of the elements in the vector () is not supported. + /// is null. + static virtual void CopyTo(TSelf vector, T[] destination, int startIndex) => TSelf.CopyTo(vector, destination.AsSpan(startIndex)); + + /// Copies a vector to a given span. + /// The vector to be copied. + /// The span to which the is copied. + /// The length of is less than . + /// The type of the elements in the vector () is not supported. + static virtual void CopyTo(TSelf vector, Span destination) + { + if (destination.Length < TSelf.Count) + { + ThrowHelper.ThrowArgumentException_DestinationTooShort(); + } + TSelf.StoreUnsafe(vector, ref MemoryMarshal.GetReference(destination)); + } + + /// Creates a new vector with all elements initialized to the specified value. + /// The value that all elements will be initialized to. + /// A new vector with all elements initialized to . + /// The type of the elements in the vector () is not supported. + static abstract TSelf Create(T value); + + /// Creates a new vector from a given array. + /// The array from which the vector is created. + /// A new vector with its elements set to the first elements from . + /// The length of is less than . + /// The type of the elements in the vector () is not supported. + /// is null. + static virtual TSelf Create(T[] values) => TSelf.Create(values.AsSpan()); + + /// Creates a new vector from a given array. + /// The array from which the vector is created. + /// The index in at which to being reading elements. + /// A new vector with its elements set to the first elements from . + /// The length of , starting from , is less than . + /// The type of the elements in the vector () is not supported. + /// is null. + static virtual TSelf Create(T[] values, int index) => TSelf.Create(values.AsSpan(index)); + + /// Creates a new vector from a given readonly span. + /// The readonly span from which the vector is created. + /// A new vector with its elements set to the first elements from . + /// The length of is less than . + /// The type of the elements in the vector () is not supported. + static virtual TSelf Create(ReadOnlySpan values) + { + if (values.Length < TSelf.Count) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.values); + } + return TSelf.LoadUnsafe(ref MemoryMarshal.GetReference(values)); + } + + /// Creates a new vector with the first element initialized to the specified value and the remaining elements initialized to zero. + /// The value that element 0 will be initialized to. + /// A new vector with the first element initialized to and the remaining elements initialized to zero. + /// The type of the elements in the vector () is not supported. + static virtual TSelf CreateScalar(T value) => TSelf.WithElement(TSelf.Zero, 0, value); + + /// Creates a new vector with the first element initialized to the specified value and the remaining elements left uninitialized. + /// The value that element 0 will be initialized to. + /// A new vector with the first element initialized to and the remaining elements left uninitialized. + /// The type of the elements in the vector () is not supported. + static virtual TSelf CreateScalarUnsafe(T value) + { + // This relies on us stripping the "init" flag from the ".locals" + // declaration to let the upper bits be uninitialized. + + Unsafe.SkipInit(out TSelf result); + return TSelf.WithElement(result, 0, value); + } + + /// Divides two vectors to compute their quotient. + /// The vector that will be divided by . + /// The vector that will divide . + /// The quotient of divided by . + /// The type of and () is not supported. + static virtual TSelf Divide(TSelf left, TSelf right) => left / right; + + /// Divides a vector by a scalar to compute the per-element quotient. + /// The vector that will be divided by . + /// The scalar that will divide . + /// The quotient of divided by . + static virtual TSelf Divide(TSelf left, T right) => left / right; + + /// Computes the dot product of two vectors. + /// The vector that will be dotted with . + /// The vector that will be dotted with . + /// The dot product of and . + /// The type of and () is not supported. + static abstract T Dot(TSelf left, TSelf right); + + /// Compares two vectors to determine if they are equal on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are all-bits-set or zero, depending on if the corresponding elements in and were equal. + /// The type of the elements in the vector () is not supported. + static abstract TSelf Equals(TSelf left, TSelf right); + + /// Compares two vectors to determine if all elements are equal. + /// The vector to compare with . + /// The vector to compare with . + /// true if all elements in were equal to the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool EqualsAll(TSelf left, TSelf right); + + /// Compares two vectors to determine if any elements are equal. + /// The vector to compare with . + /// The vector to compare with . + /// true if any elements in was equal to the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool EqualsAny(TSelf left, TSelf right); + + /// Computes the floor of each element in a vector. + /// The vector that will have its floor computed. + /// A vector whose elements are the floor of the elements in . + static abstract TSelf Floor(TSelf vector); + + /// Gets the element at the specified index. + /// The vector to get the element from. + /// The index of the element to get. + /// The value of the element at . + /// was less than zero or greater than the number of elements. + /// The type of the elements in the vector () is not supported. + static abstract T GetElement(TSelf vector, int index); + + /// Compares two vectors to determine which is greater on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are all-bits-set or zero, depending on if which of the corresponding elements in and were greater. + /// The type of the elements in the vector () is not supported. + static abstract TSelf GreaterThan(TSelf left, TSelf right); + + /// Compares two vectors to determine if all elements are greater. + /// The vector to compare with . + /// The vector to compare with . + /// true if all elements in were greater than the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool GreaterThanAll(TSelf left, TSelf right); + + /// Compares two vectors to determine if any elements are greater. + /// The vector to compare with . + /// The vector to compare with . + /// true if any elements in was greater than the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool GreaterThanAny(TSelf left, TSelf right); + + /// Compares two vectors to determine which is greater or equal on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are all-bits-set or zero, depending on if which of the corresponding elements in and were greater or equal. + /// The type of the elements in the vector () is not supported. + static abstract TSelf GreaterThanOrEqual(TSelf left, TSelf right); + + /// Compares two vectors to determine if all elements are greater or equal. + /// The vector to compare with . + /// The vector to compare with . + /// true if all elements in were greater than or equal to the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool GreaterThanOrEqualAll(TSelf left, TSelf right); + + /// Compares two vectors to determine if any elements are greater or equal. + /// The vector to compare with . + /// The vector to compare with . + /// true if any elements in was greater than or equal to the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool GreaterThanOrEqualAny(TSelf left, TSelf right); + + /// Compares two vectors to determine which is less on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are all-bits-set or zero, depending on if which of the corresponding elements in and were less. + /// The type of the elements in the vector () is not supported. + static abstract TSelf LessThan(TSelf left, TSelf right); + + /// Compares two vectors to determine if all elements are less. + /// The vector to compare with . + /// The vector to compare with . + /// true if all elements in were less than the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool LessThanAll(TSelf left, TSelf right); + + /// Compares two vectors to determine if any elements are less. + /// The vector to compare with . + /// The vector to compare with . + /// true if any elements in was less than the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool LessThanAny(TSelf left, TSelf right); + + /// Compares two vectors to determine which is less or equal on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are all-bits-set or zero, depending on if which of the corresponding elements in and were less or equal. + /// The type of the elements in the vector () is not supported. + static abstract TSelf LessThanOrEqual(TSelf left, TSelf right); + + /// Compares two vectors to determine if all elements are less or equal. + /// The vector to compare with . + /// The vector to compare with . + /// true if all elements in were less than or equal to the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool LessThanOrEqualAll(TSelf left, TSelf right); + + /// Compares two vectors to determine if any elements are less or equal. + /// The vector to compare with . + /// The vector to compare with . + /// true if any elements in was less than or equal to the corresponding element in . + /// The type of the elements in the vector () is not supported. + static abstract bool LessThanOrEqualAny(TSelf left, TSelf right); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// Loads a vector from the given source. + /// The source from which the vector will be loaded. + /// The vector loaded from . + /// The type of () is not supported. + static virtual TSelf Load(T* source) => TSelf.LoadUnsafe(ref *source); + + /// Loads a vector from the given aligned source. + /// The aligned source from which the vector will be loaded. + /// The vector loaded from . + /// The type of () is not supported. + static virtual TSelf LoadAligned(T* source) + { + if (((nuint)(source) % (uint)(TSelf.Alignment)) != 0) + { + ThrowHelper.ThrowAccessViolationException(); + } + return TSelf.LoadUnsafe(ref *source); + } + + /// Loads a vector from the given aligned source. + /// The aligned source from which the vector will be loaded. + /// The vector loaded from . + /// This method may bypass the cache on certain platforms. + /// The type of () is not supported. + static virtual TSelf LoadAlignedNonTemporal(T* source) => TSelf.LoadAligned(source); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// Loads a vector from the given source. + /// The source from which the vector will be loaded. + /// The vector loaded from . + /// The type of the elements in the vector () is not supported. + static virtual TSelf LoadUnsafe(ref readonly T source) => TSelf.LoadUnsafe(in source, elementOffset: 0); + + /// Loads a vector from the given source and element offset. + /// The source to which will be added before loading the vector. + /// The element offset from from which the vector will be loaded. + /// The vector loaded from plus . + /// The type of the elements in the vector () is not supported. + static abstract TSelf LoadUnsafe(ref readonly T source, nuint elementOffset); + + /// Computes the maximum of two vectors on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are the maximum of the corresponding elements in and . + /// The type of the elements in the vector () is not supported. + static abstract TSelf Max(TSelf left, TSelf right); + + /// Computes the minimum of two vectors on a per-element basis. + /// The vector to compare with . + /// The vector to compare with . + /// A vector whose elements are the minimum of the corresponding elements in and . + /// The type of the elements in the vector () is not supported. + static abstract TSelf Min(TSelf left, TSelf right); + + /// Multiplies two vectors to compute their element-wise product. + /// The vector to multiply with . + /// The vector to multiply with . + /// The element-wise product of and . + /// The type of and () is not supported. + static virtual TSelf Multiply(TSelf left, TSelf right) => left * right; + + /// Multiplies a vector by a scalar to compute their product. + /// The vector to multiply with . + /// The scalar to multiply with . + /// The product of and . + /// The type of and () is not supported. + static virtual TSelf Multiply(TSelf left, T right) => left * right; + + /// Negates a vector. + /// The vector to negate. + /// A vector whose elements are the negation of the corresponding elements in . + /// The type of () is not supported. + static virtual TSelf Negate(TSelf vector) => -vector; + + /// Computes the ones-complement of a vector. + /// The vector whose ones-complement is to be computed. + /// A vector whose elements are the ones-complement of the corresponding elements in . + /// The type of () is not supported. + static virtual TSelf OnesComplement(TSelf vector) => ~vector; + + /// Shifts each element of a vector left by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted left by . + static virtual TSelf ShiftLeft(TSelf vector, int shiftCount) => vector << shiftCount; + + /// Shifts (signed) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + static virtual TSelf ShiftRightArithmetic(TSelf vector, int shiftCount) => vector >> shiftCount; + + /// Shifts (unsigned) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + static virtual TSelf ShiftRightLogical(TSelf vector, int shiftCount) => vector >>> shiftCount; + + /// Computes the square root of a vector on a per-element basis. + /// The vector whose square root is to be computed. + /// A vector whose elements are the square root of the corresponding elements in . + /// The type of () is not supported. + static abstract TSelf Sqrt(TSelf vector); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// Stores a vector at the given destination. + /// The vector that will be stored. + /// The destination at which will be stored. + /// The type of () is not supported. + static virtual void Store(TSelf source, T* destination) => TSelf.StoreUnsafe(source, ref *destination); + + /// Stores a vector at the given aligned destination. + /// The vector that will be stored. + /// The aligned destination at which will be stored. + /// The type of () is not supported. + static virtual void StoreAligned(TSelf source, T* destination) + { + if (((nuint)(destination) % (uint)(TSelf.Alignment)) != 0) + { + ThrowHelper.ThrowAccessViolationException(); + } + TSelf.StoreUnsafe(source, ref *destination); + } + + /// Stores a vector at the given aligned destination. + /// The vector that will be stored. + /// The aligned destination at which will be stored. + /// This method may bypass the cache on certain platforms. + /// The type of () is not supported. + static virtual void StoreAlignedNonTemporal(TSelf source, T* destination) => TSelf.StoreAligned(source, destination); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// Stores a vector at the given destination. + /// The vector that will be stored. + /// The destination at which will be stored. + /// The type of the elements in the vector () is not supported. + static virtual void StoreUnsafe(TSelf vector, ref T destination) => TSelf.StoreUnsafe(vector, ref destination, elementOffset: 0); + + /// Stores a vector at the given destination. + /// The vector that will be stored. + /// The destination to which will be added before the vector will be stored. + /// The element offset from from which the vector will be stored. + /// The type of the elements in the vector () is not supported. + static abstract void StoreUnsafe(TSelf vector, ref T destination, nuint elementOffset); + + /// Subtracts two vectors to compute their difference. + /// The vector from which will be subtracted. + /// The vector to subtract from . + /// The difference of and . + /// The type of and () is not supported. + static virtual TSelf Subtract(TSelf left, TSelf right) => left - right; + + /// Computes the sum of all elements in a vector. + /// The vector whose elements will be summed. + /// The sum of all elements in . + /// The type of () is not supported. + static abstract T Sum(TSelf vector); + + /// Converts the given vector to a scalar containing the value of the first element. + /// The vector to get the first element from. + /// A scalar containing the value of the first element. + /// The type of the elements in the vector () is not supported. + static virtual T ToScalar(TSelf vector) => TSelf.GetElement(vector, 0); + + /// Tries to copy a to a given span. + /// The vector to copy. + /// The span to which is copied. + /// true if was successfully copied to ; otherwise, false if the length of is less than . + /// The type of the elements in the vector () is not supported. + static virtual bool TryCopyTo(TSelf vector, Span destination) + { + if (destination.Length < TSelf.Count) + { + return false; + } + + TSelf.StoreUnsafe(vector, ref MemoryMarshal.GetReference(destination)); + return true; + } + + /// Creates a new vector with the element at the specified index set to the specified value and the remaining elements set to the same value as that in the given vector. + /// The vector to get the remaining elements from. + /// The index of the element to set. + /// The value to set the element to. + /// A vector with the value of the element at set to and the remaining elements set to the same value as that in . + /// was less than zero or greater than the number of elements. + /// The type of the elements in the vector () is not supported. + static abstract TSelf WithElement(TSelf vector, int index, T value); + + /// Computes the exclusive-or of two vectors. + /// The vector to exclusive-or with . + /// The vector to exclusive-or with . + /// The exclusive-or of and . + /// The type of and () is not supported. + static virtual TSelf Xor(TSelf left, TSelf right) => left ^ right; + + // + // New Surface Area + // + + static abstract int IndexOfLastMatch(TSelf vector); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/SimdVectorExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/SimdVectorExtensions.cs new file mode 100644 index 0000000000000..f483b9ccb6d36 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/SimdVectorExtensions.cs @@ -0,0 +1,171 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Runtime.Intrinsics +{ + internal static unsafe class SimdVectorExtensions + { + // TODO: As + + /// Copies a vector to a given array. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to be copied. + /// The array to which is copied. + /// The length of is less than . + /// is null. + /// The type of the elements in the vector () is not supported. + public static void CopyTo(this TVector vector, T[] destination) + where TVector : ISimdVector + { + TVector.CopyTo(vector, destination); + } + + /// Copies a vector to a given array starting at the specified index. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to be copied. + /// The array to which is copied. + /// The starting index of which will be copied to. + /// The length of is less than . + /// is negative or greater than the length of . + /// is null. + /// The type of the elements in the vector () is not supported. + public static void CopyTo(this TVector vector, T[] destination, int startIndex) + where TVector : ISimdVector + { + TVector.CopyTo(vector, destination, startIndex); + } + + /// Copies a vector to a given span. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to be copied. + /// The span to which the is copied. + /// The length of is less than . + /// The type of the elements in the vector () is not supported. + public static void CopyTo(this TVector vector, Span destination) + where TVector : ISimdVector + { + TVector.CopyTo(vector, destination); + } + + /// Gets the element at the specified index. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to get the element from. + /// The index of the element to get. + /// The value of the element at . + /// was less than zero or greater than the number of elements. + /// The type of () is not supported. + public static T GetElement(this TVector vector, int index) + where TVector : ISimdVector + { + return TVector.GetElement(vector, index); + } + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// Stores a vector at the given destination. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector that will be stored. + /// The destination at which will be stored. + /// The type of () is not supported. + public static void Store(this TVector source, T* destination) + where TVector : ISimdVector + { + TVector.Store(source, destination); + } + + /// Stores a vector at the given aligned destination. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector that will be stored. + /// The aligned destination at which will be stored. + /// The type of () is not supported. + public static void StoreAligned(this TVector source, T* destination) + where TVector : ISimdVector + { + TVector.StoreAligned(source, destination); + } + + /// Stores a vector at the given aligned destination. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector that will be stored. + /// The aligned destination at which will be stored. + /// This method may bypass the cache on certain platforms. + /// The type of () is not supported. + public static void StoreAlignedNonTemporal(this TVector source, T* destination) + where TVector : ISimdVector + { + TVector.StoreAlignedNonTemporal(source, destination); + } +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// Stores a vector at the given destination. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector that will be stored. + /// The destination at which the vector will be stored. + /// The type of the elements in the vector () is not supported. + public static void StoreUnsafe(this TVector vector, ref T destination) + where TVector : ISimdVector + { + TVector.StoreUnsafe(vector, ref destination); + } + + /// Stores a vector at the given destination. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector that will be stored. + /// The destination to which will be added before the vector will be stored. + /// The element offset from from which the vector will be stored. + /// The type of the elements in the vector () is not supported. + public static void StoreUnsafe(this TVector vector, ref T destination, nuint elementOffset) + where TVector : ISimdVector + { + TVector.StoreUnsafe(vector, ref destination, elementOffset); + } + + /// Converts the given vector to a scalar containing the value of the first element. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to get the first element from. + /// A scalar containing the value of the first element. + /// The type of the elements in the vector () is not supported. + public static T ToScalar(this TVector vector) + where TVector : ISimdVector + { + return TVector.ToScalar(vector); + } + + /// Tries to copy a vector to a given span. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to copy. + /// The span to which is copied. + /// true if was successfully copied to ; otherwise, false if the length of is less than . + /// The type of the elements in the vector () is not supported. + public static bool TryCopyTo(this TVector vector, Span destination) + where TVector : ISimdVector + { + return TVector.TryCopyTo(vector, destination); + } + + /// Creates a new vector with the element at the specified index set to the specified value and the remaining elements set to the same value as that in the given vector. + /// The type of the vector. + /// The type of the elements in the vector. + /// The vector to get the remaining elements from. + /// The index of the element to set. + /// The value to set the element to. + /// A vector with the value of the element at set to and the remaining elements set to the same value as that in . + /// was less than zero or greater than the number of elements. + /// The type of the elements in the vector () is not supported. + public static TVector WithElement(this TVector vector, int index, T value) + where TVector : ISimdVector + { + return TVector.WithElement(vector, index, value); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index 90ef3cf84b008..3f57e5c50f444 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -344,10 +344,9 @@ public static Vector AsVector(this Vector128 value) /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Ceiling(Vector128 vector) + internal static Vector128 Ceiling(Vector128 vector) { return Create( Vector64.Ceiling(vector._lower), @@ -355,19 +354,21 @@ public static Vector128 Ceiling(Vector128 vector) ); } + /// Computes the ceiling of each element in a vector. + /// The vector that will have its ceiling computed. + /// A vector whose elements are the ceiling of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Ceiling(Vector128 vector) => Ceiling(vector); + /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Ceiling(Vector128 vector) - { - return Create( - Vector64.Ceiling(vector._lower), - Vector64.Ceiling(vector._upper) - ); - } + public static Vector128 Ceiling(Vector128 vector) => Ceiling(vector); /// Conditionally selects a value from two vectors on a bitwise basis. /// The type of the elements in the vector. @@ -1443,10 +1444,9 @@ public static uint ExtractMostSignificantBits(this Vector128 vector) /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Floor(Vector128 vector) + internal static Vector128 Floor(Vector128 vector) { return Create( Vector64.Floor(vector._lower), @@ -1454,19 +1454,21 @@ public static Vector128 Floor(Vector128 vector) ); } + /// Computes the floor of each element in a vector. + /// The vector that will have its floor computed. + /// A vector whose elements are the floor of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Floor(Vector128 vector) => Floor(vector); + /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Floor(Vector128 vector) - { - return Create( - Vector64.Floor(vector._lower), - Vector64.Floor(vector._upper) - ); - } + public static Vector128 Floor(Vector128 vector) => Floor(vector); /// Gets the element at the specified index. /// The type of the elements in the vector. @@ -1967,13 +1969,7 @@ public static unsafe Vector128 Narrow(Vector128 lower, Vector128
    A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + internal static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -1981,13 +1977,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -1995,13 +1985,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2009,13 +1993,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2023,13 +2001,15 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; + + /// Shifts each element of a vector left by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted left by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2038,13 +2018,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2053,13 +2027,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2068,13 +2036,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2083,13 +2045,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCou [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2098,13 +2054,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftLeft(vector._lower, shiftCount), - Vector64.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftLeft(Vector128 vector, int shiftCount) => vector << shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2112,13 +2062,7 @@ public static Vector128 ShiftLeft(Vector128 vector, int shiftCount /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightArithmetic(vector._lower, shiftCount), - Vector64.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + internal static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2126,13 +2070,7 @@ public static Vector128 ShiftRightArithmetic(Vector128 vector, int /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightArithmetic(vector._lower, shiftCount), - Vector64.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2140,13 +2078,7 @@ public static Vector128 ShiftRightArithmetic(Vector128 vector, int shi /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightArithmetic(vector._lower, shiftCount), - Vector64.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2154,13 +2086,15 @@ public static Vector128 ShiftRightArithmetic(Vector128 vector, int s /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightArithmetic(vector._lower, shiftCount), - Vector64.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; + + /// Shifts (signed) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2169,13 +2103,7 @@ public static Vector128 ShiftRightArithmetic(Vector128 vector, int s [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightArithmetic(vector._lower, shiftCount), - Vector64.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightArithmetic(Vector128 vector, int shiftCount) => vector >> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2183,13 +2111,7 @@ public static Vector128 ShiftRightArithmetic(Vector128 vector, int /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + internal static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2197,13 +2119,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int shif /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2211,13 +2127,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int sh /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2225,13 +2135,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int shiftC /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2239,13 +2143,15 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int shif /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; + + /// Shifts (unsigned) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2254,13 +2160,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int shif [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2269,13 +2169,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int sh [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2284,13 +2178,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int sh [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2299,13 +2187,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2314,13 +2196,7 @@ public static Vector128 ShiftRightLogical(Vector128 vector, int shif [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) - { - return Create( - Vector64.ShiftRightLogical(vector._lower, shiftCount), - Vector64.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector128 ShiftRightLogical(Vector128 vector, int shiftCount) => vector >>> shiftCount; /// Creates a new vector by selecting values from an input vector using a set of indices. /// The input vector from which values are selected. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs index 7248a4795bff0..ced4fc6e09401 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs @@ -29,7 +29,7 @@ namespace System.Runtime.Intrinsics [DebuggerDisplay("{DisplayString,nq}")] [DebuggerTypeProxy(typeof(Vector128DebugView<>))] [StructLayout(LayoutKind.Sequential, Size = Vector128.Size)] - public readonly struct Vector128 : IEquatable> + public readonly unsafe struct Vector128 : ISimdVector, T> { internal readonly Vector64 _lower; internal readonly Vector64 _upper; @@ -467,5 +467,215 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri return sb.ToString(); } + + // + // ISimdVector + // + + /// + static int ISimdVector, T>.Alignment => Vector128.Alignment; + + /// + static bool ISimdVector, T>.IsHardwareAccelerated => Vector128.IsHardwareAccelerated; + + /// + static Vector128 ISimdVector, T>.Abs(Vector128 vector) => Vector128.Abs(vector); + + /// + static Vector128 ISimdVector, T>.Add(Vector128 left, Vector128 right) => Vector128.Add(left, right); + + /// + static Vector128 ISimdVector, T>.AndNot(Vector128 left, Vector128 right) => Vector128.AndNot(left, right); + + /// + static Vector128 ISimdVector, T>.BitwiseAnd(Vector128 left, Vector128 right) => Vector128.BitwiseAnd(left, right); + + /// + static Vector128 ISimdVector, T>.BitwiseOr(Vector128 left, Vector128 right) => Vector128.BitwiseOr(left, right); + + /// + static Vector128 ISimdVector, T>.Ceiling(Vector128 vector) => Vector128.Ceiling(vector); + + /// + static Vector128 ISimdVector, T>.ConditionalSelect(Vector128 condition, Vector128 left, Vector128 right) => Vector128.ConditionalSelect(condition, left, right); + + /// + static void ISimdVector, T>.CopyTo(Vector128 vector, T[] destination) => vector.CopyTo(destination); + + /// + static void ISimdVector, T>.CopyTo(Vector128 vector, T[] destination, int startIndex) => vector.CopyTo(destination, startIndex); + + /// + static void ISimdVector, T>.CopyTo(Vector128 vector, Span destination) => vector.CopyTo(destination); + + /// + static Vector128 ISimdVector, T>.Create(T value) => Vector128.Create(value); + + /// + static Vector128 ISimdVector, T>.Create(T[] values) => Vector128.Create(values); + + /// + static Vector128 ISimdVector, T>.Create(T[] values, int index) => Vector128.Create(values, index); + + /// + static Vector128 ISimdVector, T>.Create(ReadOnlySpan values) => Vector128.Create(values); + + /// + static Vector128 ISimdVector, T>.CreateScalar(T value) => Vector128.CreateScalar(value); + + /// + static Vector128 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector128.CreateScalarUnsafe(value); + + /// + static Vector128 ISimdVector, T>.Divide(Vector128 left, Vector128 right) => Vector128.Divide(left, right); + + /// + static Vector128 ISimdVector, T>.Divide(Vector128 left, T right) => Vector128.Divide(left, right); + + /// + static T ISimdVector, T>.Dot(Vector128 left, Vector128 right) => Vector128.Dot(left, right); + + /// + static Vector128 ISimdVector, T>.Equals(Vector128 left, Vector128 right) => Vector128.Equals(left, right); + + /// + static bool ISimdVector, T>.EqualsAll(Vector128 left, Vector128 right) => Vector128.EqualsAll(left, right); + + /// + static bool ISimdVector, T>.EqualsAny(Vector128 left, Vector128 right) => Vector128.EqualsAny(left, right); + + /// + static Vector128 ISimdVector, T>.Floor(Vector128 vector) => Vector128.Floor(vector); + + /// + static T ISimdVector, T>.GetElement(Vector128 vector, int index) => Vector128.GetElement(vector, index); + + /// + static Vector128 ISimdVector, T>.GreaterThan(Vector128 left, Vector128 right) => Vector128.GreaterThan(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAll(Vector128 left, Vector128 right) => Vector128.GreaterThanAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAny(Vector128 left, Vector128 right) => Vector128.GreaterThanAny(left, right); + + /// + static Vector128 ISimdVector, T>.GreaterThanOrEqual(Vector128 left, Vector128 right) => Vector128.GreaterThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector128 left, Vector128 right) => Vector128.GreaterThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector128 left, Vector128 right) => Vector128.GreaterThanOrEqualAny(left, right); + + /// + static Vector128 ISimdVector, T>.LessThan(Vector128 left, Vector128 right) => Vector128.LessThan(left, right); + + /// + static bool ISimdVector, T>.LessThanAll(Vector128 left, Vector128 right) => Vector128.LessThanAll(left, right); + + /// + static bool ISimdVector, T>.LessThanAny(Vector128 left, Vector128 right) => Vector128.LessThanAny(left, right); + + /// + static Vector128 ISimdVector, T>.LessThanOrEqual(Vector128 left, Vector128 right) => Vector128.LessThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAll(Vector128 left, Vector128 right) => Vector128.LessThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAny(Vector128 left, Vector128 right) => Vector128.LessThanOrEqualAny(left, right); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static Vector128 ISimdVector, T>.Load(T* source) => Vector128.Load(source); + + /// + static Vector128 ISimdVector, T>.LoadAligned(T* source) => Vector128.LoadAligned(source); + + /// + static Vector128 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector128.LoadAlignedNonTemporal(source); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static Vector128 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector128.LoadUnsafe(in source); + + /// + static Vector128 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector128.LoadUnsafe(in source, elementOffset); + + /// + static Vector128 ISimdVector, T>.Max(Vector128 left, Vector128 right) => Vector128.Max(left, right); + + /// + static Vector128 ISimdVector, T>.Min(Vector128 left, Vector128 right) => Vector128.Min(left, right); + + /// + static Vector128 ISimdVector, T>.Multiply(Vector128 left, Vector128 right) => Vector128.Multiply(left, right); + + /// + static Vector128 ISimdVector, T>.Multiply(Vector128 left, T right) => Vector128.Multiply(left, right); + + /// + static Vector128 ISimdVector, T>.Negate(Vector128 vector) => Vector128.Negate(vector); + + /// + static Vector128 ISimdVector, T>.OnesComplement(Vector128 vector) => Vector128.OnesComplement(vector); + + /// + static Vector128 ISimdVector, T>.ShiftLeft(Vector128 vector, int shiftCount) => Vector128.ShiftLeft(vector, shiftCount); + + /// + static Vector128 ISimdVector, T>.ShiftRightArithmetic(Vector128 vector, int shiftCount) => Vector128.ShiftRightArithmetic(vector, shiftCount); + + /// + static Vector128 ISimdVector, T>.ShiftRightLogical(Vector128 vector, int shiftCount) => Vector128.ShiftRightLogical(vector, shiftCount); + + /// + static Vector128 ISimdVector, T>.Sqrt(Vector128 vector) => Vector128.Sqrt(vector); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static void ISimdVector, T>.Store(Vector128 source, T* destination) => Vector128.Store(source, destination); + + /// + static void ISimdVector, T>.StoreAligned(Vector128 source, T* destination) => Vector128.StoreAligned(source, destination); + + /// + static void ISimdVector, T>.StoreAlignedNonTemporal(Vector128 source, T* destination) => Vector128.StoreAlignedNonTemporal(source, destination); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static void ISimdVector, T>.StoreUnsafe(Vector128 vector, ref T destination) => Vector128.StoreUnsafe(vector, ref destination); + + /// + static void ISimdVector, T>.StoreUnsafe(Vector128 vector, ref T destination, nuint elementOffset) => Vector128.StoreUnsafe(vector, ref destination, elementOffset); + + /// + static Vector128 ISimdVector, T>.Subtract(Vector128 left, Vector128 right) => Vector128.Subtract(left, right); + + /// + static T ISimdVector, T>.Sum(Vector128 vector) => Vector128.Sum(vector); + + /// + static T ISimdVector, T>.ToScalar(Vector128 vector) => Vector128.ToScalar(vector); + + /// + static bool ISimdVector, T>.TryCopyTo(Vector128 vector, Span destination) => Vector128.TryCopyTo(vector, destination); + + /// + static Vector128 ISimdVector, T>.WithElement(Vector128 vector, int index, T value) => Vector128.WithElement(vector, index, value); + + /// + static Vector128 ISimdVector, T>.Xor(Vector128 left, Vector128 right) => Vector128.Xor(left, right); + + // + // New Surface Area + // + + static int ISimdVector, T>.IndexOfLastMatch(Vector128 vector) + { + uint mask = vector.ExtractMostSignificantBits(); + return 31 - BitOperations.LeadingZeroCount(mask); // 31 = 32 (bits in Int32) - 1 (indexing from zero) + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs index f8f62bdd7fea3..1c92fb6d54dc5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs @@ -275,10 +275,9 @@ public static Vector AsVector(this Vector256 value) /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Ceiling(Vector256 vector) + internal static Vector256 Ceiling(Vector256 vector) { return Create( Vector128.Ceiling(vector._lower), @@ -286,19 +285,21 @@ public static Vector256 Ceiling(Vector256 vector) ); } + /// Computes the ceiling of each element in a vector. + /// The vector that will have its ceiling computed. + /// A vector whose elements are the ceiling of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Ceiling(Vector256 vector) => Ceiling(vector); + /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Ceiling(Vector256 vector) - { - return Create( - Vector128.Ceiling(vector._lower), - Vector128.Ceiling(vector._upper) - ); - } + public static Vector256 Ceiling(Vector256 vector) => Ceiling(vector); /// Conditionally selects a value from two vectors on a bitwise basis. /// The type of the elements in the vector. @@ -1419,10 +1420,9 @@ public static uint ExtractMostSignificantBits(this Vector256 vector) /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Floor(Vector256 vector) + internal static Vector256 Floor(Vector256 vector) { return Create( Vector128.Floor(vector._lower), @@ -1430,19 +1430,21 @@ public static Vector256 Floor(Vector256 vector) ); } + /// Computes the floor of each element in a vector. + /// The vector that will have its floor computed. + /// A vector whose elements are the floor of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Floor(Vector256 vector) => Floor(vector); + /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Floor(Vector256 vector) - { - return Create( - Vector128.Floor(vector._lower), - Vector128.Floor(vector._upper) - ); - } + public static Vector256 Floor(Vector256 vector) => Floor(vector); /// Gets the element at the specified index. /// The type of the input vector. @@ -1947,13 +1949,7 @@ public static Vector256 OnesComplement(Vector256 vector) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + internal static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -1961,13 +1957,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -1975,13 +1965,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -1989,13 +1973,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2003,13 +1981,15 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; + + /// Shifts each element of a vector left by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted left by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2018,13 +1998,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2033,13 +2007,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2048,13 +2016,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2063,13 +2025,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCou [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2078,13 +2034,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftLeft(vector._lower, shiftCount), - Vector128.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftLeft(Vector256 vector, int shiftCount) => vector << shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2092,13 +2042,7 @@ public static Vector256 ShiftLeft(Vector256 vector, int shiftCount /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightArithmetic(vector._lower, shiftCount), - Vector128.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + internal static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2106,13 +2050,7 @@ public static Vector256 ShiftRightArithmetic(Vector256 vector, int /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightArithmetic(vector._lower, shiftCount), - Vector128.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2120,13 +2058,7 @@ public static Vector256 ShiftRightArithmetic(Vector256 vector, int shi /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightArithmetic(vector._lower, shiftCount), - Vector128.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2134,13 +2066,15 @@ public static Vector256 ShiftRightArithmetic(Vector256 vector, int s /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightArithmetic(vector._lower, shiftCount), - Vector128.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; + + /// Shifts (signed) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2149,13 +2083,7 @@ public static Vector256 ShiftRightArithmetic(Vector256 vector, int s [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightArithmetic(vector._lower, shiftCount), - Vector128.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightArithmetic(Vector256 vector, int shiftCount) => vector >> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2163,13 +2091,7 @@ public static Vector256 ShiftRightArithmetic(Vector256 vector, int /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + internal static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2177,13 +2099,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int shif /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2191,13 +2107,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int sh /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2205,13 +2115,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int shiftC /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2219,13 +2123,15 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int shif /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; + + /// Shifts (unsigned) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2234,13 +2140,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int shif [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2249,13 +2149,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int sh [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2264,13 +2158,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int sh [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2279,13 +2167,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2294,13 +2176,7 @@ public static Vector256 ShiftRightLogical(Vector256 vector, int shif [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) - { - return Create( - Vector128.ShiftRightLogical(vector._lower, shiftCount), - Vector128.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector256 ShiftRightLogical(Vector256 vector, int shiftCount) => vector >>> shiftCount; /// Creates a new vector by selecting values from an input vector using a set of indices. /// The input vector from which values are selected. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs index 3be1048d9f145..c8c317ada0857 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -27,7 +28,7 @@ namespace System.Runtime.Intrinsics [DebuggerDisplay("{DisplayString,nq}")] [DebuggerTypeProxy(typeof(Vector256DebugView<>))] [StructLayout(LayoutKind.Sequential, Size = Vector256.Size)] - public readonly struct Vector256 : IEquatable> + public readonly unsafe struct Vector256 : ISimdVector, T> { internal readonly Vector128 _lower; internal readonly Vector128 _upper; @@ -456,5 +457,215 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri return sb.ToString(); } + + // + // ISimdVector + // + + /// + static int ISimdVector, T>.Alignment => Vector256.Alignment; + + /// + static bool ISimdVector, T>.IsHardwareAccelerated => Vector256.IsHardwareAccelerated; + + /// + static Vector256 ISimdVector, T>.Abs(Vector256 vector) => Vector256.Abs(vector); + + /// + static Vector256 ISimdVector, T>.Add(Vector256 left, Vector256 right) => Vector256.Add(left, right); + + /// + static Vector256 ISimdVector, T>.AndNot(Vector256 left, Vector256 right) => Vector256.AndNot(left, right); + + /// + static Vector256 ISimdVector, T>.BitwiseAnd(Vector256 left, Vector256 right) => Vector256.BitwiseAnd(left, right); + + /// + static Vector256 ISimdVector, T>.BitwiseOr(Vector256 left, Vector256 right) => Vector256.BitwiseOr(left, right); + + /// + static Vector256 ISimdVector, T>.Ceiling(Vector256 vector) => Vector256.Ceiling(vector); + + /// + static Vector256 ISimdVector, T>.ConditionalSelect(Vector256 condition, Vector256 left, Vector256 right) => Vector256.ConditionalSelect(condition, left, right); + + /// + static void ISimdVector, T>.CopyTo(Vector256 vector, T[] destination) => vector.CopyTo(destination); + + /// + static void ISimdVector, T>.CopyTo(Vector256 vector, T[] destination, int startIndex) => vector.CopyTo(destination, startIndex); + + /// + static void ISimdVector, T>.CopyTo(Vector256 vector, Span destination) => vector.CopyTo(destination); + + /// + static Vector256 ISimdVector, T>.Create(T value) => Vector256.Create(value); + + /// + static Vector256 ISimdVector, T>.Create(T[] values) => Vector256.Create(values); + + /// + static Vector256 ISimdVector, T>.Create(T[] values, int index) => Vector256.Create(values, index); + + /// + static Vector256 ISimdVector, T>.Create(ReadOnlySpan values) => Vector256.Create(values); + + /// + static Vector256 ISimdVector, T>.CreateScalar(T value) => Vector256.CreateScalar(value); + + /// + static Vector256 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector256.CreateScalarUnsafe(value); + + /// + static Vector256 ISimdVector, T>.Divide(Vector256 left, Vector256 right) => Vector256.Divide(left, right); + + /// + static Vector256 ISimdVector, T>.Divide(Vector256 left, T right) => Vector256.Divide(left, right); + + /// + static T ISimdVector, T>.Dot(Vector256 left, Vector256 right) => Vector256.Dot(left, right); + + /// + static Vector256 ISimdVector, T>.Equals(Vector256 left, Vector256 right) => Vector256.Equals(left, right); + + /// + static bool ISimdVector, T>.EqualsAll(Vector256 left, Vector256 right) => Vector256.EqualsAll(left, right); + + /// + static bool ISimdVector, T>.EqualsAny(Vector256 left, Vector256 right) => Vector256.EqualsAny(left, right); + + /// + static Vector256 ISimdVector, T>.Floor(Vector256 vector) => Vector256.Floor(vector); + + /// + static T ISimdVector, T>.GetElement(Vector256 vector, int index) => Vector256.GetElement(vector, index); + + /// + static Vector256 ISimdVector, T>.GreaterThan(Vector256 left, Vector256 right) => Vector256.GreaterThan(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAll(Vector256 left, Vector256 right) => Vector256.GreaterThanAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAny(Vector256 left, Vector256 right) => Vector256.GreaterThanAny(left, right); + + /// + static Vector256 ISimdVector, T>.GreaterThanOrEqual(Vector256 left, Vector256 right) => Vector256.GreaterThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector256 left, Vector256 right) => Vector256.GreaterThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector256 left, Vector256 right) => Vector256.GreaterThanOrEqualAny(left, right); + + /// + static Vector256 ISimdVector, T>.LessThan(Vector256 left, Vector256 right) => Vector256.LessThan(left, right); + + /// + static bool ISimdVector, T>.LessThanAll(Vector256 left, Vector256 right) => Vector256.LessThanAll(left, right); + + /// + static bool ISimdVector, T>.LessThanAny(Vector256 left, Vector256 right) => Vector256.LessThanAny(left, right); + + /// + static Vector256 ISimdVector, T>.LessThanOrEqual(Vector256 left, Vector256 right) => Vector256.LessThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAll(Vector256 left, Vector256 right) => Vector256.LessThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAny(Vector256 left, Vector256 right) => Vector256.LessThanOrEqualAny(left, right); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static Vector256 ISimdVector, T>.Load(T* source) => Vector256.Load(source); + + /// + static Vector256 ISimdVector, T>.LoadAligned(T* source) => Vector256.LoadAligned(source); + + /// + static Vector256 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector256.LoadAlignedNonTemporal(source); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static Vector256 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector256.LoadUnsafe(in source); + + /// + static Vector256 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector256.LoadUnsafe(in source, elementOffset); + + /// + static Vector256 ISimdVector, T>.Max(Vector256 left, Vector256 right) => Vector256.Max(left, right); + + /// + static Vector256 ISimdVector, T>.Min(Vector256 left, Vector256 right) => Vector256.Min(left, right); + + /// + static Vector256 ISimdVector, T>.Multiply(Vector256 left, Vector256 right) => Vector256.Multiply(left, right); + + /// + static Vector256 ISimdVector, T>.Multiply(Vector256 left, T right) => Vector256.Multiply(left, right); + + /// + static Vector256 ISimdVector, T>.Negate(Vector256 vector) => Vector256.Negate(vector); + + /// + static Vector256 ISimdVector, T>.OnesComplement(Vector256 vector) => Vector256.OnesComplement(vector); + + /// + static Vector256 ISimdVector, T>.ShiftLeft(Vector256 vector, int shiftCount) => Vector256.ShiftLeft(vector, shiftCount); + + /// + static Vector256 ISimdVector, T>.ShiftRightArithmetic(Vector256 vector, int shiftCount) => Vector256.ShiftRightArithmetic(vector, shiftCount); + + /// + static Vector256 ISimdVector, T>.ShiftRightLogical(Vector256 vector, int shiftCount) => Vector256.ShiftRightLogical(vector, shiftCount); + + /// + static Vector256 ISimdVector, T>.Sqrt(Vector256 vector) => Vector256.Sqrt(vector); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static void ISimdVector, T>.Store(Vector256 source, T* destination) => Vector256.Store(source, destination); + + /// + static void ISimdVector, T>.StoreAligned(Vector256 source, T* destination) => Vector256.StoreAligned(source, destination); + + /// + static void ISimdVector, T>.StoreAlignedNonTemporal(Vector256 source, T* destination) => Vector256.StoreAlignedNonTemporal(source, destination); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static void ISimdVector, T>.StoreUnsafe(Vector256 vector, ref T destination) => Vector256.StoreUnsafe(vector, ref destination); + + /// + static void ISimdVector, T>.StoreUnsafe(Vector256 vector, ref T destination, nuint elementOffset) => Vector256.StoreUnsafe(vector, ref destination, elementOffset); + + /// + static Vector256 ISimdVector, T>.Subtract(Vector256 left, Vector256 right) => Vector256.Subtract(left, right); + + /// + static T ISimdVector, T>.Sum(Vector256 vector) => Vector256.Sum(vector); + + /// + static T ISimdVector, T>.ToScalar(Vector256 vector) => Vector256.ToScalar(vector); + + /// + static bool ISimdVector, T>.TryCopyTo(Vector256 vector, Span destination) => Vector256.TryCopyTo(vector, destination); + + /// + static Vector256 ISimdVector, T>.WithElement(Vector256 vector, int index, T value) => Vector256.WithElement(vector, index, value); + + /// + static Vector256 ISimdVector, T>.Xor(Vector256 left, Vector256 right) => Vector256.Xor(left, right); + + // + // New Surface Area + // + + static int ISimdVector, T>.IndexOfLastMatch(Vector256 vector) + { + uint mask = vector.ExtractMostSignificantBits(); + return 31 - BitOperations.LeadingZeroCount(mask); // 31 = 32 (bits in Int32) - 1 (indexing from zero) + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs index eca02cdc981c2..99124a4522172 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs @@ -275,10 +275,9 @@ public static Vector AsVector(this Vector512 value) /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Ceiling(Vector512 vector) + internal static Vector512 Ceiling(Vector512 vector) { return Create( Vector256.Ceiling(vector._lower), @@ -286,19 +285,21 @@ public static Vector512 Ceiling(Vector512 vector) ); } + /// Computes the ceiling of each element in a vector. + /// The vector that will have its ceiling computed. + /// A vector whose elements are the ceiling of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Ceiling(Vector512 vector) => Ceiling(vector); + /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Ceiling(Vector512 vector) - { - return Create( - Vector256.Ceiling(vector._lower), - Vector256.Ceiling(vector._upper) - ); - } + public static Vector512 Ceiling(Vector512 vector) => Ceiling(vector); /// Conditionally selects a value from two vectors on a bitwise basis. /// The type of the elements in the vector. @@ -1472,10 +1473,9 @@ public static ulong ExtractMostSignificantBits(this Vector512 vector) /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Floor(Vector512 vector) + internal static Vector512 Floor(Vector512 vector) { return Create( Vector256.Floor(vector._lower), @@ -1483,19 +1483,21 @@ public static Vector512 Floor(Vector512 vector) ); } + /// Computes the floor of each element in a vector. + /// The vector that will have its floor computed. + /// A vector whose elements are the floor of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Floor(Vector512 vector) => Floor(vector); + /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Floor(Vector512 vector) - { - return Create( - Vector256.Floor(vector._lower), - Vector256.Floor(vector._upper) - ); - } + public static Vector512 Floor(Vector512 vector) => Floor(vector); /// Gets the element at the specified index. /// The type of the input vector. @@ -2000,13 +2002,7 @@ public static Vector512 OnesComplement(Vector512 vector) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + internal static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2014,13 +2010,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2028,13 +2018,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2042,13 +2026,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2056,13 +2034,15 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) /// A vector whose elements where shifted left by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; + + /// Shifts each element of a vector left by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted left by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2071,13 +2051,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2086,13 +2060,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2101,13 +2069,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2116,13 +2078,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCou [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. @@ -2131,13 +2087,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftLeft(vector._lower, shiftCount), - Vector256.ShiftLeft(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftLeft(Vector512 vector, int shiftCount) => vector << shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2145,13 +2095,7 @@ public static Vector512 ShiftLeft(Vector512 vector, int shiftCount /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightArithmetic(vector._lower, shiftCount), - Vector256.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + internal static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2159,13 +2103,7 @@ public static Vector512 ShiftRightArithmetic(Vector512 vector, int /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightArithmetic(vector._lower, shiftCount), - Vector256.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2173,13 +2111,7 @@ public static Vector512 ShiftRightArithmetic(Vector512 vector, int shi /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightArithmetic(vector._lower, shiftCount), - Vector256.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2187,13 +2119,15 @@ public static Vector512 ShiftRightArithmetic(Vector512 vector, int s /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightArithmetic(vector._lower, shiftCount), - Vector256.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; + + /// Shifts (signed) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2202,13 +2136,7 @@ public static Vector512 ShiftRightArithmetic(Vector512 vector, int s [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightArithmetic(vector._lower, shiftCount), - Vector256.ShiftRightArithmetic(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightArithmetic(Vector512 vector, int shiftCount) => vector >> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2216,13 +2144,7 @@ public static Vector512 ShiftRightArithmetic(Vector512 vector, int /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + internal static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2230,13 +2152,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int shif /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2244,13 +2160,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int sh /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2258,13 +2168,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int shiftC /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2272,13 +2176,15 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int shif /// A vector whose elements where shifted right by . [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; + + /// Shifts (unsigned) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2287,13 +2193,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int shif [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2302,13 +2202,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int sh [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2317,13 +2211,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int sh [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2332,13 +2220,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. @@ -2347,13 +2229,7 @@ public static Vector512 ShiftRightLogical(Vector512 vector, int shif [Intrinsic] [CLSCompliant(false)] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) - { - return Create( - Vector256.ShiftRightLogical(vector._lower, shiftCount), - Vector256.ShiftRightLogical(vector._upper, shiftCount) - ); - } + public static Vector512 ShiftRightLogical(Vector512 vector, int shiftCount) => vector >>> shiftCount; /// Creates a new vector by selecting values from an input vector using a set of indices. /// The input vector from which values are selected. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs index 37927cac3f355..c5306d1b0b93d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -27,7 +28,7 @@ namespace System.Runtime.Intrinsics [DebuggerDisplay("{DisplayString,nq}")] [DebuggerTypeProxy(typeof(Vector512DebugView<>))] [StructLayout(LayoutKind.Sequential, Size = Vector512.Size)] - public readonly struct Vector512 : IEquatable> + public readonly unsafe struct Vector512 : ISimdVector, T> { internal readonly Vector256 _lower; internal readonly Vector256 _upper; @@ -456,5 +457,215 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri return sb.ToString(); } + + // + // ISimdVector + // + + /// + static int ISimdVector, T>.Alignment => Vector512.Alignment; + + /// + static bool ISimdVector, T>.IsHardwareAccelerated => Vector512.IsHardwareAccelerated; + + /// + static Vector512 ISimdVector, T>.Abs(Vector512 vector) => Vector512.Abs(vector); + + /// + static Vector512 ISimdVector, T>.Add(Vector512 left, Vector512 right) => Vector512.Add(left, right); + + /// + static Vector512 ISimdVector, T>.AndNot(Vector512 left, Vector512 right) => Vector512.AndNot(left, right); + + /// + static Vector512 ISimdVector, T>.BitwiseAnd(Vector512 left, Vector512 right) => Vector512.BitwiseAnd(left, right); + + /// + static Vector512 ISimdVector, T>.BitwiseOr(Vector512 left, Vector512 right) => Vector512.BitwiseOr(left, right); + + /// + static Vector512 ISimdVector, T>.Ceiling(Vector512 vector) => Vector512.Ceiling(vector); + + /// + static Vector512 ISimdVector, T>.ConditionalSelect(Vector512 condition, Vector512 left, Vector512 right) => Vector512.ConditionalSelect(condition, left, right); + + /// + static void ISimdVector, T>.CopyTo(Vector512 vector, T[] destination) => vector.CopyTo(destination); + + /// + static void ISimdVector, T>.CopyTo(Vector512 vector, T[] destination, int startIndex) => vector.CopyTo(destination, startIndex); + + /// + static void ISimdVector, T>.CopyTo(Vector512 vector, Span destination) => vector.CopyTo(destination); + + /// + static Vector512 ISimdVector, T>.Create(T value) => Vector512.Create(value); + + /// + static Vector512 ISimdVector, T>.Create(T[] values) => Vector512.Create(values); + + /// + static Vector512 ISimdVector, T>.Create(T[] values, int index) => Vector512.Create(values, index); + + /// + static Vector512 ISimdVector, T>.Create(ReadOnlySpan values) => Vector512.Create(values); + + /// + static Vector512 ISimdVector, T>.CreateScalar(T value) => Vector512.CreateScalar(value); + + /// + static Vector512 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector512.CreateScalarUnsafe(value); + + /// + static Vector512 ISimdVector, T>.Divide(Vector512 left, Vector512 right) => Vector512.Divide(left, right); + + /// + static Vector512 ISimdVector, T>.Divide(Vector512 left, T right) => Vector512.Divide(left, right); + + /// + static T ISimdVector, T>.Dot(Vector512 left, Vector512 right) => Vector512.Dot(left, right); + + /// + static Vector512 ISimdVector, T>.Equals(Vector512 left, Vector512 right) => Vector512.Equals(left, right); + + /// + static bool ISimdVector, T>.EqualsAll(Vector512 left, Vector512 right) => Vector512.EqualsAll(left, right); + + /// + static bool ISimdVector, T>.EqualsAny(Vector512 left, Vector512 right) => Vector512.EqualsAny(left, right); + + /// + static Vector512 ISimdVector, T>.Floor(Vector512 vector) => Vector512.Floor(vector); + + /// + static T ISimdVector, T>.GetElement(Vector512 vector, int index) => Vector512.GetElement(vector, index); + + /// + static Vector512 ISimdVector, T>.GreaterThan(Vector512 left, Vector512 right) => Vector512.GreaterThan(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAll(Vector512 left, Vector512 right) => Vector512.GreaterThanAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAny(Vector512 left, Vector512 right) => Vector512.GreaterThanAny(left, right); + + /// + static Vector512 ISimdVector, T>.GreaterThanOrEqual(Vector512 left, Vector512 right) => Vector512.GreaterThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector512 left, Vector512 right) => Vector512.GreaterThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector512 left, Vector512 right) => Vector512.GreaterThanOrEqualAny(left, right); + + /// + static Vector512 ISimdVector, T>.LessThan(Vector512 left, Vector512 right) => Vector512.LessThan(left, right); + + /// + static bool ISimdVector, T>.LessThanAll(Vector512 left, Vector512 right) => Vector512.LessThanAll(left, right); + + /// + static bool ISimdVector, T>.LessThanAny(Vector512 left, Vector512 right) => Vector512.LessThanAny(left, right); + + /// + static Vector512 ISimdVector, T>.LessThanOrEqual(Vector512 left, Vector512 right) => Vector512.LessThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAll(Vector512 left, Vector512 right) => Vector512.LessThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAny(Vector512 left, Vector512 right) => Vector512.LessThanOrEqualAny(left, right); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static Vector512 ISimdVector, T>.Load(T* source) => Vector512.Load(source); + + /// + static Vector512 ISimdVector, T>.LoadAligned(T* source) => Vector512.LoadAligned(source); + + /// + static Vector512 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector512.LoadAlignedNonTemporal(source); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static Vector512 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector512.LoadUnsafe(in source); + + /// + static Vector512 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector512.LoadUnsafe(in source, elementOffset); + + /// + static Vector512 ISimdVector, T>.Max(Vector512 left, Vector512 right) => Vector512.Max(left, right); + + /// + static Vector512 ISimdVector, T>.Min(Vector512 left, Vector512 right) => Vector512.Min(left, right); + + /// + static Vector512 ISimdVector, T>.Multiply(Vector512 left, Vector512 right) => Vector512.Multiply(left, right); + + /// + static Vector512 ISimdVector, T>.Multiply(Vector512 left, T right) => Vector512.Multiply(left, right); + + /// + static Vector512 ISimdVector, T>.Negate(Vector512 vector) => Vector512.Negate(vector); + + /// + static Vector512 ISimdVector, T>.OnesComplement(Vector512 vector) => Vector512.OnesComplement(vector); + + /// + static Vector512 ISimdVector, T>.ShiftLeft(Vector512 vector, int shiftCount) => Vector512.ShiftLeft(vector, shiftCount); + + /// + static Vector512 ISimdVector, T>.ShiftRightArithmetic(Vector512 vector, int shiftCount) => Vector512.ShiftRightArithmetic(vector, shiftCount); + + /// + static Vector512 ISimdVector, T>.ShiftRightLogical(Vector512 vector, int shiftCount) => Vector512.ShiftRightLogical(vector, shiftCount); + + /// + static Vector512 ISimdVector, T>.Sqrt(Vector512 vector) => Vector512.Sqrt(vector); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static void ISimdVector, T>.Store(Vector512 source, T* destination) => Vector512.Store(source, destination); + + /// + static void ISimdVector, T>.StoreAligned(Vector512 source, T* destination) => Vector512.StoreAligned(source, destination); + + /// + static void ISimdVector, T>.StoreAlignedNonTemporal(Vector512 source, T* destination) => Vector512.StoreAlignedNonTemporal(source, destination); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static void ISimdVector, T>.StoreUnsafe(Vector512 vector, ref T destination) => Vector512.StoreUnsafe(vector, ref destination); + + /// + static void ISimdVector, T>.StoreUnsafe(Vector512 vector, ref T destination, nuint elementOffset) => Vector512.StoreUnsafe(vector, ref destination, elementOffset); + + /// + static Vector512 ISimdVector, T>.Subtract(Vector512 left, Vector512 right) => Vector512.Subtract(left, right); + + /// + static T ISimdVector, T>.Sum(Vector512 vector) => Vector512.Sum(vector); + + /// + static T ISimdVector, T>.ToScalar(Vector512 vector) => Vector512.ToScalar(vector); + + /// + static bool ISimdVector, T>.TryCopyTo(Vector512 vector, Span destination) => Vector512.TryCopyTo(vector, destination); + + /// + static Vector512 ISimdVector, T>.WithElement(Vector512 vector, int index, T value) => Vector512.WithElement(vector, index, value); + + /// + static Vector512 ISimdVector, T>.Xor(Vector512 left, Vector512 right) => Vector512.Xor(left, right); + + // + // New Surface Area + // + + static int ISimdVector, T>.IndexOfLastMatch(Vector512 vector) + { + ulong mask = vector.ExtractMostSignificantBits(); + return 63 - BitOperations.LeadingZeroCount(mask); // 63 = 64 (bits in Int64) - 1 (indexing from zero) + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs index 4f314511bc948..44a03c3f2a6a2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs @@ -236,40 +236,52 @@ public static Vector64 As(this Vector64 vector) /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector64 Ceiling(Vector64 vector) + internal static Vector64 Ceiling(Vector64 vector) { - Unsafe.SkipInit(out Vector64 result); - - for (int index = 0; index < Vector64.Count; index++) + if ((typeof(T) == typeof(byte)) + || (typeof(T) == typeof(short)) + || (typeof(T) == typeof(int)) + || (typeof(T) == typeof(long)) + || (typeof(T) == typeof(nint)) + || (typeof(T) == typeof(nuint)) + || (typeof(T) == typeof(sbyte)) + || (typeof(T) == typeof(ushort)) + || (typeof(T) == typeof(uint)) + || (typeof(T) == typeof(ulong))) { - float value = Scalar.Ceiling(vector.GetElementUnsafe(index)); - result.SetElementUnsafe(index, value); + return vector; } + else + { + Unsafe.SkipInit(out Vector64 result); - return result; + for (int index = 0; index < Vector64.Count; index++) + { + T value = Scalar.Ceiling(vector.GetElementUnsafe(index)); + result.SetElementUnsafe(index, value); + } + + return result; + } } /// Computes the ceiling of each element in a vector. /// The vector that will have its ceiling computed. /// A vector whose elements are the ceiling of the elements in . - /// + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector64 Ceiling(Vector64 vector) - { - Unsafe.SkipInit(out Vector64 result); + public static Vector64 Ceiling(Vector64 vector) => Ceiling(vector); - for (int index = 0; index < Vector64.Count; index++) - { - double value = Scalar.Ceiling(vector.GetElementUnsafe(index)); - result.SetElementUnsafe(index, value); - } - - return result; - } + /// Computes the ceiling of each element in a vector. + /// The vector that will have its ceiling computed. + /// A vector whose elements are the ceiling of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 Ceiling(Vector64 vector) => Ceiling(vector); /// Conditionally selects a value from two vectors on a bitwise basis. /// The type of the elements in the vector. @@ -1168,40 +1180,52 @@ public static uint ExtractMostSignificantBits(this Vector64 vector) /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . - /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector64 Floor(Vector64 vector) + internal static Vector64 Floor(Vector64 vector) { - Unsafe.SkipInit(out Vector64 result); - - for (int index = 0; index < Vector64.Count; index++) + if ((typeof(T) == typeof(byte)) + || (typeof(T) == typeof(short)) + || (typeof(T) == typeof(int)) + || (typeof(T) == typeof(long)) + || (typeof(T) == typeof(nint)) + || (typeof(T) == typeof(nuint)) + || (typeof(T) == typeof(sbyte)) + || (typeof(T) == typeof(ushort)) + || (typeof(T) == typeof(uint)) + || (typeof(T) == typeof(ulong))) { - float value = Scalar.Floor(vector.GetElementUnsafe(index)); - result.SetElementUnsafe(index, value); + return vector; } + else + { + Unsafe.SkipInit(out Vector64 result); - return result; + for (int index = 0; index < Vector64.Count; index++) + { + T value = Scalar.Floor(vector.GetElementUnsafe(index)); + result.SetElementUnsafe(index, value); + } + + return result; + } } /// Computes the floor of each element in a vector. /// The vector that will have its floor computed. /// A vector whose elements are the floor of the elements in . - /// + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector64 Floor(Vector64 vector) - { - Unsafe.SkipInit(out Vector64 result); - - for (int index = 0; index < Vector64.Count; index++) - { - double value = Scalar.Floor(vector.GetElementUnsafe(index)); - result.SetElementUnsafe(index, value); - } + public static Vector64 Floor(Vector64 vector) => Floor(vector); - return result; - } + /// Computes the floor of each element in a vector. + /// The vector that will have its floor computed. + /// A vector whose elements are the floor of the elements in . + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 Floor(Vector64 vector) => Floor(vector); /// Gets the element at the specified index. /// The type of the elements in the vector. @@ -1816,6 +1840,14 @@ public static unsafe Vector64 Narrow(Vector64 lower, Vector64 OnesComplement(Vector64 vector) => ~vector; + /// Shifts each element of a vector left by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted left by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Vector64 ShiftLeft(Vector64 vector, int shiftCount) => vector << shiftCount; + /// Shifts each element of a vector left by the specified amount. /// The vector whose elements are to be shifted. /// The number of bits by which to shift each element. @@ -1901,6 +1933,14 @@ public static unsafe Vector64 Narrow(Vector64 lower, Vector64 ShiftLeft(Vector64 vector, int shiftCount) => vector << shiftCount; + /// Shifts (signed) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Vector64 ShiftRightArithmetic(Vector64 vector, int shiftCount) => vector >> shiftCount; + /// Shifts (signed) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. /// The number of bits by which to shift each element. @@ -1942,6 +1982,14 @@ public static unsafe Vector64 Narrow(Vector64 lower, Vector64 ShiftRightArithmetic(Vector64 vector, int shiftCount) => vector >> shiftCount; + /// Shifts (unsigned) each element of a vector right by the specified amount. + /// The vector whose elements are to be shifted. + /// The number of bits by which to shift each element. + /// A vector whose elements where shifted right by . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Vector64 ShiftRightLogical(Vector64 vector, int shiftCount) => vector >>> shiftCount; + /// Shifts (unsigned) each element of a vector right by the specified amount. /// The vector whose elements are to be shifted. /// The number of bits by which to shift each element. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs index 175a6b0320774..d4ab526e562af 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs @@ -28,7 +28,7 @@ namespace System.Runtime.Intrinsics [DebuggerDisplay("{DisplayString,nq}")] [DebuggerTypeProxy(typeof(Vector64DebugView<>))] [StructLayout(LayoutKind.Sequential, Size = Vector64.Size)] - public readonly struct Vector64 : IEquatable> + public readonly unsafe struct Vector64 : ISimdVector, T> { // This field allows the debug view to work https://github.com/dotnet/runtime/issues/9495) internal readonly ulong _00; @@ -532,5 +532,215 @@ private string ToString([StringSyntax(StringSyntaxAttribute.NumericFormat)] stri return sb.ToString(); } + + // + // ISimdVector + // + + /// + static int ISimdVector, T>.Alignment => Vector64.Alignment; + + /// + static bool ISimdVector, T>.IsHardwareAccelerated => Vector64.IsHardwareAccelerated; + + /// + static Vector64 ISimdVector, T>.Abs(Vector64 vector) => Vector64.Abs(vector); + + /// + static Vector64 ISimdVector, T>.Add(Vector64 left, Vector64 right) => Vector64.Add(left, right); + + /// + static Vector64 ISimdVector, T>.AndNot(Vector64 left, Vector64 right) => Vector64.AndNot(left, right); + + /// + static Vector64 ISimdVector, T>.BitwiseAnd(Vector64 left, Vector64 right) => Vector64.BitwiseAnd(left, right); + + /// + static Vector64 ISimdVector, T>.BitwiseOr(Vector64 left, Vector64 right) => Vector64.BitwiseOr(left, right); + + /// + static Vector64 ISimdVector, T>.Ceiling(Vector64 vector) => Vector64.Ceiling(vector); + + /// + static Vector64 ISimdVector, T>.ConditionalSelect(Vector64 condition, Vector64 left, Vector64 right) => Vector64.ConditionalSelect(condition, left, right); + + /// + static void ISimdVector, T>.CopyTo(Vector64 vector, T[] destination) => vector.CopyTo(destination); + + /// + static void ISimdVector, T>.CopyTo(Vector64 vector, T[] destination, int startIndex) => vector.CopyTo(destination, startIndex); + + /// + static void ISimdVector, T>.CopyTo(Vector64 vector, Span destination) => vector.CopyTo(destination); + + /// + static Vector64 ISimdVector, T>.Create(T value) => Vector64.Create(value); + + /// + static Vector64 ISimdVector, T>.Create(T[] values) => Vector64.Create(values); + + /// + static Vector64 ISimdVector, T>.Create(T[] values, int index) => Vector64.Create(values, index); + + /// + static Vector64 ISimdVector, T>.Create(ReadOnlySpan values) => Vector64.Create(values); + + /// + static Vector64 ISimdVector, T>.CreateScalar(T value) => Vector64.CreateScalar(value); + + /// + static Vector64 ISimdVector, T>.CreateScalarUnsafe(T value) => Vector64.CreateScalarUnsafe(value); + + /// + static Vector64 ISimdVector, T>.Divide(Vector64 left, Vector64 right) => Vector64.Divide(left, right); + + /// + static Vector64 ISimdVector, T>.Divide(Vector64 left, T right) => Vector64.Divide(left, right); + + /// + static T ISimdVector, T>.Dot(Vector64 left, Vector64 right) => Vector64.Dot(left, right); + + /// + static Vector64 ISimdVector, T>.Equals(Vector64 left, Vector64 right) => Vector64.Equals(left, right); + + /// + static bool ISimdVector, T>.EqualsAll(Vector64 left, Vector64 right) => Vector64.EqualsAll(left, right); + + /// + static bool ISimdVector, T>.EqualsAny(Vector64 left, Vector64 right) => Vector64.EqualsAny(left, right); + + /// + static Vector64 ISimdVector, T>.Floor(Vector64 vector) => Vector64.Floor(vector); + + /// + static T ISimdVector, T>.GetElement(Vector64 vector, int index) => Vector64.GetElement(vector, index); + + /// + static Vector64 ISimdVector, T>.GreaterThan(Vector64 left, Vector64 right) => Vector64.GreaterThan(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAll(Vector64 left, Vector64 right) => Vector64.GreaterThanAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanAny(Vector64 left, Vector64 right) => Vector64.GreaterThanAny(left, right); + + /// + static Vector64 ISimdVector, T>.GreaterThanOrEqual(Vector64 left, Vector64 right) => Vector64.GreaterThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAll(Vector64 left, Vector64 right) => Vector64.GreaterThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.GreaterThanOrEqualAny(Vector64 left, Vector64 right) => Vector64.GreaterThanOrEqualAny(left, right); + + /// + static Vector64 ISimdVector, T>.LessThan(Vector64 left, Vector64 right) => Vector64.LessThan(left, right); + + /// + static bool ISimdVector, T>.LessThanAll(Vector64 left, Vector64 right) => Vector64.LessThanAll(left, right); + + /// + static bool ISimdVector, T>.LessThanAny(Vector64 left, Vector64 right) => Vector64.LessThanAny(left, right); + + /// + static Vector64 ISimdVector, T>.LessThanOrEqual(Vector64 left, Vector64 right) => Vector64.LessThanOrEqual(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAll(Vector64 left, Vector64 right) => Vector64.LessThanOrEqualAll(left, right); + + /// + static bool ISimdVector, T>.LessThanOrEqualAny(Vector64 left, Vector64 right) => Vector64.LessThanOrEqualAny(left, right); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static Vector64 ISimdVector, T>.Load(T* source) => Vector64.Load(source); + + /// + static Vector64 ISimdVector, T>.LoadAligned(T* source) => Vector64.LoadAligned(source); + + /// + static Vector64 ISimdVector, T>.LoadAlignedNonTemporal(T* source) => Vector64.LoadAlignedNonTemporal(source); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static Vector64 ISimdVector, T>.LoadUnsafe(ref readonly T source) => Vector64.LoadUnsafe(in source); + + /// + static Vector64 ISimdVector, T>.LoadUnsafe(ref readonly T source, nuint elementOffset) => Vector64.LoadUnsafe(in source, elementOffset); + + /// + static Vector64 ISimdVector, T>.Max(Vector64 left, Vector64 right) => Vector64.Max(left, right); + + /// + static Vector64 ISimdVector, T>.Min(Vector64 left, Vector64 right) => Vector64.Min(left, right); + + /// + static Vector64 ISimdVector, T>.Multiply(Vector64 left, Vector64 right) => Vector64.Multiply(left, right); + + /// + static Vector64 ISimdVector, T>.Multiply(Vector64 left, T right) => Vector64.Multiply(left, right); + + /// + static Vector64 ISimdVector, T>.Negate(Vector64 vector) => Vector64.Negate(vector); + + /// + static Vector64 ISimdVector, T>.OnesComplement(Vector64 vector) => Vector64.OnesComplement(vector); + + /// + static Vector64 ISimdVector, T>.ShiftLeft(Vector64 vector, int shiftCount) => Vector64.ShiftLeft(vector, shiftCount); + + /// + static Vector64 ISimdVector, T>.ShiftRightArithmetic(Vector64 vector, int shiftCount) => Vector64.ShiftRightArithmetic(vector, shiftCount); + + /// + static Vector64 ISimdVector, T>.ShiftRightLogical(Vector64 vector, int shiftCount) => Vector64.ShiftRightLogical(vector, shiftCount); + + /// + static Vector64 ISimdVector, T>.Sqrt(Vector64 vector) => Vector64.Sqrt(vector); + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + /// + static void ISimdVector, T>.Store(Vector64 source, T* destination) => Vector64.Store(source, destination); + + /// + static void ISimdVector, T>.StoreAligned(Vector64 source, T* destination) => Vector64.StoreAligned(source, destination); + + /// + static void ISimdVector, T>.StoreAlignedNonTemporal(Vector64 source, T* destination) => Vector64.StoreAlignedNonTemporal(source, destination); +#pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type ('T') + + /// + static void ISimdVector, T>.StoreUnsafe(Vector64 vector, ref T destination) => Vector64.StoreUnsafe(vector, ref destination); + + /// + static void ISimdVector, T>.StoreUnsafe(Vector64 vector, ref T destination, nuint elementOffset) => Vector64.StoreUnsafe(vector, ref destination, elementOffset); + + /// + static Vector64 ISimdVector, T>.Subtract(Vector64 left, Vector64 right) => Vector64.Subtract(left, right); + + /// + static T ISimdVector, T>.Sum(Vector64 vector) => Vector64.Sum(vector); + + /// + static T ISimdVector, T>.ToScalar(Vector64 vector) => Vector64.ToScalar(vector); + + /// + static bool ISimdVector, T>.TryCopyTo(Vector64 vector, Span destination) => Vector64.TryCopyTo(vector, destination); + + /// + static Vector64 ISimdVector, T>.WithElement(Vector64 vector, int index, T value) => Vector64.WithElement(vector, index, value); + + /// + static Vector64 ISimdVector, T>.Xor(Vector64 left, Vector64 right) => Vector64.Xor(left, right); + + // + // New Surface Area + // + + static int ISimdVector, T>.IndexOfLastMatch(Vector64 vector) + { + uint mask = vector.ExtractMostSignificantBits(); + return 31 - BitOperations.LeadingZeroCount(mask); // 31 = 32 (bits in Int32) - 1 (indexing from zero) + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index d6295705c038a..3f10526ec96bb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -2475,90 +2475,49 @@ private static int LastIndexOfValueType(ref TValue searchSpace } else if (Vector512.IsHardwareAccelerated && length >= Vector512.Count) { - Vector512 current, values = Vector512.Create(value); - nint offset = length - Vector512.Count; - - // Loop until either we've finished all elements -or- there's one or less than a vector's-worth remaining. - while (offset > 0) - { - current = Vector512.LoadUnsafe(ref searchSpace, (nuint)(offset)); - - if (TNegator.HasMatch(values, current)) - { - return ComputeLastIndex(offset, TNegator.GetMatchMask(values, current)); - } - - offset -= Vector512.Count; - } - - // Process the first vector in the search space. - - current = Vector512.LoadUnsafe(ref searchSpace); - - if (TNegator.HasMatch(values, current)) - { - return ComputeLastIndex(offset: 0, TNegator.GetMatchMask(values, current)); - } + return SimdImpl>(ref searchSpace, value, length); } else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) { - Vector256 equals, values = Vector256.Create(value); - nint offset = length - Vector256.Count; - - // Loop until either we've finished all elements -or- there's one or less than a vector's-worth remaining. - while (offset > 0) - { - equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset)))); - - if (equals == Vector256.Zero) - { - offset -= Vector256.Count; - continue; - } - - return ComputeLastIndex(offset, equals); - } - - // Process the first vector in the search space. - - equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace))); - - if (equals != Vector256.Zero) - { - return ComputeLastIndex(offset: 0, equals); - } + return SimdImpl>(ref searchSpace, value, length); } else { - Vector128 equals, values = Vector128.Create(value); - nint offset = length - Vector128.Count; + return SimdImpl>(ref searchSpace, value, length); + } + + static int SimdImpl(ref TValue searchSpace, TValue value, int length) + where TVector : struct, ISimdVector + { + TVector current; + TVector values = TVector.Create(value); + + int offset = length - TVector.Count; // Loop until either we've finished all elements -or- there's one or less than a vector's-worth remaining. while (offset > 0) { - equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset)))); + current = TVector.LoadUnsafe(ref searchSpace, (uint)(offset)); - if (equals == Vector128.Zero) + if (TNegator.HasMatch(values, current)) { - offset -= Vector128.Count; - continue; + return offset + TVector.IndexOfLastMatch(TNegator.GetMatchMask(values, current)); } - return ComputeLastIndex(offset, equals); + offset -= TVector.Count; } - // Process the first vector in the search space. - equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace))); + current = TVector.LoadUnsafe(ref searchSpace); - if (equals != Vector128.Zero) + if (TNegator.HasMatch(values, current)) { - return ComputeLastIndex(offset: 0, equals); + return TVector.IndexOfLastMatch(TNegator.GetMatchMask(values, current)); } - } - return -1; + return -1; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -3402,39 +3361,62 @@ internal interface INegator where T : struct static abstract Vector256 NegateIfNeeded(Vector256 equals); static abstract Vector512 NegateIfNeeded(Vector512 equals); - // The V512 APIs assume use for IndexOf where `DontNegate` is + // The generic vector APIs assume use for IndexOf where `DontNegate` is // for `IndexOfAny` and `Negate` is for `IndexOfAnyExcept` - static abstract bool HasMatch(Vector512 left, Vector512 right); - static abstract Vector512 GetMatchMask(Vector512 left, Vector512 right); + static abstract bool HasMatch(TVector left, TVector right) + where TVector : struct, ISimdVector; + + static abstract TVector GetMatchMask(TVector left, TVector right) + where TVector : struct, ISimdVector; } - internal readonly struct DontNegate : INegator where T : struct + internal readonly struct DontNegate : INegator + where T : struct { public static bool NegateIfNeeded(bool equals) => equals; public static Vector128 NegateIfNeeded(Vector128 equals) => equals; public static Vector256 NegateIfNeeded(Vector256 equals) => equals; public static Vector512 NegateIfNeeded(Vector512 equals) => equals; - // The V512 APIs assume use for `IndexOfAny` where we + // The generic vector APIs assume use for `IndexOfAny` where we // want "HasMatch" to mean any of the two elements match. - public static bool HasMatch(Vector512 left, Vector512 right) => Vector512.EqualsAny(left, right); - public static Vector512 GetMatchMask(Vector512 left, Vector512 right) => Vector512.Equals(left, right); + public static bool HasMatch(TVector left, TVector right) + where TVector : struct, ISimdVector + { + return TVector.EqualsAny(left, right); + } + + public static TVector GetMatchMask(TVector left, TVector right) + where TVector : struct, ISimdVector + { + return TVector.Equals(left, right); + } } - internal readonly struct Negate : INegator where T : struct + internal readonly struct Negate : INegator + where T : struct { public static bool NegateIfNeeded(bool equals) => !equals; public static Vector128 NegateIfNeeded(Vector128 equals) => ~equals; public static Vector256 NegateIfNeeded(Vector256 equals) => ~equals; public static Vector512 NegateIfNeeded(Vector512 equals) => ~equals; - // The V512 APIs assume use for `IndexOfAnyExcept` where we + // The generic vector APIs assume use for `IndexOfAnyExcept` where we // want "HasMatch" to mean any of the two elements don't match - public static bool HasMatch(Vector512 left, Vector512 right) => !Vector512.EqualsAll(left, right); - public static Vector512 GetMatchMask(Vector512 left, Vector512 right) => ~Vector512.Equals(left, right); + public static bool HasMatch(TVector left, TVector right) + where TVector : struct, ISimdVector + { + return !TVector.EqualsAll(left, right); + } + + public static TVector GetMatchMask(TVector left, TVector right) + where TVector : struct, ISimdVector + { + return ~TVector.Equals(left, right); + } } internal static int IndexOfAnyInRange(ref T searchSpace, T lowInclusive, T highInclusive, int length) diff --git a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs index 2d4700576196c..92c09c00b81e9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @@ -701,6 +701,18 @@ internal static void IfNullAndNullsAreIllegalThenThrow(object? value, Excepti ThrowArgumentNullException(argName); } + + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void ThrowForUnsupportedSimdVectorBaseType() + where TVector : ISimdVector + { + if (!TVector.IsSupported) + { + ThrowNotSupportedException(ExceptionResource.Arg_TypeNotSupported); + } + } + // Throws if 'T' is disallowed in Vector in the Numerics namespace. // If 'T' is allowed, no-ops. JIT will elide the method entirely if 'T' // is supported and we're on an optimized release build.