SuperDex Physics C++ API
Loading...
Searching...
No Matches
x64_simd_inl.h
Go to the documentation of this file.
1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#pragma once
18
19/**
20 This file contains Simd specializations for x64 CPUs with AVX2 support.
21 Used for both Intel and AMD CPUs.
22*/
23
24#include "../../simd.h" // for IntelliSense
25
26#if MOCHI_USE_SIMD && MOCHI_ARCH_X64_AVX2
27
28#include <immintrin.h>
29#include <xmmintrin.h>
30#include <cstring>
31
32/***********************************************************************************************
33 Simd Specializations for x64 Architecture
34*/
35
36// NOTE: Order of these headers matters in some cases. Do not sort alphabetically.
37
38#include "x64_simd_tables_inl.h"
39
40#include "x64_simd_int_4_inl.h"
41
42#include "x64_simd_int_8_inl.h"
43
44#include "x64_simd_int64_2_inl.h"
45
46#include "x64_simd_int64_4_inl.h"
47
48#include "x64_simd_double_2_inl.h"
49
50#include "x64_simd_double_4_inl.h"
51
52#include "x64_simd_float_4_inl.h"
53
54#include "x64_simd_float_8_inl.h"
55
56/***********************************************************************************************
57 Simd Utlities for x64 Architecture
58*/
59
60namespace superdex {
61
62// clang-format off
63template <> MOCHI_FORCE_INLINE Simd<double, 2> ReinterpretCast<Simd<double, 2>, Simd<int64_t, 2>>(Simd<int64_t, 2> const& a) { return _mm_castsi128_pd(a.raw); } // SSE2
64template <> MOCHI_FORCE_INLINE Simd<int64_t, 2> ReinterpretCast<Simd<int64_t, 2>, Simd<double, 2>>(Simd<double, 2> const& a) { return _mm_castpd_si128(a.raw); } // SSE2
65// clang-format on
66
67template <>
68MOCHI_FORCE_INLINE Simd<int64_t, 2> StaticCast<Simd<int64_t, 2>, Simd<double, 2>>(
69 Simd<double, 2> const& a) {
70 // Native support for _mm_cvttpd_epi64 requires AVX512
71 return {_mm_cvttsd_si64(a.raw), _mm_cvttsd_si64(_mm_unpackhi_pd(a.raw, a.raw))};
72} // SSE2
73template <>
74MOCHI_FORCE_INLINE Simd<double, 2> StaticCast<Simd<double, 2>, Simd<int64_t, 2>>(
75 Simd<int64_t, 2> const& a) {
76 // Native support for _mm_cvtepi64_pd requires AVX512
77 return {
78 static_cast<double>(_mm_cvtsi128_si64(a.raw)),
79 static_cast<double>(_mm_cvtsi128_si64(_mm_unpackhi_epi64(a.raw, a.raw)))};
80} // SSE2
81
82// clang-format off
83template <> MOCHI_FORCE_INLINE Simd<double, 2> ReinterpretCast<Simd<double, 2>, Simd<float, 4>>(Simd<float, 4> const& a) { return _mm_castps_pd(a.raw); } // SSE2
84template <> MOCHI_FORCE_INLINE Simd<float, 4> ReinterpretCast<Simd<float, 4>, Simd<double, 2>>(Simd<double, 2> const& a) { return _mm_castpd_ps(a.raw); } // SSE2
85
86template <> MOCHI_FORCE_INLINE Simd<double, 2> ReinterpretCast<Simd<double, 2>, Simd<int, 4>>(Simd<int, 4> const& a) { return _mm_castsi128_pd(a.raw); } // SSE2
87template <> MOCHI_FORCE_INLINE Simd<int, 4> ReinterpretCast<Simd<int, 4>, Simd<double, 2>>(Simd<double, 2> const& a) { return _mm_castpd_si128(a.raw); } // SSE2
88
89template <> MOCHI_FORCE_INLINE Simd<int64_t, 2> ReinterpretCast<Simd<int64_t, 2>, Simd<float, 4>>(Simd<float, 4> const& a) { return _mm_castps_si128(a.raw); } // SSE2
90template <> MOCHI_FORCE_INLINE Simd<float, 4> ReinterpretCast<Simd<float, 4>, Simd<int64_t, 2>>(Simd<int64_t, 2> const& a) { return _mm_castsi128_ps(a.raw); } // SSE2
91
92template <> MOCHI_FORCE_INLINE Simd<int64_t, 2> ReinterpretCast<Simd<int64_t, 2>, Simd<int, 4>>(Simd<int, 4> const& a) { return a.raw; }
93template <> MOCHI_FORCE_INLINE Simd<int, 4> ReinterpretCast<Simd<int, 4>, Simd<int64_t, 2>>(Simd<int64_t, 2> const& a) { return a.raw; }
94
95template <> MOCHI_FORCE_INLINE Simd<float, 4> StaticCast<Simd<float, 4>, Simd<double, 4>>(Simd<double, 4> const& a) { return {_mm256_cvtpd_ps(a.raw)}; } // AVX
96template <> MOCHI_FORCE_INLINE Simd<double, 4> StaticCast<Simd<double, 4>, Simd<float, 4>>(Simd<float, 4> const& a) { return {_mm256_cvtps_pd(a.raw)}; } // AVX
97
98template <> MOCHI_FORCE_INLINE Simd<int, 4> StaticCast<Simd<int, 4>, Simd<double, 4>>(Simd<double, 4> const& a) { return {_mm256_cvttpd_epi32(a.raw)}; } // AVX
99template <> MOCHI_FORCE_INLINE Simd<double, 4> StaticCast<Simd<double, 4>, Simd<int, 4>>(Simd<int, 4> const& a) { return {_mm256_cvtepi32_pd(a.raw)}; } // AVX
100
101template <> MOCHI_FORCE_INLINE Simd<double, 4> ReinterpretCast<Simd<double, 4>, Simd<int64_t, 4>>(Simd<int64_t, 4> const& a) { return _mm256_castsi256_pd(a.raw); } // AVX
102template <> MOCHI_FORCE_INLINE Simd<int64_t, 4> ReinterpretCast<Simd<int64_t, 4>, Simd<double, 4>>(Simd<double, 4> const& a) { return _mm256_castpd_si256(a.raw); } // AVX
103// clang-format on
104
105template <>
106MOCHI_FORCE_INLINE Simd<int64_t, 4> StaticCast<Simd<int64_t, 4>, Simd<double, 4>>(
107 Simd<double, 4> const& a) {
108 /// Native support is provided in AVX512 '_mm256_cvttpd_epi64'
109 using VType = Simd<double, 4>;
110 return {
111 static_cast<int64_t>(VType::Get<0>(a)),
112 static_cast<int64_t>(VType::Get<1>(a)),
113 static_cast<int64_t>(VType::Get<2>(a)),
114 static_cast<int64_t>(VType::Get<3>(a))};
115}
116
117template <>
118MOCHI_FORCE_INLINE Simd<double, 4> StaticCast<Simd<double, 4>, Simd<int64_t, 4>>(
119 Simd<int64_t, 4> const& a) {
120 /// Native support is provided in AVX512 '_mm256_cvtepi64_pd'
121 using VType = Simd<int64_t, 4>;
122 return {
123 static_cast<double>(VType::Get<0>(a)),
124 static_cast<double>(VType::Get<1>(a)),
125 static_cast<double>(VType::Get<2>(a)),
126 static_cast<double>(VType::Get<3>(a))};
127}
128
129// clang-format off
130template <> MOCHI_FORCE_INLINE Simd<double, 4> ReinterpretCast<Simd<double, 4>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_castps_pd(a.raw); } // AVX
131template <> MOCHI_FORCE_INLINE Simd<float, 8> ReinterpretCast<Simd<float, 8>, Simd<double, 4>>(Simd<double, 4> const& a) { return _mm256_castpd_ps(a.raw); } // AVX
132
133template <> MOCHI_FORCE_INLINE Simd<double, 4> ReinterpretCast<Simd<double, 4>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm256_castsi256_pd(a.raw); } // AVX
134template <> MOCHI_FORCE_INLINE Simd<int, 8> ReinterpretCast<Simd<int, 8>, Simd<double, 4>>(Simd<double, 4> const& a) { return _mm256_castpd_si256(a.raw); } // AVX
135
136template <> MOCHI_FORCE_INLINE Simd<float, 4> ReinterpretCast<Simd<float, 4>, Simd<int, 4>>(Simd<int, 4> const& a) { return _mm_castsi128_ps(a.raw); } // SSE2
137template <> MOCHI_FORCE_INLINE Simd<int, 4> ReinterpretCast<Simd<int, 4>, Simd<float, 4>>(Simd<float, 4> const& a) { return _mm_castps_si128(a.raw); } // SSE2
138
139template <> MOCHI_FORCE_INLINE Simd<int, 4> StaticCast<Simd<int, 4>, Simd<float, 4>>(Simd<float, 4> const& a) { return {_mm_cvttps_epi32(a.raw)}; } // SSE2
140template <> MOCHI_FORCE_INLINE Simd<float, 4> StaticCast<Simd<float, 4>, Simd<int, 4>>(Simd<int, 4> const& a) { return {_mm_cvtepi32_ps(a.raw)}; } // SSE2
141// clang-format on
142
143template <>
144MOCHI_FORCE_INLINE Simd<int64_t, 4> StaticCast<Simd<int64_t, 4>, Simd<float, 4>>(
145 Simd<float, 4> const& a) {
146 // Native support '_mm256_cvttps_epi64' requires AVX512
147 using VType = Simd<float, 4>;
148 return {
149 static_cast<int64_t>(VType::Get<0>(a)),
150 static_cast<int64_t>(VType::Get<1>(a)),
151 static_cast<int64_t>(VType::Get<2>(a)),
152 static_cast<int64_t>(VType::Get<3>(a))};
153}
154
155template <>
156MOCHI_FORCE_INLINE Simd<float, 4> StaticCast<Simd<float, 4>, Simd<int64_t, 4>>(
157 Simd<int64_t, 4> const& a) {
158 // Native support '_mm256_cvtepi64_ps' requires AVX512
159 using VType = Simd<int64_t, 4>;
160 return {
161 static_cast<float>(VType::Get<0>(a)),
162 static_cast<float>(VType::Get<1>(a)),
163 static_cast<float>(VType::Get<2>(a)),
164 static_cast<float>(VType::Get<3>(a))};
165}
166
167template <>
168MOCHI_FORCE_INLINE Simd<int, 4> StaticCast<Simd<int, 4>, Simd<int64_t, 4>>(
169 Simd<int64_t, 4> const& a) {
170 // Native support '_mm256_cvtepi64_epi32' requires AVX512
171 using VType = Simd<int64_t, 4>;
172 return {
173 static_cast<int>(VType::Get<0>(a)),
174 static_cast<int>(VType::Get<1>(a)),
175 static_cast<int>(VType::Get<2>(a)),
176 static_cast<int>(VType::Get<3>(a))};
177}
178template <>
179MOCHI_FORCE_INLINE Simd<int64_t, 4> StaticCast<Simd<int64_t, 4>, Simd<int, 4>>(
180 Simd<int, 4> const& a) {
181 return _mm256_cvtepi32_epi64(a.raw); // AVX2
182}
183
184// clang-format off
185template <> MOCHI_FORCE_INLINE Simd<float, 8> ReinterpretCast<Simd<float, 8>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm256_castsi256_ps(a.raw); } // AVX
186template <> MOCHI_FORCE_INLINE Simd<int, 8> ReinterpretCast<Simd<int, 8>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_castps_si256(a.raw); } // AVX
187template <> MOCHI_FORCE_INLINE Simd<int, 8> StaticCast<Simd<int, 8>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_cvttps_epi32(a.raw); } // AVX
188template <> MOCHI_FORCE_INLINE Simd<float, 8> StaticCast<Simd<float, 8>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm256_cvtepi32_ps(a.raw); } // AVX
189
190template <> MOCHI_FORCE_INLINE Simd<float, 8> ReinterpretCast<Simd<float, 8>, Simd<int64_t, 4>>(Simd<int64_t, 4> const& a) { return _mm256_castsi256_ps(a.raw); } // AVX
191template <> MOCHI_FORCE_INLINE Simd<int64_t, 4> ReinterpretCast<Simd<int64_t, 4>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_castps_si256(a.raw); } // AVX
192
193template <> MOCHI_FORCE_INLINE Simd<int, 8> ReinterpretCast<Simd<int, 8>, Simd<int64_t, 4>>(Simd<int64_t, 4> const& a) { return a.raw; } // AVX
194template <> MOCHI_FORCE_INLINE Simd<int64_t, 4> ReinterpretCast<Simd<int64_t, 4>, Simd<int, 8>>(Simd<int, 8> const& a) { return a.raw; } // AVX
195// clang-format on
196
197} // namespace superdex
198
199#endif // MOCHI_USE_SIMD && MOCHI_ARCH_X64_AVX2
MOCHI_FORCE_INLINE
#define MOCHI_FORCE_INLINE
Definition mochi_platform.h:340
superdex::StaticCast
constexpr To StaticCast(From const &a)
Definition basic_utils.h:79
x64_simd_double_2_inl.h
x64_simd_double_4_inl.h
x64_simd_float_4_inl.h
x64_simd_float_8_inl.h
x64_simd_int64_2_inl.h
x64_simd_int64_4_inl.h
x64_simd_int_4_inl.h
x64_simd_int_8_inl.h
x64_simd_tables_inl.h