SuperDex Physics C++ API
Loading...
Searching...
No Matches
x64_simd_inl.h
Go to the documentation of this file.
1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#pragma once
18
19/**
20 This file contains Simd specializations for x64 CPUs with AVX2 support.
21 Used for both Intel and AMD CPUs.
22*/
23
24#include "../../simd.h" // for IntelliSense
25
26#if MOCHI_USE_SIMD && MOCHI_ARCH_X64_AVX2
27
28#include <immintrin.h>
29#include <xmmintrin.h>
30#include <cstring>
31
32/***********************************************************************************************
33 Simd Specializations for x64 Architecture
34*/
35
36// NOTE: Order of these headers matters in some cases. Do not sort alphabetically.
37
38#include "x64_simd_tables_inl.h"
39
40#include "x64_simd_int_4_inl.h"
41
42#include "x64_simd_int_8_inl.h"
43
44#include "x64_simd_int_16_inl.h"
45
46#include "x64_simd_int64_2_inl.h"
47
48#include "x64_simd_int64_4_inl.h"
49
50#include "x64_simd_int64_8_inl.h"
51
52#include "x64_simd_double_2_inl.h"
53
54#include "x64_simd_double_4_inl.h"
55
56#include "x64_simd_double_8_inl.h"
57
58#include "x64_simd_float_4_inl.h"
59
60#include "x64_simd_float_8_inl.h"
61
62#include "x64_simd_float_16_inl.h"
63
64/***********************************************************************************************
65 Simd Utlities for x64 Architecture
66*/
67
68namespace superdex {
69
70// clang-format off
71template <> MOCHI_FORCE_INLINE Simd<double, 2> ReinterpretCast<Simd<double, 2>, Simd<int64_t, 2>>(Simd<int64_t, 2> const& a) { return _mm_castsi128_pd(a.raw); } // SSE2
72template <> MOCHI_FORCE_INLINE Simd<int64_t, 2> ReinterpretCast<Simd<int64_t, 2>, Simd<double, 2>>(Simd<double, 2> const& a) { return _mm_castpd_si128(a.raw); } // SSE2
73// clang-format on
74
75template <>
76MOCHI_FORCE_INLINE Simd<int64_t, 2> StaticCast<Simd<int64_t, 2>, Simd<double, 2>>(
77 Simd<double, 2> const& a) {
78#if MOCHI_ARCH_X64_AVX512
79 return _mm_cvttpd_epi64(a.raw); // AVX512VL
80#else
81 return {_mm_cvttsd_si64(a.raw), _mm_cvttsd_si64(_mm_unpackhi_pd(a.raw, a.raw))};
82#endif
83}
84template <>
85MOCHI_FORCE_INLINE Simd<double, 2> StaticCast<Simd<double, 2>, Simd<int64_t, 2>>(
86 Simd<int64_t, 2> const& a) {
87#if MOCHI_ARCH_X64_AVX512
88 return _mm_cvtepi64_pd(a.raw); // AVX512VL
89#else
90 return {
91 static_cast<double>(_mm_cvtsi128_si64(a.raw)),
92 static_cast<double>(_mm_cvtsi128_si64(_mm_unpackhi_epi64(a.raw, a.raw)))};
93#endif
94}
95
96// clang-format off
97template <> MOCHI_FORCE_INLINE Simd<double, 2> ReinterpretCast<Simd<double, 2>, Simd<float, 4>>(Simd<float, 4> const& a) { return _mm_castps_pd(a.raw); } // SSE2
98template <> MOCHI_FORCE_INLINE Simd<float, 4> ReinterpretCast<Simd<float, 4>, Simd<double, 2>>(Simd<double, 2> const& a) { return _mm_castpd_ps(a.raw); } // SSE2
99
100template <> MOCHI_FORCE_INLINE Simd<double, 2> ReinterpretCast<Simd<double, 2>, Simd<int, 4>>(Simd<int, 4> const& a) { return _mm_castsi128_pd(a.raw); } // SSE2
101template <> MOCHI_FORCE_INLINE Simd<int, 4> ReinterpretCast<Simd<int, 4>, Simd<double, 2>>(Simd<double, 2> const& a) { return _mm_castpd_si128(a.raw); } // SSE2
102
103template <> MOCHI_FORCE_INLINE Simd<int64_t, 2> ReinterpretCast<Simd<int64_t, 2>, Simd<float, 4>>(Simd<float, 4> const& a) { return _mm_castps_si128(a.raw); } // SSE2
104template <> MOCHI_FORCE_INLINE Simd<float, 4> ReinterpretCast<Simd<float, 4>, Simd<int64_t, 2>>(Simd<int64_t, 2> const& a) { return _mm_castsi128_ps(a.raw); } // SSE2
105
106template <> MOCHI_FORCE_INLINE Simd<int64_t, 2> ReinterpretCast<Simd<int64_t, 2>, Simd<int, 4>>(Simd<int, 4> const& a) { return a.raw; }
107template <> MOCHI_FORCE_INLINE Simd<int, 4> ReinterpretCast<Simd<int, 4>, Simd<int64_t, 2>>(Simd<int64_t, 2> const& a) { return a.raw; }
108
109template <> MOCHI_FORCE_INLINE Simd<float, 4> StaticCast<Simd<float, 4>, Simd<double, 4>>(Simd<double, 4> const& a) { return {_mm256_cvtpd_ps(a.raw)}; } // AVX
110template <> MOCHI_FORCE_INLINE Simd<double, 4> StaticCast<Simd<double, 4>, Simd<float, 4>>(Simd<float, 4> const& a) { return {_mm256_cvtps_pd(a.raw)}; } // AVX
111
112template <> MOCHI_FORCE_INLINE Simd<int, 4> StaticCast<Simd<int, 4>, Simd<double, 4>>(Simd<double, 4> const& a) { return {_mm256_cvttpd_epi32(a.raw)}; } // AVX
113template <> MOCHI_FORCE_INLINE Simd<double, 4> StaticCast<Simd<double, 4>, Simd<int, 4>>(Simd<int, 4> const& a) { return {_mm256_cvtepi32_pd(a.raw)}; } // AVX
114
115template <> MOCHI_FORCE_INLINE Simd<double, 4> ReinterpretCast<Simd<double, 4>, Simd<int64_t, 4>>(Simd<int64_t, 4> const& a) { return _mm256_castsi256_pd(a.raw); } // AVX
116template <> MOCHI_FORCE_INLINE Simd<int64_t, 4> ReinterpretCast<Simd<int64_t, 4>, Simd<double, 4>>(Simd<double, 4> const& a) { return _mm256_castpd_si256(a.raw); } // AVX
117// clang-format on
118
119template <>
120MOCHI_FORCE_INLINE Simd<int64_t, 4> StaticCast<Simd<int64_t, 4>, Simd<double, 4>>(
121 Simd<double, 4> const& a) {
122#if MOCHI_ARCH_X64_AVX512
123 return _mm256_cvttpd_epi64(a.raw); // AVX512VL
124#else
125 using VType = Simd<double, 4>;
126 return {
127 static_cast<int64_t>(VType::Get<0>(a)),
128 static_cast<int64_t>(VType::Get<1>(a)),
129 static_cast<int64_t>(VType::Get<2>(a)),
130 static_cast<int64_t>(VType::Get<3>(a))};
131#endif
132}
133
134template <>
135MOCHI_FORCE_INLINE Simd<double, 4> StaticCast<Simd<double, 4>, Simd<int64_t, 4>>(
136 Simd<int64_t, 4> const& a) {
137#if MOCHI_ARCH_X64_AVX512
138 return _mm256_cvtepi64_pd(a.raw); // AVX512VL
139#else
140 using VType = Simd<int64_t, 4>;
141 return {
142 static_cast<double>(VType::Get<0>(a)),
143 static_cast<double>(VType::Get<1>(a)),
144 static_cast<double>(VType::Get<2>(a)),
145 static_cast<double>(VType::Get<3>(a))};
146#endif
147}
148
149// clang-format off
150template <> MOCHI_FORCE_INLINE Simd<double, 4> ReinterpretCast<Simd<double, 4>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_castps_pd(a.raw); } // AVX
151template <> MOCHI_FORCE_INLINE Simd<float, 8> ReinterpretCast<Simd<float, 8>, Simd<double, 4>>(Simd<double, 4> const& a) { return _mm256_castpd_ps(a.raw); } // AVX
152
153template <> MOCHI_FORCE_INLINE Simd<double, 4> ReinterpretCast<Simd<double, 4>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm256_castsi256_pd(a.raw); } // AVX
154template <> MOCHI_FORCE_INLINE Simd<int, 8> ReinterpretCast<Simd<int, 8>, Simd<double, 4>>(Simd<double, 4> const& a) { return _mm256_castpd_si256(a.raw); } // AVX
155
156template <> MOCHI_FORCE_INLINE Simd<float, 4> ReinterpretCast<Simd<float, 4>, Simd<int, 4>>(Simd<int, 4> const& a) { return _mm_castsi128_ps(a.raw); } // SSE2
157template <> MOCHI_FORCE_INLINE Simd<int, 4> ReinterpretCast<Simd<int, 4>, Simd<float, 4>>(Simd<float, 4> const& a) { return _mm_castps_si128(a.raw); } // SSE2
158
159template <> MOCHI_FORCE_INLINE Simd<int, 4> StaticCast<Simd<int, 4>, Simd<float, 4>>(Simd<float, 4> const& a) { return {_mm_cvttps_epi32(a.raw)}; } // SSE2
160template <> MOCHI_FORCE_INLINE Simd<float, 4> StaticCast<Simd<float, 4>, Simd<int, 4>>(Simd<int, 4> const& a) { return {_mm_cvtepi32_ps(a.raw)}; } // SSE2
161// clang-format on
162
163template <>
164MOCHI_FORCE_INLINE Simd<int64_t, 4> StaticCast<Simd<int64_t, 4>, Simd<float, 4>>(
165 Simd<float, 4> const& a) {
166#if MOCHI_ARCH_X64_AVX512
167 return _mm256_cvttps_epi64(a.raw); // AVX512VL
168#else
169 using VType = Simd<float, 4>;
170 return {
171 static_cast<int64_t>(VType::Get<0>(a)),
172 static_cast<int64_t>(VType::Get<1>(a)),
173 static_cast<int64_t>(VType::Get<2>(a)),
174 static_cast<int64_t>(VType::Get<3>(a))};
175#endif
176}
177
178template <>
179MOCHI_FORCE_INLINE Simd<float, 4> StaticCast<Simd<float, 4>, Simd<int64_t, 4>>(
180 Simd<int64_t, 4> const& a) {
181#if MOCHI_ARCH_X64_AVX512
182 return _mm256_cvtepi64_ps(a.raw); // AVX512VL
183#else
184 using VType = Simd<int64_t, 4>;
185 return {
186 static_cast<float>(VType::Get<0>(a)),
187 static_cast<float>(VType::Get<1>(a)),
188 static_cast<float>(VType::Get<2>(a)),
189 static_cast<float>(VType::Get<3>(a))};
190#endif
191}
192
193template <>
194MOCHI_FORCE_INLINE Simd<int, 4> StaticCast<Simd<int, 4>, Simd<int64_t, 4>>(
195 Simd<int64_t, 4> const& a) {
196#if MOCHI_ARCH_X64_AVX512
197 return _mm256_cvtepi64_epi32(a.raw); // AVX512VL
198#else
199 using VType = Simd<int64_t, 4>;
200 return {
201 static_cast<int>(VType::Get<0>(a)),
202 static_cast<int>(VType::Get<1>(a)),
203 static_cast<int>(VType::Get<2>(a)),
204 static_cast<int>(VType::Get<3>(a))};
205#endif
206}
207template <>
208MOCHI_FORCE_INLINE Simd<int64_t, 4> StaticCast<Simd<int64_t, 4>, Simd<int, 4>>(
209 Simd<int, 4> const& a) {
210 return _mm256_cvtepi32_epi64(a.raw); // AVX2
211}
212
213// clang-format off
214template <> MOCHI_FORCE_INLINE Simd<float, 8> ReinterpretCast<Simd<float, 8>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm256_castsi256_ps(a.raw); } // AVX
215template <> MOCHI_FORCE_INLINE Simd<int, 8> ReinterpretCast<Simd<int, 8>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_castps_si256(a.raw); } // AVX
216template <> MOCHI_FORCE_INLINE Simd<int, 8> StaticCast<Simd<int, 8>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_cvttps_epi32(a.raw); } // AVX
217template <> MOCHI_FORCE_INLINE Simd<float, 8> StaticCast<Simd<float, 8>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm256_cvtepi32_ps(a.raw); } // AVX
218
219template <> MOCHI_FORCE_INLINE Simd<float, 8> ReinterpretCast<Simd<float, 8>, Simd<int64_t, 4>>(Simd<int64_t, 4> const& a) { return _mm256_castsi256_ps(a.raw); } // AVX
220template <> MOCHI_FORCE_INLINE Simd<int64_t, 4> ReinterpretCast<Simd<int64_t, 4>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm256_castps_si256(a.raw); } // AVX
221
222template <> MOCHI_FORCE_INLINE Simd<int, 8> ReinterpretCast<Simd<int, 8>, Simd<int64_t, 4>>(Simd<int64_t, 4> const& a) { return a.raw; } // AVX
223template <> MOCHI_FORCE_INLINE Simd<int64_t, 4> ReinterpretCast<Simd<int64_t, 4>, Simd<int, 8>>(Simd<int, 8> const& a) { return a.raw; } // AVX
224// clang-format on
225
226#if MOCHI_ARCH_X64_AVX512
227// clang-format off
228template <> MOCHI_FORCE_INLINE Simd<float, 16> ReinterpretCast<Simd<float, 16>, Simd<int, 16>>(Simd<int, 16> const& a) { return _mm512_castsi512_ps(a.raw); }
229template <> MOCHI_FORCE_INLINE Simd<int, 16> ReinterpretCast<Simd<int, 16>, Simd<float, 16>>(Simd<float, 16> const& a) { return _mm512_castps_si512(a.raw); }
230template <> MOCHI_FORCE_INLINE Simd<double, 8> ReinterpretCast<Simd<double, 8>, Simd<int64_t, 8>>(Simd<int64_t, 8> const& a) { return _mm512_castsi512_pd(a.raw); }
231template <> MOCHI_FORCE_INLINE Simd<int64_t, 8> ReinterpretCast<Simd<int64_t, 8>, Simd<double, 8>>(Simd<double, 8> const& a) { return _mm512_castpd_si512(a.raw); }
232template <> MOCHI_FORCE_INLINE Simd<double, 8> ReinterpretCast<Simd<double, 8>, Simd<float, 16>>(Simd<float, 16> const& a) { return _mm512_castps_pd(a.raw); }
233template <> MOCHI_FORCE_INLINE Simd<float, 16> ReinterpretCast<Simd<float, 16>, Simd<double, 8>>(Simd<double, 8> const& a) { return _mm512_castpd_ps(a.raw); }
234template <> MOCHI_FORCE_INLINE Simd<double, 8> ReinterpretCast<Simd<double, 8>, Simd<int, 16>>(Simd<int, 16> const& a) { return _mm512_castsi512_pd(a.raw); }
235template <> MOCHI_FORCE_INLINE Simd<int, 16> ReinterpretCast<Simd<int, 16>, Simd<double, 8>>(Simd<double, 8> const& a) { return _mm512_castpd_si512(a.raw); }
236template <> MOCHI_FORCE_INLINE Simd<float, 16> ReinterpretCast<Simd<float, 16>, Simd<int64_t, 8>>(Simd<int64_t, 8> const& a) { return _mm512_castsi512_ps(a.raw); }
237template <> MOCHI_FORCE_INLINE Simd<int64_t, 8> ReinterpretCast<Simd<int64_t, 8>, Simd<float, 16>>(Simd<float, 16> const& a) { return _mm512_castps_si512(a.raw); }
238template <> MOCHI_FORCE_INLINE Simd<int, 16> ReinterpretCast<Simd<int, 16>, Simd<int64_t, 8>>(Simd<int64_t, 8> const& a) { return a.raw; }
239template <> MOCHI_FORCE_INLINE Simd<int64_t, 8> ReinterpretCast<Simd<int64_t, 8>, Simd<int, 16>>(Simd<int, 16> const& a) { return a.raw; }
240
241template <> MOCHI_FORCE_INLINE Simd<int, 16> StaticCast<Simd<int, 16>, Simd<float, 16>>(Simd<float, 16> const& a) { return _mm512_cvttps_epi32(a.raw); }
242template <> MOCHI_FORCE_INLINE Simd<float, 16> StaticCast<Simd<float, 16>, Simd<int, 16>>(Simd<int, 16> const& a) { return _mm512_cvtepi32_ps(a.raw); }
243template <> MOCHI_FORCE_INLINE Simd<int64_t, 8> StaticCast<Simd<int64_t, 8>, Simd<double, 8>>(Simd<double, 8> const& a) { return _mm512_cvttpd_epi64(a.raw); }
244template <> MOCHI_FORCE_INLINE Simd<double, 8> StaticCast<Simd<double, 8>, Simd<int64_t, 8>>(Simd<int64_t, 8> const& a) { return _mm512_cvtepi64_pd(a.raw); }
245template <> MOCHI_FORCE_INLINE Simd<float, 8> StaticCast<Simd<float, 8>, Simd<double, 8>>(Simd<double, 8> const& a) { return _mm512_cvtpd_ps(a.raw); }
246template <> MOCHI_FORCE_INLINE Simd<double, 8> StaticCast<Simd<double, 8>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm512_cvtps_pd(a.raw); }
247template <> MOCHI_FORCE_INLINE Simd<int, 8> StaticCast<Simd<int, 8>, Simd<double, 8>>(Simd<double, 8> const& a) { return _mm512_cvttpd_epi32(a.raw); }
248template <> MOCHI_FORCE_INLINE Simd<double, 8> StaticCast<Simd<double, 8>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm512_cvtepi32_pd(a.raw); }
249template <> MOCHI_FORCE_INLINE Simd<int64_t, 8> StaticCast<Simd<int64_t, 8>, Simd<float, 8>>(Simd<float, 8> const& a) { return _mm512_cvttps_epi64(a.raw); }
250template <> MOCHI_FORCE_INLINE Simd<float, 8> StaticCast<Simd<float, 8>, Simd<int64_t, 8>>(Simd<int64_t, 8> const& a) { return _mm512_cvtepi64_ps(a.raw); }
251template <> MOCHI_FORCE_INLINE Simd<int, 8> StaticCast<Simd<int, 8>, Simd<int64_t, 8>>(Simd<int64_t, 8> const& a) { return _mm512_cvtepi64_epi32(a.raw); }
252template <> MOCHI_FORCE_INLINE Simd<int64_t, 8> StaticCast<Simd<int64_t, 8>, Simd<int, 8>>(Simd<int, 8> const& a) { return _mm512_cvtepi32_epi64(a.raw); }
253// clang-format on
254#endif
255
256} // namespace superdex
257
258#endif // MOCHI_USE_SIMD && MOCHI_ARCH_X64_AVX2
MOCHI_FORCE_INLINE
#define MOCHI_FORCE_INLINE
Definition mochi_platform.h:357
superdex::StaticCast
constexpr To StaticCast(From const &a)
Definition basic_utils.h:79
x64_simd_double_2_inl.h
x64_simd_double_4_inl.h
x64_simd_double_8_inl.h
x64_simd_float_16_inl.h
x64_simd_float_4_inl.h
x64_simd_float_8_inl.h
x64_simd_int64_2_inl.h
x64_simd_int64_4_inl.h
x64_simd_int64_8_inl.h
x64_simd_int_16_inl.h
x64_simd_int_4_inl.h
x64_simd_int_8_inl.h
x64_simd_tables_inl.h