Medial Code Documentation
Loading...
Searching...
No Matches
Complex.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_COMPLEX_AVX_H
11#define EIGEN_COMPLEX_AVX_H
12
13namespace Eigen {
14
15namespace internal {
16
17//---------- float ----------
18struct Packet4cf
19{
20 EIGEN_STRONG_INLINE Packet4cf() {}
21 EIGEN_STRONG_INLINE explicit Packet4cf(const __m256& a) : v(a) {}
22 __m256 v;
23};
24
25#ifndef EIGEN_VECTORIZE_AVX512
26template<> struct packet_traits<std::complex<float> > : default_packet_traits
27{
28 typedef Packet4cf type;
29 typedef Packet2cf half;
30 enum {
31 Vectorizable = 1,
32 AlignedOnScalar = 1,
33 size = 4,
34 HasHalfPacket = 1,
35
36 HasAdd = 1,
37 HasSub = 1,
38 HasMul = 1,
39 HasDiv = 1,
40 HasNegate = 1,
41 HasSqrt = 1,
42 HasAbs = 0,
43 HasAbs2 = 0,
44 HasMin = 0,
45 HasMax = 0,
46 HasSetLinear = 0
47 };
48};
49#endif
50
51template<> struct unpacket_traits<Packet4cf> {
52 typedef std::complex<float> type;
53 typedef Packet2cf half;
54 typedef Packet8f as_real;
55 enum {
56 size=4,
57 alignment=Aligned32,
58 vectorizable=true,
59 masked_load_available=false,
60 masked_store_available=false
61 };
62};
63
64template<> EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_add_ps(a.v,b.v)); }
65template<> EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_sub_ps(a.v,b.v)); }
66template<> EIGEN_STRONG_INLINE Packet4cf pnegate(const Packet4cf& a)
67{
68 return Packet4cf(pnegate(a.v));
69}
70template<> EIGEN_STRONG_INLINE Packet4cf pconj(const Packet4cf& a)
71{
72 const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000));
73 return Packet4cf(_mm256_xor_ps(a.v,mask));
74}
75
76template<> EIGEN_STRONG_INLINE Packet4cf pmul<Packet4cf>(const Packet4cf& a, const Packet4cf& b)
77{
78 __m256 tmp1 = _mm256_mul_ps(_mm256_moveldup_ps(a.v), b.v);
79 __m256 tmp2 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1)));
80 __m256 result = _mm256_addsub_ps(tmp1, tmp2);
81 return Packet4cf(result);
82}
83
84template <>
85EIGEN_STRONG_INLINE Packet4cf pcmp_eq(const Packet4cf& a, const Packet4cf& b) {
86 __m256 eq = _mm256_cmp_ps(a.v, b.v, _CMP_EQ_OQ);
87 return Packet4cf(_mm256_and_ps(eq, _mm256_permute_ps(eq, 0xb1)));
88}
89
90template<> EIGEN_STRONG_INLINE Packet4cf ptrue<Packet4cf>(const Packet4cf& a) { return Packet4cf(ptrue(Packet8f(a.v))); }
91template<> EIGEN_STRONG_INLINE Packet4cf pand <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_and_ps(a.v,b.v)); }
92template<> EIGEN_STRONG_INLINE Packet4cf por <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_or_ps(a.v,b.v)); }
93template<> EIGEN_STRONG_INLINE Packet4cf pxor <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_xor_ps(a.v,b.v)); }
94template<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(b.v,a.v)); }
95
96template<> EIGEN_STRONG_INLINE Packet4cf pload <Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(pload<Packet8f>(&numext::real_ref(*from))); }
97template<> EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(ploadu<Packet8f>(&numext::real_ref(*from))); }
98
99
100template<> EIGEN_STRONG_INLINE Packet4cf pset1<Packet4cf>(const std::complex<float>& from)
101{
102 const float re = std::real(from);
103 const float im = std::imag(from);
104 return Packet4cf(_mm256_set_ps(im, re, im, re, im, re, im, re));
105}
106
107template<> EIGEN_STRONG_INLINE Packet4cf ploaddup<Packet4cf>(const std::complex<float>* from)
108{
109 // FIXME The following might be optimized using _mm256_movedup_pd
110 Packet2cf a = ploaddup<Packet2cf>(from);
111 Packet2cf b = ploaddup<Packet2cf>(from+1);
112 return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1));
113}
114
115template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); }
116template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); }
117
118template<> EIGEN_DEVICE_FUNC inline Packet4cf pgather<std::complex<float>, Packet4cf>(const std::complex<float>* from, Index stride)
119{
120 return Packet4cf(_mm256_set_ps(std::imag(from[3*stride]), std::real(from[3*stride]),
121 std::imag(from[2*stride]), std::real(from[2*stride]),
122 std::imag(from[1*stride]), std::real(from[1*stride]),
123 std::imag(from[0*stride]), std::real(from[0*stride])));
124}
125
126template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from, Index stride)
127{
128 __m128 low = _mm256_extractf128_ps(from.v, 0);
129 to[stride*0] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)),
130 _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1)));
131 to[stride*1] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)),
132 _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3)));
133
134 __m128 high = _mm256_extractf128_ps(from.v, 1);
135 to[stride*2] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)),
136 _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1)));
137 to[stride*3] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)),
138 _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3)));
139
140}
141
142template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet4cf>(const Packet4cf& a)
143{
144 return pfirst(Packet2cf(_mm256_castps256_ps128(a.v)));
145}
146
147template<> EIGEN_STRONG_INLINE Packet4cf preverse(const Packet4cf& a) {
148 __m128 low = _mm256_extractf128_ps(a.v, 0);
149 __m128 high = _mm256_extractf128_ps(a.v, 1);
150 __m128d lowd = _mm_castps_pd(low);
151 __m128d highd = _mm_castps_pd(high);
152 low = _mm_castpd_ps(_mm_shuffle_pd(lowd,lowd,0x1));
153 high = _mm_castpd_ps(_mm_shuffle_pd(highd,highd,0x1));
154 __m256 result = _mm256_setzero_ps();
155 result = _mm256_insertf128_ps(result, low, 1);
156 result = _mm256_insertf128_ps(result, high, 0);
157 return Packet4cf(result);
158}
159
160template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet4cf>(const Packet4cf& a)
161{
162 return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v,0)),
163 Packet2cf(_mm256_extractf128_ps(a.v,1))));
164}
165
166template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet4cf>(const Packet4cf& a)
167{
168 return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)),
169 Packet2cf(_mm256_extractf128_ps(a.v, 1))));
170}
171
172
173EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cf,Packet8f)
174
175template<> EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(const Packet4cf& a, const Packet4cf& b)
176{
177 return pdiv_complex(a, b);
178}
179
180template<> EIGEN_STRONG_INLINE Packet4cf pcplxflip<Packet4cf>(const Packet4cf& x)
181{
182 return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1)));
183}
184
185//---------- double ----------
186struct Packet2cd
187{
188 EIGEN_STRONG_INLINE Packet2cd() {}
189 EIGEN_STRONG_INLINE explicit Packet2cd(const __m256d& a) : v(a) {}
190 __m256d v;
191};
192
193#ifndef EIGEN_VECTORIZE_AVX512
194template<> struct packet_traits<std::complex<double> > : default_packet_traits
195{
196 typedef Packet2cd type;
197 typedef Packet1cd half;
198 enum {
199 Vectorizable = 1,
200 AlignedOnScalar = 0,
201 size = 2,
202 HasHalfPacket = 1,
203
204 HasAdd = 1,
205 HasSub = 1,
206 HasMul = 1,
207 HasDiv = 1,
208 HasNegate = 1,
209 HasSqrt = 1,
210 HasAbs = 0,
211 HasAbs2 = 0,
212 HasMin = 0,
213 HasMax = 0,
214 HasSetLinear = 0
215 };
216};
217#endif
218
219template<> struct unpacket_traits<Packet2cd> {
220 typedef std::complex<double> type;
221 typedef Packet1cd half;
222 typedef Packet4d as_real;
223 enum {
224 size=2,
225 alignment=Aligned32,
226 vectorizable=true,
227 masked_load_available=false,
228 masked_store_available=false
229 };
230};
231
232template<> EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_add_pd(a.v,b.v)); }
233template<> EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_sub_pd(a.v,b.v)); }
234template<> EIGEN_STRONG_INLINE Packet2cd pnegate(const Packet2cd& a) { return Packet2cd(pnegate(a.v)); }
235template<> EIGEN_STRONG_INLINE Packet2cd pconj(const Packet2cd& a)
236{
237 const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0));
238 return Packet2cd(_mm256_xor_pd(a.v,mask));
239}
240
241template<> EIGEN_STRONG_INLINE Packet2cd pmul<Packet2cd>(const Packet2cd& a, const Packet2cd& b)
242{
243 __m256d tmp1 = _mm256_shuffle_pd(a.v,a.v,0x0);
244 __m256d even = _mm256_mul_pd(tmp1, b.v);
245 __m256d tmp2 = _mm256_shuffle_pd(a.v,a.v,0xF);
246 __m256d tmp3 = _mm256_shuffle_pd(b.v,b.v,0x5);
247 __m256d odd = _mm256_mul_pd(tmp2, tmp3);
248 return Packet2cd(_mm256_addsub_pd(even, odd));
249}
250
251template <>
252EIGEN_STRONG_INLINE Packet2cd pcmp_eq(const Packet2cd& a, const Packet2cd& b) {
253 __m256d eq = _mm256_cmp_pd(a.v, b.v, _CMP_EQ_OQ);
254 return Packet2cd(pand(eq, _mm256_permute_pd(eq, 0x5)));
255}
256
257template<> EIGEN_STRONG_INLINE Packet2cd ptrue<Packet2cd>(const Packet2cd& a) { return Packet2cd(ptrue(Packet4d(a.v))); }
258template<> EIGEN_STRONG_INLINE Packet2cd pand <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_and_pd(a.v,b.v)); }
259template<> EIGEN_STRONG_INLINE Packet2cd por <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_or_pd(a.v,b.v)); }
260template<> EIGEN_STRONG_INLINE Packet2cd pxor <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_xor_pd(a.v,b.v)); }
261template<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(b.v,a.v)); }
262
263template<> EIGEN_STRONG_INLINE Packet2cd pload <Packet2cd>(const std::complex<double>* from)
264{ EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(pload<Packet4d>((const double*)from)); }
265template<> EIGEN_STRONG_INLINE Packet2cd ploadu<Packet2cd>(const std::complex<double>* from)
266{ EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(ploadu<Packet4d>((const double*)from)); }
267
268template<> EIGEN_STRONG_INLINE Packet2cd pset1<Packet2cd>(const std::complex<double>& from)
269{
270 // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though)
271// return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from));
272 return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from));
273}
274
275template<> EIGEN_STRONG_INLINE Packet2cd ploaddup<Packet2cd>(const std::complex<double>* from) { return pset1<Packet2cd>(*from); }
276
277template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); }
278template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); }
279
280template<> EIGEN_DEVICE_FUNC inline Packet2cd pgather<std::complex<double>, Packet2cd>(const std::complex<double>* from, Index stride)
281{
282 return Packet2cd(_mm256_set_pd(std::imag(from[1*stride]), std::real(from[1*stride]),
283 std::imag(from[0*stride]), std::real(from[0*stride])));
284}
285
286template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from, Index stride)
287{
288 __m128d low = _mm256_extractf128_pd(from.v, 0);
289 to[stride*0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1)));
290 __m128d high = _mm256_extractf128_pd(from.v, 1);
291 to[stride*1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)));
292}
293
294template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2cd>(const Packet2cd& a)
295{
296 __m128d low = _mm256_extractf128_pd(a.v, 0);
297 EIGEN_ALIGN16 double res[2];
298 _mm_store_pd(res, low);
299 return std::complex<double>(res[0],res[1]);
300}
301
302template<> EIGEN_STRONG_INLINE Packet2cd preverse(const Packet2cd& a) {
303 __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1);
304 return Packet2cd(result);
305}
306
307template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet2cd>(const Packet2cd& a)
308{
309 return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v,0)),
310 Packet1cd(_mm256_extractf128_pd(a.v,1))));
311}
312
313template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet2cd>(const Packet2cd& a)
314{
315 return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v,0)),
316 Packet1cd(_mm256_extractf128_pd(a.v,1))));
317}
318
319EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cd,Packet4d)
320
321template<> EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(const Packet2cd& a, const Packet2cd& b)
322{
323 return pdiv_complex(a, b);
324}
325
326template<> EIGEN_STRONG_INLINE Packet2cd pcplxflip<Packet2cd>(const Packet2cd& x)
327{
328 return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5));
329}
330
331EIGEN_DEVICE_FUNC inline void
332ptranspose(PacketBlock<Packet4cf,4>& kernel) {
333 __m256d P0 = _mm256_castps_pd(kernel.packet[0].v);
334 __m256d P1 = _mm256_castps_pd(kernel.packet[1].v);
335 __m256d P2 = _mm256_castps_pd(kernel.packet[2].v);
336 __m256d P3 = _mm256_castps_pd(kernel.packet[3].v);
337
338 __m256d T0 = _mm256_shuffle_pd(P0, P1, 15);
339 __m256d T1 = _mm256_shuffle_pd(P0, P1, 0);
340 __m256d T2 = _mm256_shuffle_pd(P2, P3, 15);
341 __m256d T3 = _mm256_shuffle_pd(P2, P3, 0);
342
343 kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32));
344 kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49));
345 kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32));
346 kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49));
347}
348
349EIGEN_DEVICE_FUNC inline void
350ptranspose(PacketBlock<Packet2cd,2>& kernel) {
351 __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0+(2<<4));
352 kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1+(3<<4));
353 kernel.packet[0].v = tmp;
354}
355
356template<> EIGEN_STRONG_INLINE Packet2cd psqrt<Packet2cd>(const Packet2cd& a) {
357 return psqrt_complex<Packet2cd>(a);
358}
359
360template<> EIGEN_STRONG_INLINE Packet4cf psqrt<Packet4cf>(const Packet4cf& a) {
361 return psqrt_complex<Packet4cf>(a);
362}
363
364} // end namespace internal
365
366} // end namespace Eigen
367
368#endif // EIGEN_COMPLEX_AVX_H
Eigen::MatrixBase
Base class for all dense matrices, vectors, and expressions.
Definition MatrixBase.h:50
Eigen::Aligned32
@ Aligned32
Data pointer is aligned on a 32 bytes boundary.
Definition Constants.h:236
Eigen
Namespace containing all symbols from the Eigen library.
Definition LDLT.h:16
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:74
std
Definition BFloat16.h:88
Eigen::half
Definition Half.h:140
Eigen::internal::Packet1cd
Definition Complex.h:338
Eigen::internal::Packet2cd
Definition Complex.h:187
Eigen::internal::Packet2cf
Definition Complex.h:33
Eigen::internal::Packet4cf
Definition Complex.h:19
Eigen::internal::default_packet_traits
Definition GenericPacketMath.h:43
Eigen::internal::packet_traits
Definition GenericPacketMath.h:107
Eigen::internal::unpacket_traits
Definition GenericPacketMath.h:133
Generated on Thu Dec 18 2025 18:56:06 for Medial Code Documentation by doxygen 1.9.8