Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Wave Computing, Inc.
5// Written by:
6// Chris Larsen
7// Alexey Frunze (afrunze@wavecomp.com)
8//
9// This Source Code Form is subject to the terms of the Mozilla
10// Public License v. 2.0. If a copy of the MPL was not distributed
11// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
12// SPDX-License-Identifier: MPL-2.0
13
14#ifndef EIGEN_PACKET_MATH_MSA_H
15#define EIGEN_PACKET_MATH_MSA_H
16
17#include <iostream>
18#include <string>
19
20// IWYU pragma: private
21#include "../../InternalHeaderCheck.h"
22
23namespace Eigen {
24
25namespace internal {
26
27#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
28#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
29#endif
30
31#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
32#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
33#endif
34
35#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
36#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
37#endif
38
39#define EIGEN_MSA_DEBUG
40
41#define EIGEN_MSA_SHF_I8(a, b, c, d) (((d) << 6) | ((c) << 4) | ((b) << 2) | (a))
42
43using Packet4f = v4f32;
44using Packet4i = v4i32;
45using Packet4ui = v4u32;
46
47#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = {X, X, X, X}
48#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = {X, X, X, X}
49#define EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = {X, X, X, X}
50
51inline std::ostream& operator<<(std::ostream& os, const Packet4f& value) {
52 os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]";
53 return os;
54}
55
56inline std::ostream& operator<<(std::ostream& os, const Packet4i& value) {
57 os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]";
58 return os;
59}
60
61inline std::ostream& operator<<(std::ostream& os, const Packet4ui& value) {
62 os << "[ " << value[0] << ", " << value[1] << ", " << value[2] << ", " << value[3] << " ]";
63 return os;
64}
65
66template <>
67struct packet_traits<float> : default_packet_traits {
68 using type = Packet4f;
69 using half = Packet4f; // Packet2f intrinsics not implemented yet
70 enum {
71 Vectorizable = 1,
72 AlignedOnScalar = 1,
73 size = 4,
74 // FIXME: verify the Has* flags.
75 HasDiv = 1,
76 HasSin = EIGEN_FAST_MATH,
77 HasCos = EIGEN_FAST_MATH,
78 HasTanh = EIGEN_FAST_MATH,
79 HasErf = EIGEN_FAST_MATH,
80 HasLog = 1,
81 HasExp = 1,
82 HasSqrt = 1,
83 HasRsqrt = 1,
84 };
85};
86
87template <>
88struct packet_traits<int32_t> : default_packet_traits {
89 using type = Packet4i;
90 using half = Packet4i; // Packet2i intrinsics not implemented yet
91 enum {
92 Vectorizable = 1,
93 AlignedOnScalar = 1,
94 size = 4,
95 // FIXME: verify the Has* flags.
96 HasDiv = 1,
97 };
98};
99
100template <>
101struct unpacket_traits<Packet4f> {
102 using type = float;
103 enum {
104 size = 4,
105 alignment = Aligned16,
106 vectorizable = true,
107 masked_load_available = false,
108 masked_store_available = false
109 };
110 using half = Packet4f;
111 using integer_packet = Packet4i;
112};
113
114template <>
115struct unpacket_traits<Packet4i> {
116 using type = int32_t;
117 enum {
118 size = 4,
119 alignment = Aligned16,
120 vectorizable = true,
121 masked_load_available = false,
122 masked_store_available = false
123 };
124 using half = Packet4i;
125};
126
127template <>
128EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
129 EIGEN_MSA_DEBUG;
130
131 Packet4f v = {from, from, from, from};
132 return v;
133}
134
135template <>
136EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) {
137 EIGEN_MSA_DEBUG;
138
139 return __builtin_msa_fill_w(from);
140}
141
142template <>
143EIGEN_STRONG_INLINE Packet4f pload1<Packet4f>(const float* from) {
144 EIGEN_MSA_DEBUG;
145
146 float f = *from;
147 Packet4f v = {f, f, f, f};
148 return v;
149}
150
151template <>
152EIGEN_STRONG_INLINE Packet4i pload1<Packet4i>(const int32_t* from) {
153 EIGEN_MSA_DEBUG;
154
155 return __builtin_msa_fill_w(*from);
156}
157
158template <>
159EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
160 EIGEN_MSA_DEBUG;
161
162 return __builtin_msa_fadd_w(a, b);
163}
164
165template <>
166EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
167 EIGEN_MSA_DEBUG;
168
169 return __builtin_msa_addv_w(a, b);
170}
171
172template <>
173EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
174 EIGEN_MSA_DEBUG;
175
176 static const Packet4f countdown = {0.0f, 1.0f, 2.0f, 3.0f};
177 return padd(pset1<Packet4f>(a), countdown);
178}
179
180template <>
181EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) {
182 EIGEN_MSA_DEBUG;
183
184 static const Packet4i countdown = {0, 1, 2, 3};
185 return padd(pset1<Packet4i>(a), countdown);
186}
187
188template <>
189EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
190 EIGEN_MSA_DEBUG;
191
192 return __builtin_msa_fsub_w(a, b);
193}
194
195template <>
196EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
197 EIGEN_MSA_DEBUG;
198
199 return __builtin_msa_subv_w(a, b);
200}
201
202template <>
203EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
204 EIGEN_MSA_DEBUG;
205
206 return (Packet4f)__builtin_msa_bnegi_w((v4u32)a, 31);
207}
208
209template <>
210EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
211 EIGEN_MSA_DEBUG;
212
213 return __builtin_msa_addvi_w((v4i32)__builtin_msa_nori_b((v16u8)a, 0), 1);
214}
215
216template <>
217EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
218 EIGEN_MSA_DEBUG;
219
220 return __builtin_msa_fmul_w(a, b);
221}
222
223template <>
224EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
225 EIGEN_MSA_DEBUG;
226
227 return __builtin_msa_mulv_w(a, b);
228}
229
230template <>
231EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
232 EIGEN_MSA_DEBUG;
233
234 return __builtin_msa_fdiv_w(a, b);
235}
236
237template <>
238EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
239 EIGEN_MSA_DEBUG;
240
241 return __builtin_msa_div_s_w(a, b);
242}
243
244template <>
245struct is_fused_madd<Packet4f> : true_type {};
246template <>
247EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
248 EIGEN_MSA_DEBUG;
249
250 return __builtin_msa_fmadd_w(c, a, b);
251}
252
253template <>
254EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
255 EIGEN_MSA_DEBUG;
256
257 // Use "asm" construct to avoid __builtin_msa_maddv_w GNU C bug.
258 Packet4i value = c;
259 __asm__("maddv.w %w[value], %w[a], %w[b]\n"
260 // Outputs
261 : [value] "+f"(value)
262 // Inputs
263 : [a] "f"(a), [b] "f"(b));
264 return value;
265}
266
267template <>
268EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
269 EIGEN_MSA_DEBUG;
270
271 return (Packet4f)__builtin_msa_and_v((v16u8)a, (v16u8)b);
272}
273
274template <>
275EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
276 EIGEN_MSA_DEBUG;
277
278 return (Packet4i)__builtin_msa_and_v((v16u8)a, (v16u8)b);
279}
280
281template <>
282EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
283 EIGEN_MSA_DEBUG;
284
285 return (Packet4f)__builtin_msa_or_v((v16u8)a, (v16u8)b);
286}
287
288template <>
289EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
290 EIGEN_MSA_DEBUG;
291
292 return (Packet4i)__builtin_msa_or_v((v16u8)a, (v16u8)b);
293}
294
295template <>
296EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
297 EIGEN_MSA_DEBUG;
298
299 return (Packet4f)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
300}
301
302template <>
303EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
304 EIGEN_MSA_DEBUG;
305
306 return (Packet4i)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
307}
308
309template <>
310EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
311 EIGEN_MSA_DEBUG;
312
313 return pand(a, (Packet4f)__builtin_msa_xori_b((v16u8)b, 255));
314}
315
316template <>
317EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
318 EIGEN_MSA_DEBUG;
319
320 return pand(a, (Packet4i)__builtin_msa_xori_b((v16u8)b, 255));
321}
322
323template <>
324EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
325 EIGEN_MSA_DEBUG;
326
327#if EIGEN_FAST_MATH
328 // This prefers numbers to NaNs.
329 return __builtin_msa_fmin_w(a, b);
330#else
331 // This prefers NaNs to numbers.
332 Packet4i aNaN = __builtin_msa_fcun_w(a, a);
333 Packet4i aMinOrNaN = por(__builtin_msa_fclt_w(a, b), aNaN);
334 return (Packet4f)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a);
335#endif
336}
337
338template <>
339EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
340 EIGEN_MSA_DEBUG;
341
342 return __builtin_msa_min_s_w(a, b);
343}
344
345template <>
346EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
347 EIGEN_MSA_DEBUG;
348
349#if EIGEN_FAST_MATH
350 // This prefers numbers to NaNs.
351 return __builtin_msa_fmax_w(a, b);
352#else
353 // This prefers NaNs to numbers.
354 Packet4i aNaN = __builtin_msa_fcun_w(a, a);
355 Packet4i aMaxOrNaN = por(__builtin_msa_fclt_w(b, a), aNaN);
356 return (Packet4f)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a);
357#endif
358}
359
360template <>
361EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
362 EIGEN_MSA_DEBUG;
363
364 return __builtin_msa_max_s_w(a, b);
365}
366
367template <>
368EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
369 EIGEN_MSA_DEBUG;
370
371 EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__builtin_msa_ld_w(const_cast<float*>(from), 0);
372}
373
374template <>
375EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) {
376 EIGEN_MSA_DEBUG;
377
378 EIGEN_DEBUG_ALIGNED_LOAD return __builtin_msa_ld_w(const_cast<int32_t*>(from), 0);
379}
380
381template <>
382EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
383 EIGEN_MSA_DEBUG;
384
385 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f)__builtin_msa_ld_w(const_cast<float*>(from), 0);
386}
387
388template <>
389EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) {
390 EIGEN_MSA_DEBUG;
391
392 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4i)__builtin_msa_ld_w(const_cast<int32_t*>(from), 0);
393}
394
395template <>
396EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
397 EIGEN_MSA_DEBUG;
398
399 float f0 = from[0], f1 = from[1];
400 Packet4f v0 = {f0, f0, f0, f0};
401 Packet4f v1 = {f1, f1, f1, f1};
402 return (Packet4f)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0);
403}
404
405template <>
406EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) {
407 EIGEN_MSA_DEBUG;
408
409 int32_t i0 = from[0], i1 = from[1];
410 Packet4i v0 = {i0, i0, i0, i0};
411 Packet4i v1 = {i1, i1, i1, i1};
412 return (Packet4i)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0);
413}
414
415template <>
416EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
417 EIGEN_MSA_DEBUG;
418
419 EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0);
420}
421
422template <>
423EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) {
424 EIGEN_MSA_DEBUG;
425
426 EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w(from, to, 0);
427}
428
429template <>
430EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
431 EIGEN_MSA_DEBUG;
432
433 EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0);
434}
435
436template <>
437EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) {
438 EIGEN_MSA_DEBUG;
439
440 EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w(from, to, 0);
441}
442
443template <>
444EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
445 EIGEN_MSA_DEBUG;
446
447 float f = *from;
448 Packet4f v = {f, f, f, f};
449 v[1] = from[stride];
450 v[2] = from[2 * stride];
451 v[3] = from[3 * stride];
452 return v;
453}
454
455template <>
456EIGEN_DEVICE_FUNC inline Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) {
457 EIGEN_MSA_DEBUG;
458
459 int32_t i = *from;
460 Packet4i v = {i, i, i, i};
461 v[1] = from[stride];
462 v[2] = from[2 * stride];
463 v[3] = from[3 * stride];
464 return v;
465}
466
467template <>
468EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
469 EIGEN_MSA_DEBUG;
470
471 *to = from[0];
472 to += stride;
473 *to = from[1];
474 to += stride;
475 *to = from[2];
476 to += stride;
477 *to = from[3];
478}
479
480template <>
481EIGEN_DEVICE_FUNC inline void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from, Index stride) {
482 EIGEN_MSA_DEBUG;
483
484 *to = from[0];
485 to += stride;
486 *to = from[1];
487 to += stride;
488 *to = from[2];
489 to += stride;
490 *to = from[3];
491}
492
493template <>
494EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
495 EIGEN_MSA_DEBUG;
496
497 __builtin_prefetch(addr);
498}
499
500template <>
501EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) {
502 EIGEN_MSA_DEBUG;
503
504 __builtin_prefetch(addr);
505}
506
507template <>
508EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
509 EIGEN_MSA_DEBUG;
510
511 return a[0];
512}
513
514template <>
515EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) {
516 EIGEN_MSA_DEBUG;
517
518 return a[0];
519}
520
521template <>
522EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
523 EIGEN_MSA_DEBUG;
524
525 return (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(3, 2, 1, 0));
526}
527
528template <>
529EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
530 EIGEN_MSA_DEBUG;
531
532 return __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(3, 2, 1, 0));
533}
534
535template <>
536EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) {
537 EIGEN_MSA_DEBUG;
538
539 return (Packet4f)__builtin_msa_bclri_w((v4u32)a, 31);
540}
541
542template <>
543EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) {
544 EIGEN_MSA_DEBUG;
545
546 Packet4i zero = __builtin_msa_ldi_w(0);
547 return __builtin_msa_add_a_w(zero, a);
548}
549
550template <int N>
551EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
552 return __builtin_msa_srai_w(a, N);
553}
554
555template <int N>
556EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
557 return __builtin_msa_srli_w(a, N);
558}
559
560template <int N>
561EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
562 return __builtin_msa_slli_w(a, N);
563}
564
565template <>
566EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
567 return __builtin_msa_ftrunc_s_w(a);
568}
569
570template <>
571EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
572 return pldexp_generic(a, exponent);
573}
574
575template <>
576EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
577 EIGEN_MSA_DEBUG;
578
579 Packet4f s = padd(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
580 s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
581 return s[0];
582}
583
584template <>
585EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) {
586 return __builtin_msa_bnz_v((v16u8)a);
587}
588
589template <>
590EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) {
591 EIGEN_MSA_DEBUG;
592
593 Packet4i s = padd(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
594 s = padd(s, __builtin_msa_shf_w(s, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
595 return s[0];
596}
597
598template <>
599EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) {
600 return __builtin_msa_bnz_v((v16u8)a);
601}
602
603// Other reduction functions:
604// mul
605template <>
606EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
607 EIGEN_MSA_DEBUG;
608
609 Packet4f p = pmul(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
610 p = pmul(p, (Packet4f)__builtin_msa_shf_w((v4i32)p, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
611 return p[0];
612}
613
614template <>
615EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) {
616 EIGEN_MSA_DEBUG;
617
618 Packet4i p = pmul(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
619 p = pmul(p, __builtin_msa_shf_w(p, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
620 return p[0];
621}
622
623// min
624template <>
625EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
626 EIGEN_MSA_DEBUG;
627
628 // Swap 64-bit halves of a.
629 Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
630#if !EIGEN_FAST_MATH
631 // Detect presence of NaNs from pairs a[0]-a[2] and a[1]-a[3] as two 32-bit
632 // masks of all zeroes/ones in low 64 bits.
633 v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped);
634 // Combine the two masks into one: 64 ones if no NaNs, otherwise 64 zeroes.
635 unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0);
636#endif
637 // Continue with min computation.
638 Packet4f v = __builtin_msa_fmin_w(a, swapped);
639 v = __builtin_msa_fmin_w(v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
640#if !EIGEN_FAST_MATH
641 // Based on the mask select between v and 4 qNaNs.
642 v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000);
643 v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v);
644#endif
645 return v[0];
646}
647
648template <>
649EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) {
650 EIGEN_MSA_DEBUG;
651
652 Packet4i m = pmin(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
653 m = pmin(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
654 return m[0];
655}
656
657// max
658template <>
659EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
660 EIGEN_MSA_DEBUG;
661
662 // Swap 64-bit halves of a.
663 Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
664#if !EIGEN_FAST_MATH
665 // Detect presence of NaNs from pairs a[0]-a[2] and a[1]-a[3] as two 32-bit
666 // masks of all zeroes/ones in low 64 bits.
667 v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped);
668 // Combine the two masks into one: 64 ones if no NaNs, otherwise 64 zeroes.
669 unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0);
670#endif
671 // Continue with max computation.
672 Packet4f v = __builtin_msa_fmax_w(a, swapped);
673 v = __builtin_msa_fmax_w(v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
674#if !EIGEN_FAST_MATH
675 // Based on the mask select between v and 4 qNaNs.
676 v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000);
677 v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v);
678#endif
679 return v[0];
680}
681
682template <>
683EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) {
684 EIGEN_MSA_DEBUG;
685
686 Packet4i m = pmax(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
687 m = pmax(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
688 return m[0];
689}
690
691inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet4f, 4>& value) {
692 os << "[ " << value.packet[0] << "," << std::endl
693 << " " << value.packet[1] << "," << std::endl
694 << " " << value.packet[2] << "," << std::endl
695 << " " << value.packet[3] << " ]";
696 return os;
697}
698
699EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
700 EIGEN_MSA_DEBUG;
701
702 v4i32 tmp1, tmp2, tmp3, tmp4;
703
704 tmp1 = __builtin_msa_ilvr_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]);
705 tmp2 = __builtin_msa_ilvr_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]);
706 tmp3 = __builtin_msa_ilvl_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]);
707 tmp4 = __builtin_msa_ilvl_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]);
708
709 kernel.packet[0] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1);
710 kernel.packet[1] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1);
711 kernel.packet[2] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3);
712 kernel.packet[3] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3);
713}
714
715inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet4i, 4>& value) {
716 os << "[ " << value.packet[0] << "," << std::endl
717 << " " << value.packet[1] << "," << std::endl
718 << " " << value.packet[2] << "," << std::endl
719 << " " << value.packet[3] << " ]";
720 return os;
721}
722
723EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
724 EIGEN_MSA_DEBUG;
725
726 v4i32 tmp1, tmp2, tmp3, tmp4;
727
728 tmp1 = __builtin_msa_ilvr_w(kernel.packet[1], kernel.packet[0]);
729 tmp2 = __builtin_msa_ilvr_w(kernel.packet[3], kernel.packet[2]);
730 tmp3 = __builtin_msa_ilvl_w(kernel.packet[1], kernel.packet[0]);
731 tmp4 = __builtin_msa_ilvl_w(kernel.packet[3], kernel.packet[2]);
732
733 kernel.packet[0] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1);
734 kernel.packet[1] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1);
735 kernel.packet[2] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3);
736 kernel.packet[3] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3);
737}
738
739template <>
740EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
741 EIGEN_MSA_DEBUG;
742
743 return __builtin_msa_fsqrt_w(a);
744}
745
746template <>
747EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) {
748 EIGEN_MSA_DEBUG;
749
750#if EIGEN_FAST_MATH
751 return __builtin_msa_frsqrt_w(a);
752#else
753 Packet4f ones = __builtin_msa_ffint_s_w(__builtin_msa_ldi_w(1));
754 return pdiv(ones, psqrt(a));
755#endif
756}
757
758template <>
759EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
760 Packet4f v = a;
761 int32_t old_mode, new_mode;
762 asm volatile(
763 "cfcmsa %[old_mode], $1\n"
764 "ori %[new_mode], %[old_mode], 3\n" // 3 = round towards -INFINITY.
765 "ctcmsa $1, %[new_mode]\n"
766 "frint.w %w[v], %w[v]\n"
767 "ctcmsa $1, %[old_mode]\n"
768 : // outputs
769 [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
770 [v] "+f"(v)
771 : // inputs
772 : // clobbers
773 );
774 return v;
775}
776
777template <>
778EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
779 Packet4f v = a;
780 int32_t old_mode, new_mode;
781 asm volatile(
782 "cfcmsa %[old_mode], $1\n"
783 "ori %[new_mode], %[old_mode], 3\n"
784 "xori %[new_mode], %[new_mode], 1\n" // 2 = round towards +INFINITY.
785 "ctcmsa $1, %[new_mode]\n"
786 "frint.w %w[v], %w[v]\n"
787 "ctcmsa $1, %[old_mode]\n"
788 : // outputs
789 [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
790 [v] "+f"(v)
791 : // inputs
792 : // clobbers
793 );
794 return v;
795}
796
797template <>
798EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
799 // frint.w uses the current rounding mode (default: round to nearest, ties to even).
800 Packet4f v = a;
801 asm volatile("frint.w %w[v], %w[v]\n" : [v] "+f"(v));
802 return v;
803}
804
805template <>
806EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
807 Packet4f v = a;
808 int32_t old_mode, new_mode;
809 asm volatile(
810 "cfcmsa %[old_mode], $1\n"
811 "ori %[new_mode], %[old_mode], 3\n"
812 "xori %[new_mode], %[new_mode], 2\n" // 1 = round toward zero.
813 "ctcmsa $1, %[new_mode]\n"
814 "frint.w %w[v], %w[v]\n"
815 "ctcmsa $1, %[old_mode]\n"
816 : // outputs
817 [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
818 [v] "+f"(v)
819 : // inputs
820 : // clobbers
821 );
822 return v;
823}
824
825template <>
826EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) {
827 return (Packet4f)__builtin_msa_fcult_w(a, b);
828}
829
830//---------- double ----------
831
832using Packet2d = v2f64;
833using Packet2l = v2i64;
834using Packet2ul = v2u64;
835
836#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = {X, X}
837#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) const Packet2l p2l_##NAME = {X, X}
838#define EIGEN_DECLARE_CONST_Packet2ul(NAME, X) const Packet2ul p2ul_##NAME = {X, X}
839
840inline std::ostream& operator<<(std::ostream& os, const Packet2d& value) {
841 os << "[ " << value[0] << ", " << value[1] << " ]";
842 return os;
843}
844
845inline std::ostream& operator<<(std::ostream& os, const Packet2l& value) {
846 os << "[ " << value[0] << ", " << value[1] << " ]";
847 return os;
848}
849
850inline std::ostream& operator<<(std::ostream& os, const Packet2ul& value) {
851 os << "[ " << value[0] << ", " << value[1] << " ]";
852 return os;
853}
854
855template <>
856struct packet_traits<double> : default_packet_traits {
857 using type = Packet2d;
858 using half = Packet2d;
859 enum {
860 Vectorizable = 1,
861 AlignedOnScalar = 1,
862 size = 2,
863 // FIXME: verify the Has* flags.
864 HasDiv = 1,
865 HasExp = 1,
866 HasSqrt = 1,
867 HasRsqrt = 1,
868 };
869};
870
871template <>
872struct unpacket_traits<Packet2d> {
873 using type = double;
874 enum {
875 size = 2,
876 alignment = Aligned16,
877 vectorizable = true,
878 masked_load_available = false,
879 masked_store_available = false
880 };
881 using half = Packet2d;
882 using integer_packet = Packet2l;
883};
884
885template <>
886struct unpacket_traits<Packet2l> {
887 using type = int64_t;
888 enum {
889 size = 2,
890 alignment = Aligned16,
891 vectorizable = true,
892 masked_load_available = false,
893 masked_store_available = false
894 };
895 using half = Packet2l;
896};
897
898template <>
899EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
900 return __builtin_msa_fill_d(from);
901}
902
903template <int N>
904EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
905 return __builtin_msa_srli_d(a, N);
906}
907
908template <int N>
909EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
910 return __builtin_msa_slli_d(a, N);
911}
912
913template <>
914EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
915 return __builtin_msa_ftrunc_s_d(a);
916}
917
918template <>
919EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
920 return (Packet2l)__builtin_msa_and_v((v16u8)a, (v16u8)b);
921}
922
923template <>
924EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
925 return (Packet2l)__builtin_msa_or_v((v16u8)a, (v16u8)b);
926}
927
928template <>
929EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
930 return (Packet2l)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
931}
932
933template <>
934EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
935 return pand(a, (Packet2l)__builtin_msa_xori_b((v16u8)b, 255));
936}
937
938template <>
939EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
940 EIGEN_MSA_DEBUG;
941
942 Packet2d value = {from, from};
943 return value;
944}
945
946template <>
947EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
948 EIGEN_MSA_DEBUG;
949
950 return __builtin_msa_fadd_d(a, b);
951}
952
953template <>
954EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
955 EIGEN_MSA_DEBUG;
956
957 static const Packet2d countdown = {0.0, 1.0};
958 return padd(pset1<Packet2d>(a), countdown);
959}
960
961template <>
962EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
963 EIGEN_MSA_DEBUG;
964
965 return __builtin_msa_fsub_d(a, b);
966}
967
968template <>
969EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
970 EIGEN_MSA_DEBUG;
971
972 return (Packet2d)__builtin_msa_bnegi_d((v2u64)a, 63);
973}
974
975template <>
976EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
977 EIGEN_MSA_DEBUG;
978
979 return __builtin_msa_fmul_d(a, b);
980}
981
982template <>
983EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
984 EIGEN_MSA_DEBUG;
985
986 return __builtin_msa_fdiv_d(a, b);
987}
988
989template <>
990struct is_fused_madd<Packet2d> : true_type {};
991template <>
992EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
993 EIGEN_MSA_DEBUG;
994
995 return __builtin_msa_fmadd_d(c, a, b);
996}
997
998// Logical Operations are not supported for float, so we have to reinterpret casts using MSA
999// intrinsics
1000template <>
1001EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
1002 EIGEN_MSA_DEBUG;
1003
1004 return (Packet2d)__builtin_msa_and_v((v16u8)a, (v16u8)b);
1005}
1006
1007template <>
1008EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
1009 EIGEN_MSA_DEBUG;
1010
1011 return (Packet2d)__builtin_msa_or_v((v16u8)a, (v16u8)b);
1012}
1013
1014template <>
1015EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
1016 EIGEN_MSA_DEBUG;
1017
1018 return (Packet2d)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
1019}
1020
1021template <>
1022EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
1023 EIGEN_MSA_DEBUG;
1024
1025 return pand(a, (Packet2d)__builtin_msa_xori_b((v16u8)b, 255));
1026}
1027
1028template <>
1029EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
1030 EIGEN_MSA_DEBUG;
1031
1032 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__builtin_msa_ld_d(const_cast<double*>(from), 0);
1033}
1034
1035template <>
1036EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
1037 EIGEN_MSA_DEBUG;
1038
1039#if EIGEN_FAST_MATH
1040 // This prefers numbers to NaNs.
1041 return __builtin_msa_fmin_d(a, b);
1042#else
1043 // This prefers NaNs to numbers.
1044 v2i64 aNaN = __builtin_msa_fcun_d(a, a);
1045 v2i64 aMinOrNaN = por(__builtin_msa_fclt_d(a, b), aNaN);
1046 return (Packet2d)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a);
1047#endif
1048}
1049
1050template <>
1051EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
1052 EIGEN_MSA_DEBUG;
1053
1054#if EIGEN_FAST_MATH
1055 // This prefers numbers to NaNs.
1056 return __builtin_msa_fmax_d(a, b);
1057#else
1058 // This prefers NaNs to numbers.
1059 v2i64 aNaN = __builtin_msa_fcun_d(a, a);
1060 v2i64 aMaxOrNaN = por(__builtin_msa_fclt_d(b, a), aNaN);
1061 return (Packet2d)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a);
1062#endif
1063}
1064
1065template <>
1066EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
1067 EIGEN_MSA_DEBUG;
1068
1069 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__builtin_msa_ld_d(const_cast<double*>(from), 0);
1070}
1071
1072template <>
1073EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) {
1074 EIGEN_MSA_DEBUG;
1075
1076 Packet2d value = {*from, *from};
1077 return value;
1078}
1079
1080template <>
1081EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
1082 EIGEN_MSA_DEBUG;
1083
1084 EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0);
1085}
1086
1087template <>
1088EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
1089 EIGEN_MSA_DEBUG;
1090
1091 EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0);
1092}
1093
1094template <>
1095EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
1096 EIGEN_MSA_DEBUG;
1097
1098 Packet2d value;
1099 value[0] = *from;
1100 from += stride;
1101 value[1] = *from;
1102 return value;
1103}
1104
1105template <>
1106EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
1107 EIGEN_MSA_DEBUG;
1108
1109 *to = from[0];
1110 to += stride;
1111 *to = from[1];
1112}
1113
1114template <>
1115EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1116 EIGEN_MSA_DEBUG;
1117
1118 __builtin_prefetch(addr);
1119}
1120
1121template <>
1122EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
1123 EIGEN_MSA_DEBUG;
1124
1125 return a[0];
1126}
1127
1128template <>
1129EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
1130 EIGEN_MSA_DEBUG;
1131
1132 return (Packet2d)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
1133}
1134
1135template <>
1136EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) {
1137 EIGEN_MSA_DEBUG;
1138
1139 return (Packet2d)__builtin_msa_bclri_d((v2u64)a, 63);
1140}
1141
1142template <>
1143EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
1144 EIGEN_MSA_DEBUG;
1145
1146 Packet2d s = padd(a, preverse(a));
1147 return s[0];
1148}
1149
1150template <>
1151EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
1152 return __builtin_msa_bnz_v((v16u8)a);
1153}
1154
1155// Other reduction functions:
1156// mul
1157template <>
1158EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
1159 EIGEN_MSA_DEBUG;
1160
1161 Packet2d p = pmul(a, preverse(a));
1162 return p[0];
1163}
1164
1165// min
1166template <>
1167EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
1168 EIGEN_MSA_DEBUG;
1169
1170#if EIGEN_FAST_MATH
1171 Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
1172 Packet2d v = __builtin_msa_fmin_d(a, swapped);
1173 return v[0];
1174#else
1175 double a0 = a[0], a1 = a[1];
1176 return ((numext::isnan)(a0) || a0 < a1) ? a0 : a1;
1177#endif
1178}
1179
1180// max
1181template <>
1182EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
1183 EIGEN_MSA_DEBUG;
1184
1185#if EIGEN_FAST_MATH
1186 Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
1187 Packet2d v = __builtin_msa_fmax_d(a, swapped);
1188 return v[0];
1189#else
1190 double a0 = a[0], a1 = a[1];
1191 return ((numext::isnan)(a0) || a0 > a1) ? a0 : a1;
1192#endif
1193}
1194
1195template <>
1196EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& a) {
1197 EIGEN_MSA_DEBUG;
1198
1199 return __builtin_msa_fsqrt_d(a);
1200}
1201
1202template <>
1203EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) {
1204 EIGEN_MSA_DEBUG;
1205
1206#if EIGEN_FAST_MATH
1207 return __builtin_msa_frsqrt_d(a);
1208#else
1209 Packet2d ones = __builtin_msa_ffint_s_d(__builtin_msa_ldi_d(1));
1210 return pdiv(ones, psqrt(a));
1211#endif
1212}
1213
1214inline std::ostream& operator<<(std::ostream& os, const PacketBlock<Packet2d, 2>& value) {
1215 os << "[ " << value.packet[0] << "," << std::endl << " " << value.packet[1] << " ]";
1216 return os;
1217}
1218
1219EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
1220 EIGEN_MSA_DEBUG;
1221
1222 Packet2d trn1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]);
1223 Packet2d trn2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]);
1224 kernel.packet[0] = trn1;
1225 kernel.packet[1] = trn2;
1226}
1227
1228template <>
1229EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
1230 Packet2d v = a;
1231 int32_t old_mode, new_mode;
1232 asm volatile(
1233 "cfcmsa %[old_mode], $1\n"
1234 "ori %[new_mode], %[old_mode], 3\n" // 3 = round towards -INFINITY.
1235 "ctcmsa $1, %[new_mode]\n"
1236 "frint.d %w[v], %w[v]\n"
1237 "ctcmsa $1, %[old_mode]\n"
1238 : // outputs
1239 [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
1240 [v] "+f"(v)
1241 : // inputs
1242 : // clobbers
1243 );
1244 return v;
1245}
1246
1247template <>
1248EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
1249 Packet2d v = a;
1250 int32_t old_mode, new_mode;
1251 asm volatile(
1252 "cfcmsa %[old_mode], $1\n"
1253 "ori %[new_mode], %[old_mode], 3\n"
1254 "xori %[new_mode], %[new_mode], 1\n" // 2 = round towards +INFINITY.
1255 "ctcmsa $1, %[new_mode]\n"
1256 "frint.d %w[v], %w[v]\n"
1257 "ctcmsa $1, %[old_mode]\n"
1258 : // outputs
1259 [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
1260 [v] "+f"(v)
1261 : // inputs
1262 : // clobbers
1263 );
1264 return v;
1265}
1266
1267template <>
1268EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
1269 // frint.d uses the current rounding mode (default: round to nearest, ties to even).
1270 Packet2d v = a;
1271 asm volatile("frint.d %w[v], %w[v]\n" : [v] "+f"(v));
1272 return v;
1273}
1274
1275template <>
1276EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
1277 Packet2d v = a;
1278 int32_t old_mode, new_mode;
1279 asm volatile(
1280 "cfcmsa %[old_mode], $1\n"
1281 "ori %[new_mode], %[old_mode], 3\n"
1282 "xori %[new_mode], %[new_mode], 2\n" // 1 = round toward zero.
1283 "ctcmsa $1, %[new_mode]\n"
1284 "frint.d %w[v], %w[v]\n"
1285 "ctcmsa $1, %[old_mode]\n"
1286 : // outputs
1287 [old_mode] "=r"(old_mode), [new_mode] "=r"(new_mode),
1288 [v] "+f"(v)
1289 : // inputs
1290 : // clobbers
1291 );
1292 return v;
1293}
1294
1295template <>
1296EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(const Packet2d& a, const Packet2d& b) {
1297 return (Packet2d)__builtin_msa_fcult_d(a, b);
1298}
1299
1300template <>
1301EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
1302 return pldexp_generic(a, exponent);
1303}
1304
1305} // end namespace internal
1306
1307} // end namespace Eigen
1308
1309#endif // EIGEN_PACKET_MATH_MSA_H
@ Aligned16
Definition Constants.h:238