-
Notifications
You must be signed in to change notification settings - Fork 708
/
vectorization.h
5625 lines (5092 loc) · 175 KB
/
vectorization.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// ---------------------------------------------------------------------
//
// Copyright (C) 2011 - 2021 by the deal.II authors
//
// This file is part of the deal.II library.
//
// The deal.II library is free software; you can use it, redistribute
// it, and/or modify it under the terms of the GNU Lesser General
// Public License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// The full text of the license can be found in the file LICENSE.md at
// the top level directory of deal.II.
//
// ---------------------------------------------------------------------
#ifndef dealii_vectorization_h
#define dealii_vectorization_h
#include <deal.II/base/config.h>
#include <deal.II/base/exceptions.h>
#include <deal.II/base/template_constraints.h>
#include <array>
#include <cmath>
// Note:
// The flag DEAL_II_VECTORIZATION_WIDTH_IN_BITS is essentially constructed
// according to the following scheme (on x86-based architectures)
// #ifdef __AVX512F__
// #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 512
// #elif defined (__AVX__)
// #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 256
// #elif defined (__SSE2__)
// #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 128
// #else
// #define DEAL_II_VECTORIZATION_WIDTH_IN_BITS 0
// #endif
// In addition to checking the flags __AVX512F__, __AVX__ and __SSE2__, a CMake
// test, 'check_01_cpu_features.cmake', ensures that these feature are not only
// present in the compilation unit but also working properly.
#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS > 0
// These error messages try to detect the case that deal.II was compiled with
// a wider instruction set extension as the current compilation unit, for
// example because deal.II was compiled with AVX, but a user project does not
// add -march=native or similar flags, making it fall to SSE2. This leads to
// very strange errors as the size of data structures differs between the
// compiled deal.II code sitting in libdeal_II.so and the user code if not
// detected.
# if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256 && !defined(__AVX__)
# error \
"Mismatch in vectorization capabilities: AVX was detected during configuration of deal.II and switched on, but it is apparently not available for the file you are trying to compile at the moment. Check compilation flags controlling the instruction set, such as -march=native."
# endif
# if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512 && !defined(__AVX512F__)
# error \
"Mismatch in vectorization capabilities: AVX-512F was detected during configuration of deal.II and switched on, but it is apparently not available for the file you are trying to compile at the moment. Check compilation flags controlling the instruction set, such as -march=native."
# endif
# ifdef _MSC_VER
# include <intrin.h>
# elif defined(__ALTIVEC__)
# include <altivec.h>
// altivec.h defines vector, pixel, bool, but we do not use them, so undefine
// them before they make trouble
# undef vector
# undef pixel
# undef bool
# else
# include <x86intrin.h>
# endif
#endif
DEAL_II_NAMESPACE_OPEN
// Enable the EnableIfScalar type trait for VectorizedArray<Number> such
// that it can be used as a Number type in Tensor<rank,dim,Number>, etc.
template <typename Number, std::size_t width>
struct EnableIfScalar<VectorizedArray<Number, width>>
{
using type = VectorizedArray<typename EnableIfScalar<Number>::type, width>;
};
/**
* An iterator for VectorizedArray.
*/
template <typename T>
class VectorizedArrayIterator
{
public:
/**
* Constructor.
*
* @param data The actual VectorizedArray.
* @param lane A pointer to the current lane.
*/
VectorizedArrayIterator(T &data, const std::size_t lane)
: data(&data)
, lane(lane)
{}
/**
* Compare for equality.
*/
bool
operator==(const VectorizedArrayIterator<T> &other) const
{
Assert(this->data == other.data,
ExcMessage(
"You are trying to compare iterators into different arrays."));
return this->lane == other.lane;
}
/**
* Compare for inequality.
*/
bool
operator!=(const VectorizedArrayIterator<T> &other) const
{
Assert(this->data == other.data,
ExcMessage(
"You are trying to compare iterators into different arrays."));
return this->lane != other.lane;
}
/**
* Copy assignment.
*/
VectorizedArrayIterator<T> &
operator=(const VectorizedArrayIterator<T> &other) = default;
/**
* Dereferencing operator (const version): returns the value of the current
* lane.
*/
const typename T::value_type &
operator*() const
{
AssertIndexRange(lane, T::size());
return (*data)[lane];
}
/**
* Dereferencing operator (non-@p const version): returns the value of the
* current lane.
*/
template <typename U = T>
std::enable_if_t<!std::is_same<U, const U>::value, typename T::value_type> &
operator*()
{
AssertIndexRange(lane, T::size());
return (*data)[lane];
}
/**
* Prefix <tt>++</tt> operator: <tt>++iterator</tt>. This operator advances
* the iterator to the next lane and returns a reference to
* <tt>*this</tt>.
*/
VectorizedArrayIterator<T> &
operator++()
{
AssertIndexRange(lane + 1, T::size() + 1);
lane++;
return *this;
}
/**
* This operator advances the iterator by @p offset lanes and returns a
* reference to <tt>*this</tt>.
*/
VectorizedArrayIterator<T> &
operator+=(const std::size_t offset)
{
AssertIndexRange(lane + offset, T::size() + 1);
lane += offset;
return *this;
}
/**
* Prefix <tt>--</tt> operator: <tt>--iterator</tt>. This operator advances
* the iterator to the previous lane and returns a reference to
* <tt>*this</tt>.
*/
VectorizedArrayIterator<T> &
operator--()
{
Assert(
lane > 0,
ExcMessage(
"You can't decrement an iterator that is already at the beginning of the range."));
--lane;
return *this;
}
/**
* Create new iterator, which is shifted by @p offset.
*/
VectorizedArrayIterator<T>
operator+(const std::size_t &offset) const
{
AssertIndexRange(lane + offset, T::size() + 1);
return VectorizedArrayIterator<T>(*data, lane + offset);
}
/**
* Compute distance between this iterator and iterator @p other.
*/
std::ptrdiff_t
operator-(const VectorizedArrayIterator<T> &other) const
{
return static_cast<std::ptrdiff_t>(lane) -
static_cast<ptrdiff_t>(other.lane);
}
private:
/**
* Pointer to the actual VectorizedArray.
*/
T *data;
/**
* Pointer to the current lane.
*/
std::size_t lane;
};
/**
* A base class for the various VectorizedArray template specializations,
* containing common functionalities.
*
* @tparam T Type of the actual vectorized array. We are using the
* Couriously Recurring Template Pattern (see
* https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern) in this
* class to avoid having to resort to `virtual` member functions.
*/
template <typename T, std::size_t width>
class VectorizedArrayBase
{
public:
/**
* Default constructor.
*/
VectorizedArrayBase() = default;
/**
* Construct an array with the given initializer list.
*/
template <typename U>
VectorizedArrayBase(const std::initializer_list<U> &list)
{
auto i0 = this->begin();
auto i1 = list.begin();
for (; i1 != list.end(); ++i0, ++i1)
{
Assert(
i0 != this->end(),
ExcMessage(
"Initializer list exceeds size of this VectorizedArray object."));
*i0 = *i1;
}
for (; i0 != this->end(); ++i0)
{
*i0 = 0.0;
}
}
/**
* Return the number of elements in the array.
*/
static constexpr std::size_t
size()
{
return width;
}
/**
* @return An iterator pointing to the beginning of the underlying data.
*/
VectorizedArrayIterator<T>
begin()
{
return VectorizedArrayIterator<T>(static_cast<T &>(*this), 0);
}
/**
* @return An iterator pointing to the beginning of the underlying data (`const`
* version).
*/
VectorizedArrayIterator<const T>
begin() const
{
return VectorizedArrayIterator<const T>(static_cast<const T &>(*this), 0);
}
/**
* @return An iterator pointing to the end of the underlying data.
*/
VectorizedArrayIterator<T>
end()
{
return VectorizedArrayIterator<T>(static_cast<T &>(*this), width);
}
/**
* @return An iterator pointing to the end of the underlying data (`const`
* version).
*/
VectorizedArrayIterator<const T>
end() const
{
return VectorizedArrayIterator<const T>(static_cast<const T &>(*this),
width);
}
};
/**
* This generic class defines a unified interface to a vectorized data type.
* For general template arguments, this class simply corresponds to the
* template argument. For example, VectorizedArray<long double> is nothing
* else but a wrapper around <tt>long double</tt> with exactly one data field
* of type <tt>long double</tt> and overloaded arithmetic operations. This
* means that <tt>VectorizedArray<ComplicatedType></tt> has a similar layout
* as ComplicatedType, provided that ComplicatedType defines basic arithmetic
* operations. For floats and doubles, an array of numbers are packed together
* with the goal to be processed in a single-instruction/multiple-data (SIMD)
* fashion. In the SIMD context, the elements of such a short vector are often
* called lanes. The number of elements packed together, i.e., the number of
* lanes, depends on the computer system and compiler flags that are used for
* compilation of deal.II. The fundamental idea of these packed data types is
* to use one single CPU instruction to perform arithmetic operations on the
* whole array using the processor's vector (SIMD) units. Most computer
* systems by 2010 standards will use an array of two doubles or four floats,
* respectively (this corresponds to the SSE/SSE2 data sets) when compiling
* deal.II on 64-bit operating systems. On Intel Sandy Bridge processors and
* newer or AMD Bulldozer processors and newer, four doubles or eight floats
* are used when deal.II is configured using gcc with \--with-cpu=native
* or \--with-cpu=corei7-avx. On compilations with AVX-512 support (e.g.,
* Intel Skylake Server from 2017), eight doubles or sixteen floats are used.
*
* This behavior of this class is made similar to the basic data types double
* and float. The definition of a vectorized array does not initialize the
* data field but rather leaves it undefined, as is the case for double and
* float. However, when calling something like `VectorizedArray<double> a =
* VectorizedArray<double>()` or `VectorizedArray<double> a = 0.`, it sets all
* numbers in this field to zero. This class is of standard layout type
* according to the C++11 standard, which means that there is an equivalent C
* representation and the class can e.g. be safely copied with std::memcpy.
* (See also https://en.cppreference.com/w/cpp/named_req/StandardLayoutType.)
* The standard layout is also necessary for ensuring correct alignment of
* data with address boundaries when collected in a vector (i.e., when the
* first element in a vector is properly aligned, all subsequent elements will
* be correctly aligned, too).
*
* Note that for proper functioning of this class, certain data alignment
* rules must be respected. This is because the computer expects the starting
* address of a VectorizedArray<double> field at specific addresses in memory
* (usually, the address of the vectorized array should be a multiple of the
* length of the array in bytes). Otherwise, a segmentation fault or a severe
* loss of performance might occur. When creating a single data field on the
* stack like `VectorizedArray<double> a = 5.;`, the compiler will take care
* of data alignment automatically. However, when allocating a long vector of
* VectorizedArray<double> data, one needs to respect these rules. Use the
* class AlignedVector or data containers based on AlignedVector (such as
* Table) for this purpose. It is a class very similar to std::vector
* otherwise but always makes sure that data is correctly aligned.
*
* The user can explicitly control the width of a particular instruction set
* architecture (ISA) extension by specifying the number of lanes via the second
* template parameter of this wrapper class. For example on Intel Skylake
* Server, you have the following options for the data type double:
* - VectorizedArray<double, 1> // no vectorization (auto-optimization)
* - VectorizedArray<double, 2> // SSE2
* - VectorizedArray<double, 4> // AVX
* - VectorizedArray<double, 8> // AVX-512 (default)
*
* and for Intel Sandy Bridge, Haswell, Broadwell, AMD Bulldozer and Zen/Ryzen:
* - VectorizedArray<double, 1> // no vectorization (auto-optimization)
* - VectorizedArray<double, 2> // SSE2
* - VectorizedArray<double, 4> // AVX (default)
*
* and for processors with AltiVec support:
* - VectorizedArray<double, 1>
* - VectorizedArray<double, 2>
*
* for older x86 processors or in case no processor-specific compilation flags
* were added (i.e., without `-D CMAKE_CXX_FLAGS=-march=native` or similar
* flags):
* - VectorizedArray<double, 1> // no vectorization (auto-optimization)
* - VectorizedArray<double, 2> // SSE2
*
* Similar considerations also apply to the data type `float`.
*
* Wrongly selecting the width, e.g., width=3 or width=8 on a processor which
* does not support AVX-512 leads to a static assert.
*
* @tparam Number underlying data type
* @tparam width vector length (optional; if not set, the maximal width of the
* architecture is used)
*/
template <typename Number, std::size_t width>
class VectorizedArray
: public VectorizedArrayBase<VectorizedArray<Number, width>, 1>
{
public:
/**
* This gives the type of the array elements.
*/
using value_type = Number;
static_assert(width == 1,
"You specified an illegal width that is not supported.");
/**
* Default empty constructor, leaving the data in an uninitialized state
* similar to float/double.
*/
VectorizedArray() = default;
/**
* Construct an array with the given scalar broadcast to all lanes.
*/
VectorizedArray(const Number scalar)
{
this->operator=(scalar);
}
/**
* Construct an array with the given initializer list.
*/
template <typename U>
VectorizedArray(const std::initializer_list<U> &list)
: VectorizedArrayBase<VectorizedArray<Number, width>, 1>(list)
{}
/**
* This function assigns a scalar to this class.
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray &
operator=(const Number scalar)
{
data = scalar;
return *this;
}
/**
* Access operator (only valid with component 0 in the base class without
* specialization).
*/
DEAL_II_ALWAYS_INLINE
Number &
operator[](const unsigned int comp)
{
(void)comp;
AssertIndexRange(comp, 1);
return data;
}
/**
* Constant access operator (only valid with component 0 in the base class
* without specialization).
*/
DEAL_II_ALWAYS_INLINE
const Number &
operator[](const unsigned int comp) const
{
(void)comp;
AssertIndexRange(comp, 1);
return data;
}
/**
* Addition
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray &
operator+=(const VectorizedArray &vec)
{
data += vec.data;
return *this;
}
/**
* Subtraction
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray &
operator-=(const VectorizedArray &vec)
{
data -= vec.data;
return *this;
}
/**
* Multiplication
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray &
operator*=(const VectorizedArray &vec)
{
data *= vec.data;
return *this;
}
/**
* Division
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray &
operator/=(const VectorizedArray &vec)
{
data /= vec.data;
return *this;
}
/**
* Load size() data items from memory into the calling class, starting at
* the given address. The pointer `ptr` needs not be aligned by the amount
* of bytes in the vectorized array, as opposed to casting a double address
* to VectorizedArray<double>*.
*/
DEAL_II_ALWAYS_INLINE
void
load(const Number *ptr)
{
data = *ptr;
}
/**
* Write the content of the calling class into memory in form of
* size() data items to the given address. The pointer `ptr` needs not be
* aligned by the amount of bytes in the vectorized array, as opposed to
* casting a double address to VectorizedArray<double>*.
*/
DEAL_II_ALWAYS_INLINE
void
store(Number *ptr) const
{
*ptr = data;
}
/**
* Write the content of the calling class into memory in form of
* size() data items to the given address using non-temporal stores that
* bypass the processor's caches, using @p _mm_stream_pd store intrinsics on
* supported CPUs. The destination of the store @p ptr must be aligned by
* the amount of bytes in the vectorized array.
*
* This store operation can be faster than usual store operations in case
* the store is streaming because it avoids the read-for-ownership transfer
* typically invoked in standard stores. This approximately works as follows
* (see the literature on computer architecture for details): When an
* algorithm stores some results to a memory address, a processor typically
* wants to move it into some of its caches as it expects the data to be
* re-used again at some point. Since caches are organized in lines of sizes
* either 64 byte or 128 byte but writes are usually smaller, a processor
* must first load in the destination cache line upon a write because only
* part of the cache line is overwritten initially. If a series of stores
* write data in a chunk bigger than any of its caches could handle, the
* data finally has to be moved out from the caches to main memory. But
* since all addressed have first been read, this doubles the load on main
* memory, which can incur a performance penalty. Furthermore, the
* organization of caches in a multicore context also requires reading an
* address before something can be written to cache to that address, see
* e.g. the <a href="https://en.wikipedia.org/wiki/MESI_protocol">Wikipedia
* article on the MESI protocol</a> for details. The instruction underlying
* this function call signals to the processor that these two prerequisites
* on a store are relaxed: Firstly, one expects the whole cache line to be
* overwritten (meaning that the memory subsystem makes sure that
* consecutive stores that together span a cache line are merged, and
* appropriately handling the case where only part of a cache line is
* written), so there is no need to first read the "remainder" of the cache
* line. Secondly, the data behind that particular memory will not be
* subject to cache coherency protocol as it will be in main memory both
* when the same processor wants to access it again as well as any other
* processors in a multicore chip. Due to this particular setup, any
* subsequent access to the data written by this function will need to query
* main memory, which is slower than an access from a cache both
* latency-wise and throughput-wise. Thus, this command should only be used
* for storing large arrays that will collectively not fit into caches, as
* performance will be degraded otherwise. For a typical use case, see also
* <a href="https://blogs.fau.de/hager/archives/2103">this blog article</a>.
*
* Note that streaming stores are only available in the specialized SSE/AVX
* classes of VectorizedArray of type @p double or @p float, not in the
* generic base class.
*/
DEAL_II_ALWAYS_INLINE
void
streaming_store(Number *ptr) const
{
*ptr = data;
}
/**
* Load size() data items from memory into the calling class, starting at
* the given address and with given offsets, each entry from the offset
* providing one element of the vectorized array.
*
* This operation corresponds to the following code (but uses a more
* efficient implementation in case the hardware allows for that):
* @code
* for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
* this->operator[](v) = base_ptr[offsets[v]];
* @endcode
*/
DEAL_II_ALWAYS_INLINE
void
gather(const Number *base_ptr, const unsigned int *offsets)
{
data = base_ptr[offsets[0]];
}
/**
* Write the content of the calling class into memory in form of
* size() data items to the given address and the given offsets, filling the
* elements of the vectorized array into each offset.
*
* This operation corresponds to the following code (but uses a more
* efficient implementation in case the hardware allows for that):
* @code
* for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
* base_ptr[offsets[v]] = this->operator[](v);
* @endcode
*/
DEAL_II_ALWAYS_INLINE
void
scatter(const unsigned int *offsets, Number *base_ptr) const
{
base_ptr[offsets[0]] = data;
}
/**
* Actual data field. To be consistent with the standard layout type and to
* enable interaction with external SIMD functionality, this member is
* declared public.
*/
Number data;
private:
/**
* Return the square root of this field. Not for use in user code. Use
* sqrt(x) instead.
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray
get_sqrt() const
{
VectorizedArray res;
res.data = std::sqrt(data);
return res;
}
/**
* Return the absolute value of this field. Not for use in user code. Use
* abs(x) instead.
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray
get_abs() const
{
VectorizedArray res;
res.data = std::fabs(data);
return res;
}
/**
* Return the component-wise maximum of this field and another one. Not for
* use in user code. Use max(x,y) instead.
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray
get_max(const VectorizedArray &other) const
{
VectorizedArray res;
res.data = std::max(data, other.data);
return res;
}
/**
* Return the component-wise minimum of this field and another one. Not for
* use in user code. Use min(x,y) instead.
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray
get_min(const VectorizedArray &other) const
{
VectorizedArray res;
res.data = std::min(data, other.data);
return res;
}
// Make a few functions friends.
template <typename Number2, std::size_t width2>
friend VectorizedArray<Number2, width2>
std::sqrt(const VectorizedArray<Number2, width2> &);
template <typename Number2, std::size_t width2>
friend VectorizedArray<Number2, width2>
std::abs(const VectorizedArray<Number2, width2> &);
template <typename Number2, std::size_t width2>
friend VectorizedArray<Number2, width2>
std::max(const VectorizedArray<Number2, width2> &,
const VectorizedArray<Number2, width2> &);
template <typename Number2, std::size_t width2>
friend VectorizedArray<Number2, width2>
std::min(const VectorizedArray<Number2, width2> &,
const VectorizedArray<Number2, width2> &);
};
/**
* @name Packing and unpacking of a VectorizedArray
*/
//@{
/**
* Create a vectorized array that sets all entries in the array to the given
* scalar, i.e., broadcasts the scalar to all array elements.
*
* @relatesalso VectorizedArray
*/
template <typename Number,
std::size_t width =
internal::VectorizedArrayWidthSpecifier<Number>::max_width>
inline DEAL_II_ALWAYS_INLINE VectorizedArray<Number, width>
make_vectorized_array(const Number &u)
{
VectorizedArray<Number, width> result = u;
return result;
}
/**
* Create a vectorized array of given type and broadcast the scalar value
* to all array elements.
*
* @relatesalso VectorizedArray
*/
template <typename VectorizedArrayType>
inline DEAL_II_ALWAYS_INLINE VectorizedArrayType
make_vectorized_array(const typename VectorizedArrayType::value_type &u)
{
static_assert(
std::is_same<VectorizedArrayType,
VectorizedArray<typename VectorizedArrayType::value_type,
VectorizedArrayType::size()>>::value,
"VectorizedArrayType is not a VectorizedArray.");
VectorizedArrayType result = u;
return result;
}
/**
* Load size() data items from memory into the VectorizedArray @p out,
* starting at the given addresses and with given offset, each entry from the
* offset providing one element of the vectorized array.
*
* This operation corresponds to the following code:
* @code
* for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
* out.data[v] = ptrs[v][offset];
* @endcode
*/
template <typename Number, std::size_t width>
inline DEAL_II_ALWAYS_INLINE void
gather(VectorizedArray<Number, width> & out,
const std::array<Number *, width> &ptrs,
const unsigned int offset)
{
for (unsigned int v = 0; v < width; ++v)
out.data[v] = ptrs[v][offset];
}
/**
* This method loads VectorizedArray::size() data streams from the
* given array @p in. The offsets to the input array are given by the array @p
* offsets. From each stream, n_entries are read. The data is then transposed
* and stored it into an array of VectorizedArray type. The output array @p
* out is expected to be an array of size @p n_entries. This method operates
* on plain arrays, so no checks for valid data access are made. It is the
* user's responsibility to ensure that the given arrays are valid according
* to the access layout below.
*
* This operation corresponds to a transformation of an array-of-struct
* (input) into a struct-of-array (output) according to the following formula:
*
* @code
* for (unsigned int i=0; i<n_entries; ++i)
* for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
* out[i][v] = in[offsets[v]+i];
* @endcode
*
* A more optimized version of this code will be used for supported types.
*
* This is the inverse operation to vectorized_transpose_and_store().
*
* @relatesalso VectorizedArray
*/
template <typename Number, std::size_t width>
inline DEAL_II_ALWAYS_INLINE void
vectorized_load_and_transpose(const unsigned int n_entries,
const Number * in,
const unsigned int * offsets,
VectorizedArray<Number, width> *out)
{
for (unsigned int i = 0; i < n_entries; ++i)
for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
out[i][v] = in[offsets[v] + i];
}
/**
* The same as above with the difference that an array of pointers are
* passed in as input argument @p in.
*
* In analogy to the function above, one can consider that
* `in+offset[v]` is precomputed and passed as input argument.
*
* However, this function can also be used if some function returns an array
* of pointers and no assumption can be made that they belong to the same array,
* i.e., they can have their origin in different memory allocations.
*/
template <typename Number, std::size_t width>
inline DEAL_II_ALWAYS_INLINE void
vectorized_load_and_transpose(const unsigned int n_entries,
const std::array<Number *, width> &in,
VectorizedArray<Number, width> * out)
{
for (unsigned int i = 0; i < n_entries; ++i)
for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
out[i][v] = in[v][i];
}
/**
* This method stores the vectorized arrays in transposed form into the given
* output array @p out with the given offsets @p offsets. This operation
* corresponds to a transformation of a struct-of-array (input) into an array-
* of-struct (output). This method operates on plain array, so no checks for
* valid data access are made. It is the user's responsibility to ensure that
* the given arrays are valid according to the access layout below.
*
* This method assumes that the specified offsets do not overlap. Otherwise,
* the behavior is undefined in the vectorized case. It is the user's
* responsibility to make sure that the access does not overlap and avoid
* undefined behavior.
*
* The argument @p add_into selects where the entries should only be written
* into the output arrays or the result should be added into the existing
* entries in the output. For <code>add_into == false</code>, the following
* code is assumed:
*
* @code
* for (unsigned int i=0; i<n_entries; ++i)
* for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
* out[offsets[v]+i] = in[i][v];
* @endcode
*
* For <code>add_into == true</code>, the code implements the following
* action:
* @code
* for (unsigned int i=0; i<n_entries; ++i)
* for (unsigned int v=0; v<VectorizedArray<Number>::size(); ++v)
* out[offsets[v]+i] += in[i][v];
* @endcode
*
* A more optimized version of this code will be used for supported types.
*
* This is the inverse operation to vectorized_load_and_transpose().
*
* @relatesalso VectorizedArray
*/
template <typename Number, std::size_t width>
inline DEAL_II_ALWAYS_INLINE void
vectorized_transpose_and_store(const bool add_into,
const unsigned int n_entries,
const VectorizedArray<Number, width> *in,
const unsigned int * offsets,
Number * out)
{
if (add_into)
for (unsigned int i = 0; i < n_entries; ++i)
for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
out[offsets[v] + i] += in[i][v];
else
for (unsigned int i = 0; i < n_entries; ++i)
for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
out[offsets[v] + i] = in[i][v];
}
/**
* The same as above with the difference that an array of pointers are
* passed in as input argument @p out.
*
* In analogy to the function above, one can consider that
* `out+offset[v]` is precomputed and passed as input argument.
*
* However, this function can also be used if some function returns an array
* of pointers and no assumption can be made that they belong to the same array,
* i.e., they can have their origin in different memory allocations.
*/
template <typename Number, std::size_t width>
inline DEAL_II_ALWAYS_INLINE void
vectorized_transpose_and_store(const bool add_into,
const unsigned int n_entries,
const VectorizedArray<Number, width> *in,
std::array<Number *, width> & out)
{
if (add_into)
for (unsigned int i = 0; i < n_entries; ++i)
for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
out[v][i] += in[i][v];
else
for (unsigned int i = 0; i < n_entries; ++i)
for (unsigned int v = 0; v < VectorizedArray<Number, width>::size(); ++v)
out[v][i] = in[i][v];
}
//@}
#ifndef DOXYGEN
// for safety, also check that __AVX512F__ is defined in case the user manually
// set some conflicting compile flags which prevent compilation
# if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512 && defined(__AVX512F__)
/**
* Specialization of VectorizedArray class for double and AVX-512.
*/
template <>
class VectorizedArray<double, 8>
: public VectorizedArrayBase<VectorizedArray<double, 8>, 8>
{
public:
/**
* This gives the type of the array elements.
*/
using value_type = double;
/**
* Default empty constructor, leaving the data in an uninitialized state
* similar to float/double.
*/
VectorizedArray() = default;
/**
* Construct an array with the given scalar broadcast to all lanes.
*/
VectorizedArray(const double scalar)
{
this->operator=(scalar);
}
/**
* Construct an array with the given initializer list.
*/
template <typename U>
VectorizedArray(const std::initializer_list<U> &list)
: VectorizedArrayBase<VectorizedArray<double, 8>, 8>(list)
{}
/**
* This function can be used to set all data fields to a given scalar.
*/
DEAL_II_ALWAYS_INLINE
VectorizedArray &
operator=(const double x)
{
data = _mm512_set1_pd(x);
return *this;
}