-
Notifications
You must be signed in to change notification settings - Fork 11.6k
/
ScopInfo.h
2898 lines (2393 loc) · 109 KB
/
ScopInfo.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//===- polly/ScopInfo.h -----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Store the polyhedral model representation of a static control flow region,
// also called SCoP (Static Control Part).
//
// This representation is shared among several tools in the polyhedral
// community, which are e.g. CLooG, Pluto, Loopo, Graphite.
//
//===----------------------------------------------------------------------===//
#ifndef POLLY_SCOPINFO_H
#define POLLY_SCOPINFO_H
#include "polly/ScopDetection.h"
#include "polly/Support/SCEVAffinator.h"
#include "polly/Support/ScopHelper.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/RegionPass.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Pass.h"
#include "isl/isl-noexceptions.h"
#include <cassert>
#include <cstddef>
#include <forward_list>
namespace llvm {
void initializeScopInfoRegionPassPass(PassRegistry &);
void initializeScopInfoWrapperPassPass(PassRegistry &);
} // end namespace llvm
namespace polly {
using llvm::AnalysisInfoMixin;
using llvm::ArrayRef;
using llvm::AssertingVH;
using llvm::AssumptionCache;
using llvm::cast;
using llvm::DataLayout;
using llvm::DenseMap;
using llvm::DenseSet;
using llvm::function_ref;
using llvm::isa;
using llvm::iterator_range;
using llvm::LoadInst;
using llvm::make_range;
using llvm::MapVector;
using llvm::MemIntrinsic;
using llvm::Optional;
using llvm::PassInfoMixin;
using llvm::PHINode;
using llvm::RegionNode;
using llvm::RegionPass;
using llvm::RGPassManager;
using llvm::SetVector;
using llvm::SmallPtrSetImpl;
using llvm::SmallVector;
using llvm::SmallVectorImpl;
using llvm::StringMap;
using llvm::Type;
using llvm::Use;
using llvm::Value;
using llvm::ValueToValueMap;
class MemoryAccess;
//===---------------------------------------------------------------------===//
extern bool UseInstructionNames;
// The maximal number of basic sets we allow during domain construction to
// be created. More complex scops will result in very high compile time and
// are also unlikely to result in good code.
extern int const MaxDisjunctsInDomain;
/// The different memory kinds used in Polly.
///
/// We distinguish between arrays and various scalar memory objects. We use
/// the term ``array'' to describe memory objects that consist of a set of
/// individual data elements arranged in a multi-dimensional grid. A scalar
/// memory object describes an individual data element and is used to model
/// the definition and uses of llvm::Values.
///
/// The polyhedral model does traditionally not reason about SSA values. To
/// reason about llvm::Values we model them "as if" they were zero-dimensional
/// memory objects, even though they were not actually allocated in (main)
/// memory. Memory for such objects is only alloca[ed] at CodeGeneration
/// time. To relate the memory slots used during code generation with the
/// llvm::Values they belong to the new names for these corresponding stack
/// slots are derived by appending suffixes (currently ".s2a" and ".phiops")
/// to the name of the original llvm::Value. To describe how def/uses are
/// modeled exactly we use these suffixes here as well.
///
/// There are currently four different kinds of memory objects:
enum class MemoryKind {
/// MemoryKind::Array: Models a one or multi-dimensional array
///
/// A memory object that can be described by a multi-dimensional array.
/// Memory objects of this type are used to model actual multi-dimensional
/// arrays as they exist in LLVM-IR, but they are also used to describe
/// other objects:
/// - A single data element allocated on the stack using 'alloca' is
/// modeled as a one-dimensional, single-element array.
/// - A single data element allocated as a global variable is modeled as
/// one-dimensional, single-element array.
/// - Certain multi-dimensional arrays with variable size, which in
/// LLVM-IR are commonly expressed as a single-dimensional access with a
/// complicated access function, are modeled as multi-dimensional
/// memory objects (grep for "delinearization").
Array,
/// MemoryKind::Value: Models an llvm::Value
///
/// Memory objects of type MemoryKind::Value are used to model the data flow
/// induced by llvm::Values. For each llvm::Value that is used across
/// BasicBlocks, one ScopArrayInfo object is created. A single memory WRITE
/// stores the llvm::Value at its definition into the memory object and at
/// each use of the llvm::Value (ignoring trivial intra-block uses) a
/// corresponding READ is added. For instance, the use/def chain of a
/// llvm::Value %V depicted below
/// ______________________
/// |DefBB: |
/// | %V = float op ... |
/// ----------------------
/// | |
/// _________________ _________________
/// |UseBB1: | |UseBB2: |
/// | use float %V | | use float %V |
/// ----------------- -----------------
///
/// is modeled as if the following memory accesses occurred:
///
/// __________________________
/// |entry: |
/// | %V.s2a = alloca float |
/// --------------------------
/// |
/// ___________________________________
/// |DefBB: |
/// | store %float %V, float* %V.s2a |
/// -----------------------------------
/// | |
/// ____________________________________ ___________________________________
/// |UseBB1: | |UseBB2: |
/// | %V.reload1 = load float* %V.s2a | | %V.reload2 = load float* %V.s2a|
/// | use float %V.reload1 | | use float %V.reload2 |
/// ------------------------------------ -----------------------------------
///
Value,
/// MemoryKind::PHI: Models PHI nodes within the SCoP
///
/// Besides the MemoryKind::Value memory object used to model the normal
/// llvm::Value dependences described above, PHI nodes require an additional
/// memory object of type MemoryKind::PHI to describe the forwarding of values
/// to
/// the PHI node.
///
/// As an example, a PHIInst instructions
///
/// %PHI = phi float [ %Val1, %IncomingBlock1 ], [ %Val2, %IncomingBlock2 ]
///
/// is modeled as if the accesses occurred this way:
///
/// _______________________________
/// |entry: |
/// | %PHI.phiops = alloca float |
/// -------------------------------
/// | |
/// __________________________________ __________________________________
/// |IncomingBlock1: | |IncomingBlock2: |
/// | ... | | ... |
/// | store float %Val1 %PHI.phiops | | store float %Val2 %PHI.phiops |
/// | br label % JoinBlock | | br label %JoinBlock |
/// ---------------------------------- ----------------------------------
/// \ /
/// \ /
/// _________________________________________
/// |JoinBlock: |
/// | %PHI = load float, float* PHI.phiops |
/// -----------------------------------------
///
/// Note that there can also be a scalar write access for %PHI if used in a
/// different BasicBlock, i.e. there can be a memory object %PHI.phiops as
/// well as a memory object %PHI.s2a.
PHI,
/// MemoryKind::ExitPHI: Models PHI nodes in the SCoP's exit block
///
/// For PHI nodes in the Scop's exit block a special memory object kind is
/// used. The modeling used is identical to MemoryKind::PHI, with the
/// exception
/// that there are no READs from these memory objects. The PHINode's
/// llvm::Value is treated as a value escaping the SCoP. WRITE accesses
/// write directly to the escaping value's ".s2a" alloca.
ExitPHI
};
/// Maps from a loop to the affine function expressing its backedge taken count.
/// The backedge taken count already enough to express iteration domain as we
/// only allow loops with canonical induction variable.
/// A canonical induction variable is:
/// an integer recurrence that starts at 0 and increments by one each time
/// through the loop.
using LoopBoundMapType = std::map<const Loop *, const SCEV *>;
using AccFuncVector = std::vector<std::unique_ptr<MemoryAccess>>;
/// A class to store information about arrays in the SCoP.
///
/// Objects are accessible via the ScoP, MemoryAccess or the id associated with
/// the MemoryAccess access function.
///
class ScopArrayInfo {
public:
/// Construct a ScopArrayInfo object.
///
/// @param BasePtr The array base pointer.
/// @param ElementType The type of the elements stored in the array.
/// @param IslCtx The isl context used to create the base pointer id.
/// @param DimensionSizes A vector containing the size of each dimension.
/// @param Kind The kind of the array object.
/// @param DL The data layout of the module.
/// @param S The scop this array object belongs to.
/// @param BaseName The optional name of this memory reference.
ScopArrayInfo(Value *BasePtr, Type *ElementType, isl::ctx IslCtx,
ArrayRef<const SCEV *> DimensionSizes, MemoryKind Kind,
const DataLayout &DL, Scop *S, const char *BaseName = nullptr);
/// Destructor to free the isl id of the base pointer.
~ScopArrayInfo();
/// Update the element type of the ScopArrayInfo object.
///
/// Memory accesses referencing this ScopArrayInfo object may use
/// different element sizes. This function ensures the canonical element type
/// stored is small enough to model accesses to the current element type as
/// well as to @p NewElementType.
///
/// @param NewElementType An element type that is used to access this array.
void updateElementType(Type *NewElementType);
/// Update the sizes of the ScopArrayInfo object.
///
/// A ScopArrayInfo object may be created without all outer dimensions being
/// available. This function is called when new memory accesses are added for
/// this ScopArrayInfo object. It verifies that sizes are compatible and adds
/// additional outer array dimensions, if needed.
///
/// @param Sizes A vector of array sizes where the rightmost array
/// sizes need to match the innermost array sizes already
/// defined in SAI.
/// @param CheckConsistency Update sizes, even if new sizes are inconsistent
/// with old sizes
bool updateSizes(ArrayRef<const SCEV *> Sizes, bool CheckConsistency = true);
/// Make the ScopArrayInfo model a Fortran array.
/// It receives the Fortran array descriptor and stores this.
/// It also adds a piecewise expression for the outermost dimension
/// since this information is available for Fortran arrays at runtime.
void applyAndSetFAD(Value *FAD);
/// Get the FortranArrayDescriptor corresponding to this array if it exists,
/// nullptr otherwise.
Value *getFortranArrayDescriptor() const { return this->FAD; }
/// Set the base pointer to @p BP.
void setBasePtr(Value *BP) { BasePtr = BP; }
/// Return the base pointer.
Value *getBasePtr() const { return BasePtr; }
// Set IsOnHeap to the value in parameter.
void setIsOnHeap(bool value) { IsOnHeap = value; }
/// For indirect accesses return the origin SAI of the BP, else null.
const ScopArrayInfo *getBasePtrOriginSAI() const { return BasePtrOriginSAI; }
/// The set of derived indirect SAIs for this origin SAI.
const SmallSetVector<ScopArrayInfo *, 2> &getDerivedSAIs() const {
return DerivedSAIs;
}
/// Return the number of dimensions.
unsigned getNumberOfDimensions() const {
if (Kind == MemoryKind::PHI || Kind == MemoryKind::ExitPHI ||
Kind == MemoryKind::Value)
return 0;
return DimensionSizes.size();
}
/// Return the size of dimension @p dim as SCEV*.
//
// Scalars do not have array dimensions and the first dimension of
// a (possibly multi-dimensional) array also does not carry any size
// information, in case the array is not newly created.
const SCEV *getDimensionSize(unsigned Dim) const {
assert(Dim < getNumberOfDimensions() && "Invalid dimension");
return DimensionSizes[Dim];
}
/// Return the size of dimension @p dim as isl::pw_aff.
//
// Scalars do not have array dimensions and the first dimension of
// a (possibly multi-dimensional) array also does not carry any size
// information, in case the array is not newly created.
isl::pw_aff getDimensionSizePw(unsigned Dim) const {
assert(Dim < getNumberOfDimensions() && "Invalid dimension");
return DimensionSizesPw[Dim];
}
/// Get the canonical element type of this array.
///
/// @returns The canonical element type of this array.
Type *getElementType() const { return ElementType; }
/// Get element size in bytes.
int getElemSizeInBytes() const;
/// Get the name of this memory reference.
std::string getName() const;
/// Return the isl id for the base pointer.
isl::id getBasePtrId() const;
/// Return what kind of memory this represents.
MemoryKind getKind() const { return Kind; }
/// Is this array info modeling an llvm::Value?
bool isValueKind() const { return Kind == MemoryKind::Value; }
/// Is this array info modeling special PHI node memory?
///
/// During code generation of PHI nodes, there is a need for two kinds of
/// virtual storage. The normal one as it is used for all scalar dependences,
/// where the result of the PHI node is stored and later loaded from as well
/// as a second one where the incoming values of the PHI nodes are stored
/// into and reloaded when the PHI is executed. As both memories use the
/// original PHI node as virtual base pointer, we have this additional
/// attribute to distinguish the PHI node specific array modeling from the
/// normal scalar array modeling.
bool isPHIKind() const { return Kind == MemoryKind::PHI; }
/// Is this array info modeling an MemoryKind::ExitPHI?
bool isExitPHIKind() const { return Kind == MemoryKind::ExitPHI; }
/// Is this array info modeling an array?
bool isArrayKind() const { return Kind == MemoryKind::Array; }
/// Is this array allocated on heap
///
/// This property is only relevant if the array is allocated by Polly instead
/// of pre-existing. If false, it is allocated using alloca instead malloca.
bool isOnHeap() const { return IsOnHeap; }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Dump a readable representation to stderr.
void dump() const;
#endif
/// Print a readable representation to @p OS.
///
/// @param SizeAsPwAff Print the size as isl::pw_aff
void print(raw_ostream &OS, bool SizeAsPwAff = false) const;
/// Access the ScopArrayInfo associated with an access function.
static const ScopArrayInfo *getFromAccessFunction(isl::pw_multi_aff PMA);
/// Access the ScopArrayInfo associated with an isl Id.
static const ScopArrayInfo *getFromId(isl::id Id);
/// Get the space of this array access.
isl::space getSpace() const;
/// If the array is read only
bool isReadOnly();
/// Verify that @p Array is compatible to this ScopArrayInfo.
///
/// Two arrays are compatible if their dimensionality, the sizes of their
/// dimensions, and their element sizes match.
///
/// @param Array The array to compare against.
///
/// @returns True, if the arrays are compatible, False otherwise.
bool isCompatibleWith(const ScopArrayInfo *Array) const;
private:
void addDerivedSAI(ScopArrayInfo *DerivedSAI) {
DerivedSAIs.insert(DerivedSAI);
}
/// For indirect accesses this is the SAI of the BP origin.
const ScopArrayInfo *BasePtrOriginSAI;
/// For origin SAIs the set of derived indirect SAIs.
SmallSetVector<ScopArrayInfo *, 2> DerivedSAIs;
/// The base pointer.
AssertingVH<Value> BasePtr;
/// The canonical element type of this array.
///
/// The canonical element type describes the minimal accessible element in
/// this array. Not all elements accessed, need to be of the very same type,
/// but the allocation size of the type of the elements loaded/stored from/to
/// this array needs to be a multiple of the allocation size of the canonical
/// type.
Type *ElementType;
/// The isl id for the base pointer.
isl::id Id;
/// True if the newly allocated array is on heap.
bool IsOnHeap = false;
/// The sizes of each dimension as SCEV*.
SmallVector<const SCEV *, 4> DimensionSizes;
/// The sizes of each dimension as isl::pw_aff.
SmallVector<isl::pw_aff, 4> DimensionSizesPw;
/// The type of this scop array info object.
///
/// We distinguish between SCALAR, PHI and ARRAY objects.
MemoryKind Kind;
/// The data layout of the module.
const DataLayout &DL;
/// The scop this SAI object belongs to.
Scop &S;
/// If this array models a Fortran array, then this points
/// to the Fortran array descriptor.
Value *FAD = nullptr;
};
/// Represent memory accesses in statements.
class MemoryAccess {
friend class Scop;
friend class ScopStmt;
friend class ScopBuilder;
public:
/// The access type of a memory access
///
/// There are three kind of access types:
///
/// * A read access
///
/// A certain set of memory locations are read and may be used for internal
/// calculations.
///
/// * A must-write access
///
/// A certain set of memory locations is definitely written. The old value is
/// replaced by a newly calculated value. The old value is not read or used at
/// all.
///
/// * A may-write access
///
/// A certain set of memory locations may be written. The memory location may
/// contain a new value if there is actually a write or the old value may
/// remain, if no write happens.
enum AccessType {
READ = 0x1,
MUST_WRITE = 0x2,
MAY_WRITE = 0x3,
};
/// Reduction access type
///
/// Commutative and associative binary operations suitable for reductions
enum ReductionType {
RT_NONE, ///< Indicate no reduction at all
RT_ADD, ///< Addition
RT_MUL, ///< Multiplication
RT_BOR, ///< Bitwise Or
RT_BXOR, ///< Bitwise XOr
RT_BAND, ///< Bitwise And
};
using SubscriptsTy = SmallVector<const SCEV *, 4>;
private:
/// A unique identifier for this memory access.
///
/// The identifier is unique between all memory accesses belonging to the same
/// scop statement.
isl::id Id;
/// What is modeled by this MemoryAccess.
/// @see MemoryKind
MemoryKind Kind;
/// Whether it a reading or writing access, and if writing, whether it
/// is conditional (MAY_WRITE).
enum AccessType AccType;
/// Reduction type for reduction like accesses, RT_NONE otherwise
///
/// An access is reduction like if it is part of a load-store chain in which
/// both access the same memory location (use the same LLVM-IR value
/// as pointer reference). Furthermore, between the load and the store there
/// is exactly one binary operator which is known to be associative and
/// commutative.
///
/// TODO:
///
/// We can later lift the constraint that the same LLVM-IR value defines the
/// memory location to handle scops such as the following:
///
/// for i
/// for j
/// sum[i+j] = sum[i] + 3;
///
/// Here not all iterations access the same memory location, but iterations
/// for which j = 0 holds do. After lifting the equality check in ScopBuilder,
/// subsequent transformations do not only need check if a statement is
/// reduction like, but they also need to verify that that the reduction
/// property is only exploited for statement instances that load from and
/// store to the same data location. Doing so at dependence analysis time
/// could allow us to handle the above example.
ReductionType RedType = RT_NONE;
/// Parent ScopStmt of this access.
ScopStmt *Statement;
/// The domain under which this access is not modeled precisely.
///
/// The invalid domain for an access describes all parameter combinations
/// under which the statement looks to be executed but is in fact not because
/// some assumption/restriction makes the access invalid.
isl::set InvalidDomain;
// Properties describing the accessed array.
// TODO: It might be possible to move them to ScopArrayInfo.
// @{
/// The base address (e.g., A for A[i+j]).
///
/// The #BaseAddr of a memory access of kind MemoryKind::Array is the base
/// pointer of the memory access.
/// The #BaseAddr of a memory access of kind MemoryKind::PHI or
/// MemoryKind::ExitPHI is the PHI node itself.
/// The #BaseAddr of a memory access of kind MemoryKind::Value is the
/// instruction defining the value.
AssertingVH<Value> BaseAddr;
/// Type a single array element wrt. this access.
Type *ElementType;
/// Size of each dimension of the accessed array.
SmallVector<const SCEV *, 4> Sizes;
// @}
// Properties describing the accessed element.
// @{
/// The access instruction of this memory access.
///
/// For memory accesses of kind MemoryKind::Array the access instruction is
/// the Load or Store instruction performing the access.
///
/// For memory accesses of kind MemoryKind::PHI or MemoryKind::ExitPHI the
/// access instruction of a load access is the PHI instruction. The access
/// instruction of a PHI-store is the incoming's block's terminator
/// instruction.
///
/// For memory accesses of kind MemoryKind::Value the access instruction of a
/// load access is nullptr because generally there can be multiple
/// instructions in the statement using the same llvm::Value. The access
/// instruction of a write access is the instruction that defines the
/// llvm::Value.
Instruction *AccessInstruction = nullptr;
/// Incoming block and value of a PHINode.
SmallVector<std::pair<BasicBlock *, Value *>, 4> Incoming;
/// The value associated with this memory access.
///
/// - For array memory accesses (MemoryKind::Array) it is the loaded result
/// or the stored value. If the access instruction is a memory intrinsic it
/// the access value is also the memory intrinsic.
/// - For accesses of kind MemoryKind::Value it is the access instruction
/// itself.
/// - For accesses of kind MemoryKind::PHI or MemoryKind::ExitPHI it is the
/// PHI node itself (for both, READ and WRITE accesses).
///
AssertingVH<Value> AccessValue;
/// Are all the subscripts affine expression?
bool IsAffine = true;
/// Subscript expression for each dimension.
SubscriptsTy Subscripts;
/// Relation from statement instances to the accessed array elements.
///
/// In the common case this relation is a function that maps a set of loop
/// indices to the memory address from which a value is loaded/stored:
///
/// for i
/// for j
/// S: A[i + 3 j] = ...
///
/// => { S[i,j] -> A[i + 3j] }
///
/// In case the exact access function is not known, the access relation may
/// also be a one to all mapping { S[i,j] -> A[o] } describing that any
/// element accessible through A might be accessed.
///
/// In case of an access to a larger element belonging to an array that also
/// contains smaller elements, the access relation models the larger access
/// with multiple smaller accesses of the size of the minimal array element
/// type:
///
/// short *A;
///
/// for i
/// S: A[i] = *((double*)&A[4 * i]);
///
/// => { S[i] -> A[i]; S[i] -> A[o] : 4i <= o <= 4i + 3 }
isl::map AccessRelation;
/// Updated access relation read from JSCOP file.
isl::map NewAccessRelation;
/// Fortran arrays whose sizes are not statically known are stored in terms
/// of a descriptor struct. This maintains a raw pointer to the memory,
/// along with auxiliary fields with information such as dimensions.
/// We hold a reference to the descriptor corresponding to a MemoryAccess
/// into a Fortran array. FAD for "Fortran Array Descriptor"
AssertingVH<Value> FAD;
// @}
isl::basic_map createBasicAccessMap(ScopStmt *Statement);
isl::set assumeNoOutOfBound();
/// Compute bounds on an over approximated access relation.
///
/// @param ElementSize The size of one element accessed.
void computeBoundsOnAccessRelation(unsigned ElementSize);
/// Get the original access function as read from IR.
isl::map getOriginalAccessRelation() const;
/// Return the space in which the access relation lives in.
isl::space getOriginalAccessRelationSpace() const;
/// Get the new access function imported or set by a pass
isl::map getNewAccessRelation() const;
/// Fold the memory access to consider parametric offsets
///
/// To recover memory accesses with array size parameters in the subscript
/// expression we post-process the delinearization results.
///
/// We would normally recover from an access A[exp0(i) * N + exp1(i)] into an
/// array A[][N] the 2D access A[exp0(i)][exp1(i)]. However, another valid
/// delinearization is A[exp0(i) - 1][exp1(i) + N] which - depending on the
/// range of exp1(i) - may be preferable. Specifically, for cases where we
/// know exp1(i) is negative, we want to choose the latter expression.
///
/// As we commonly do not have any information about the range of exp1(i),
/// we do not choose one of the two options, but instead create a piecewise
/// access function that adds the (-1, N) offsets as soon as exp1(i) becomes
/// negative. For a 2D array such an access function is created by applying
/// the piecewise map:
///
/// [i,j] -> [i, j] : j >= 0
/// [i,j] -> [i-1, j+N] : j < 0
///
/// We can generalize this mapping to arbitrary dimensions by applying this
/// piecewise mapping pairwise from the rightmost to the leftmost access
/// dimension. It would also be possible to cover a wider range by introducing
/// more cases and adding multiple of Ns to these cases. However, this has
/// not yet been necessary.
/// The introduction of different cases necessarily complicates the memory
/// access function, but cases that can be statically proven to not happen
/// will be eliminated later on.
void foldAccessRelation();
/// Create the access relation for the underlying memory intrinsic.
void buildMemIntrinsicAccessRelation();
/// Assemble the access relation from all available information.
///
/// In particular, used the information passes in the constructor and the
/// parent ScopStmt set by setStatment().
///
/// @param SAI Info object for the accessed array.
void buildAccessRelation(const ScopArrayInfo *SAI);
/// Carry index overflows of dimensions with constant size to the next higher
/// dimension.
///
/// For dimensions that have constant size, modulo the index by the size and
/// add up the carry (floored division) to the next higher dimension. This is
/// how overflow is defined in row-major order.
/// It happens e.g. when ScalarEvolution computes the offset to the base
/// pointer and would algebraically sum up all lower dimensions' indices of
/// constant size.
///
/// Example:
/// float (*A)[4];
/// A[1][6] -> A[2][2]
void wrapConstantDimensions();
public:
/// Create a new MemoryAccess.
///
/// @param Stmt The parent statement.
/// @param AccessInst The instruction doing the access.
/// @param BaseAddr The accessed array's address.
/// @param ElemType The type of the accessed array elements.
/// @param AccType Whether read or write access.
/// @param IsAffine Whether the subscripts are affine expressions.
/// @param Kind The kind of memory accessed.
/// @param Subscripts Subscript expressions
/// @param Sizes Dimension lengths of the accessed array.
MemoryAccess(ScopStmt *Stmt, Instruction *AccessInst, AccessType AccType,
Value *BaseAddress, Type *ElemType, bool Affine,
ArrayRef<const SCEV *> Subscripts, ArrayRef<const SCEV *> Sizes,
Value *AccessValue, MemoryKind Kind);
/// Create a new MemoryAccess that corresponds to @p AccRel.
///
/// Along with @p Stmt and @p AccType it uses information about dimension
/// lengths of the accessed array, the type of the accessed array elements,
/// the name of the accessed array that is derived from the object accessible
/// via @p AccRel.
///
/// @param Stmt The parent statement.
/// @param AccType Whether read or write access.
/// @param AccRel The access relation that describes the memory access.
MemoryAccess(ScopStmt *Stmt, AccessType AccType, isl::map AccRel);
MemoryAccess(const MemoryAccess &) = delete;
MemoryAccess &operator=(const MemoryAccess &) = delete;
~MemoryAccess();
/// Add a new incoming block/value pairs for this PHI/ExitPHI access.
///
/// @param IncomingBlock The PHI's incoming block.
/// @param IncomingValue The value when reaching the PHI from the @p
/// IncomingBlock.
void addIncoming(BasicBlock *IncomingBlock, Value *IncomingValue) {
assert(!isRead());
assert(isAnyPHIKind());
Incoming.emplace_back(std::make_pair(IncomingBlock, IncomingValue));
}
/// Return the list of possible PHI/ExitPHI values.
///
/// After code generation moves some PHIs around during region simplification,
/// we cannot reliably locate the original PHI node and its incoming values
/// anymore. For this reason we remember these explicitly for all PHI-kind
/// accesses.
ArrayRef<std::pair<BasicBlock *, Value *>> getIncoming() const {
assert(isAnyPHIKind());
return Incoming;
}
/// Get the type of a memory access.
enum AccessType getType() { return AccType; }
/// Is this a reduction like access?
bool isReductionLike() const { return RedType != RT_NONE; }
/// Is this a read memory access?
bool isRead() const { return AccType == MemoryAccess::READ; }
/// Is this a must-write memory access?
bool isMustWrite() const { return AccType == MemoryAccess::MUST_WRITE; }
/// Is this a may-write memory access?
bool isMayWrite() const { return AccType == MemoryAccess::MAY_WRITE; }
/// Is this a write memory access?
bool isWrite() const { return isMustWrite() || isMayWrite(); }
/// Is this a memory intrinsic access (memcpy, memset, memmove)?
bool isMemoryIntrinsic() const {
return isa<MemIntrinsic>(getAccessInstruction());
}
/// Check if a new access relation was imported or set by a pass.
bool hasNewAccessRelation() const { return !NewAccessRelation.is_null(); }
/// Return the newest access relation of this access.
///
/// There are two possibilities:
/// 1) The original access relation read from the LLVM-IR.
/// 2) A new access relation imported from a json file or set by another
/// pass (e.g., for privatization).
///
/// As 2) is by construction "newer" than 1) we return the new access
/// relation if present.
///
isl::map getLatestAccessRelation() const {
return hasNewAccessRelation() ? getNewAccessRelation()
: getOriginalAccessRelation();
}
/// Old name of getLatestAccessRelation().
isl::map getAccessRelation() const { return getLatestAccessRelation(); }
/// Get an isl map describing the memory address accessed.
///
/// In most cases the memory address accessed is well described by the access
/// relation obtained with getAccessRelation. However, in case of arrays
/// accessed with types of different size the access relation maps one access
/// to multiple smaller address locations. This method returns an isl map that
/// relates each dynamic statement instance to the unique memory location
/// that is loaded from / stored to.
///
/// For an access relation { S[i] -> A[o] : 4i <= o <= 4i + 3 } this method
/// will return the address function { S[i] -> A[4i] }.
///
/// @returns The address function for this memory access.
isl::map getAddressFunction() const;
/// Return the access relation after the schedule was applied.
isl::pw_multi_aff
applyScheduleToAccessRelation(isl::union_map Schedule) const;
/// Get an isl string representing the access function read from IR.
std::string getOriginalAccessRelationStr() const;
/// Get an isl string representing a new access function, if available.
std::string getNewAccessRelationStr() const;
/// Get an isl string representing the latest access relation.
std::string getAccessRelationStr() const;
/// Get the original base address of this access (e.g. A for A[i+j]) when
/// detected.
///
/// This address may differ from the base address referenced by the original
/// ScopArrayInfo to which this array belongs, as this memory access may
/// have been canonicalized to a ScopArrayInfo which has a different but
/// identically-valued base pointer in case invariant load hoisting is
/// enabled.
Value *getOriginalBaseAddr() const { return BaseAddr; }
/// Get the detection-time base array isl::id for this access.
isl::id getOriginalArrayId() const;
/// Get the base array isl::id for this access, modifiable through
/// setNewAccessRelation().
isl::id getLatestArrayId() const;
/// Old name of getOriginalArrayId().
isl::id getArrayId() const { return getOriginalArrayId(); }
/// Get the detection-time ScopArrayInfo object for the base address.
const ScopArrayInfo *getOriginalScopArrayInfo() const;
/// Get the ScopArrayInfo object for the base address, or the one set
/// by setNewAccessRelation().
const ScopArrayInfo *getLatestScopArrayInfo() const;
/// Legacy name of getOriginalScopArrayInfo().
const ScopArrayInfo *getScopArrayInfo() const {
return getOriginalScopArrayInfo();
}
/// Return a string representation of the access's reduction type.
const std::string getReductionOperatorStr() const;
/// Return a string representation of the reduction type @p RT.
static const std::string getReductionOperatorStr(ReductionType RT);
/// Return the element type of the accessed array wrt. this access.
Type *getElementType() const { return ElementType; }
/// Return the access value of this memory access.
Value *getAccessValue() const { return AccessValue; }
/// Return llvm::Value that is stored by this access, if available.
///
/// PHI nodes may not have a unique value available that is stored, as in
/// case of region statements one out of possibly several llvm::Values
/// might be stored. In this case nullptr is returned.
Value *tryGetValueStored() {
assert(isWrite() && "Only write statement store values");
if (isAnyPHIKind()) {
if (Incoming.size() == 1)
return Incoming[0].second;
return nullptr;
}
return AccessValue;
}
/// Return the access instruction of this memory access.
Instruction *getAccessInstruction() const { return AccessInstruction; }
/// Return an iterator range containing the subscripts.
iterator_range<SubscriptsTy::const_iterator> subscripts() const {
return make_range(Subscripts.begin(), Subscripts.end());
}
/// Return the number of access function subscript.
unsigned getNumSubscripts() const { return Subscripts.size(); }
/// Return the access function subscript in the dimension @p Dim.
const SCEV *getSubscript(unsigned Dim) const { return Subscripts[Dim]; }
/// Compute the isl representation for the SCEV @p E wrt. this access.
///
/// Note that this function will also adjust the invalid context accordingly.
isl::pw_aff getPwAff(const SCEV *E);
/// Get the invalid domain for this access.
isl::set getInvalidDomain() const { return InvalidDomain; }
/// Get the invalid context for this access.
isl::set getInvalidContext() const { return getInvalidDomain().params(); }
/// Get the stride of this memory access in the specified Schedule. Schedule
/// is a map from the statement to a schedule where the innermost dimension is
/// the dimension of the innermost loop containing the statement.
isl::set getStride(isl::map Schedule) const;
/// Get the FortranArrayDescriptor corresponding to this memory access if
/// it exists, and nullptr otherwise.
Value *getFortranArrayDescriptor() const { return this->FAD; }
/// Is the stride of the access equal to a certain width? Schedule is a map
/// from the statement to a schedule where the innermost dimension is the
/// dimension of the innermost loop containing the statement.
bool isStrideX(isl::map Schedule, int StrideWidth) const;
/// Is consecutive memory accessed for a given statement instance set?
/// Schedule is a map from the statement to a schedule where the innermost
/// dimension is the dimension of the innermost loop containing the
/// statement.
bool isStrideOne(isl::map Schedule) const;
/// Is always the same memory accessed for a given statement instance set?
/// Schedule is a map from the statement to a schedule where the innermost
/// dimension is the dimension of the innermost loop containing the
/// statement.
bool isStrideZero(isl::map Schedule) const;
/// Return the kind when this access was first detected.
MemoryKind getOriginalKind() const {
assert(!getOriginalScopArrayInfo() /* not yet initialized */ ||
getOriginalScopArrayInfo()->getKind() == Kind);
return Kind;
}
/// Return the kind considering a potential setNewAccessRelation.
MemoryKind getLatestKind() const {
return getLatestScopArrayInfo()->getKind();
}
/// Whether this is an access of an explicit load or store in the IR.
bool isOriginalArrayKind() const {
return getOriginalKind() == MemoryKind::Array;
}
/// Whether storage memory is either an custom .s2a/.phiops alloca
/// (false) or an existing pointer into an array (true).
bool isLatestArrayKind() const {
return getLatestKind() == MemoryKind::Array;
}
/// Old name of isOriginalArrayKind.
bool isArrayKind() const { return isOriginalArrayKind(); }
/// Whether this access is an array to a scalar memory object, without
/// considering changes by setNewAccessRelation.
///
/// Scalar accesses are accesses to MemoryKind::Value, MemoryKind::PHI or
/// MemoryKind::ExitPHI.
bool isOriginalScalarKind() const {
return getOriginalKind() != MemoryKind::Array;
}
/// Whether this access is an array to a scalar memory object, also
/// considering changes by setNewAccessRelation.
bool isLatestScalarKind() const {
return getLatestKind() != MemoryKind::Array;
}
/// Old name of isOriginalScalarKind.