diff --git a/docs/Runtime.md b/docs/Runtime.md index aa2155923f805..bcac83cac2b49 100644 --- a/docs/Runtime.md +++ b/docs/Runtime.md @@ -73,6 +73,8 @@ Rename with a non-`stdlib` naming scheme. 000000000001cb30 T _swift_allocBox 000000000001c990 T _swift_allocObject 000000000001ca60 T _swift_bufferAllocate +000000000001ca70 T _swift_bufferAllocateOnStack +000000000001ca80 T _swift_bufferDeallocateFromStack 000000000001ca90 T _swift_bufferHeaderSize 000000000001cd30 T _swift_deallocBox 000000000001d490 T _swift_deallocClassInstance diff --git a/docs/SIL.rst b/docs/SIL.rst index acbc6963ccd95..4444e7cc0200f 100644 --- a/docs/SIL.rst +++ b/docs/SIL.rst @@ -1828,24 +1828,16 @@ alloc_ref ````````` :: - sil-instruction ::= 'alloc_ref' - ('[' 'objc' ']')? - ('[' 'stack' ']')? - ('[' 'tail_elems' sil-type '*' sil-operand ']')* - sil-type + sil-instruction ::= 'alloc_ref' ('[' 'objc' ']')? ('[' 'stack' ']')? sil-type %1 = alloc_ref [stack] $T - %1 = alloc_ref [tail_elems $E * %2 : Builtin.Word] $T // $T must be a reference type // %1 has type $T - // $E is the type of the tail-allocated elements - // %2 must be of a builtin integer type Allocates an object of reference type ``T``. The object will be initialized with retain count 1; its state will be otherwise uninitialized. The optional ``objc`` attribute indicates that the object should be allocated using Objective-C's allocation methods (``+allocWithZone:``). - The optional ``stack`` attribute indicates that the object can be allocated on the stack instead on the heap. In this case the instruction must have balanced with a ``dealloc_ref [stack]`` instruction to mark the end of the @@ -1855,15 +1847,6 @@ possible. The final decision on stack allocation is done during llvm IR generation. This is because the decision also depends on the object size, which is not necessarily known at SIL level. -The optional ``tail_elems`` attributes specifies the amount of space to be -reserved for tail-allocated arrays of given element types and element counts. -If there are more than one ``tail_elems`` attributes then the tail arrays are -allocated in the specified order. -The count-operand must be of a builtin integer type. -The instructions ``ref_tail_addr`` and ``tail_addr`` can be used to project -the tail elements. -The ``objc`` attribute cannot be used together with ``tail_elems``. - alloc_ref_dynamic ````````````````` :: @@ -2346,31 +2329,6 @@ special behavior in this regard, unlike ``char*`` or ``void*`` in C.) It is also undefined behavior to index out of bounds of an array, except to index the "past-the-end" address of the array. -tail_addr -````````` -:: - - sil-instruction ::= 'tail_addr' sil-operand ',' sil-operand ',' sil-type - - %2 = tail_addr %0 : $*T, %1 : $Builtin.Int, $E - // %0 must be of an address type $*T - // %1 must be of a builtin integer type - // %2 will be of type $*E - -Given an address of an array of ``%1`` values, returns the address of an -element which is tail-allocated after the array. -This instruction is equivalent to ``index_addr`` except that the resulting -address is aligned-up to the tail-element type ``$E``. - -This instruction is used to project the N-th tail-allocated array from an -object which is created by an ``alloc_ref`` with multiple ``tail_elems``. -The first operand is the address of an element of the (N-1)-th array, usually -the first element. The second operand is the number of elements until the end -of that array. The result is the address of the first element of the N-th array. - -It is undefined behavior if the provided address, count and type do not match -the actual layout of tail-allocated arrays of the underlying object. - index_raw_pointer ````````````````` :: @@ -3298,23 +3256,6 @@ Given an instance of a class, derives the address of a physical instance variable inside the instance. It is undefined behavior if the class value is null. -ref_tail_addr -````````````` -:: - - sil-instruction ::= 'ref_tail_addr' sil-operand ',' sil-type - - %1 = ref_tail_addr %0 : $C, $E - // %0 must be a value of class type $C with tail-allocated elements $E - // %1 will be of type $*E - -Given an instance of a class, which is created with tail-allocated array(s), -derives the address of the first element of the first tail-allocated array. -This instruction is used to project the first tail-allocated element from an -object which is created by an ``alloc_ref`` with ``tail_elems``. -It is undefined behavior if the class instance does not have tail-allocated -arrays or if the element-types do not match. - Enums ~~~~~ diff --git a/include/swift/AST/Builtins.def b/include/swift/AST/Builtins.def index e97dff05924c5..54fa738e9b789 100644 --- a/include/swift/AST/Builtins.def +++ b/include/swift/AST/Builtins.def @@ -252,23 +252,9 @@ BUILTIN_SIL_OPERATION(ReinterpretCast, "reinterpretCast", Special) /// valid within the scope of the statement for logical lvalues. BUILTIN_SIL_OPERATION(AddressOf, "addressof", Special) -/// GepRaw(Builtin.RawPointer, Builtin.Word) -> Builtin.RawPointer -/// -/// Adds index bytes to a base pointer. -BUILTIN_SIL_OPERATION(GepRaw, "gepRaw", Integer) - -/// Gep (Builtin.RawPointer, Builtin.Word, T.Type) -> Builtin.RawPointer -/// -/// Like the GepRaw-bultin, but multiplies the index by stride-of type 'T'. +/// GetElementPtr has type (Builtin.RawPointer, T) -> Builtin.RawPointer BUILTIN_SIL_OPERATION(Gep, "gep", Integer) -/// getTailAddr(Builtin.RawPointer, -/// Builtin.Word, T.Type, E.Type) -> Builtin.RawPointer -/// -/// Like the Gep-builtin, but rounds up the resulting address to a tail- -/// allocated element type 'E'. -BUILTIN_SIL_OPERATION(GetTailAddr, "getTailAddr", Integer) - /// condfail(Int1) -> () /// Triggers a runtime failure if the condition is true. BUILTIN_SIL_OPERATION(CondFail, "condfail", Special) @@ -328,18 +314,6 @@ BUILTIN_SIL_OPERATION(IsUniqueOrPinned_native, "isUniqueOrPinned_native", /// bindMemory : (Builtin.RawPointer, Builtin.Word, T.Type) -> () BUILTIN_SIL_OPERATION(BindMemory, "bindMemory", Special) -/// allocWithTailElems_(C.Type, -/// Builtin.Word, E1.Type, ... , Builtin.Word, En.Type) -> C\ -/// -/// The integer suffix specifies the number of tail-allocated arrays. -/// Each tail-allocated array adds a counter and an element meta-type parameter. -BUILTIN_SIL_OPERATION(AllocWithTailElems, "allocWithTailElems", Special) - -/// projectTailElems : (C) -> Builtin.RawPointer -/// -/// Projects the first tail-allocated element of type E from a class C. -BUILTIN_SIL_OPERATION(ProjectTailElems, "projectTailElems", Special) - #undef BUILTIN_SIL_OPERATION // BUILTIN_RUNTIME_CALL - A call into a runtime function. diff --git a/include/swift/AST/DiagnosticsParse.def b/include/swift/AST/DiagnosticsParse.def index d95189e0c2392..4fe42647f4819 100644 --- a/include/swift/AST/DiagnosticsParse.def +++ b/include/swift/AST/DiagnosticsParse.def @@ -511,8 +511,6 @@ ERROR(sil_too_many_substitutions,none, "too many substitutions", ()) ERROR(sil_dbg_unknown_key,none, "unknown key '%0' in debug variable declaration", (StringRef)) -ERROR(sil_objc_with_tail_elements,none, - "alloc_ref [objc] cannot have tail allocated elements", ()) // SIL Basic Blocks ERROR(expected_sil_block_name,none, diff --git a/include/swift/LLVMPasses/Passes.h b/include/swift/LLVMPasses/Passes.h index 0985528c4df57..a05a406b9d0e4 100644 --- a/include/swift/LLVMPasses/Passes.h +++ b/include/swift/LLVMPasses/Passes.h @@ -87,6 +87,14 @@ namespace swift { SwiftARCContract() : llvm::FunctionPass(ID) {} }; + class SwiftStackPromotion : public llvm::FunctionPass { + virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override; + virtual bool runOnFunction(llvm::Function &F) override; + public: + static char ID; + SwiftStackPromotion() : llvm::FunctionPass(ID) {} + }; + class InlineTreePrinter : public llvm::ModulePass { virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const override; virtual bool runOnModule(llvm::Module &M) override; diff --git a/include/swift/LLVMPasses/PassesFwd.h b/include/swift/LLVMPasses/PassesFwd.h index 83067a7114315..cb14c9fbfb617 100644 --- a/include/swift/LLVMPasses/PassesFwd.h +++ b/include/swift/LLVMPasses/PassesFwd.h @@ -23,6 +23,7 @@ namespace llvm { void initializeSwiftRCIdentityPass(PassRegistry &); void initializeSwiftARCOptPass(PassRegistry &); void initializeSwiftARCContractPass(PassRegistry &); + void initializeSwiftStackPromotionPass(PassRegistry &); void initializeInlineTreePrinterPass(PassRegistry &); void initializeSwiftMergeFunctionsPass(PassRegistry &); } @@ -30,6 +31,7 @@ namespace llvm { namespace swift { llvm::FunctionPass *createSwiftARCOptPass(); llvm::FunctionPass *createSwiftARCContractPass(); + llvm::FunctionPass *createSwiftStackPromotionPass(); llvm::ModulePass *createInlineTreePrinterPass(); llvm::ModulePass *createSwiftMergeFunctionsPass(); llvm::ImmutablePass *createSwiftAAWrapperPass(); diff --git a/include/swift/SIL/InstructionUtils.h b/include/swift/SIL/InstructionUtils.h index fbcbe8f25ee04..c5a8cb0915d74 100644 --- a/include/swift/SIL/InstructionUtils.h +++ b/include/swift/SIL/InstructionUtils.h @@ -20,14 +20,6 @@ namespace swift { /// Strip off casts/indexing insts/address projections from V until there is /// nothing left to strip. SILValue getUnderlyingObject(SILValue V); - -/// Strip off indexing and address projections. -/// -/// This is similar to getUnderlyingObject, except that it does not strip any -/// object-to-address projections, like ref_element_addr. In other words, the -/// result is always an address value. -SILValue getUnderlyingAddressRoot(SILValue V); - SILValue getUnderlyingObjectStopAtMarkDependence(SILValue V); SILValue stripSinglePredecessorArgs(SILValue V); diff --git a/include/swift/SIL/Projection.h b/include/swift/SIL/Projection.h index de59e30dade65..e2bbddb22cde2 100644 --- a/include/swift/SIL/Projection.h +++ b/include/swift/SIL/Projection.h @@ -91,9 +91,8 @@ enum class ProjectionKind : unsigned { Upcast = 0, RefCast = 1, BitwiseCast = 2, - TailElems = 3, FirstPointerKind = Upcast, - LastPointerKind = TailElems, + LastPointerKind = BitwiseCast, // Index Projection Kinds FirstIndexKind = 7, @@ -125,7 +124,6 @@ static inline bool isCastProjectionKind(ProjectionKind Kind) { case ProjectionKind::Class: case ProjectionKind::Enum: case ProjectionKind::Box: - case ProjectionKind::TailElems: return false; } } @@ -160,12 +158,6 @@ struct ProjectionIndex { Aggregate = REA->getOperand(); break; } - case ValueKind::RefTailAddrInst: { - RefTailAddrInst *REA = cast(V); - Index = 0; - Aggregate = REA->getOperand(); - break; - } case ValueKind::ProjectBoxInst: { ProjectBoxInst *PBI = cast(V); // A box has only a single payload. @@ -301,7 +293,6 @@ class Projection { case ProjectionKind::Upcast: case ProjectionKind::RefCast: case ProjectionKind::BitwiseCast: - case ProjectionKind::TailElems: return getCastType(BaseType); case ProjectionKind::Index: // Index types do not change the underlying type. @@ -332,19 +323,13 @@ class Projection { SILType getCastType(SILType BaseType) const { assert(isValid()); + assert(getKind() == ProjectionKind::Upcast || + getKind() == ProjectionKind::RefCast || + getKind() == ProjectionKind::BitwiseCast); auto *Ty = getPointer(); assert(Ty->isCanonical()); - switch (getKind()) { - case ProjectionKind::Upcast: - case ProjectionKind::RefCast: - case ProjectionKind::BitwiseCast: - return SILType::getPrimitiveType(Ty->getCanonicalType(), - BaseType.getCategory()); - case ProjectionKind::TailElems: - return SILType::getPrimitiveAddressType(Ty->getCanonicalType()); - default: - llvm_unreachable("unknown cast projection type"); - } + return SILType::getPrimitiveType(Ty->getCanonicalType(), + BaseType.getCategory()); } bool operator<(const Projection &Other) const; @@ -377,7 +362,6 @@ class Projection { } case ValueKind::StructElementAddrInst: case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::ProjectBoxInst: case ValueKind::TupleElementAddrInst: case ValueKind::UncheckedTakeEnumDataAddrInst: @@ -401,8 +385,7 @@ class Projection { /// Returns true if this instruction projects from an object type into an /// address subtype. static bool isObjectToAddressProjection(SILValue V) { - return isa(V) || isa(V) || - isa(V); + return isa(V) || isa(V); } /// Given a specific SILType, return all first level projections if it is an @@ -427,7 +410,6 @@ class Projection { case ProjectionKind::Tuple: case ProjectionKind::Index: case ProjectionKind::Class: - case ProjectionKind::TailElems: case ProjectionKind::Enum: case ProjectionKind::Box: return false; @@ -446,7 +428,6 @@ class Projection { case ProjectionKind::Tuple: case ProjectionKind::Upcast: case ProjectionKind::Box: - case ProjectionKind::TailElems: return false; } } diff --git a/include/swift/SIL/SILBuilder.h b/include/swift/SIL/SILBuilder.h index 9c6321d0858c9..276a1215b69d3 100644 --- a/include/swift/SIL/SILBuilder.h +++ b/include/swift/SIL/SILBuilder.h @@ -255,26 +255,13 @@ class SILBuilder { Var)); } - AllocRefInst *createAllocRef(SILLocation Loc, SILType ObjectType, - bool objc, bool canAllocOnStack) { + AllocRefInst *createAllocRef(SILLocation Loc, SILType elementType, bool objc, + bool canAllocOnStack) { // AllocRefInsts expand to function calls and can therefore not be // counted towards the function prologue. assert(!Loc.isInPrologue()); - return insert(AllocRefInst::create(getSILDebugLocation(Loc), F, ObjectType, + return insert(AllocRefInst::create(getSILDebugLocation(Loc), elementType, F, objc, canAllocOnStack, - {}, {}, OpenedArchetypes)); - } - - AllocRefInst *createAllocRef(SILLocation Loc, SILType ObjectType, - bool canAllocOnStack, - ArrayRef ElementTypes, - ArrayRef ElementCountOperands) { - // AllocRefInsts expand to function calls and can therefore not be - // counted towards the function prologue. - assert(!Loc.isInPrologue()); - return insert(AllocRefInst::create(getSILDebugLocation(Loc), - F, ObjectType, false, canAllocOnStack, - ElementTypes, ElementCountOperands, OpenedArchetypes)); } @@ -911,12 +898,6 @@ class SILBuilder { return createRefElementAddr(Loc, Operand, Field, ResultTy); } - RefTailAddrInst *createRefTailAddr(SILLocation Loc, SILValue Ref, - SILType ResultTy) { - return insert(new (F.getModule()) RefTailAddrInst(getSILDebugLocation(Loc), - Ref, ResultTy)); - } - ClassMethodInst *createClassMethod(SILLocation Loc, SILValue Operand, SILDeclRef Member, SILType MethodTy, bool Volatile = false) { @@ -1262,12 +1243,6 @@ class SILBuilder { Operand, Index)); } - TailAddrInst *createTailAddr(SILLocation Loc, SILValue Operand, - SILValue Count, SILType ResultTy) { - return insert(new (F.getModule()) TailAddrInst(getSILDebugLocation(Loc), - Operand, Count, ResultTy)); - } - IndexRawPointerInst *createIndexRawPointer(SILLocation Loc, SILValue Operand, SILValue Index) { return insert(new (F.getModule()) IndexRawPointerInst( diff --git a/include/swift/SIL/SILCloner.h b/include/swift/SIL/SILCloner.h index e4eadb93e9818..87fe69854098a 100644 --- a/include/swift/SIL/SILCloner.h +++ b/include/swift/SIL/SILCloner.h @@ -473,23 +473,10 @@ template void SILCloner::visitAllocRefInst(AllocRefInst *Inst) { getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); - SILInstruction *NewInst = nullptr; - if (Inst->isObjC()) { - NewInst = getBuilder().createAllocRef(getOpLocation(Inst->getLoc()), - getOpType(Inst->getType()), - true, Inst->canAllocOnStack()); - } else { - auto CountArgs = getOpValueArray<8>(OperandValueArrayRef(Inst->getTailAllocatedCounts())); - SmallVector ElemTypes; - for (SILType OrigElemType : Inst->getTailAllocatedTypes()) { - ElemTypes.push_back(getOpType(OrigElemType)); - } - NewInst = getBuilder().createAllocRef(getOpLocation(Inst->getLoc()), - getOpType(Inst->getType()), - Inst->canAllocOnStack(), - ElemTypes, CountArgs); - } - doPostProcess(Inst, NewInst); + doPostProcess(Inst, + getBuilder().createAllocRef(getOpLocation(Inst->getLoc()), + getOpType(Inst->getType()), + Inst->isObjC(), Inst->canAllocOnStack())); } template @@ -1304,16 +1291,6 @@ SILCloner::visitRefElementAddrInst(RefElementAddrInst *Inst) { getOpType(Inst->getType()))); } -template -void -SILCloner::visitRefTailAddrInst(RefTailAddrInst *Inst) { - getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); - doPostProcess(Inst, - getBuilder().createRefTailAddr(getOpLocation(Inst->getLoc()), - getOpValue(Inst->getOperand()), - getOpType(Inst->getType()))); -} - template void SILCloner::visitClassMethodInst(ClassMethodInst *Inst) { @@ -1740,17 +1717,6 @@ SILCloner::visitIndexAddrInst(IndexAddrInst *Inst) { getOpValue(Inst->getIndex()))); } -template -void -SILCloner::visitTailAddrInst(TailAddrInst *Inst) { - getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); - doPostProcess(Inst, - getBuilder().createTailAddr(getOpLocation(Inst->getLoc()), - getOpValue(Inst->getBase()), - getOpValue(Inst->getIndex()), - getOpType(Inst->getType()))); -} - template void SILCloner::visitIndexRawPointerInst(IndexRawPointerInst *Inst) { diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index beb9d0c308c20..b466a37700c15 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -676,74 +676,44 @@ class AllocStackInst final /// AllocRefInst - This represents the primitive allocation of an instance /// of a reference type. Aside from the reference count, the instance is /// returned uninitialized. -/// Optionally, the allocated instance contains space for one or more tail- -/// allocated arrays. class AllocRefInst final : public AllocationInst, - public StackPromotable { + public StackPromotable, + private llvm::TrailingObjects { + friend TrailingObjects; friend class SILBuilder; + unsigned NumOperands : 31; + bool ObjC : 1; - // Number of tail-allocated arrays. - unsigned short NumTailTypes; - - bool ObjC; - - /// The first NumTailTypes operands are counts for the tail allocated - /// elements, the remaining operands are opened archetype operands. - TailAllocatedOperandList<0> Operands; - - SILType *getTypeStorage() { - return reinterpret_cast(Operands.asArray().end()); - } + AllocRefInst(SILDebugLocation Loc, SILType type, SILFunction &F, bool objc, + bool canBeOnStack, ArrayRef TypeDependentOperands); - const SILType *getTypeStorage() const { - return reinterpret_cast(Operands.asArray().end()); - } - - AllocRefInst(SILDebugLocation DebugLoc, SILFunction &F, - SILType ObjectType, - bool objc, bool canBeOnStack, - ArrayRef ElementTypes, - ArrayRef AllOperands); - - static AllocRefInst *create(SILDebugLocation DebugLoc, SILFunction &F, - SILType ObjectType, - bool objc, bool canBeOnStack, - ArrayRef ElementTypes, - ArrayRef ElementCountOperands, + static AllocRefInst *create(SILDebugLocation Loc, SILType type, + SILFunction &F, bool objc, bool canBeOnStack, SILOpenedArchetypesState &OpenedArchetypes); public: - ArrayRef getTailAllocatedTypes() const { - return {getTypeStorage(), NumTailTypes}; - } - - MutableArrayRef getTailAllocatedTypes() { - return {getTypeStorage(), NumTailTypes}; - } - - ArrayRef getTailAllocatedCounts() const { - return getAllOperands().slice(0, NumTailTypes); - } - - MutableArrayRef getTailAllocatedCounts() { - return getAllOperands().slice(0, NumTailTypes); + ~AllocRefInst() { + Operand *Operands = getTrailingObjects(); + for (unsigned i = 0, end = NumOperands; i < end; ++i) { + Operands[i].~Operand(); + } } ArrayRef getAllOperands() const { - return Operands.asArray(); + return { getTrailingObjects(), NumOperands }; } MutableArrayRef getAllOperands() { - return Operands.asArray(); + return { getTrailingObjects(), NumOperands }; } ArrayRef getTypeDependentOperands() const { - return getAllOperands().slice(NumTailTypes); + return getAllOperands(); } MutableArrayRef getTypeDependentOperands() { - return getAllOperands().slice(NumTailTypes); + return getAllOperands(); } /// Whether to use Objective-C's allocation mechanism (+allocWithZone:). @@ -3524,26 +3494,6 @@ class RefElementAddrInst } }; -/// RefTailAddrInst - Derive the address of the first element of the first -/// tail-allocated array in a reference type instance. -class RefTailAddrInst - : public UnaryInstructionBase -{ - friend class SILBuilder; - - RefTailAddrInst(SILDebugLocation DebugLoc, SILValue Operand, SILType ResultTy) - : UnaryInstructionBase(DebugLoc, Operand, ResultTy) {} - -public: - ClassDecl *getClassDecl() const { - auto s = getOperand()->getType().getClassOrBoundGenericClass(); - assert(s); - return s; - } - - SILType getTailType() const { return getType().getObjectType(); } -}; - /// MethodInst - Abstract base for instructions that implement dynamic /// method lookup. class MethodInst : public SILInstruction { @@ -4329,11 +4279,6 @@ class IndexingInst : public SILInstruction { : SILInstruction(Kind, DebugLoc, Operand->getType()), Operands{this, Operand, Index} {} - IndexingInst(ValueKind Kind, SILDebugLocation DebugLoc, SILValue Operand, - SILValue Index, SILType ResultTy) - : SILInstruction(Kind, DebugLoc, ResultTy), - Operands{this, Operand, Index} {} - SILValue getBase() const { return Operands[Base].get(); } SILValue getIndex() const { return Operands[Index].get(); } @@ -4363,24 +4308,6 @@ class IndexAddrInst : public IndexingInst { } }; -/// TailAddrInst - like IndexingInst, but aligns-up the resulting address to a -/// tail-allocated element type. -class TailAddrInst : public IndexingInst { - friend class SILBuilder; - - TailAddrInst(SILDebugLocation DebugLoc, SILValue Operand, SILValue Count, - SILType ResultTy) - : IndexingInst(ValueKind::TailAddrInst, DebugLoc, Operand, Count, - ResultTy) {} - -public: - static bool classof(const ValueBase *V) { - return V->getKind() == ValueKind::TailAddrInst; - } - - SILType getTailType() const { return getType().getObjectType(); } -}; - /// IndexRawPointerInst /// %2 : $Builtin.RawPointer \ /// = index_raw_pointer %0 : $Builtin.RawPointer, %1 : $Builtin.Word diff --git a/include/swift/SIL/SILNodes.def b/include/swift/SIL/SILNodes.def index f02595ade8cb8..5fd2eaf43e5a6 100644 --- a/include/swift/SIL/SILNodes.def +++ b/include/swift/SIL/SILNodes.def @@ -109,7 +109,6 @@ ABSTRACT_VALUE(SILInstruction, ValueBase) INST(ProjectExistentialBoxInst, SILInstruction, None, DoesNotRelease) ABSTRACT_VALUE(IndexingInst, SILInstruction) INST(IndexAddrInst, IndexingInst, None, DoesNotRelease) - INST(TailAddrInst, IndexingInst, None, DoesNotRelease) INST(IndexRawPointerInst, IndexingInst, None, DoesNotRelease) VALUE_RANGE(IndexingInst, IndexAddrInst, IndexRawPointerInst) @@ -186,7 +185,6 @@ ABSTRACT_VALUE(SILInstruction, ValueBase) INST(StructExtractInst, SILInstruction, None, DoesNotRelease) INST(StructElementAddrInst, SILInstruction, None, DoesNotRelease) INST(RefElementAddrInst, SILInstruction, None, DoesNotRelease) - INST(RefTailAddrInst, SILInstruction, None, DoesNotRelease) // Enums INST(EnumInst, SILInstruction, None, DoesNotRelease) diff --git a/include/swift/Serialization/ModuleFormat.h b/include/swift/Serialization/ModuleFormat.h index b40ec6bffb665..875f31223fd36 100644 --- a/include/swift/Serialization/ModuleFormat.h +++ b/include/swift/Serialization/ModuleFormat.h @@ -53,7 +53,7 @@ const uint16_t VERSION_MAJOR = 0; /// in source control, you should also update the comment to briefly /// describe what change you made. The content of this comment isn't important; /// it just ensures a conflict if two people change the module format. -const uint16_t VERSION_MINOR = 267; // Last change: tail-alloc instructions +const uint16_t VERSION_MINOR = 266; // Last change: pattern binding init contexts using DeclID = PointerEmbeddedInt; using DeclIDField = BCFixed<31>; diff --git a/lib/AST/Builtins.cpp b/lib/AST/Builtins.cpp index be4a024eb6155..a53734a0eef87 100644 --- a/lib/AST/Builtins.cpp +++ b/lib/AST/Builtins.cpp @@ -250,7 +250,7 @@ getBuiltinGenericFunction(Identifier Id, } /// Build a getelementptr operation declaration. -static ValueDecl *getGepRawOperation(Identifier Id, Type ArgType) { +static ValueDecl *getGepOperation(Identifier Id, Type ArgType) { auto &Context = ArgType->getASTContext(); // This is always "(i8*, IntTy) -> i8*" @@ -427,11 +427,6 @@ static ValueDecl *getCastOperation(ASTContext &Context, Identifier Id, static const char * const GenericParamNames[] = { "T", "U", - "V", - "W", - "X", - "Y", - "Z" }; static std::pair @@ -685,53 +680,6 @@ static ValueDecl *getBindMemoryOperation(ASTContext &Context, Identifier Id) { return builder.build(Id); } -static ValueDecl *getAllocWithTailElemsOperation(ASTContext &Context, - Identifier Id, - int NumTailTypes) { - if (NumTailTypes < 1 || - 1 + NumTailTypes > (int)llvm::array_lengthof(GenericParamNames)) - return nullptr; - GenericSignatureBuilder builder(Context, 1 + NumTailTypes); - builder.addParameter(makeMetatype(makeGenericParam(0))); - for (int Idx = 0; Idx < NumTailTypes; ++Idx) { - builder.addParameter(makeConcrete(BuiltinIntegerType::getWordType(Context))); - builder.addParameter(makeMetatype(makeGenericParam(Idx + 1))); - } - builder.setResult(makeGenericParam(0)); - return builder.build(Id); -} - -static ValueDecl *getProjectTailElemsOperation(ASTContext &Context, - Identifier Id) { - GenericSignatureBuilder builder(Context, 2); - builder.addParameter(makeGenericParam(0)); - builder.addParameter(makeMetatype(makeGenericParam(1))); - builder.setResult(makeConcrete(Context.TheRawPointerType)); - return builder.build(Id); -} - -/// Build a getelementptr operation declaration. -static ValueDecl *getGepOperation(ASTContext &Context, Identifier Id, - Type ArgType) { - GenericSignatureBuilder builder(Context, 1); - builder.addParameter(makeConcrete(Context.TheRawPointerType)); - builder.addParameter(makeConcrete(ArgType)); - builder.addParameter(makeMetatype(makeGenericParam(0))); - builder.setResult(makeConcrete(Context.TheRawPointerType)); - return builder.build(Id); -} - -static ValueDecl *getGetTailAddrOperation(ASTContext &Context, Identifier Id, - Type ArgType) { - GenericSignatureBuilder builder(Context, 2); - builder.addParameter(makeConcrete(Context.TheRawPointerType)); - builder.addParameter(makeConcrete(ArgType)); - builder.addParameter(makeMetatype(makeGenericParam(0))); - builder.addParameter(makeMetatype(makeGenericParam(1))); - builder.setResult(makeConcrete(Context.TheRawPointerType)); - return builder.build(Id); -} - static ValueDecl *getSizeOrAlignOfOperation(ASTContext &Context, Identifier Id) { GenericSignatureBuilder builder(Context); @@ -1504,14 +1452,6 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { return nullptr; return getAtomicStoreOperation(Context, Id, T); } - if (OperationName.startswith("allocWithTailElems_")) { - OperationName = OperationName.drop_front(strlen("allocWithTailElems_")); - int NumTailTypes = 0; - if (OperationName.getAsInteger(10, NumTailTypes)) - return nullptr; - - return getAllocWithTailElemsOperation(Context, Id, NumTailTypes); - } BuiltinValueKind BV = llvm::StringSwitch(OperationName) #define BUILTIN(id, name, Attrs) \ @@ -1533,21 +1473,12 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { case BuiltinValueKind::AtomicRMW: case BuiltinValueKind::AtomicLoad: case BuiltinValueKind::AtomicStore: - case BuiltinValueKind::AllocWithTailElems: llvm_unreachable("Handled above"); case BuiltinValueKind::None: return nullptr; - case BuiltinValueKind::GepRaw: - if (Types.size() != 1) return nullptr; - return getGepRawOperation(Id, Types[0]); - case BuiltinValueKind::Gep: if (Types.size() != 1) return nullptr; - return getGepOperation(Context, Id, Types[0]); - - case BuiltinValueKind::GetTailAddr: - if (Types.size() != 1) return nullptr; - return getGetTailAddrOperation(Context, Id, Types[0]); + return getGepOperation(Id, Types[0]); #define BUILTIN(id, name, Attrs) #define BUILTIN_BINARY_OPERATION(id, name, attrs, overload) case BuiltinValueKind::id: @@ -1625,10 +1556,6 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { if (!Types.empty()) return nullptr; return getBindMemoryOperation(Context, Id); - case BuiltinValueKind::ProjectTailElems: - if (!Types.empty()) return nullptr; - return getProjectTailElemsOperation(Context, Id); - case BuiltinValueKind::Sizeof: case BuiltinValueKind::Strideof: case BuiltinValueKind::Alignof: diff --git a/lib/IRGen/GenClass.cpp b/lib/IRGen/GenClass.cpp index 18ac4ba0ea6e8..979a3f4ec066b 100644 --- a/lib/IRGen/GenClass.cpp +++ b/lib/IRGen/GenClass.cpp @@ -531,111 +531,9 @@ irgen::getPhysicalClassMemberAccessStrategy(IRGenModule &IGM, llvm_unreachable("bad field-access strategy"); } -Address irgen::emitTailProjection(IRGenFunction &IGF, llvm::Value *Base, - SILType ClassType, - SILType TailType) { - const ClassTypeInfo &classTI = IGF.getTypeInfo(ClassType).as(); - - llvm::Value *Offset = nullptr; - auto &layout = classTI.getLayout(IGF.IGM, ClassType); - Alignment HeapObjAlign = IGF.IGM.TargetInfo.HeapObjectAlignment; - Alignment Align; - - // Get the size of the class instance. - if (layout.isFixedLayout()) { - Size ClassSize = layout.getSize(); - Offset = llvm::ConstantInt::get(IGF.IGM.SizeTy, ClassSize.getValue()); - Align = HeapObjAlign.alignmentAtOffset(ClassSize); - } else { - llvm::Value *metadata = emitHeapMetadataRefForHeapObject(IGF, Base, - ClassType); - Offset = emitClassFragileInstanceSizeAndAlignMask(IGF, - ClassType.getClassOrBoundGenericClass(), - metadata).first; - } - // Align up to the TailType. - assert(TailType.isObject()); - const TypeInfo &TailTI = IGF.getTypeInfo(TailType); - llvm::Value *AlignMask = TailTI.getAlignmentMask(IGF, TailType); - Offset = IGF.Builder.CreateAdd(Offset, AlignMask); - llvm::Value *InvertedMask = IGF.Builder.CreateNot(AlignMask); - Offset = IGF.Builder.CreateAnd(Offset, InvertedMask); - - llvm::Value *Addr = IGF.emitByteOffsetGEP(Base, Offset, - TailTI.getStorageType(), "tailaddr"); - - if (auto *OffsetConst = dyn_cast(Offset)) { - // Try to get an accurate alignment (only possible if the Offset is a - // constant). - Size TotalOffset(OffsetConst->getZExtValue()); - Align = HeapObjAlign.alignmentAtOffset(TotalOffset); - } - return Address(Addr, Align); -} - -/// Try to stack promote a class instance with possible tail allocated arrays. -/// -/// Returns the alloca if successfull, or nullptr otherwise. -static llvm::Value *stackPromote(IRGenFunction &IGF, - const StructLayout &ClassLayout, - int &StackAllocSize, - ArrayRef> TailArrays) { - if (StackAllocSize < 0) - return nullptr; - if (!ClassLayout.isFixedLayout()) - return nullptr; - - // Calculate the total size needed. - // The first part is the size of the class itself. - Alignment ClassAlign = ClassLayout.getAlignment(); - Size TotalSize = ClassLayout.getSize(); - - // Add size for tail-allocated arrays. - for (const auto &TailArray : TailArrays) { - SILType ElemTy = TailArray.first; - llvm::Value *Count = TailArray.second; - - // We can only calculate a constant size if the tail-count is constant. - auto *CI = dyn_cast(Count); - if (!CI) - return nullptr; - - const TypeInfo &ElemTI = IGF.getTypeInfo(ElemTy); - if (!ElemTI.isFixedSize()) - return nullptr; - - const FixedTypeInfo &ElemFTI = ElemTI.as(); - Alignment ElemAlign = ElemFTI.getFixedAlignment(); - - // This should not happen - just to be save. - if (ElemAlign > ClassAlign) - return nullptr; - - TotalSize = TotalSize.roundUpToAlignment(ElemAlign); - TotalSize += ElemFTI.getFixedStride() * CI->getValue().getZExtValue(); - } - if (TotalSize > Size(StackAllocSize)) - return nullptr; - StackAllocSize = TotalSize.getValue(); - - if (TotalSize == ClassLayout.getSize()) { - // No tail-allocated arrays: we can use the llvm class type for alloca. - llvm::Type *ClassTy = ClassLayout.getType(); - Address Alloca = IGF.createAlloca(ClassTy, ClassAlign, "reference.raw"); - return Alloca.getAddress(); - } - // Use a byte-array as type for alloca. - llvm::Value *SizeVal = llvm::ConstantInt::get(IGF.IGM.Int32Ty, - TotalSize.getValue()); - Address Alloca = IGF.createAlloca(IGF.IGM.Int8Ty, SizeVal, ClassAlign, - "reference.raw"); - return Alloca.getAddress(); -} - /// Emit an allocation of a class. llvm::Value *irgen::emitClassAllocation(IRGenFunction &IGF, SILType selfType, - bool objc, int &StackAllocSize, - ArrayRef> TailArrays) { + bool objc, int &StackAllocSize) { auto &classTI = IGF.getTypeInfo(selfType).as(); auto classType = selfType.getSwiftRValueType(); @@ -660,33 +558,22 @@ llvm::Value *irgen::emitClassAllocation(IRGenFunction &IGF, SILType selfType, selfType.getClassOrBoundGenericClass(), metadata); - const StructLayout &layout = classTI.getLayout(IGF.IGM, selfType); + auto &layout = classTI.getLayout(IGF.IGM, selfType); llvm::Type *destType = layout.getType()->getPointerTo(); llvm::Value *val = nullptr; - if (llvm::Value *Promoted = stackPromote(IGF, layout, StackAllocSize, - TailArrays)) { - val = IGF.Builder.CreateBitCast(Promoted, IGF.IGM.RefCountedPtrTy); + if (layout.isFixedLayout() && + (int)layout.getSize().getValue() < StackAllocSize) { + // Allocate the object on the stack. + auto *Ty = layout.getType(); + auto Alloca = IGF.createAlloca(Ty, layout.getAlignment(), + "reference.raw"); + val = Alloca.getAddress(); + assert(val->getType() == destType); + val = IGF.Builder.CreateBitCast(val, IGF.IGM.RefCountedPtrTy); val = IGF.emitInitStackObjectCall(metadata, val, "reference.new"); + StackAllocSize = layout.getSize().getValue(); } else { // Allocate the object on the heap. - - for (const auto &TailArray : TailArrays) { - SILType ElemTy = TailArray.first; - llvm::Value *Count = TailArray.second; - - const TypeInfo &ElemTI = IGF.getTypeInfo(ElemTy); - - // Align up to the tail-allocated array. - llvm::Value *ElemStride = ElemTI.getStride(IGF, ElemTy); - llvm::Value *AlignMask = ElemTI.getAlignmentMask(IGF, ElemTy); - size = IGF.Builder.CreateAdd(size, AlignMask); - llvm::Value *InvertedMask = IGF.Builder.CreateNot(AlignMask); - size = IGF.Builder.CreateAnd(size, InvertedMask); - - // Add the size of the tail allocated array. - llvm::Value *AllocSize = IGF.Builder.CreateMul(ElemStride, Count); - size = IGF.Builder.CreateAdd(size, AllocSize); - } val = IGF.emitAllocObjectCall(metadata, size, alignMask, "reference.new"); StackAllocSize = -1; } diff --git a/lib/IRGen/GenClass.h b/lib/IRGen/GenClass.h index c0fdd14081666..c9c7236132986 100644 --- a/lib/IRGen/GenClass.h +++ b/lib/IRGen/GenClass.h @@ -18,7 +18,6 @@ #define SWIFT_IRGEN_GENCLASS_H #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/ArrayRef.h" namespace llvm { class Constant; @@ -41,7 +40,6 @@ namespace irgen { class IRGenModule; class MemberAccessStrategy; class OwnedAddress; - class Address; class Size; enum class ReferenceCounting : unsigned char; @@ -78,11 +76,6 @@ namespace irgen { llvm::Constant *emitCategoryData(IRGenModule &IGM, ExtensionDecl *ext); llvm::Constant *emitObjCProtocolData(IRGenModule &IGM, ProtocolDecl *ext); - /// Emit a projection from a class instance to the first tail allocated - /// element. - Address emitTailProjection(IRGenFunction &IGF, llvm::Value *Base, - SILType ClassType, SILType TailType); - /// Emit an allocation of a class. /// The \p StackAllocSize is an in- and out-parameter. The passed value /// specifies the maximum object size for stack allocation. A negative value @@ -90,8 +83,7 @@ namespace irgen { /// The returned \p StackAllocSize value is the actual size if the object is /// allocated on the stack or -1, if the object is allocated on the heap. llvm::Value *emitClassAllocation(IRGenFunction &IGF, SILType selfType, - bool objc, int &StackAllocSize, - llvm::ArrayRef> TailArrays); + bool objc, int &StackAllocSize); /// Emit an allocation of a class using a metadata value. llvm::Value *emitClassAllocationDynamic(IRGenFunction &IGF, diff --git a/lib/IRGen/GenDecl.cpp b/lib/IRGen/GenDecl.cpp index 09f6ebf28eae8..4895bdd463d9d 100644 --- a/lib/IRGen/GenDecl.cpp +++ b/lib/IRGen/GenDecl.cpp @@ -2898,17 +2898,6 @@ Address IRGenFunction::createAlloca(llvm::Type *type, return Address(alloca, alignment); } -/// Create an allocation of an array on the stack. -Address IRGenFunction::createAlloca(llvm::Type *type, - llvm::Value *ArraySize, - Alignment alignment, - const llvm::Twine &name) { - llvm::AllocaInst *alloca = new llvm::AllocaInst(type, ArraySize, - alignment.getValue(), name, - AllocaIP); - return Address(alloca, alignment); -} - /// Allocate a fixed-size buffer on the stack. Address IRGenFunction::createFixedSizeBufferAlloca(const llvm::Twine &name) { return createAlloca(IGM.getFixedBufferTy(), diff --git a/lib/IRGen/GenValueWitness.cpp b/lib/IRGen/GenValueWitness.cpp index 31e7f07b24fe6..a0d5043181dd5 100644 --- a/lib/IRGen/GenValueWitness.cpp +++ b/lib/IRGen/GenValueWitness.cpp @@ -1539,26 +1539,6 @@ Address TypeInfo::indexArray(IRGenFunction &IGF, Address base, } } -Address TypeInfo::roundUpToTypeAlignment(IRGenFunction &IGF, Address base, - SILType T) const { - Alignment Align = base.getAlignment(); - llvm::Value *TyAlignMask = getAlignmentMask(IGF, T); - if (auto *TyAlignMaskConst = dyn_cast(TyAlignMask)) { - Alignment TyAlign(TyAlignMaskConst->getZExtValue() + 1); - - // No need to align if the base is already aligned. - if (TyAlign <= Align) - return base; - } - llvm::Value *Addr = base.getAddress(); - Addr = IGF.Builder.CreatePtrToInt(Addr, IGF.IGM.IntPtrTy); - Addr = IGF.Builder.CreateNUWAdd(Addr, TyAlignMask); - llvm::Value *InvertedMask = IGF.Builder.CreateNot(TyAlignMask); - Addr = IGF.Builder.CreateAnd(Addr, InvertedMask); - Addr = IGF.Builder.CreateIntToPtr(Addr, base.getAddress()->getType()); - return Address(Addr, Align); -} - void TypeInfo::destroyArray(IRGenFunction &IGF, Address array, llvm::Value *count, SILType T) const { if (isPOD(ResilienceExpansion::Maximal)) diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index 1e7547551ae25..cf8f5e6b43a34 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -88,6 +88,12 @@ static void addSwiftContractPass(const PassManagerBuilder &Builder, PM.add(createSwiftARCContractPass()); } +static void addSwiftStackPromotionPass(const PassManagerBuilder &Builder, + PassManagerBase &PM) { + if (Builder.OptLevel > 0) + PM.add(createSwiftStackPromotionPass()); +} + static void addSwiftMergeFunctionsPass(const PassManagerBuilder &Builder, PassManagerBase &PM) { if (Builder.OptLevel > 0) @@ -156,6 +162,9 @@ void swift::performLLVMOptimizations(IRGenOptions &Opts, llvm::Module *Module, llvm::createAlwaysInlinerPass(/*insertlifetime*/false); } + PMBuilder.addExtension(PassManagerBuilder::EP_ModuleOptimizerEarly, + addSwiftStackPromotionPass); + // If the optimizer is enabled, we run the ARCOpt pass in the scalar optimizer // and the Contract pass as late as possible. if (!Opts.DisableLLVMARCOpts) { diff --git a/lib/IRGen/IRGenFunction.h b/lib/IRGen/IRGenFunction.h index d0367ee3f39c4..999d66c263ede 100644 --- a/lib/IRGen/IRGenFunction.h +++ b/lib/IRGen/IRGenFunction.h @@ -124,8 +124,6 @@ class IRGenFunction { public: Address createAlloca(llvm::Type *ty, Alignment align, const llvm::Twine &name); - Address createAlloca(llvm::Type *ty, llvm::Value *ArraySize, Alignment align, - const llvm::Twine &name); Address createFixedSizeBufferAlloca(const llvm::Twine &name); llvm::BasicBlock *createBasicBlock(const llvm::Twine &Name); diff --git a/lib/IRGen/IRGenSIL.cpp b/lib/IRGen/IRGenSIL.cpp index b705208fb5e6f..e5c10cd31b8c2 100644 --- a/lib/IRGen/IRGenSIL.cpp +++ b/lib/IRGen/IRGenSIL.cpp @@ -824,7 +824,6 @@ class IRGenSILFunction : void visitStructExtractInst(StructExtractInst *i); void visitStructElementAddrInst(StructElementAddrInst *i); void visitRefElementAddrInst(RefElementAddrInst *i); - void visitRefTailAddrInst(RefTailAddrInst *i); void visitClassMethodInst(ClassMethodInst *i); void visitSuperMethodInst(SuperMethodInst *i); @@ -909,7 +908,6 @@ class IRGenSILFunction : void visitIsNonnullInst(IsNonnullInst *i); void visitIndexAddrInst(IndexAddrInst *i); - void visitTailAddrInst(TailAddrInst *i); void visitIndexRawPointerInst(IndexRawPointerInst *i); void visitUnreachableInst(UnreachableInst *i); @@ -3162,15 +3160,6 @@ void IRGenSILFunction::visitRefElementAddrInst(swift::RefElementAddrInst *i) { setLoweredAddress(i, field); } -void IRGenSILFunction::visitRefTailAddrInst(RefTailAddrInst *i) { - SILValue Ref = i->getOperand(); - llvm::Value *RefValue = getLoweredExplosion(Ref).claimNext(); - - Address TailAddr = emitTailProjection(*this, RefValue, Ref->getType(), - i->getTailType()); - setLoweredAddress(i, TailAddr); -} - void IRGenSILFunction::visitLoadInst(swift::LoadInst *i) { Explosion lowered; Address source = getLoweredAddress(i->getOperand()); @@ -3557,17 +3546,8 @@ void IRGenSILFunction::visitAllocRefInst(swift::AllocRefInst *i) { // Is there enough space for stack allocation? StackAllocSize = IGM.IRGen.Opts.StackPromotionSizeLimit - EstimatedStackSize; } - SmallVector, 4> TailArrays; - - auto Types = i->getTailAllocatedTypes(); - auto Counts = i->getTailAllocatedCounts(); - for (unsigned Idx = 0, NumTypes = Types.size(); Idx < NumTypes; ++Idx) { - llvm::Value *ElemCount = getLoweredExplosion(Counts[Idx].get()).claimNext(); - TailArrays.push_back({Types[Idx], ElemCount}); - } - llvm::Value *alloced = emitClassAllocation(*this, i->getType(), i->isObjC(), - StackAllocSize, TailArrays); + StackAllocSize); if (StackAllocSize >= 0) { // Remember that this alloc_ref allocates the object on the stack. @@ -4293,22 +4273,6 @@ void IRGenSILFunction::visitIndexAddrInst(swift::IndexAddrInst *i) { setLoweredAddress(i, dest); } -void IRGenSILFunction::visitTailAddrInst(swift::TailAddrInst *i) { - Address base = getLoweredAddress(i->getBase()); - Explosion indexValues = getLoweredExplosion(i->getIndex()); - llvm::Value *index = indexValues.claimNext(); - - SILType baseTy = i->getBase()->getType(); - const TypeInfo &baseTI = getTypeInfo(baseTy); - - Address dest = baseTI.indexArray(*this, base, index, baseTy); - const TypeInfo &TailTI = getTypeInfo(i->getTailType()); - dest = TailTI.roundUpToTypeAlignment(*this, dest, i->getTailType()); - llvm::Type *destType = TailTI.getStorageType()->getPointerTo(); - dest = Builder.CreateBitCast(dest, destType); - setLoweredAddress(i, dest); -} - void IRGenSILFunction::visitIndexRawPointerInst(swift::IndexRawPointerInst *i) { Explosion baseValues = getLoweredExplosion(i->getBase()); llvm::Value *base = baseValues.claimNext(); diff --git a/lib/IRGen/TypeInfo.h b/lib/IRGen/TypeInfo.h index e2139267c9e2d..91994eb5e25a8 100644 --- a/lib/IRGen/TypeInfo.h +++ b/lib/IRGen/TypeInfo.h @@ -450,11 +450,7 @@ class TypeInfo { /// Index into an array of objects of this type. Address indexArray(IRGenFunction &IGF, Address base, llvm::Value *offset, SILType T) const; - - /// Round up the address value \p base to the alignment of type \p T. - Address roundUpToTypeAlignment(IRGenFunction &IGF, Address base, - SILType T) const; - + /// Destroy an array of objects of this type in memory. virtual void destroyArray(IRGenFunction &IGF, Address base, llvm::Value *count, SILType T) const; diff --git a/lib/LLVMPasses/CMakeLists.txt b/lib/LLVMPasses/CMakeLists.txt index 54af8cd1c52b4..20ea2970cfe59 100644 --- a/lib/LLVMPasses/CMakeLists.txt +++ b/lib/LLVMPasses/CMakeLists.txt @@ -4,6 +4,7 @@ add_swift_library(swiftLLVMPasses STATIC LLVMARCOpts.cpp LLVMARCContract.cpp LLVMInlineTree.cpp + LLVMStackPromotion.cpp LLVMMergeFunctions.cpp LLVM_COMPONENT_DEPENDS diff --git a/lib/LLVMPasses/LLVMStackPromotion.cpp b/lib/LLVMPasses/LLVMStackPromotion.cpp new file mode 100644 index 0000000000000..667d73c8244d8 --- /dev/null +++ b/lib/LLVMPasses/LLVMStackPromotion.cpp @@ -0,0 +1,272 @@ +//===--- LLVMStackPromotion.cpp - Replace allocation calls with alloca ----===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This pass performs the last part of stack promotion for array buffers. +// The SIL StackPromotion pass generates a pair of swift_bufferAllocateOnStack +// and swift_bufferDeallocateFromStack calls. In this pass the final decision +// is made if stack promotion should be done. If yes, the +// swift_bufferAllocateOnStack is replace with an alloca plus a call to +// swift_initStackObject and the swift_bufferDeallocateFromStack is removed. +// TODO: This is a hack and eventually this pass should not be required at all. +// For details see the comments for the SIL StackPromoter. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "swift-stack-promotion" +#include "swift/LLVMPasses/Passes.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/Pass.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace swift; + +STATISTIC(NumBufferAllocsPromoted, + "Number of swift_bufferAllocate promoted"); + +cl::opt LimitOpt("stack-promotion-limit", + llvm::cl::init(1024), llvm::cl::Hidden); + +//===----------------------------------------------------------------------===// +// SwiftStackPromotion Pass +//===----------------------------------------------------------------------===// + +char SwiftStackPromotion::ID = 0; + +INITIALIZE_PASS_BEGIN(SwiftStackPromotion, + "swift-stack-promotion", "Swift stack promotion pass", + false, false) +INITIALIZE_PASS_END(SwiftStackPromotion, + "swift-stack-promotion", "Swift stack promotion pass", + false, false) + +llvm::FunctionPass *swift::createSwiftStackPromotionPass() { + initializeSwiftStackPromotionPass(*llvm::PassRegistry::getPassRegistry()); + return new SwiftStackPromotion(); +} + +void SwiftStackPromotion::getAnalysisUsage(llvm::AnalysisUsage &AU) const { + AU.setPreservesCFG(); +} + +/// Checks if we can promote a buffer and returns the size of the buffer. +/// The \a align parameter is set to the alignment of the buffer. +int canPromote(CallInst *CI, unsigned &align, int maxSize) { + if (CI->getNumArgOperands() != 3) + return 0; + + auto *SizeConst = dyn_cast(CI->getArgOperand(1)); + if (!SizeConst) + return 0; + + auto *AlignMaskConst = dyn_cast(CI->getArgOperand(2)); + if (!AlignMaskConst) + return 0; + + int size = SizeConst->getValue().getSExtValue(); + if (size > maxSize) + return 0; + + align = AlignMaskConst->getValue().getZExtValue() + 1; + + return size; +} + +/// Remove redundant runtime calls for stack allocated buffers. +/// If a buffer is allocated on the stack it's not needed to explicitly set +/// the RC_DEALLOCATING_FLAG flag (except there is code which may depend on it). +/// Also the a call to swift_deallocClassInstance (which stems from an inlined +/// deallocator) is not needed. +/// +/// %0 = alloca +/// ... +/// call @swift_setDeallocating(%0) // not needed +/// // code which does not depend on the RC_DEALLOCATING_FLAG flag. +/// call @swift_deallocClassInstance(%0) // not needed +/// call @llvm.lifetime.end(%0) +/// +static void removeRedundantRTCalls(CallInst *DeallocCall) { + BasicBlock::iterator Iter(DeallocCall); + BasicBlock::iterator Begin = DeallocCall->getParent()->begin(); + Value *Buffer = DeallocCall->getArgOperand(0); + CallInst *RedundantDealloc = nullptr; + CallInst *RedundantSetFlag = nullptr; + SmallVector ToDelete; + while (Iter != Begin) { + --Iter; + Instruction *I = &*Iter; + if (auto *CI = dyn_cast(I)) { + + // Check if we have a runtime function with the buffer as argument. + if (CI->getNumArgOperands() < 1) + break; + if (CI->getArgOperand(0)->stripPointerCasts() != Buffer) + break; + auto *Callee = dyn_cast(CI->getCalledValue()); + if (!Callee) + break; + + // The callee function my be a bitcast constant expression. + if (auto *U = dyn_cast(Callee)) { + if (U->getOpcode() == Instruction::BitCast) + Callee = U->getOperand(0); + } + auto *RTFunc = dyn_cast(Callee); + if (!RTFunc) + break; + + if (RTFunc->getName() == "swift_setDeallocating") { + assert(RedundantDealloc && "dealloc call must follow setDeallocating"); + assert(!RedundantSetFlag && "multiple setDeallocating calls"); + RedundantSetFlag = CI; + continue; + } + if (RTFunc->getName() == "swift_deallocClassInstance") { + assert(!RedundantSetFlag && "dealloc call must follow setDeallocating"); + assert(!RedundantDealloc && "multiple deallocClassInstance calls"); + RedundantDealloc = CI; + continue; + } + break; + } + // Bail if we have an instruction which may read the RC_DEALLOCATING_FLAG + // flag. + if (I->mayReadFromMemory()) + break; + } + if (RedundantDealloc) + RedundantDealloc->eraseFromParent(); + if (RedundantSetFlag) + RedundantSetFlag->eraseFromParent(); +} + +bool SwiftStackPromotion::runOnFunction(Function &F) { + + bool Changed = false; + Constant *allocFunc = nullptr; + Constant *initFunc = nullptr; + int maxSize = LimitOpt; + Module *M = F.getParent(); + const DataLayout &DL = M->getDataLayout(); + IntegerType *AllocType = nullptr; + IntegerType *IntType = nullptr; + + SmallVector BufferAllocs; + SmallPtrSet PromotedAllocs; + SmallVector BufferDeallocs; + + // Search for allocation- and deallocation-calls in the function. + for (BasicBlock &BB : F) { + for (auto Iter = BB.begin(); Iter != BB.end(); ) { + Instruction *I = &*Iter; + Iter++; + + if (auto *AI = dyn_cast(I)) { + int Size = 1; + if (auto *SizeConst = dyn_cast(AI->getArraySize())) + Size = SizeConst->getValue().getSExtValue(); + + // Count the existing alloca sizes against the limit. + maxSize -= DL.getTypeAllocSize(AI->getAllocatedType()) * Size; + } + + auto *CI = dyn_cast(I); + if (!CI) + continue; + + Function *Callee = CI->getCalledFunction(); + if (!Callee) + continue; + + if (Callee->getName() == "swift_bufferAllocateOnStack") { + BufferAllocs.push_back(CI); + } else if (Callee->getName() == "swift_bufferDeallocateFromStack") { + BufferDeallocs.push_back(CI); + } + } + } + + // First handle allocations. + for (CallInst *CI : BufferAllocs) { + Function *Callee = CI->getCalledFunction(); + assert(Callee); + unsigned align = 0; + if (int size = canPromote(CI, align, maxSize)) { + maxSize -= size; + if (!AllocType) { + // Create the swift_initStackObject function and all required types. + AllocType = IntegerType::get(M->getContext(), 8); + IntType = IntegerType::get(M->getContext(), 32); + auto *OrigFT = Callee->getFunctionType(); + auto *HeapObjTy = OrigFT->getReturnType(); + auto *MetaDataTy = OrigFT->getParamType(0); + auto *NewFTy = FunctionType::get(HeapObjTy, + {MetaDataTy, HeapObjTy}, + false); + initFunc = M->getOrInsertFunction("swift_initStackObject", NewFTy); + if (llvm::Triple(M->getTargetTriple()).isOSBinFormatCOFF()) + if (auto *F = dyn_cast(initFunc)) + F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + } + // Replace the allocation call with an alloca. + Value *AllocA = new AllocaInst(AllocType, ConstantInt::get(IntType, size), + align, "buffer", &*F.front().begin()); + // And initialize it with a call to swift_initStackObject. + IRBuilder<> B(CI); + Value *casted = B.CreateBitCast(AllocA, CI->getType()); + CallInst *initCall = B.CreateCall(initFunc, + {CI->getArgOperand(0), casted}); + CI->replaceAllUsesWith(initCall); + CI->eraseFromParent(); + PromotedAllocs.insert(initCall); + ++NumBufferAllocsPromoted; + } else { + // We don't do stack promotion. Replace the call with a call to the + // regular swift_bufferAllocate. + if (!allocFunc) { + allocFunc = M->getOrInsertFunction("swift_bufferAllocate", + Callee->getFunctionType()); + if (llvm::Triple(M->getTargetTriple()).isOSBinFormatCOFF()) + if (auto *F = dyn_cast(allocFunc)) + F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + } + CI->setCalledFunction(allocFunc); + } + Changed = true; + } + + // After we made the decision for all allocations we can handle the + // deallocations. + for (CallInst *CI : BufferDeallocs) { + CallInst *Alloc = dyn_cast(CI->getArgOperand(0)); + assert(Alloc && "alloc buffer obfuscated"); + if (PromotedAllocs.count(Alloc)) { + + removeRedundantRTCalls(CI); + + IRBuilder<> B(CI); + // This has two purposes: + // 1. Tell LLVM the lifetime of the allocated stack memory. + // 2. Avoid tail-call optimization which may convert the call to the final + // release to a jump, which is done after the stack frame is + // destructed. + B.CreateLifetimeEnd(CI->getArgOperand(0)); + } + // Other than inserting the end-of-lifetime, the deallocation is a no-op. + CI->eraseFromParent(); + Changed = true; + } + return Changed; +} diff --git a/lib/Parse/ParseSIL.cpp b/lib/Parse/ParseSIL.cpp index 975c51ed92ea8..65918f626a4c3 100644 --- a/lib/Parse/ParseSIL.cpp +++ b/lib/Parse/ParseSIL.cpp @@ -1261,7 +1261,6 @@ bool SILParser::parseSILOpcode(ValueKind &Opcode, SourceLoc &OpcodeLoc, .Case("existential_metatype", ValueKind::ExistentialMetatypeInst) .Case("raw_pointer_to_ref", ValueKind::RawPointerToRefInst) .Case("ref_element_addr", ValueKind::RefElementAddrInst) - .Case("ref_tail_addr", ValueKind::RefTailAddrInst) .Case("ref_to_bridge_object", ValueKind::RefToBridgeObjectInst) .Case("ref_to_raw_pointer", ValueKind::RefToRawPointerInst) .Case("ref_to_unmanaged", ValueKind::RefToUnmanagedInst) @@ -1291,7 +1290,6 @@ bool SILParser::parseSILOpcode(ValueKind &Opcode, SourceLoc &OpcodeLoc, .Case("switch_enum_addr", ValueKind::SwitchEnumAddrInst) .Case("switch_value", ValueKind::SwitchValueInst) - .Case("tail_addr", ValueKind::TailAddrInst) .Case("try_apply", ValueKind::TryApplyInst) .Case("unchecked_enum_data", ValueKind::UncheckedEnumDataInst) .Case("unchecked_addr_cast", ValueKind::UncheckedAddrCastInst) @@ -2564,76 +2562,40 @@ bool SILParser::parseSILInstruction(SILBasicBlock *BB, SILBuilder &B) { break; } case ValueKind::AllocStackInst: + case ValueKind::AllocRefInst: case ValueKind::MetatypeInst: { - - SILType Ty; - if (parseSILType(Ty)) - return true; - - if (Opcode == ValueKind::AllocStackInst) { - SILDebugVariable VarInfo; - if (parseSILDebugVar(VarInfo) || - parseSILDebugLocation(InstLoc, B)) - return true; - ResultVal = B.createAllocStack(InstLoc, Ty, VarInfo); - } else { - assert(Opcode == ValueKind::MetatypeInst); - if (parseSILDebugLocation(InstLoc, B)) - return true; - ResultVal = B.createMetatype(InstLoc, Ty); - } - break; - } - case ValueKind::AllocRefInst: { bool IsObjC = false; bool OnStack = false; - SmallVector ElementTypes; - SmallVector ElementCounts; StringRef Optional; - while (P.consumeIf(tok::l_square)) { - Identifier Id; - parseSILIdentifier(Id, diag::expected_in_attribute_list); - StringRef Optional = Id.str(); + while (parseSILOptional(Optional, *this)) { if (Optional == "objc") { IsObjC = true; } else if (Optional == "stack") { OnStack = true; - } else if (Optional == "tail_elems") { - SILType ElemTy; - if (parseSILType(ElemTy) || - !P.Tok.isAnyOperator() || - P.Tok.getText() != "*") - return true; - P.consumeToken(); - - SILValue ElemCount; - if (parseTypedValueRef(ElemCount, B)) - return true; - - ElementTypes.push_back(ElemTy); - ElementCounts.push_back(ElemCount); } else { return true; } - P.parseToken(tok::r_square, diag::expected_in_attribute_list); } - SILType ObjectType; - if (parseSILType(ObjectType)) - return true; - - if (parseSILDebugLocation(InstLoc, B)) + SILType Ty; + if (parseSILType(Ty)) return true; - if (ElementTypes.size() == 0) { - ResultVal = B.createAllocRef(InstLoc, ObjectType, IsObjC, OnStack); + if (Opcode == ValueKind::AllocStackInst) { + SILDebugVariable VarInfo; + if (parseSILDebugVar(VarInfo) || + parseSILDebugLocation(InstLoc, B)) + return true; + ResultVal = B.createAllocStack(InstLoc, Ty, VarInfo); + } else if (Opcode == ValueKind::AllocRefInst) { + if (parseSILDebugLocation(InstLoc, B)) + return true; + ResultVal = B.createAllocRef(InstLoc, Ty, IsObjC, OnStack); } else { - if (IsObjC) { - P.diagnose(P.Tok, diag::sil_objc_with_tail_elements); + assert(Opcode == ValueKind::MetatypeInst); + if (parseSILDebugLocation(InstLoc, B)) return true; - } - ResultVal = B.createAllocRef(InstLoc, ObjectType, OnStack, - ElementTypes, ElementCounts); + ResultVal = B.createMetatype(InstLoc, Ty); } break; } @@ -3203,18 +3165,6 @@ bool SILParser::parseSILInstruction(SILBasicBlock *BB, SILBuilder &B) { ResultVal = B.createRefElementAddr(InstLoc, Val, Field, ResultTy); break; } - case ValueKind::RefTailAddrInst: { - SourceLoc NameLoc; - SILType ResultObjTy; - if (parseTypedValueRef(Val, B) || - P.parseToken(tok::comma, diag::expected_tok_in_sil_instr, ",") || - parseSILType(ResultObjTy) || - parseSILDebugLocation(InstLoc, B)) - return true; - SILType ResultTy = ResultObjTy.getAddressType(); - ResultVal = B.createRefTailAddr(InstLoc, Val, ResultTy); - break; - } case ValueKind::IsNonnullInst: { SourceLoc Loc; if (parseTypedValueRef(Val, Loc, B) || @@ -3233,20 +3183,6 @@ bool SILParser::parseSILInstruction(SILBasicBlock *BB, SILBuilder &B) { ResultVal = B.createIndexAddr(InstLoc, Val, IndexVal); break; } - case ValueKind::TailAddrInst: { - SILValue IndexVal; - SILType ResultObjTy; - if (parseTypedValueRef(Val, B) || - P.parseToken(tok::comma, diag::expected_tok_in_sil_instr, ",") || - parseTypedValueRef(IndexVal, B) || - P.parseToken(tok::comma, diag::expected_tok_in_sil_instr, ",") || - parseSILType(ResultObjTy) || - parseSILDebugLocation(InstLoc, B)) - return true; - SILType ResultTy = ResultObjTy.getAddressType(); - ResultVal = B.createTailAddr(InstLoc, Val, IndexVal, ResultTy); - break; - } case ValueKind::IndexRawPointerInst: { SILValue IndexVal; if (parseTypedValueRef(Val, B) || diff --git a/lib/SIL/InstructionUtils.cpp b/lib/SIL/InstructionUtils.cpp index 6e95973c4dad4..7c7716d4bd944 100644 --- a/lib/SIL/InstructionUtils.cpp +++ b/lib/SIL/InstructionUtils.cpp @@ -30,25 +30,6 @@ SILValue swift::getUnderlyingObject(SILValue V) { } } -SILValue swift::getUnderlyingAddressRoot(SILValue V) { - while (true) { - SILValue V2 = stripIndexingInsts(stripCasts(V)); - switch (V2->getKind()) { - case ValueKind::StructElementAddrInst: - case ValueKind::TupleElementAddrInst: - case ValueKind::UncheckedTakeEnumDataAddrInst: - V2 = cast(V2)->getOperand(0); - break; - default: - break; - } - if (V2 == V) - return V2; - V = V2; - } -} - - SILValue swift::getUnderlyingObjectStopAtMarkDependence(SILValue V) { while (true) { SILValue V2 = stripIndexingInsts(stripAddressProjections(stripCastsWithoutMarkDependence(V))); diff --git a/lib/SIL/Projection.cpp b/lib/SIL/Projection.cpp index 6e01f5d4daa50..423049ddb9616 100644 --- a/lib/SIL/Projection.cpp +++ b/lib/SIL/Projection.cpp @@ -95,13 +95,6 @@ Projection::Projection(SILInstruction *I) : Value() { REAI->getType()); break; } - case ValueKind::RefTailAddrInst: { - auto *RTAI = cast(I); - auto *Ty = RTAI->getTailType().getSwiftRValueType().getPointer(); - Value = ValueTy(ProjectionKind::TailElems, Ty); - assert(getKind() == ProjectionKind::TailElems); - break; - } case ValueKind::ProjectBoxInst: { auto *PBI = cast(I); Value = ValueTy(ProjectionKind::Box, (unsigned)0); @@ -154,9 +147,11 @@ Projection::Projection(SILInstruction *I) : Value() { // updated and a MaxLargeIndex will need to be used here. Currently we // represent large Indexes using a 64 bit integer, so we don't need to mess // with anything. - unsigned NewIndex = 0; + unsigned NewIndex = ~0; auto *IAI = cast(I); if (getIntegerIndex(IAI->getIndex(), NewIndex)) { + assert(NewIndex != unsigned(~0) && "NewIndex should have been changed " + "by getIntegerIndex?!"); Value = ValueTy(ProjectionKind::Index, NewIndex); assert(getKind() == ProjectionKind::Index); assert(getIndex() == NewIndex); @@ -217,8 +212,6 @@ Projection::createObjectProjection(SILBuilder &B, SILLocation Loc, return B.createUncheckedEnumData(Loc, Base, getEnumElementDecl(BaseTy)); case ProjectionKind::Class: return nullptr; - case ProjectionKind::TailElems: - return nullptr; case ProjectionKind::Box: return nullptr; case ProjectionKind::Upcast: @@ -260,8 +253,6 @@ Projection::createAddressProjection(SILBuilder &B, SILLocation Loc, getEnumElementDecl(BaseTy)); case ProjectionKind::Class: return B.createRefElementAddr(Loc, Base, getVarDecl(BaseTy)); - case ProjectionKind::TailElems: - return B.createRefTailAddr(Loc, Base, getCastType(BaseTy)); case ProjectionKind::Box: return B.createProjectBox(Loc, Base); case ProjectionKind::Upcast: @@ -834,7 +825,6 @@ SILValue Projection::getOperandForAggregate(SILInstruction *I) const { } break; case ProjectionKind::Class: - case ProjectionKind::TailElems: case ProjectionKind::Box: case ProjectionKind::Upcast: case ProjectionKind::RefCast: diff --git a/lib/SIL/SILInstruction.cpp b/lib/SIL/SILInstruction.cpp index 1ec57495b30e9..a4e30c0756b66 100644 --- a/lib/SIL/SILInstruction.cpp +++ b/lib/SIL/SILInstruction.cpp @@ -238,15 +238,6 @@ namespace { } bool visitAllocRefInst(const AllocRefInst *RHS) { - auto *LHSInst = cast(LHS); - auto LHSTypes = LHSInst->getTailAllocatedTypes(); - auto RHSTypes = RHS->getTailAllocatedTypes(); - unsigned NumTypes = LHSTypes.size(); - assert(NumTypes == RHSTypes.size()); - for (unsigned Idx = 0; Idx < NumTypes; ++Idx) { - if (LHSTypes[Idx] != RHSTypes[Idx]) - return false; - } return true; } @@ -357,13 +348,6 @@ namespace { return true; } - bool visitRefTailAddrInst(RefTailAddrInst *RHS) { - auto *X = cast(LHS); - if (X->getTailType() != RHS->getTailType()) - return false; - return true; - } - bool visitStructElementAddrInst(const StructElementAddrInst *RHS) { // We have already checked that the operands of our struct_element_addrs // match. Thus we only need to check the field/struct decl which are not @@ -435,13 +419,6 @@ namespace { return true; } - bool visitTailAddrInst(TailAddrInst *RHS) { - auto *X = cast(LHS); - if (X->getTailType() != RHS->getTailType()) - return false; - return true; - } - bool visitCondFailInst(CondFailInst *RHS) { // We have already compared the operands/types, so we should have equality // at this point. diff --git a/lib/SIL/SILInstructions.cpp b/lib/SIL/SILInstructions.cpp index 917ad6ee1db26..9251f97166554 100644 --- a/lib/SIL/SILInstructions.cpp +++ b/lib/SIL/SILInstructions.cpp @@ -156,46 +156,27 @@ VarDecl *AllocStackInst::getDecl() const { return getLoc().getAsASTNode(); } -AllocRefInst::AllocRefInst(SILDebugLocation Loc, SILFunction &F, - SILType ObjectType, - bool objc, bool canBeOnStack, - ArrayRef ElementTypes, - ArrayRef AllOperands) - : AllocationInst(ValueKind::AllocRefInst, Loc, ObjectType), +AllocRefInst::AllocRefInst(SILDebugLocation Loc, SILType elementType, + SILFunction &F, bool objc, bool canBeOnStack, + ArrayRef TypeDependentOperands) + : AllocationInst(ValueKind::AllocRefInst, Loc, elementType), StackPromotable(canBeOnStack), - NumTailTypes(ElementTypes.size()), - ObjC(objc), - Operands(this, AllOperands) { - static_assert(IsTriviallyCopyable::value, - "assuming SILType is trivially copyable"); - assert(!objc || ElementTypes.size() == 0); - assert(AllOperands.size() >= ElementTypes.size()); - - memcpy(getTypeStorage(), ElementTypes.begin(), - sizeof(SILType) * ElementTypes.size()); -} - -AllocRefInst *AllocRefInst::create(SILDebugLocation Loc, SILFunction &F, - SILType ObjectType, - bool objc, bool canBeOnStack, - ArrayRef ElementTypes, - ArrayRef ElementCountOperands, - SILOpenedArchetypesState &OpenedArchetypes) { - assert(ElementTypes.size() == ElementCountOperands.size()); - assert(!objc || ElementTypes.size() == 0); - SmallVector AllOperands(ElementCountOperands.begin(), - ElementCountOperands.end()); - for (SILType ElemType : ElementTypes) { - collectTypeDependentOperands(AllOperands, OpenedArchetypes, F, - ElemType.getSwiftRValueType()); - } + NumOperands(TypeDependentOperands.size()), ObjC(objc) { + TrailingOperandsList::InitOperandsList(getAllOperands().begin(), this, + TypeDependentOperands); +} + +AllocRefInst *AllocRefInst::create(SILDebugLocation Loc, SILType elementType, + SILFunction &F, bool objc, bool canBeOnStack, + SILOpenedArchetypesState &OpenedArchetypes) { + SmallVector TypeDependentOperands; + collectTypeDependentOperands(TypeDependentOperands, OpenedArchetypes, F, + elementType.getSwiftRValueType()); void *Buffer = F.getModule().allocateInst( - sizeof(AllocRefInst) - + decltype(Operands)::getExtraSize(AllOperands.size()) - + sizeof(SILType) * ElementTypes.size(), - alignof(AllocRefInst)); - return ::new (Buffer) AllocRefInst(Loc, F, ObjectType, objc, canBeOnStack, - ElementTypes, AllOperands); + sizeof(AllocRefInst) + sizeof(Operand) * (TypeDependentOperands.size()), + alignof(AllocRefInst)); + return ::new (Buffer) AllocRefInst(Loc, elementType, F, objc, canBeOnStack, + TypeDependentOperands); } AllocRefDynamicInst::AllocRefDynamicInst( diff --git a/lib/SIL/SILModule.cpp b/lib/SIL/SILModule.cpp index eaaa2e171edc1..5037fc021c52e 100644 --- a/lib/SIL/SILModule.cpp +++ b/lib/SIL/SILModule.cpp @@ -463,8 +463,6 @@ const BuiltinInfo &SILModule::getBuiltinInfo(Identifier ID) { Info.ID = BuiltinValueKind::AtomicLoad; else if (OperationName.startswith("atomicstore_")) Info.ID = BuiltinValueKind::AtomicStore; - else if (OperationName.startswith("allocWithTailElems_")) - Info.ID = BuiltinValueKind::AllocWithTailElems; else { // Switch through the rest of builtins. Info.ID = llvm::StringSwitch(OperationName) diff --git a/lib/SIL/SILPrinter.cpp b/lib/SIL/SILPrinter.cpp index dc2fc53c81fbf..4baac97cef75c 100644 --- a/lib/SIL/SILPrinter.cpp +++ b/lib/SIL/SILPrinter.cpp @@ -844,12 +844,6 @@ class SILPrinter : public SILVisitor { *this << "[objc] "; if (ARI->canAllocOnStack()) *this << "[stack] "; - auto Types = ARI->getTailAllocatedTypes(); - auto Counts = ARI->getTailAllocatedCounts(); - for (unsigned Idx = 0, NumTypes = Types.size(); Idx < NumTypes; ++Idx) { - *this << "[tail_elems " << Types[Idx] << " * " - << getIDAndType(Counts[Idx].get()) << "] "; - } *this << ARI->getType(); } @@ -1324,11 +1318,6 @@ class SILPrinter : public SILVisitor { *this << EI->getField()->getName().get(); } - void visitRefTailAddrInst(RefTailAddrInst *RTAI) { - *this << "ref_tail_addr " << getIDAndType(RTAI->getOperand()) << ", " - << RTAI->getTailType(); - } - void printMethodInst(MethodInst *I, SILValue Operand, StringRef Name) { *this << Name << " "; if (I->isVolatile()) @@ -1517,11 +1506,6 @@ class SILPrinter : public SILVisitor { << getIDAndType(IAI->getIndex()); } - void visitTailAddrInst(TailAddrInst *TAI) { - *this << "tail_addr " << getIDAndType(TAI->getBase()) << ", " - << getIDAndType(TAI->getIndex()) << ", " << TAI->getTailType(); - } - void visitIndexRawPointerInst(IndexRawPointerInst *IAI) { *this << "index_raw_pointer " << getIDAndType(IAI->getBase()) << ", " << getIDAndType(IAI->getIndex()); diff --git a/lib/SIL/SILVerifier.cpp b/lib/SIL/SILVerifier.cpp index 60b70c853b4b6..6283b1b5b02a7 100644 --- a/lib/SIL/SILVerifier.cpp +++ b/lib/SIL/SILVerifier.cpp @@ -679,18 +679,7 @@ class SILVerifier : public SILVerifierBase { void checkAllocRefInst(AllocRefInst *AI) { requireReferenceValue(AI, "Result of alloc_ref"); - require(AI->isObjC() || AI->getType().getClassOrBoundGenericClass(), - "alloc_ref must allocate class"); verifyOpenedArchetype(AI, AI->getType().getSwiftRValueType()); - auto Types = AI->getTailAllocatedTypes(); - auto Counts = AI->getTailAllocatedCounts(); - unsigned NumTypes = Types.size(); - require(NumTypes == Counts.size(), "Mismatching types and counts"); - for (unsigned Idx = 0; Idx < NumTypes; ++Idx) { - verifyOpenedArchetype(AI, Types[Idx].getSwiftRValueType()); - require(Counts[Idx].get()->getType().is(), - "count needs integer type"); - } } void checkAllocRefDynamicInst(AllocRefDynamicInst *ARDI) { @@ -1607,12 +1596,6 @@ class SILVerifier : public SILVerifierBase { "index_addr index must be of a builtin integer type"); } - void checkTailAddrInst(TailAddrInst *IAI) { - require(IAI->getType().isAddress(), "tail_addr must produce an address"); - require(IAI->getIndex()->getType().is(), - "tail_addr index must be of a builtin integer type"); - } - void checkIndexRawPointerInst(IndexRawPointerInst *IAI) { require(IAI->getType().is(), "index_raw_pointer must produce a RawPointer"); @@ -1718,15 +1701,6 @@ class SILVerifier : public SILVerifierBase { EI->getFieldNo(); // Make sure we can access the field without crashing. } - void checkRefTailAddrInst(RefTailAddrInst *RTAI) { - requireReferenceValue(RTAI->getOperand(), "Operand of ref_tail_addr"); - require(RTAI->getType().isAddress(), - "result of ref_tail_addr must be lvalue"); - SILType operandTy = RTAI->getOperand()->getType(); - ClassDecl *cd = operandTy.getClassOrBoundGenericClass(); - require(cd, "ref_tail_addr operand must be a class instance"); - } - SILType getMethodSelfType(CanSILFunctionType ft) { return ft->getParameters().back().getSILType(); } diff --git a/lib/SILGen/SILGenBuiltin.cpp b/lib/SILGen/SILGenBuiltin.cpp index dbc0ccd19e7e6..7fc501ad7836b 100644 --- a/lib/SILGen/SILGenBuiltin.cpp +++ b/lib/SILGen/SILGenBuiltin.cpp @@ -478,21 +478,6 @@ static ManagedValue emitBuiltinAddressOf(SILGenFunction &gen, return ManagedValue::forUnmanaged(result); } -/// Specialized emitter for Builtin.gepRaw. -static ManagedValue emitBuiltinGepRaw(SILGenFunction &gen, - SILLocation loc, - ArrayRef substitutions, - ArrayRef args, - CanFunctionType formalApplyType, - SGFContext C) { - assert(args.size() == 2 && "gepRaw should be given two arguments"); - - SILValue offsetPtr = gen.B.createIndexRawPointer(loc, - args[0].getUnmanagedValue(), - args[1].getUnmanagedValue()); - return ManagedValue::forUnmanaged(offsetPtr); -} - /// Specialized emitter for Builtin.gep. static ManagedValue emitBuiltinGep(SILGenFunction &gen, SILLocation loc, @@ -500,39 +485,12 @@ static ManagedValue emitBuiltinGep(SILGenFunction &gen, ArrayRef args, CanFunctionType formalApplyType, SGFContext C) { - assert(substitutions.size() == 1 && "gep should have two substitutions"); - assert(args.size() == 3 && "gep should be given three arguments"); - - SILType ElemTy = gen.getLoweredType(substitutions[0].getReplacement()); - SILType RawPtrType = args[0].getUnmanagedValue()->getType(); - SILValue addr = gen.B.createPointerToAddress(loc, args[0].getUnmanagedValue(), - ElemTy.getAddressType(), true); - addr = gen.B.createIndexAddr(loc, addr, args[1].getUnmanagedValue()); - addr = gen.B.createAddressToPointer(loc, addr, RawPtrType); - - return ManagedValue::forUnmanaged(addr); -} - -/// Specialized emitter for Builtin.getTailAddr. -static ManagedValue emitBuiltinGetTailAddr(SILGenFunction &gen, - SILLocation loc, - ArrayRef substitutions, - ArrayRef args, - CanFunctionType formalApplyType, - SGFContext C) { - assert(substitutions.size() == 2 && "getTailAddr should have two substitutions"); - assert(args.size() == 4 && "gep should be given four arguments"); - - SILType ElemTy = gen.getLoweredType(substitutions[0].getReplacement()); - SILType TailTy = gen.getLoweredType(substitutions[1].getReplacement()); - SILType RawPtrType = args[0].getUnmanagedValue()->getType(); - SILValue addr = gen.B.createPointerToAddress(loc, args[0].getUnmanagedValue(), - ElemTy.getAddressType(), true); - addr = gen.B.createTailAddr(loc, addr, args[1].getUnmanagedValue(), - TailTy.getAddressType()); - addr = gen.B.createAddressToPointer(loc, addr, RawPtrType); - - return ManagedValue::forUnmanaged(addr); + assert(args.size() == 2 && "gep should be given two arguments"); + + SILValue offsetPtr = gen.B.createIndexRawPointer(loc, + args[0].getUnmanagedValue(), + args[1].getUnmanagedValue()); + return ManagedValue::forUnmanaged(offsetPtr); } /// Specialized emitter for Builtin.condfail. @@ -846,55 +804,6 @@ static ManagedValue emitBuiltinBindMemory(SILGenFunction &gen, return ManagedValue::forUnmanaged(gen.emitEmptyTuple(loc)); } -static ManagedValue emitBuiltinAllocWithTailElems(SILGenFunction &gen, - SILLocation loc, - ArrayRef subs, - ArrayRef args, - CanFunctionType formalApplyType, - SGFContext C) { - unsigned NumTailTypes = subs.size() - 1; - assert(args.size() == NumTailTypes * 2 + 1 && - "wrong number of substitutions for allocWithTailElems"); - - // The substitution determines the element type for bound memory. - SILType RefType = gen.getLoweredType(subs[0].getReplacement()-> - getCanonicalType()).getObjectType(); - - SmallVector Counts; - SmallVector ElemTypes; - for (unsigned Idx = 0; Idx < NumTailTypes; ++Idx) { - Counts.push_back(args[Idx * 2 + 1].getValue()); - ElemTypes.push_back(gen.getLoweredType(subs[Idx+1].getReplacement()-> - getCanonicalType()).getObjectType()); - } - - SILValue result = gen.B.createAllocRef(loc, RefType, false, ElemTypes, Counts); - - return ManagedValue::forUnmanaged(result); -} - -static ManagedValue emitBuiltinProjectTailElems(SILGenFunction &gen, - SILLocation loc, - ArrayRef subs, - ArrayRef args, - CanFunctionType formalApplyType, - SGFContext C) { - assert(subs.size() == 2 && - "allocWithTailElems should have two substitutions"); - assert(args.size() == 2 && - "allocWithTailElems should have three arguments"); - - // The substitution determines the element type for bound memory. - SILType ElemType = gen.getLoweredType(subs[1].getReplacement()-> - getCanonicalType()).getObjectType(); - - SILValue result = gen.B.createRefTailAddr(loc, args[0].getValue(), - ElemType.getAddressType()); - SILType rawPointerType = SILType::getRawPointerType(gen.F.getASTContext()); - result = gen.B.createAddressToPointer(loc, result, rawPointerType); - return ManagedValue::forUnmanaged(result); -} - /// Specialized emitter for type traits. template diff --git a/lib/SILOptimizer/Analysis/ARCAnalysis.cpp b/lib/SILOptimizer/Analysis/ARCAnalysis.cpp index af3351ae7e96c..c3930c067662c 100644 --- a/lib/SILOptimizer/Analysis/ARCAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/ARCAnalysis.cpp @@ -178,7 +178,6 @@ bool swift::canNeverUseValues(SILInstruction *Inst) { case ValueKind::TupleElementAddrInst: case ValueKind::UncheckedTakeEnumDataAddrInst: case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::UncheckedEnumDataInst: case ValueKind::IndexAddrInst: case ValueKind::IndexRawPointerInst: diff --git a/lib/SILOptimizer/Analysis/AliasAnalysis.cpp b/lib/SILOptimizer/Analysis/AliasAnalysis.cpp index 108bf13eee4e9..1144e299b66b1 100644 --- a/lib/SILOptimizer/Analysis/AliasAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/AliasAnalysis.cpp @@ -305,7 +305,6 @@ AliasResult AliasAnalysis::aliasAddressProjection(SILValue V1, SILValue V2, static bool isTypedAccessOracle(SILInstruction *I) { switch (I->getKind()) { case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::StructElementAddrInst: case ValueKind::TupleElementAddrInst: case ValueKind::UncheckedTakeEnumDataAddrInst: @@ -341,9 +340,7 @@ static bool isAddressRootTBAASafe(SILValue V) { default: return false; case ValueKind::AllocStackInst: - case ValueKind::ProjectBoxInst: - case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: + case ValueKind::AllocBoxInst: return true; } } @@ -369,7 +366,7 @@ static SILType findTypedAccessType(SILValue V) { } SILType swift::computeTBAAType(SILValue V) { - if (isAddressRootTBAASafe(getUnderlyingAddressRoot(V))) + if (isAddressRootTBAASafe(getUnderlyingObject(V))) return findTypedAccessType(V); // FIXME: add ref_element_addr check here. TBAA says that objects cannot be diff --git a/lib/SILOptimizer/Analysis/ArraySemantic.cpp b/lib/SILOptimizer/Analysis/ArraySemantic.cpp index 36b3ff4c66a08..3be2b35c80f3c 100644 --- a/lib/SILOptimizer/Analysis/ArraySemantic.cpp +++ b/lib/SILOptimizer/Analysis/ArraySemantic.cpp @@ -90,7 +90,8 @@ bool swift::ArraySemanticsCall::isValidSignature() { return false; StringRef AllocFuncName = AllocFn->getName(); - if (AllocFuncName != "swift_bufferAllocate") + if (AllocFuncName != "swift_bufferAllocate" && + AllocFuncName != "swift_bufferAllocateOnStack") return false; if (!hasOneNonDebugUse(AllocBufferAI)) @@ -578,8 +579,7 @@ SILValue swift::ArraySemanticsCall::getInitializationCount() const { // argument. The count is the second argument. // A call to _allocateUninitialized has the count as first argument. SILValue Arg0 = SemanticsCall->getArgument(0); - if (Arg0->getType().isExistentialType() || - Arg0->getType().hasReferenceSemantics()) + if (Arg0->getType().isExistentialType()) return SemanticsCall->getArgument(1); else return SemanticsCall->getArgument(0); } diff --git a/lib/SILOptimizer/Analysis/EscapeAnalysis.cpp b/lib/SILOptimizer/Analysis/EscapeAnalysis.cpp index b2d97dd4bda27..d0c070e6f6215 100644 --- a/lib/SILOptimizer/Analysis/EscapeAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/EscapeAnalysis.cpp @@ -1314,7 +1314,6 @@ void EscapeAnalysis::analyzeInstruction(SILInstruction *I, case ValueKind::LoadWeakInst: // We treat ref_element_addr like a load (see NodeType::Content). case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::ProjectBoxInst: case ValueKind::InitExistentialAddrInst: case ValueKind::OpenExistentialAddrInst: diff --git a/lib/SILOptimizer/Analysis/SideEffectAnalysis.cpp b/lib/SILOptimizer/Analysis/SideEffectAnalysis.cpp index 93419bd20600a..aff45dcac90c8 100644 --- a/lib/SILOptimizer/Analysis/SideEffectAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/SideEffectAnalysis.cpp @@ -94,7 +94,6 @@ static SILValue skipAddrProjections(SILValue V) { case ValueKind::StructElementAddrInst: case ValueKind::TupleElementAddrInst: case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::ProjectBoxInst: case ValueKind::UncheckedTakeEnumDataAddrInst: case ValueKind::PointerToAddressInst: diff --git a/lib/SILOptimizer/Transforms/CSE.cpp b/lib/SILOptimizer/Transforms/CSE.cpp index fb8ae098399a0..9413e8da169c8 100644 --- a/lib/SILOptimizer/Transforms/CSE.cpp +++ b/lib/SILOptimizer/Transforms/CSE.cpp @@ -133,10 +133,6 @@ class HashVisitor : public SILInstructionVisitor { return llvm::hash_combine(X->getKind(), X->getOperand(), X->getField()); } - hash_code visitRefTailAddrInst(RefTailAddrInst *X) { - return llvm::hash_combine(X->getKind(), X->getOperand()); - } - hash_code visitProjectBoxInst(ProjectBoxInst *X) { return llvm::hash_combine(X->getKind(), X->getOperand()); } @@ -871,7 +867,6 @@ bool CSE::canHandle(SILInstruction *Inst) { case ValueKind::ValueMetatypeInst: case ValueKind::ObjCProtocolInst: case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::ProjectBoxInst: case ValueKind::IndexRawPointerInst: case ValueKind::IndexAddrInst: diff --git a/lib/SILOptimizer/Transforms/ReleaseDevirtualizer.cpp b/lib/SILOptimizer/Transforms/ReleaseDevirtualizer.cpp index f0579c8eb9121..b1504d2e02880 100644 --- a/lib/SILOptimizer/Transforms/ReleaseDevirtualizer.cpp +++ b/lib/SILOptimizer/Transforms/ReleaseDevirtualizer.cpp @@ -24,7 +24,6 @@ using namespace swift; namespace { /// Devirtualizes release instructions which are known to destruct the object. -/// /// This means, it replaces a sequence of /// %x = alloc_ref [stack] $X /// ... @@ -38,6 +37,10 @@ namespace { /// %a = apply %d(%x) /// dealloc_ref [stack] %x /// +/// It also works for array buffers, where the allocation/deallocation is done +/// by calls to the swift_bufferAllocateOnStack/swift_bufferDeallocateFromStack +/// functions. +/// /// The optimization is only done for stack promoted objects because they are /// known to have no associated objects (which are not explicitly released /// in the deinit method). @@ -54,6 +57,10 @@ class ReleaseDevirtualizer : public SILFunctionTransform { bool devirtualizeReleaseOfObject(SILInstruction *ReleaseInst, DeallocRefInst *DeallocInst); + /// Devirtualize releases of swift objects. + bool devirtualizeReleaseOfBuffer(SILInstruction *ReleaseInst, + ApplyInst *DeallocCall); + /// Replace the release-instruction \p ReleaseInst with an explicit call to /// the deallocating destructor of \p AllocType for \p object. bool createDeallocCall(SILType AllocType, SILInstruction *ReleaseInst, @@ -84,6 +91,11 @@ void ReleaseDevirtualizer::run() { LastRelease = nullptr; continue; } + if (auto *AI = dyn_cast(&I)) { + Changed |= devirtualizeReleaseOfBuffer(LastRelease, AI); + LastRelease = nullptr; + continue; + } } if (isa(&I) || @@ -127,6 +139,51 @@ devirtualizeReleaseOfObject(SILInstruction *ReleaseInst, return createDeallocCall(AllocType, ReleaseInst, ARI); } +bool ReleaseDevirtualizer:: +devirtualizeReleaseOfBuffer(SILInstruction *ReleaseInst, + ApplyInst *DeallocCall) { + + DEBUG(llvm::dbgs() << " try to devirtualize " << *ReleaseInst); + + // Is this a deallocation of a buffer? + SILFunction *DeallocFn = DeallocCall->getReferencedFunction(); + if (!DeallocFn || DeallocFn->getName() != "swift_bufferDeallocateFromStack") + return false; + + // Is the deallocation call paired with an allocation call? + ApplyInst *AllocAI = dyn_cast(DeallocCall->getArgument(0)); + if (!AllocAI || AllocAI->getNumArguments() < 1) + return false; + + SILFunction *AllocFunc = AllocAI->getReferencedFunction(); + if (!AllocFunc || AllocFunc->getName() != "swift_bufferAllocateOnStack") + return false; + + // Can we find the buffer type which is allocated? It's metatype is passed + // as first argument to the allocation function. + auto *IEMTI = dyn_cast(AllocAI->getArgument(0)); + if (!IEMTI) + return false; + + SILType MType = IEMTI->getOperand()->getType(); + auto *MetaType = MType.getSwiftRValueType()->getAs(); + if (!MetaType) + return false; + + // Is the allocated buffer a class type? This should always be the case. + auto *ClType = MetaType->getInstanceType()->getAs(); + if (!ClType) + return false; + + // Does the last release really release the allocated buffer? + SILValue rcRoot = RCIA->getRCIdentityRoot(ReleaseInst->getOperand(0)); + if (rcRoot != AllocAI) + return false; + + SILType SILClType = SILType::getPrimitiveObjectType(CanType(ClType)); + return createDeallocCall(SILClType, ReleaseInst, AllocAI); +} + bool ReleaseDevirtualizer::createDeallocCall(SILType AllocType, SILInstruction *ReleaseInst, SILValue object) { diff --git a/lib/SILOptimizer/Transforms/StackPromotion.cpp b/lib/SILOptimizer/Transforms/StackPromotion.cpp index 0e57b69ec85ac..50217d2949c0a 100644 --- a/lib/SILOptimizer/Transforms/StackPromotion.cpp +++ b/lib/SILOptimizer/Transforms/StackPromotion.cpp @@ -27,10 +27,23 @@ STATISTIC(NumStackPromoted, "Number of objects promoted to the stack"); using namespace swift; /// Promotes heap allocated objects to the stack. -/// -/// It handles alloc_ref instructions of native swift classes: if promoted, -/// the [stack] attribute is set in the alloc_ref and a dealloc_ref [stack] is -/// inserted at the end of the object's lifetime. +/// Following types of allocations are handled: +/// *) alloc_ref instructions of native swift classes: if promoted, the [stack] +/// attribute is set in the alloc_ref and a dealloc_ref [stack] is inserted +/// at the end of the object's lifetime. +/// *) Array buffers which are allocated by a call to swift_bufferAllocate: if +/// promoted the swift_bufferAllocate call is replaced by a call to +/// swift_bufferAllocateOnStack and a call to swift_bufferDeallocateFromStack +/// is inserted at the end of the buffer's lifetime. +/// Those calls are lowered by the LLVM SwiftStackPromotion pass. +/// TODO: This is a terrible hack, but necessary because we need constant +/// size and alignment for the final stack promotion decision. The arguments +/// to swift_bufferAllocate in SIL are not constant because they depend on +/// the not-yet-evaluatable sizeof and alignof builtins. Therefore we need +/// LLVM's constant propagation prior to deciding on stack promotion. +/// The solution to this problem is that we need native support for tail- +/// allocated arrays in SIL so that we can do the array buffer allocations +/// with alloc_ref instructions. class StackPromoter { // Some analysis we need. @@ -55,6 +68,14 @@ class StackPromoter { bool PostDomTreeValid; + // Pseudo-functions for (de-)allocating array buffers on the stack. + + SILFunction *BufferAllocFunc = nullptr; + SILFunction *BufferDeallocFunc = nullptr; + + bool ChangedInsts = false; + bool ChangedCalls = false; + /// Worklist for visiting all blocks. class WorkListType { /// The nesting depth of stack allocation instructions for each block. @@ -103,14 +124,22 @@ class StackPromoter { }; /// Tries to promote the allocation \p AI. - bool tryPromoteAlloc(AllocRefInst *ARI); + void tryPromoteAlloc(SILInstruction *AI); + + /// Creates the external declaration for swift_bufferAllocateOnStack. + SILFunction *getBufferAllocFunc(SILFunction *OrigFunc, + SILLocation Loc); + + /// Creates the external declaration for swift_bufferDeallocateFromStack. + SILFunction *getBufferDeallocFunc(SILFunction *OrigFunc, + SILLocation Loc); /// Returns true if the allocation \p AI can be promoted. /// In this case it sets the \a DeallocInsertionPoint to the instruction /// where the deallocation must be inserted. /// It optionally also sets \a AllocInsertionPoint in case the allocation /// instruction must be moved to another place. - bool canPromoteAlloc(AllocRefInst *ARI, + bool canPromoteAlloc(SILInstruction *AI, SILInstruction *&AllocInsertionPoint, SILInstruction *&DeallocInsertionPoint); @@ -170,15 +199,40 @@ class StackPromoter { F(F), ConGraph(ConGraph), DT(DT), EA(EA), PostDomTree(true), PostDomTreeValid(false) { } + /// What did the optimization change? + enum class ChangeState { + None, + Insts, + Calls + }; + SILFunction *getFunction() const { return F; } /// The main entry point for the optimization. - /// - /// Returns true if some changes were made. - bool promote(); + ChangeState promote(); }; -bool StackPromoter::promote() { +/// Returns true if instruction \p I is an allocation we can handle. +static bool isPromotableAllocInst(SILInstruction *I) { + // Check for swift object allocation. + if (auto *ARI = dyn_cast(I)) { + if (!ARI->isObjC()) + return true; + return false; + } + // Check for array buffer allocation. + auto *AI = dyn_cast(I); + if (AI && AI->getNumArguments() == 3) { + if (auto *Callee = AI->getReferencedFunction()) { + if (Callee->getName() == "swift_bufferAllocate") + return true; + } + return false; + } + return false; +} + +StackPromoter::ChangeState StackPromoter::promote() { llvm::SetVector ReachableBlocks; @@ -198,7 +252,6 @@ bool StackPromoter::promote() { ReachableBlocks.insert(Pred); } - bool Changed = false; // Search the whole function for stack promotable allocations. for (SILBasicBlock &BB : *F) { @@ -213,34 +266,105 @@ bool StackPromoter::promote() { // The allocation instruction may be moved, so increment Iter prior to // doing the optimization. SILInstruction *I = &*Iter++; - if (auto *ARI = dyn_cast(I)) { - Changed |= tryPromoteAlloc(ARI); + if (isPromotableAllocInst(I)) { + tryPromoteAlloc(I); } } } - return Changed; + if (ChangedCalls) + return ChangeState::Calls; + if (ChangedInsts) + return ChangeState::Insts; + return ChangeState::None; } -bool StackPromoter::tryPromoteAlloc(AllocRefInst *ARI) { - +void StackPromoter::tryPromoteAlloc(SILInstruction *I) { SILInstruction *AllocInsertionPoint = nullptr; SILInstruction *DeallocInsertionPoint = nullptr; - if (!canPromoteAlloc(ARI, AllocInsertionPoint, DeallocInsertionPoint)) - return false; + if (!canPromoteAlloc(I, AllocInsertionPoint, DeallocInsertionPoint)) + return; - DEBUG(llvm::dbgs() << "Promoted " << *ARI); - DEBUG(llvm::dbgs() << " in " << ARI->getFunction()->getName() << '\n'); + DEBUG(llvm::dbgs() << "Promoted " << *I); + DEBUG(llvm::dbgs() << " in " << I->getFunction()->getName() << '\n'); NumStackPromoted++; SILBuilder B(DeallocInsertionPoint); - // It's an object allocation. We set the [stack] attribute in the alloc_ref. - ARI->setStackAllocatable(); - if (AllocInsertionPoint) - ARI->moveBefore(AllocInsertionPoint); - - /// And create a dealloc_ref [stack] at the end of the object's lifetime. - B.createDeallocRef(ARI->getLoc(), ARI, true); - return true; + if (auto *ARI = dyn_cast(I)) { + // It's an object allocation. We set the [stack] attribute in the alloc_ref. + ARI->setStackAllocatable(); + if (AllocInsertionPoint) + ARI->moveBefore(AllocInsertionPoint); + + /// And create a dealloc_ref [stack] at the end of the object's lifetime. + B.createDeallocRef(I->getLoc(), I, true); + ChangedInsts = true; + return; + } + if (auto *AI = dyn_cast(I)) { + assert(!AllocInsertionPoint && "can't move call to swift_bufferAlloc"); + // It's an array buffer allocation. + auto *OldFRI = cast(AI->getCallee()); + SILFunction *OldF = OldFRI->getReferencedFunction(); + SILLocation loc = (OldF->hasLocation() ? OldF->getLocation() : AI->getLoc()); + SILFunction *DeallocFun = getBufferDeallocFunc(OldF, loc); + + // We insert a swift_bufferDeallocateFromStack at the end of the buffer's + // lifetime. + auto *DeallocFRI = B.createFunctionRef(OldFRI->getLoc(), DeallocFun); + B.createApply(loc, DeallocFRI, { AI }, false); + + // And replace the call to swift_bufferAllocate with a call to + // swift_bufferAllocateOnStack. + B.setInsertionPoint(AI); + auto *AllocFRI = B.createFunctionRef(OldFRI->getLoc(), + getBufferAllocFunc(OldF, loc)); + AI->setOperand(0, AllocFRI); + + ChangedCalls = true; + return; + } + llvm_unreachable("unhandled allocation instruction"); +} + +SILFunction *StackPromoter::getBufferAllocFunc(SILFunction *OrigFunc, + SILLocation Loc) { + if (!BufferAllocFunc) { + BufferAllocFunc = OrigFunc->getModule().getOrCreateFunction( + Loc, + "swift_bufferAllocateOnStack", + OrigFunc->getLinkage(), + OrigFunc->getLoweredFunctionType(), + OrigFunc->isBare(), IsNotTransparent, + OrigFunc->isFragile()); + } + return BufferAllocFunc; +} + +SILFunction *StackPromoter::getBufferDeallocFunc(SILFunction *OrigFunc, + SILLocation Loc) { + if (!BufferDeallocFunc) { + SILModule &M = OrigFunc->getModule(); + CanSILFunctionType OrigTy = OrigFunc->getLoweredFunctionType(); + CanType ObjectTy = OrigTy->getSILResult().getSwiftRValueType(); + + // The function type for swift_bufferDeallocateFromStack. + CanSILFunctionType FunTy = SILFunctionType::get( + OrigTy->getGenericSignature(), + OrigTy->getExtInfo(), + OrigTy->getCalleeConvention(), + { SILParameterInfo(ObjectTy, ParameterConvention::Direct_Guaranteed) }, + ArrayRef(), + OrigTy->getOptionalErrorResult(), + M.getASTContext()); + + BufferDeallocFunc = M.getOrCreateFunction( + Loc, + "swift_bufferDeallocateFromStack", + OrigFunc->getLinkage(), + FunTy, + OrigFunc->isBare(), IsNotTransparent, OrigFunc->isFragile()); + } + return BufferDeallocFunc; } namespace { @@ -325,15 +449,12 @@ template <> struct GraphTraits } -bool StackPromoter::canPromoteAlloc(AllocRefInst *ARI, +bool StackPromoter::canPromoteAlloc(SILInstruction *AI, SILInstruction *&AllocInsertionPoint, SILInstruction *&DeallocInsertionPoint) { - if (ARI->isObjC()) - return false; - AllocInsertionPoint = nullptr; DeallocInsertionPoint = nullptr; - auto *Node = ConGraph->getNodeOrNull(ARI, EA); + auto *Node = ConGraph->getNodeOrNull(AI, EA); if (!Node) return false; @@ -358,7 +479,7 @@ bool StackPromoter::canPromoteAlloc(AllocRefInst *ARI, // Try to find the point where to insert the deallocation. // This might need more than one try in case we need to move the allocation // out of a stack-alloc-dealloc pair. See findDeallocPoint(). - SILInstruction *StartInst = ARI; + SILInstruction *StartInst = AI; for (;;) { SILInstruction *RestartPoint = nullptr; DeallocInsertionPoint = findDeallocPoint(StartInst, RestartPoint, Node, @@ -369,6 +490,11 @@ bool StackPromoter::canPromoteAlloc(AllocRefInst *ARI, if (!RestartPoint) return false; + // Moving a buffer allocation call is not trivial because we would need to + // move all the parameter calculations as well. So we just don't do it. + if (!isa(AI)) + return false; + // Retry with moving the allocation up. AllocInsertionPoint = RestartPoint; StartInst = RestartPoint; @@ -556,8 +682,15 @@ class StackPromotion : public SILFunctionTransform { SILFunction *F = getFunction(); if (auto *ConGraph = EA->getConnectionGraph(F)) { StackPromoter promoter(F, ConGraph, DA->get(F), EA); - if (promoter.promote()) { - invalidateAnalysis(SILAnalysis::InvalidationKind::Instructions); + switch (promoter.promote()) { + case StackPromoter::ChangeState::None: + break; + case StackPromoter::ChangeState::Insts: + invalidateAnalysis(SILAnalysis::InvalidationKind::Instructions); + break; + case StackPromoter::ChangeState::Calls: + invalidateAnalysis(SILAnalysis::InvalidationKind::CallsAndInstructions); + break; } } } diff --git a/lib/SILOptimizer/Utils/Local.cpp b/lib/SILOptimizer/Utils/Local.cpp index 51ff3b9e4d40c..d4d5386c75d10 100644 --- a/lib/SILOptimizer/Utils/Local.cpp +++ b/lib/SILOptimizer/Utils/Local.cpp @@ -2724,7 +2724,6 @@ void swift::hoistAddressProjections(Operand &Op, SILInstruction *InsertBefore, case ValueKind::StructElementAddrInst: case ValueKind::TupleElementAddrInst: case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::UncheckedTakeEnumDataAddrInst: { auto *Inst = cast(V); // We are done once the current projection dominates the insert point. diff --git a/lib/SILOptimizer/Utils/SILInliner.cpp b/lib/SILOptimizer/Utils/SILInliner.cpp index 45edc90c575ae..b87435215c3f0 100644 --- a/lib/SILOptimizer/Utils/SILInliner.cpp +++ b/lib/SILOptimizer/Utils/SILInliner.cpp @@ -341,7 +341,6 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) { case ValueKind::DynamicMethodInst: case ValueKind::EnumInst: case ValueKind::IndexAddrInst: - case ValueKind::TailAddrInst: case ValueKind::IndexRawPointerInst: case ValueKind::InitEnumDataAddrInst: case ValueKind::InitExistentialAddrInst: @@ -359,7 +358,6 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) { case ValueKind::PartialApplyInst: case ValueKind::ExistentialMetatypeInst: case ValueKind::RefElementAddrInst: - case ValueKind::RefTailAddrInst: case ValueKind::RefToUnmanagedInst: case ValueKind::RefToUnownedInst: case ValueKind::StoreInst: diff --git a/lib/Serialization/DeserializeSIL.cpp b/lib/Serialization/DeserializeSIL.cpp index 9471b550eb5f9..31a3bbfeac30c 100644 --- a/lib/Serialization/DeserializeSIL.cpp +++ b/lib/Serialization/DeserializeSIL.cpp @@ -721,12 +721,6 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, SILBasicBlock *BB, TyID, TyCategory, ValID, TyID2, TyCategory2, ValID2); break; - case SIL_TAIL_ADDR: - SILTailAddrLayout::readRecord(scratch, OpCode, - TyID, ValID, - TyID2, ValID2, - TyID3); - break; case SIL_INST_APPLY: { unsigned IsPartial; SILInstApplyLayout::readRecord(scratch, IsPartial, NumSubs, @@ -932,30 +926,12 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, SILBasicBlock *BB, case ValueKind::AllocRefInst: { assert(RecordKind == SIL_ONE_TYPE_VALUES && "Layout should be OneTypeValues."); - unsigned NumVals = ListOfValues.size(); - assert(NumVals >= 1 && "Not enough values"); - unsigned Flags = ListOfValues[0]; - bool isObjC = (bool)(Flags & 1); - bool canAllocOnStack = (bool)((Flags >> 1) & 1); - SILType ClassTy = getSILType(MF->getType(TyID), (SILValueCategory)TyCategory); - if (NumVals == 1) { - ResultVal = Builder.createAllocRef(Loc, ClassTy, isObjC, canAllocOnStack); - } else { - assert(!isObjC); - SmallVector Counts; - SmallVector TailTypes; - for (unsigned i = 1; i < NumVals; i += 3) { - SILType TailType = getSILType(MF->getType(ListOfValues[i]), - SILValueCategory::Object); - TailTypes.push_back(TailType); - SILType CountType = getSILType(MF->getType(ListOfValues[i+2]), - SILValueCategory::Object); - SILValue CountVal = getLocalValue(ListOfValues[i+1], CountType); - Counts.push_back(CountVal); - } - ResultVal = Builder.createAllocRef(Loc, ClassTy, canAllocOnStack, - TailTypes, Counts); - } + assert(ListOfValues.size() >= 1 && "Not enough values"); + unsigned Value = ListOfValues[0]; + ResultVal = Builder.createAllocRef( + Loc, + getSILType(MF->getType(TyID), (SILValueCategory)TyCategory), + (bool)(Value & 1), (bool)((Value >> 1) & 1)); break; } case ValueKind::AllocRefDynamicInst: { @@ -1178,16 +1154,6 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, SILBasicBlock *BB, getSILType(Ty2, (SILValueCategory)TyCategory2))); break; } - case ValueKind::TailAddrInst: { - auto Ty = MF->getType(TyID); - auto Ty2 = MF->getType(TyID2); - auto ResultTy = MF->getType(TyID3); - ResultVal = Builder.createTailAddr(Loc, - getLocalValue(ValID, getSILType(Ty, SILValueCategory::Address)), - getLocalValue(ValID2, getSILType(Ty2, SILValueCategory::Object)), - getSILType(ResultTy, SILValueCategory::Address)); - break; - } case ValueKind::IndexRawPointerInst: { auto Ty = MF->getType(TyID); auto Ty2 = MF->getType(TyID2); @@ -1690,18 +1656,6 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, SILBasicBlock *BB, ResultTy); break; } - case ValueKind::RefTailAddrInst: { - assert(RecordKind == SIL_ONE_TYPE_ONE_OPERAND && - "Layout should be OneTypeOneOperand."); - assert(Attr == 0); - assert((SILValueCategory)TyCategory == SILValueCategory::Address); - ResultVal = Builder.createRefTailAddr( - Loc, - getLocalValue(ValID, getSILType(MF->getType(TyID2), - (SILValueCategory)TyCategory2)), - getSILType(MF->getType(TyID), SILValueCategory::Address)); - break; - } case ValueKind::ClassMethodInst: case ValueKind::SuperMethodInst: case ValueKind::DynamicMethodInst: { diff --git a/lib/Serialization/SILFormat.h b/lib/Serialization/SILFormat.h index 4087b77ec422b..b5cab882d0f44 100644 --- a/lib/Serialization/SILFormat.h +++ b/lib/Serialization/SILFormat.h @@ -125,7 +125,6 @@ namespace sil_block { SIL_ONE_TYPE_ONE_OPERAND, SIL_ONE_TYPE_VALUES, SIL_TWO_OPERANDS, - SIL_TAIL_ADDR, SIL_INST_APPLY, SIL_INST_NO_OPERAND, SIL_VTABLE, @@ -375,17 +374,6 @@ namespace sil_block { ValueIDField >; - // The tail_addr instruction. - using SILTailAddrLayout = BCRecordLayout< - SIL_TAIL_ADDR, - SILInstOpCodeField, - TypeIDField, // Base operand - ValueIDField, - TypeIDField, // Count operand - ValueIDField, - TypeIDField // Result type - >; - using SILGenericOuterParamsLayout = BCRecordLayout< SIL_GENERIC_OUTER_PARAMS, DeclIDField // The decl id of the outer param if any. diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp index c36403aaaa19e..2dc445f02dd69 100644 --- a/lib/Serialization/Serialization.cpp +++ b/lib/Serialization/Serialization.cpp @@ -485,7 +485,6 @@ void Serializer::writeBlockInfoBlock() { BLOCK_RECORD(sil_block, SIL_ONE_TYPE_ONE_OPERAND); BLOCK_RECORD(sil_block, SIL_ONE_TYPE_VALUES); BLOCK_RECORD(sil_block, SIL_TWO_OPERANDS); - BLOCK_RECORD(sil_block, SIL_TAIL_ADDR); BLOCK_RECORD(sil_block, SIL_INST_APPLY); BLOCK_RECORD(sil_block, SIL_INST_NO_OPERAND); BLOCK_RECORD(sil_block, SIL_VTABLE); diff --git a/lib/Serialization/SerializeSIL.cpp b/lib/Serialization/SerializeSIL.cpp index 6f1d85f9716ed..d48390259ce50 100644 --- a/lib/Serialization/SerializeSIL.cpp +++ b/lib/Serialization/SerializeSIL.cpp @@ -650,28 +650,14 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { case ValueKind::AllocRefInst: { const AllocRefInst *ARI = cast(&SI); unsigned abbrCode = SILAbbrCodes[SILOneTypeValuesLayout::Code]; - SmallVector Args; - Args.push_back((unsigned)ARI->isObjC() | - ((unsigned)ARI->canAllocOnStack() << 1)); - ArrayRef TailTypes = ARI->getTailAllocatedTypes(); - ArrayRef TailCounts = ARI->getTailAllocatedCounts(); - unsigned NumTailAllocs = TailTypes.size(); - assert(TailCounts.size() == NumTailAllocs); - for (unsigned Idx = 0; Idx < NumTailAllocs; ++Idx) { - assert(TailTypes[Idx].isObject()); - Args.push_back(S.addTypeRef(TailTypes[Idx].getSwiftRValueType())); - SILValue CountVal = TailCounts[Idx].get(); - Args.push_back(addValueRef(CountVal)); - SILType CountType = CountVal->getType(); - assert(CountType.isObject()); - Args.push_back(S.addTypeRef(CountType.getSwiftRValueType())); - } + ValueID Args[1] = { (unsigned)ARI->isObjC() | + ((unsigned)ARI->canAllocOnStack() << 1) }; SILOneTypeValuesLayout::emitRecord(Out, ScratchRecord, abbrCode, (unsigned)SI.getKind(), S.addTypeRef( ARI->getType().getSwiftRValueType()), (unsigned)ARI->getType().getCategory(), - Args); + llvm::makeArrayRef(Args)); break; } case ValueKind::AllocRefDynamicInst: { @@ -1082,18 +1068,6 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { addValueRef(operand2)); break; } - case ValueKind::TailAddrInst: { - const TailAddrInst *TAI = cast(&SI); - SILTailAddrLayout::emitRecord(Out, ScratchRecord, - SILAbbrCodes[SILTailAddrLayout::Code], - (unsigned)SI.getKind(), - S.addTypeRef(TAI->getBase()->getType().getSwiftRValueType()), - addValueRef(TAI->getBase()), - S.addTypeRef(TAI->getIndex()->getType().getSwiftRValueType()), - addValueRef(TAI->getIndex()), - S.addTypeRef(TAI->getTailType().getSwiftRValueType())); - break; - } case ValueKind::StringLiteralInst: { auto SLI = cast(&SI); StringRef Str = SLI->getValue(); @@ -1370,13 +1344,6 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { addValueRef(operand)); break; } - case ValueKind::RefTailAddrInst: { - auto *RTAI = cast(&SI); - writeOneTypeOneOperandLayout(RTAI->getKind(), 0, - RTAI->getType(), - RTAI->getOperand()); - break; - } case ValueKind::StructInst: { // Format: a type followed by a list of typed values. A typed value is // expressed by 4 IDs: TypeID, TypeCategory, ValueID, ValueResultNumber. @@ -1846,7 +1813,6 @@ void SILSerializer::writeSILBlock(const SILModule *SILMod) { registerSILAbbr(); registerSILAbbr(); registerSILAbbr(); - registerSILAbbr(); registerSILAbbr(); registerSILAbbr(); diff --git a/stdlib/public/core/Arrays.swift.gyb b/stdlib/public/core/Arrays.swift.gyb index f08c1ccb6becd..7617eadfab347 100644 --- a/stdlib/public/core/Arrays.swift.gyb +++ b/stdlib/public/core/Arrays.swift.gyb @@ -954,9 +954,12 @@ func _allocateUninitializedArray(_ builtinCount: Builtin.Word) if count > 0 { // Doing the actual buffer allocation outside of the array.uninitialized // semantics function enables stack propagation of the buffer. - let bufferObject = Builtin.allocWithTailElems_1(_ContiguousArrayStorage.self, builtinCount, Element.self) + let bufferObject = ManagedBufferPointer<_ArrayBody, Element>( + _uncheckedBufferClass: _ContiguousArrayStorage.self, + minimumCapacity: count) - let (array, ptr) = Array._adoptStorage(bufferObject, count: count) + let (array, ptr) = Array._adoptStorage( + bufferObject.buffer, count: count) return (array, ptr._rawValue) } // For an empty array no buffer allocation is needed. @@ -1107,12 +1110,16 @@ extension ${Self} : RangeReplaceableCollection, _ArrayProtocol { @_versioned @_semantics("array.uninitialized") internal static func _adoptStorage( - _ storage: _ContiguousArrayStorage, count: Int + _ storage: AnyObject, count: Int ) -> (Array, UnsafeMutablePointer) { + _sanityCheck( + storage is _ContiguousArrayStorage, "Invalid array storage type") + let innerBuffer = _ContiguousArrayBuffer( count: count, - storage: storage) + storage: unsafeDowncast( + storage, to: _ContiguousArrayStorage.self)) return ( Array( diff --git a/stdlib/public/core/ContiguousArrayBuffer.swift b/stdlib/public/core/ContiguousArrayBuffer.swift index cd2a7199eaa96..120c8ea65fc72 100644 --- a/stdlib/public/core/ContiguousArrayBuffer.swift +++ b/stdlib/public/core/ContiguousArrayBuffer.swift @@ -25,6 +25,8 @@ internal final class _EmptyArrayStorage _sanityCheckFailure("creating instance of _EmptyArrayStorage") } + var countAndCapacity: _ArrayBody + #if _runtime(_ObjC) override func _withVerbatimBridgedUnsafeBuffer( _ body: (UnsafeBufferPointer) throws -> R @@ -99,8 +101,10 @@ class _ContiguousArrayStorage1 : _ContiguousArrayStorageBase { final class _ContiguousArrayStorage : _ContiguousArrayStorage1 { deinit { - _elementPointer.deinitialize(count: countAndCapacity.count) - _fixLifetime(self) + __manager._elementPointer.deinitialize( + count: __manager._headerPointer.pointee.count) + __manager._headerPointer.deinitialize() + _fixLifetime(__manager) } #if _runtime(_ObjC) @@ -110,10 +114,10 @@ final class _ContiguousArrayStorage : _ContiguousArrayStorage1 { _ body: (UnsafeBufferPointer) throws -> Void ) rethrows { if _isBridgedVerbatimToObjectiveC(Element.self) { - let count = countAndCapacity.count - let elements = UnsafeRawPointer(_elementPointer) + let count = __manager.header.count + let elements = UnsafeRawPointer(__manager._elementPointer) .assumingMemoryBound(to: AnyObject.self) - defer { _fixLifetime(self) } + defer { _fixLifetime(__manager) } try body(UnsafeBufferPointer(start: elements, count: count)) } } @@ -125,7 +129,7 @@ final class _ContiguousArrayStorage : _ContiguousArrayStorage1 { _sanityCheck( !_isBridgedVerbatimToObjectiveC(Element.self), "Verbatim bridging should be handled separately") - return countAndCapacity.count + return __manager.header.count } /// Bridge array elements and return a new buffer that owns them. @@ -136,15 +140,15 @@ final class _ContiguousArrayStorage : _ContiguousArrayStorage1 { _sanityCheck( !_isBridgedVerbatimToObjectiveC(Element.self), "Verbatim bridging should be handled separately") - let count = countAndCapacity.count + let count = __manager.header.count let result = _HeapBuffer( _HeapBufferStorage.self, count, count) let resultPtr = result.baseAddress - let p = _elementPointer + let p = __manager._elementPointer for i in 0.. : _ContiguousArrayStorage1 { return Element.self } - internal final var _elementPointer : UnsafeMutablePointer { - return UnsafeMutablePointer(Builtin.projectTailElems(self, Element.self)) + internal // private + typealias Manager = ManagedBufferPointer<_ArrayBody, Element> + + internal // private + var __manager: Manager { + return Manager(_uncheckedUnsafeBufferObject: self) } } @@ -191,16 +199,14 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { self = _ContiguousArrayBuffer() } else { - _storage = Builtin.allocWithTailElems_1( - _ContiguousArrayStorage.self, - realMinimumCapacity._builtinWordValue, Element.self) - - let storageAddr = UnsafeMutableRawPointer(Builtin.bridgeToRawPointer(_storage)) - let endAddr = storageAddr + _swift_stdlib_malloc_size(storageAddr) - let realCapacity = endAddr.assumingMemoryBound(to: Element.self) - firstElementAddress + __bufferPointer = ManagedBufferPointer( + _uncheckedBufferClass: _ContiguousArrayStorage.self, + minimumCapacity: realMinimumCapacity) _initStorageHeader( - count: uninitializedCount, capacity: realCapacity) + count: uninitializedCount, capacity: __bufferPointer.capacity) + + _fixLifetime(__bufferPointer) } } @@ -213,13 +219,17 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { /// - Warning: storage may have been stack-allocated, so it's /// crucial not to call, e.g., `malloc_size` on it. internal init(count: Int, storage: _ContiguousArrayStorage) { - _storage = storage + __bufferPointer = ManagedBufferPointer( + _uncheckedUnsafeBufferObject: storage) _initStorageHeader(count: count, capacity: count) + + _fixLifetime(__bufferPointer) } internal init(_ storage: _ContiguousArrayStorageBase) { - _storage = storage + __bufferPointer = ManagedBufferPointer( + _uncheckedUnsafeBufferObject: storage) } /// Initialize the body part of our storage. @@ -232,12 +242,11 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { let verbatim = false #endif - // We can initialize by assignment because _ArrayBody is a trivial type, - // i.e. contains no references. - _storage.countAndCapacity = _ArrayBody( - count: count, - capacity: capacity, - elementTypeIsBridgedVerbatim: verbatim) + __bufferPointer._headerPointer.initialize(to: + _ArrayBody( + count: count, + capacity: capacity, + elementTypeIsBridgedVerbatim: verbatim)) } /// True, if the array is native and does not need a deferred type check. @@ -247,8 +256,7 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { /// A pointer to the first element. internal var firstElementAddress: UnsafeMutablePointer { - return UnsafeMutablePointer(Builtin.projectTailElems(_storage, - Element.self)) + return __bufferPointer._elementPointer } internal var firstElementAddressIfContiguous: UnsafeMutablePointer? { @@ -278,7 +286,8 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { //===--- _ArrayBufferProtocol conformance -----------------------------------===// /// Create an empty buffer. internal init() { - _storage = _emptyArrayStorage + __bufferPointer = ManagedBufferPointer( + _uncheckedUnsafeBufferObject: _emptyArrayStorage) } internal init(_buffer buffer: _ContiguousArrayBuffer, shiftedToStartIndex: Int) { @@ -337,7 +346,7 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { /// The number of elements the buffer stores. internal var count: Int { get { - return _storage.countAndCapacity.count + return __bufferPointer.header.count } nonmutating set { _sanityCheck(newValue >= 0) @@ -346,7 +355,7 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { newValue <= capacity, "Can't grow an array buffer past its capacity") - _storage.countAndCapacity.count = newValue + __bufferPointer._headerPointer.pointee.count = newValue } } @@ -355,14 +364,14 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { @inline(__always) func _checkValidSubscript(_ index : Int) { _precondition( - (index >= 0) && (index < count), + (index >= 0) && (index < __bufferPointer.header.count), "Index out of range" ) } /// The number of elements the buffer can store without reallocation. internal var capacity: Int { - return _storage.countAndCapacity.capacity + return __bufferPointer.header.capacity } /// Copy the elements in `bounds` from this buffer into uninitialized @@ -389,7 +398,7 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { internal subscript(bounds: Range) -> _SliceBuffer { get { return _SliceBuffer( - owner: _storage, + owner: __bufferPointer.buffer, subscriptBaseAddress: subscriptBaseAddress, indices: bounds, hasNativeBuffer: true) @@ -405,14 +414,14 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { /// may need to be considered, such as whether the buffer could be /// some immutable Cocoa container. internal mutating func isUniquelyReferenced() -> Bool { - return _isUnique(&_storage) + return __bufferPointer.isUniqueReference() } /// Returns `true` iff this buffer's storage is either /// uniquely-referenced or pinned. NOTE: this does not mean /// the buffer is mutable; see the comment on isUniquelyReferenced. internal mutating func isUniquelyReferencedOrPinned() -> Bool { - return _isUniqueOrPinned(&_storage) + return __bufferPointer._isUniqueOrPinnedReference() } #if _runtime(_ObjC) @@ -480,7 +489,11 @@ internal struct _ContiguousArrayBuffer : _ArrayBufferProtocol { return true } - internal var _storage: _ContiguousArrayStorageBase + internal var _storage: _ContiguousArrayStorageBase { + return Builtin.castFromNativeObject(__bufferPointer._nativeBuffer) + } + + var __bufferPointer: ManagedBufferPointer<_ArrayBody, Element> } /// Append the elements of `rhs` to `lhs`. diff --git a/stdlib/public/core/HashedCollections.swift.gyb b/stdlib/public/core/HashedCollections.swift.gyb index 77d8747b4eeb3..7624053afada0 100644 --- a/stdlib/public/core/HashedCollections.swift.gyb +++ b/stdlib/public/core/HashedCollections.swift.gyb @@ -2501,84 +2501,124 @@ internal struct _HashedContainerStorageHeader { /// Enough bytes are allocated to hold the bitmap for marking valid entries, /// keys, and values. The data layout starts with the bitmap, followed by the /// keys, followed by the values. -final internal class _Native${Self}StorageImpl<${TypeParameters}> { +final internal class _Native${Self}StorageImpl<${TypeParameters}> : + ManagedBuffer<_HashedContainerStorageHeader, UInt8> { // Note: It is intended that ${TypeParameters} // (without : Hashable) is used here - this storage must work // with non-Hashable types. + internal typealias BufferPointer = + ManagedBufferPointer<_HashedContainerStorageHeader, UInt8> internal typealias StorageImpl = _Native${Self}StorageImpl %if Self == 'Set': # Set needs these to keep signatures simple. internal typealias Key = ${TypeParameters} %end - internal var _body: _HashedContainerStorageHeader + /// Returns the bytes necessary to store a bit map of 'capacity' bytes and + /// padding to align the start to word alignment. + internal static func bytesForBitMap(capacity: Int) -> Int { + let numWords = _UnsafeBitMap.sizeInWords(forSizeInBits: capacity) + return numWords * MemoryLayout.stride + MemoryLayout.alignment + } + + /// Returns the bytes necessary to store 'capacity' keys and padding to align + /// the start to the alignment of the 'Key' type assuming a word aligned base + /// address. + internal static func bytesForKeys(capacity: Int) -> Int { + let padding = max(0, MemoryLayout.alignment - MemoryLayout.alignment) + return MemoryLayout.stride * capacity + padding + } + +%if Self == 'Dictionary': + /// Returns the bytes necessary to store 'capacity' values and padding to + /// align the start to the alignment of the 'Value' type assuming a base + /// address aligned to the maximum of the alignment of the 'Key' type and the + /// alignment of a word. + internal static func bytesForValues(capacity: Int) -> Int { + let maxPrevAlignment = max(MemoryLayout.alignment, MemoryLayout.alignment) + let padding = max(0, MemoryLayout.alignment - maxPrevAlignment) + return MemoryLayout.stride * capacity + padding + } +%end + + internal var buffer: BufferPointer { + return BufferPointer(self) + } + + // This API is unsafe and needs a `_fixLifetime` in the caller. + internal var _body: _HashedContainerStorageHeader { + unsafeAddress { + return UnsafePointer(buffer._headerPointer) + } + unsafeMutableAddress { + return buffer._headerPointer + } + } @_versioned internal var _capacity: Int { + defer { _fixLifetime(self) } return _body.capacity } @_versioned internal var _count: Int { set { + defer { _fixLifetime(self) } _body.count = newValue } get { + defer { _fixLifetime(self) } return _body.count } } internal var _maxLoadFactorInverse: Double { + defer { _fixLifetime(self) } return _body.maxLoadFactorInverse } // This API is unsafe and needs a `_fixLifetime` in the caller. internal var _initializedHashtableEntriesBitMapStorage: UnsafeMutablePointer { - return UnsafeMutablePointer(Builtin.projectTailElems(self, UInt.self)) - } - - internal var _keysRawAddr : Builtin.RawPointer { - let bitmapAddr = Builtin.projectTailElems(self, UInt.self) - let numWordsForBitmap = _UnsafeBitMap.sizeInWords(forSizeInBits: _capacity) - return Builtin.getTailAddr_Word(bitmapAddr, - numWordsForBitmap._builtinWordValue, UInt.self, Key.self) + return _roundUp(buffer._elementPointer, toAlignmentOf: UInt.self) } // This API is unsafe and needs a `_fixLifetime` in the caller. internal var _keys: UnsafeMutablePointer { - return UnsafeMutablePointer(_keysRawAddr) + let bitMapSizeInBytes = + _unsafeMultiply( + _UnsafeBitMap.sizeInWords(forSizeInBits: _body.capacity), + MemoryLayout.stride) + let start = + UnsafeMutableRawPointer(_initializedHashtableEntriesBitMapStorage) + + bitMapSizeInBytes + return _roundUp(start, toAlignmentOf: Key.self) } %if Self == 'Dictionary': // This API is unsafe and needs a `_fixLifetime` in the caller. internal var _values: UnsafeMutablePointer { - let valuesAddr = Builtin.getTailAddr_Word(_keysRawAddr, - _capacity._builtinWordValue, Key.self, Value.self) - return UnsafeMutablePointer(valuesAddr) + let keysSizeInBytes = _unsafeMultiply(_body.capacity, MemoryLayout.stride) + let start = UnsafeMutableRawPointer(_keys) + keysSizeInBytes + return _roundUp(start, toAlignmentOf: Value.self) } %end /// Create a storage instance with room for 'capacity' entries and all entries /// marked invalid. internal class func create(capacity: Int) -> StorageImpl { - let numWordsForBitmap = _UnsafeBitMap.sizeInWords(forSizeInBits: capacity) + let requiredCapacity = + bytesForBitMap(capacity: capacity) + bytesForKeys(capacity: capacity) %if Self == 'Dictionary': - let storage = Builtin.allocWithTailElems_3(StorageImpl.self, - numWordsForBitmap._builtinWordValue, UInt.self, - capacity._builtinWordValue, Key.self, - capacity._builtinWordValue, Value.self) -%else: - let storage = Builtin.allocWithTailElems_2(StorageImpl.self, - numWordsForBitmap._builtinWordValue, UInt.self, - capacity._builtinWordValue, Key.self) + + bytesForValues(capacity: capacity) %end - - // We can initialize by assignment because _HashedContainerStorageHeader - // is a trivial type, i.e. contains no references. - storage._body = _HashedContainerStorageHeader(capacity: capacity) - + + let r = super.create(minimumCapacity: requiredCapacity) { _ in + return _HashedContainerStorageHeader(capacity: capacity) + } + let storage = r as! StorageImpl let initializedEntries = _UnsafeBitMap( storage: storage._initializedHashtableEntriesBitMapStorage, bitCount: capacity) @@ -2612,12 +2652,9 @@ final internal class _Native${Self}StorageImpl<${TypeParameters}> { } } %end + buffer._headerPointer.deinitialize() _fixLifetime(self) } - - internal init(_doNotCallMe: ()) { - _sanityCheckFailure("Only initialize these by calling create") - } } @_fixed_layout diff --git a/stdlib/public/core/SwiftNativeNSArray.swift b/stdlib/public/core/SwiftNativeNSArray.swift index ca67763cb4631..4d3ab290fbc3d 100644 --- a/stdlib/public/core/SwiftNativeNSArray.swift +++ b/stdlib/public/core/SwiftNativeNSArray.swift @@ -232,12 +232,6 @@ class _SwiftNativeNSArrayWithContiguousStorage {} internal class _ContiguousArrayStorageBase : _SwiftNativeNSArrayWithContiguousStorage { - final var countAndCapacity: _ArrayBody - - init(_doNotCallMeBase: ()) { - _sanityCheckFailure("creating instance of _ContiguousArrayStorageBase") - } - #if _runtime(_ObjC) internal override func withUnsafeBufferOfObjects( _ body: (UnsafeBufferPointer) throws -> R diff --git a/stdlib/public/core/UnsafePointer.swift.gyb b/stdlib/public/core/UnsafePointer.swift.gyb index 2ec7ceb64f19d..995d17bc7bdbb 100644 --- a/stdlib/public/core/UnsafePointer.swift.gyb +++ b/stdlib/public/core/UnsafePointer.swift.gyb @@ -489,7 +489,7 @@ public func < (lhs: ${Self}, rhs: ${Self}) -> Bool { @_transparent public func + (lhs: ${Self}, rhs: Int) -> ${Self} { return ${Self}(Builtin.gep_Word( - lhs._rawValue, rhs._builtinWordValue, Pointee.self)) + lhs._rawValue, (rhs &* MemoryLayout.stride)._builtinWordValue)) } @_transparent diff --git a/stdlib/public/core/UnsafeRawPointer.swift.gyb b/stdlib/public/core/UnsafeRawPointer.swift.gyb index fb6adc02ab93f..b4189d9ec65a8 100644 --- a/stdlib/public/core/UnsafeRawPointer.swift.gyb +++ b/stdlib/public/core/UnsafeRawPointer.swift.gyb @@ -468,7 +468,7 @@ public struct Unsafe${Mutable}RawPointer : Strideable, Hashable, _Pointer { /// Returns `self + n`. public func advanced(by n: Int) -> ${Self} { - return ${Self}(Builtin.gepRaw_Word(_rawValue, n._builtinWordValue)) + return ${Self}(Builtin.gep_Word(_rawValue, n._builtinWordValue)) } } diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index e91f39bb66741..ea217b3d46feb 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -107,6 +107,28 @@ extern "C" HeapObject* swift_bufferAllocate( alignMask); } +/// \brief Another entrypoint for swift_bufferAllocate. +/// It is generated by the compiler in some corner cases, e.g. if a serialized +/// optimized module is imported into a non-optimized main module. +/// TODO: This is only a workaround. Remove this function as soon as we can +/// get rid of the llvm SwiftStackPromotion pass. +SWIFT_RUNTIME_EXPORT +extern "C" HeapObject* swift_bufferAllocateOnStack( + HeapMetadata const* bufferType, size_t size, size_t alignMask) { + return swift::SWIFT_RT_ENTRY_CALL(swift_allocObject)(bufferType, size, + alignMask); +} + +/// \brief Called at the end of the lifetime of an object returned by +/// swift_bufferAllocateOnStack. +/// It is generated by the compiler in some corner cases, e.g. if a serialized +/// optimized module is imported into a non-optimized main module. +/// TODO: This is only a workaround. Remove this function as soon as we can +/// get rid of the llvm SwiftStackPromotion pass. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_bufferDeallocateFromStack(HeapObject *) { +} + SWIFT_RUNTIME_EXPORT extern "C" intptr_t swift_bufferHeaderSize() { return sizeof(HeapObject); } diff --git a/test/DebugInfo/linetable-cleanups.swift b/test/DebugInfo/linetable-cleanups.swift index a8c60cd4bf369..aaa569bacdb4c 100644 --- a/test/DebugInfo/linetable-cleanups.swift +++ b/test/DebugInfo/linetable-cleanups.swift @@ -18,10 +18,10 @@ func main() { // CHECK: br label // CHECK: