Skip to content

Commit

Permalink
static closures
Browse files Browse the repository at this point in the history
  • Loading branch information
@tjhance
tjhance committed Mar 25, 2015
1 parent f69d1a9 commit de5fde0
Show file tree
Hide file tree
Showing 14 changed files with 225 additions and 109 deletions.
86 changes: 76 additions & 10 deletions src/analysis/scoping_analysis.cpp
View file
Expand Up @@ -103,10 +103,14 @@ class ModuleScopeInfo : public ScopeInfo {

bool usesNameLookup() override { return false; }

bool isPassedToViaClosure(InternedString name) override { return false; }

bool areLocalsFromModule() override { return true; }

DerefInfo getDerefInfo(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureOffset(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureSize() override { RELEASE_ASSERT(0, "This should never get called"); }
std::vector<std::pair<InternedString, DerefInfo>> v;
const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() override { return v; }

InternedString mangleName(InternedString id) override { return id; }
InternedString internString(llvm::StringRef s) override { abort(); }
};
Expand Down Expand Up @@ -165,10 +169,14 @@ class EvalExprScopeInfo : public ScopeInfo {

bool usesNameLookup() override { return true; }

bool isPassedToViaClosure(InternedString name) override { return false; }

bool areLocalsFromModule() override { return false; }

DerefInfo getDerefInfo(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureOffset(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureSize() override { RELEASE_ASSERT(0, "This should never get called"); }
std::vector<std::pair<InternedString, DerefInfo>> v;
const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() override { return v; }

InternedString mangleName(InternedString id) override { return id; }
InternedString internString(llvm::StringRef s) override { abort(); }
};
Expand Down Expand Up @@ -257,11 +265,22 @@ class ScopeInfoBase : public ScopeInfo {
AST* ast;
bool usesNameLookup_;

llvm::DenseMap<InternedString, size_t> closure_offsets;

std::vector<std::pair<InternedString, DerefInfo>> allDerefVarsAndInfo;
bool allDerefVarsAndInfoCached;

public:
ScopeInfoBase(ScopeInfo* parent, ScopingAnalysis::ScopeNameUsage* usage, AST* ast, bool usesNameLookup)
: parent(parent), usage(usage), ast(ast), usesNameLookup_(usesNameLookup) {
: parent(parent), usage(usage), ast(ast), usesNameLookup_(usesNameLookup), allDerefVarsAndInfoCached(false) {
assert(usage);
assert(ast);

int i = 0;
for (auto& p : usage->referenced_from_nested) {
closure_offsets[p] = i;
i++;
}
}

~ScopeInfoBase() override { delete this->usage; }
Expand Down Expand Up @@ -300,20 +319,67 @@ class ScopeInfoBase : public ScopeInfo {

bool usesNameLookup() override { return usesNameLookup_; }

bool isPassedToViaClosure(InternedString name) override {
if (isCompilerCreatedName(name))
return false;
bool areLocalsFromModule() override { return false; }

DerefInfo getDerefInfo(InternedString name) override {
assert(getScopeTypeOfName(name) == VarScopeType::DEREF);

// TODO pre-compute this?

size_t parentCounter = 0;
// Casting to a ScopeInfoBase* is okay because only a ScopeInfoBase can have a closure.
for (ScopeInfoBase* parent = static_cast<ScopeInfoBase*>(this->parent); parent != NULL;
parent = static_cast<ScopeInfoBase*>(parent->parent)) {
if (parent->createsClosure()) {
auto it = parent->closure_offsets.find(name);
if (it != parent->closure_offsets.end()) {
return DerefInfo{.num_parents_from_passed_closure = parentCounter, .offset = it->second };
}
parentCounter++;
}
}

return usage->got_from_closure.count(name) > 0 || usage->passthrough_accesses.count(name) > 0;
RELEASE_ASSERT(0, "Should not get here");
}

bool areLocalsFromModule() override { return false; }
size_t getClosureOffset(InternedString name) override {
assert(getScopeTypeOfName(name) == VarScopeType::CLOSURE);
return closure_offsets[name];
}

size_t getClosureSize() override { return closure_offsets.size(); }

InternedString mangleName(const InternedString id) override {
return pyston::mangleName(id, usage->private_name, usage->scoping->getInternedStrings());
}

InternedString internString(llvm::StringRef s) override { return usage->scoping->getInternedStrings().get(s); }

const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() override {
if (!allDerefVarsAndInfoCached) {
allDerefVarsAndInfoCached = true;

StrSet allDerefs = usage->got_from_closure;
for (InternedString name : usage->passthrough_accesses) {
if (allDerefs.find(name) != allDerefs.end()) {
allDerefs.insert(name);
}
}

for (InternedString name : allDerefs) {
allDerefVarsAndInfo.push_back({ name, getDerefInfo(name) });
}

std::sort(allDerefVarsAndInfo.begin(), allDerefVarsAndInfo.end(), derefComparator);
}
return allDerefVarsAndInfo;
}

private:
static bool derefComparator(const std::pair<InternedString, DerefInfo>& p1,
const std::pair<InternedString, DerefInfo>& p2) {
return p1.second.num_parents_from_passed_closure < p2.second.num_parents_from_passed_closure;
};
};

class NameCollectorVisitor : public ASTVisitor {
Expand Down
43 changes: 10 additions & 33 deletions src/analysis/scoping_analysis.h
View file
Expand Up @@ -25,6 +25,11 @@ class AST_Module;
class AST_Expression;
class AST_Suite;

struct DerefInfo {
size_t num_parents_from_passed_closure;
size_t offset;
};

class ScopeInfo {
public:
ScopeInfo() {}
Expand Down Expand Up @@ -75,39 +80,6 @@ class ScopeInfo {
};
virtual VarScopeType getScopeTypeOfName(InternedString name) = 0;

// Returns true if the variable should be passed via a closure to this scope.
// Note that:
// (a) This can be false even if there is an entry in the closure object
// passed to the scope, if the variable is not actually used in this
// scope or any child scopes. This can happen, because the variable
// could be in the closure to be accessed by a different function, e.g.
//
// def f();
// a = 0
// b = 0
// def g():
// print a
// def h():
// print b
// # locals() should not contain `a` even though `h` is
// # passed a closure object with `a` in it
// print locals()
//
// (b) This can be true even if it is not used in this scope, if it
// is used in a child scope. For example:
//
// def f():
// a = 0
// def g():
// def h():
// print a
// print locals() # should contain `a`
//
// This is useful because it determines whether a variable from a closure
// into the locals() dictionary.

virtual bool isPassedToViaClosure(InternedString name) = 0;

// Returns true if the scope may contain NAME variables.
// In particular, it returns true for ClassDef scope, for any scope
// with an `exec` statement or `import *` statement in it, or for any
Expand All @@ -116,6 +88,11 @@ class ScopeInfo {

virtual bool areLocalsFromModule() = 0;

virtual DerefInfo getDerefInfo(InternedString name) = 0;
virtual const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() = 0;
virtual size_t getClosureOffset(InternedString name) = 0;
virtual size_t getClosureSize() = 0;

virtual InternedString mangleName(InternedString id) = 0;
virtual InternedString internString(llvm::StringRef) = 0;
};
Expand Down
23 changes: 18 additions & 5 deletions src/codegen/ast_interpreter.cpp
View file
Expand Up @@ -228,7 +228,7 @@ void ASTInterpreter::initArguments(int nargs, BoxedClosure* _closure, BoxedGener
generator = _generator;

if (scope_info->createsClosure())
created_closure = createClosure(passed_closure);
created_closure = createClosure(passed_closure, scope_info->getClosureSize());

std::vector<Box*, StlCompatAllocator<Box*>> argsArray{ arg1, arg2, arg3 };
for (int i = 3; i < nargs; ++i)
Expand Down Expand Up @@ -354,8 +354,9 @@ void ASTInterpreter::doStore(InternedString name, Value value) {
setitem(frame_info.boxedLocals, boxString(name.str()), value.o);
} else {
sym_table[name] = value.o;
if (vst == ScopeInfo::VarScopeType::CLOSURE)
setattr(created_closure, name.c_str(), value.o);
if (vst == ScopeInfo::VarScopeType::CLOSURE) {
created_closure->elts[scope_info->getClosureOffset(name)] = value.o;
}
}
}

Expand Down Expand Up @@ -1082,8 +1083,20 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
switch (node->lookup_type) {
case ScopeInfo::VarScopeType::GLOBAL:
return getGlobal(source_info->parent_module, &node->id.str());
case ScopeInfo::VarScopeType::DEREF:
return getattr(passed_closure, node->id.c_str());
case ScopeInfo::VarScopeType::DEREF: {
DerefInfo deref_info = scope_info->getDerefInfo(node->id);
assert(passed_closure);
BoxedClosure* closure = passed_closure;
for (int i = 0; i < deref_info.num_parents_from_passed_closure; i++) {
closure = closure->parent;
}
Box* val = closure->elts[deref_info.offset];
if (val == NULL) {
raiseExcHelper(NameError, "free variable '%s' referenced before assignment in enclosing scope",
node->id.c_str());
}
return val;
}
case ScopeInfo::VarScopeType::FAST:
case ScopeInfo::VarScopeType::CLOSURE: {
SymMap::iterator it = sym_table.find(node->id);
Expand Down
6 changes: 4 additions & 2 deletions src/codegen/compvars.cpp
View file
Expand Up @@ -1781,15 +1781,17 @@ class ClosureType : public ConcreteCompilerType {

CompilerVariable* getattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var,
const std::string* attr, bool cls_only) override {
RELEASE_ASSERT(0, "should not be called\n");
/*
assert(!cls_only);
llvm::Value* bitcast = emitter.getBuilder()->CreateBitCast(var->getValue(), g.llvm_value_type_ptr);
return ConcreteCompilerVariable(UNKNOWN, bitcast, true).getattr(emitter, info, attr, cls_only);
*/
}

void setattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var, const std::string* attr,
CompilerVariable* v) override {
llvm::Value* bitcast = emitter.getBuilder()->CreateBitCast(var->getValue(), g.llvm_value_type_ptr);
ConcreteCompilerVariable(UNKNOWN, bitcast, true).setattr(emitter, info, attr, v);
RELEASE_ASSERT(0, "should not be called\n");
}

ConcreteCompilerType* getConcreteType() override { return this; }
Expand Down
44 changes: 39 additions & 5 deletions src/codegen/irgen/irgenerator.cpp
View file
Expand Up @@ -898,10 +898,29 @@ class IRGeneratorImpl : public IRGenerator {
assert(!is_kill);
assert(scope_info->takesClosure());

DerefInfo deref_info = scope_info->getDerefInfo(node->id);

static_assert(sizeof(Box) == offsetof(BoxedClosure, parent), "");
static_assert(offsetof(BoxedClosure, parent) + sizeof(BoxedClosure*) == offsetof(BoxedClosure, nelts), "");
static_assert(offsetof(BoxedClosure, nelts) + sizeof(size_t) == offsetof(BoxedClosure, elts), "");

CompilerVariable* closure = symbol_table[internString(PASSED_CLOSURE_NAME)];
assert(closure);
llvm::Value* closureValue = closure->makeConverted(emitter, CLOSURE)->getValue();
closure->decvref(emitter);
llvm::Value* gep;
for (int i = 0; i < deref_info.num_parents_from_passed_closure; i++) {
gep = emitter.getBuilder()->CreateConstInBoundsGEP2_32(closureValue, 0, 1);
closureValue = emitter.getBuilder()->CreateLoad(gep);
}
gep = emitter.getBuilder()->CreateGEP(closureValue,
{ llvm::ConstantInt::get(g.i32, 0), llvm::ConstantInt::get(g.i32, 3),
llvm::ConstantInt::get(g.i32, deref_info.offset) });
llvm::Value* lookupResult = emitter.getBuilder()->CreateLoad(gep);

return closure->getattr(emitter, getEmptyOpInfo(unw_info), &node->id.str(), false);
emitter.createCall(unw_info, g.funcs.assertDerefNameDefined,
{ lookupResult, getStringConstantPtr(node->id.str() + '\0') });

return new ConcreteCompilerVariable(UNKNOWN, lookupResult, true);
} else if (vst == ScopeInfo::VarScopeType::NAME) {
llvm::Value* boxedLocals = irstate->getBoxedLocalsVar();
llvm::Value* attr = getStringConstantPtr(node->id.str() + '\0');
Expand Down Expand Up @@ -1431,10 +1450,22 @@ class IRGeneratorImpl : public IRGenerator {
_popFake(defined_name, true);

if (vst == ScopeInfo::VarScopeType::CLOSURE) {
size_t offset = scope_info->getClosureOffset(name);

CompilerVariable* closure = symbol_table[internString(CREATED_CLOSURE_NAME)];
llvm::Value* closureValue = closure->makeConverted(emitter, CLOSURE)->getValue();
closure->decvref(emitter);
llvm::Value* gep = emitter.getBuilder()->CreateGEP(
closureValue, { llvm::ConstantInt::get(g.i32, 0), llvm::ConstantInt::get(g.i32, 3),
llvm::ConstantInt::get(g.i32, offset) });
emitter.getBuilder()->CreateStore(val->makeConverted(emitter, UNKNOWN)->getValue(), gep);

/*
CompilerVariable* closure = symbol_table[internString(CREATED_CLOSURE_NAME)];
assert(closure);
closure->setattr(emitter, getEmptyOpInfo(unw_info), &name.str(), val);
*/
}
}
}
Expand Down Expand Up @@ -1893,7 +1924,8 @@ class IRGeneratorImpl : public IRGenerator {
assert(var->getType() != BOXED_FLOAT
&& "should probably unbox it, but why is it boxed in the first place?");

// This line can never get hit right now for the same reason that the variables must already be concrete,
// This line can never get hit right now for the same reason that the variables must already be
// concrete,
// because we're over-generating phis.
ASSERT(var->isGrabbed(), "%s", p.first.c_str());
// var->ensureGrabbed(emitter);
Expand Down Expand Up @@ -2152,7 +2184,8 @@ class IRGeneratorImpl : public IRGenerator {
} else {
#ifndef NDEBUG
if (myblock->successors.size()) {
// TODO getTypeAtBlockEnd will automatically convert up to the concrete type, which we don't want
// TODO getTypeAtBlockEnd will automatically convert up to the concrete type, which we don't
// want
// here, but this is just for debugging so I guess let it happen for now:
ConcreteCompilerType* ending_type = types->getTypeAtBlockEnd(it->first, myblock);
ASSERT(it->second->canConvertTo(ending_type), "%s is supposed to be %s, but somehow is %s",
Expand Down Expand Up @@ -2352,7 +2385,8 @@ class IRGeneratorImpl : public IRGenerator {
if (!passed_closure)
passed_closure = embedConstantPtr(nullptr, g.llvm_closure_type_ptr);

llvm::Value* new_closure = emitter.getBuilder()->CreateCall(g.funcs.createClosure, passed_closure);
llvm::Value* new_closure = emitter.getBuilder()->CreateCall2(
g.funcs.createClosure, passed_closure, getConstantInt(scope_info->getClosureSize(), g.i64));
symbol_table[internString(CREATED_CLOSURE_NAME)]
= new ConcreteCompilerVariable(getCreatedClosureType(), new_closure, true);
}
Expand Down
1 change: 1 addition & 0 deletions src/codegen/runtime_hooks.cpp
View file
Expand Up @@ -216,6 +216,7 @@ void initGlobalFuncs(GlobalState& g) {
GET(raiseAttributeErrorStr);
GET(raiseNotIterableError);
GET(assertNameDefined);
GET(assertDerefNameDefined);
GET(assertFail);

GET(printFloat);
Expand Down
2 changes: 1 addition & 1 deletion src/codegen/runtime_hooks.h
View file
Expand Up @@ -41,7 +41,7 @@ struct GlobalFuncs {
*exceptionMatches, *yield, *getiterHelper, *hasnext;

llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
*assertNameDefined, *assertFail;
*assertNameDefined, *assertFail, *assertDerefNameDefined;
llvm::Value* printFloat, *listAppendInternal, *getSysStdout;
llvm::Value* runtimeCall0, *runtimeCall1, *runtimeCall2, *runtimeCall3, *runtimeCall, *runtimeCallN;
llvm::Value* callattr0, *callattr1, *callattr2, *callattr3, *callattr, *callattrN;
Expand Down
27 changes: 15 additions & 12 deletions src/codegen/unwinding.cpp
View file
Expand Up @@ -758,18 +758,21 @@ Box* fastLocalsToBoxedLocals() {

// Add the locals from the closure
// TODO in a ClassDef scope, we aren't supposed to add these
for (; closure != NULL; closure = closure->parent) {
assert(closure->cls == closure_cls);
for (auto& attr_offset : closure->attrs.hcls->getAttrOffsets()) {
const std::string& name = attr_offset.first();
int offset = attr_offset.second;
Box* val = closure->attrs.attr_list->attrs[offset];
if (val != NULL && scope_info->isPassedToViaClosure(scope_info->internString(name))) {
Box* boxedName = boxString(name);
if (d->d.count(boxedName) == 0) {
d->d[boxString(name)] = val;
}
}
size_t depth = 0;
for (auto& p : scope_info->getAllDerefVarsAndInfo()) {
InternedString name = p.first;
DerefInfo derefInfo = p.second;
while (depth < derefInfo.num_parents_from_passed_closure) {
depth++;
closure = closure->parent;
}
assert(closure != NULL);
Box* val = closure->elts[derefInfo.offset];
Box* boxedName = boxString(name.str());
if (val != NULL) {
d->d[boxedName] = val;
} else {
d->d.erase(boxedName);
}
}

Expand Down

0 comments on commit de5fde0

Please sign in to comment.