Skip to content

Commit

Permalink
Implement a preprocessor
Browse files Browse the repository at this point in the history 
Summary:
Add a preprocessing rewriter in to the strict module compiler. This is used between the analyzer and the rewriter (implemented in python and currently in another codebase) as an intermediate step that passes information.
The general strategy is to attach new decorators to classes with auto slotification information so that later the rewriter can consume that information without needing to look at the analysis result.

Reviewed By: DinoV

Differential Revision: D29617298

fbshipit-source-id: 7d76e6a
  • Loading branch information
@shiyu-wang-999 @facebook-github-bot
shiyu-wang-999 authored and facebook-github-bot committed Jul 30, 2021
1 parent f244f2c commit 1139869
Show file tree
Hide file tree
Showing 18 changed files with 652 additions and 55 deletions.
2 changes: 2 additions & 0 deletions Makefile.pre.in
View file
Expand Up @@ -519,6 +519,7 @@ STRICTM_OBJS= \
StrictModules/symbol_table.o \
StrictModules/analyzer.o \
StrictModules/ast_visitor.o \
StrictModules/ast_preprocessor.o \
StrictModules/parser_util.o \
StrictModules/strict_module_checker_interface.o \
StrictModules/scope.o \
Expand Down Expand Up @@ -1314,6 +1315,7 @@ STRICTM_PRIVATE_HEADERS= \
$(srcdir)/StrictModules/symbol_table.h \
$(srcdir)/StrictModules/ast_visitor.h \
$(srcdir)/StrictModules/analyzer.h \
$(srcdir)/StrictModules/ast_preprocessor.h \
$(srcdir)/StrictModules/parser_util.h \
$(srcdir)/StrictModules/strict_module_checker_interface.h \
$(srcdir)/StrictModules/scope.h \
Expand Down
14 changes: 10 additions & 4 deletions StrictModules/Compiler/abstract_module_loader.cpp
View file
Expand Up @@ -398,7 +398,7 @@ AnalyzedModule* ModuleLoader::analyze(std::unique_ptr<ModuleInfo> modInfo) {
kind = ModuleKind::kNonStrict;
}
AnalyzedModule* analyzedModule =
new AnalyzedModule(kind, std::move(errorSink));
new AnalyzedModule(kind, std::move(errorSink), ast);
modules_[name] = std::unique_ptr<AnalyzedModule>(analyzedModule);
lazy_modules_.erase(name);
if (analyzedModule->isStrict() || isForcedStrict(name, filename)) {
Expand Down Expand Up @@ -444,6 +444,7 @@ AnalyzedModule* ModuleLoader::analyze(std::unique_ptr<ModuleInfo> modInfo) {
modInfo->getFutureAnnotations());

analyzer.analyze();
analyzedModule->setAstToResults(analyzer.passAstToResultsMap());
}

return analyzedModule;
Expand Down Expand Up @@ -488,9 +489,9 @@ bool ModuleLoader::loadStrictModuleModule() {
auto it = modules_.find(name);
if (it == modules_.end()) {
auto strictModKind = ModuleKind::kStrict;
auto analyzedModule =
std::make_unique<AnalyzedModule>(strictModKind, errorSinkFactory_());
auto strictModModule = objects::StrictModulesModule();
auto analyzedModule = std::make_unique<AnalyzedModule>(
strictModKind, errorSinkFactory_(), nullptr);
auto strictModModule = objects::createStrictModulesModule();
strictModModule->setAttr(
"__name__",
std::make_shared<objects::StrictString>(
Expand All @@ -501,4 +502,9 @@ bool ModuleLoader::loadStrictModuleModule() {
}
return false;
}

bool ModuleLoader::isModuleLoaded(const std::string& modName) {
return modules_.find(modName) != modules_.end();
}

} // namespace strictmod::compiler
2 changes: 2 additions & 0 deletions StrictModules/Compiler/abstract_module_loader.h
View file
Expand Up @@ -145,6 +145,8 @@ class ModuleLoader {
*/
bool loadStrictModuleModule();

bool isModuleLoaded(const std::string& modName);

PyArena* getArena() {
return arena_;
}
Expand Down
18 changes: 18 additions & 0 deletions StrictModules/Compiler/analyzed_module.cpp
View file
@@ -1,5 +1,6 @@
// Copyright (c) Facebook, Inc. and its affiliates. (http://www.facebook.com)
#include "StrictModules/Compiler/analyzed_module.h"
#include "StrictModules/ast_preprocessor.h"

namespace strictmod::compiler {
bool AnalyzedModule::isStrict() const {
Expand Down Expand Up @@ -32,4 +33,21 @@ void AnalyzedModule::cleanModuleContent() {
module_->cleanContent(module_.get());
}
}

Ref<> AnalyzedModule::getPyAst(bool preprocess, PyArena* arena) {
if (ast_ == nullptr) {
return nullptr;
}
if (preprocess) {
auto preprocessor = Preprocessor(ast_, astToResults_.get(), arena);
preprocessor.preprocess();
}
Ref<> result = Ref<>::steal(PyAST_mod2obj(ast_));
return result;
}

mod_ty AnalyzedModule::getAST() {
return ast_;
}

} // namespace strictmod::compiler
28 changes: 24 additions & 4 deletions StrictModules/Compiler/analyzed_module.h
View file
Expand Up @@ -11,16 +11,24 @@ using strictmod::objects::StrictModuleObject;
enum class ModuleKind { kStrict, kStatic, kNonStrict };

class AnalyzedModule {
using astToResultT = strictmod::objects::astToResultT;

public:
AnalyzedModule(
std::unique_ptr<StrictModuleObject> module,
ModuleKind kind,
std::shared_ptr<BaseErrorSink> error)
std::shared_ptr<BaseErrorSink> error,
mod_ty ast)
: module_(std::move(module)),
moduleKind_(kind),
errorSink_(std::move(error)) {}
AnalyzedModule(ModuleKind kind, std::shared_ptr<BaseErrorSink> error)
: AnalyzedModule(nullptr, kind, std::move(error)) {}
errorSink_(std::move(error)),
ast_(ast),
astToResults_() {}
AnalyzedModule(
ModuleKind kind,
std::shared_ptr<BaseErrorSink> error,
mod_ty ast)
: AnalyzedModule(nullptr, kind, std::move(error), ast) {}
~AnalyzedModule() {
cleanModuleContent();
}
Expand All @@ -34,10 +42,22 @@ class AnalyzedModule {
void setModuleValue(std::shared_ptr<StrictModuleObject> module);
void cleanModuleContent();

void setAstToResults(std::unique_ptr<astToResultT> map) {
astToResults_ = std::move(map);
}

/** notice that this produces a fresh python AST
*/
Ref<> getPyAst(bool preprocess, PyArena* arena);

mod_ty getAST();

private:
std::shared_ptr<StrictModuleObject> module_;
ModuleKind moduleKind_;
std::shared_ptr<BaseErrorSink> errorSink_;
mod_ty ast_;
std::unique_ptr<astToResultT> astToResults_;
};
} // namespace strictmod::compiler

Expand Down
2 changes: 2 additions & 0 deletions StrictModules/Objects/base_object.h
View file
Expand Up @@ -118,6 +118,8 @@ class BaseStrictObject : public std::enable_shared_from_this<BaseStrictObject> {
};

typedef sequence_map<std::string, std::shared_ptr<BaseStrictObject>> DictType;
typedef std::unordered_map<void*, std::shared_ptr<BaseStrictObject>>
astToResultT;

// format arguments for function call
std::string formatArgs(
Expand Down
65 changes: 30 additions & 35 deletions StrictModules/Objects/objects.cpp
View file
Expand Up @@ -853,81 +853,76 @@ std::shared_ptr<StrictType> getExceptionFromString(
return it->second;
}

static std::shared_ptr<StrictModuleObject> kStrictModule =
StrictModuleObject::makeStrictModule(kModuleType, strictModName);

bool initializeStrictModulesModule();

std::shared_ptr<StrictModuleObject> StrictModulesModule() {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
[[maybe_unused]] static bool init = initializeStrictModulesModule();
return kStrictModule;
}

std::shared_ptr<BaseStrictObject> StrictModuleLooseSlots() {
std::shared_ptr<BaseStrictObject> StrictModuleLooseSlots(
std::shared_ptr<StrictModuleObject> mod) {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
static std::shared_ptr<BaseStrictObject> o(new StrictBuiltinFunctionOrMethod(
kStrictModule,
std::move(mod),
CallableWrapper(looseSlots, "loose_slots"),
nullptr,
"loose_slots"));
return o;
}

std::shared_ptr<BaseStrictObject> StrictModuleStrictSlots() {
std::shared_ptr<BaseStrictObject> StrictModuleStrictSlots(
std::shared_ptr<StrictModuleObject> mod) {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
static std::shared_ptr<BaseStrictObject> o(new StrictBuiltinFunctionOrMethod(
kStrictModule,
std::move(mod),
CallableWrapper(strictSlots, "strict_slots"),
nullptr,
"strict_slots"));
return o;
}

std::shared_ptr<BaseStrictObject> StrictModuleExtraSlot() {
std::shared_ptr<BaseStrictObject> StrictModuleExtraSlot(
std::shared_ptr<StrictModuleObject> mod) {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
static std::shared_ptr<BaseStrictObject> o(new StrictBuiltinFunctionOrMethod(
kStrictModule,
std::move(mod),
CallableWrapper(extraSlot, "extra_slot"),
nullptr,
"extra_slot"));
return o;
}

std::shared_ptr<BaseStrictObject> StrictModuleMutable() {
std::shared_ptr<BaseStrictObject> StrictModuleMutable(
std::shared_ptr<StrictModuleObject> mod) {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
static std::shared_ptr<BaseStrictObject> o(new StrictBuiltinFunctionOrMethod(
kStrictModule,
std::move(mod),
CallableWrapper(setMutable, "mutable"),
nullptr,
"mutable"));
return o;
}

std::shared_ptr<BaseStrictObject> StrictModuleMarkCachedProperty() {
std::shared_ptr<BaseStrictObject> StrictModuleMarkCachedProperty(
std::shared_ptr<StrictModuleObject> mod) {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
static std::shared_ptr<BaseStrictObject> o(new StrictBuiltinFunctionOrMethod(
kStrictModule,
std::move(mod),
CallableWrapper(markCachedProperty, "_mark_cached_property"),
nullptr,
"_mark_cached_property"));
return o;
}

bool initializeStrictModulesModule() {
static bool initialized = false;
if (!initialized) {
initialized = true;
DictType* dict = new DictType({
{"loose_slots", StrictModuleLooseSlots()},
{"strict_slots", StrictModuleStrictSlots()},
{"extra_slot", StrictModuleExtraSlot()},
{"mutable", StrictModuleMutable()},
{"_mark_cached_property", StrictModuleMarkCachedProperty()},
});
kStrictModule->setDict(std::shared_ptr<DictType>(dict));
}
return initialized;
std::shared_ptr<StrictModuleObject> createStrictModulesModule() {
[[maybe_unused]] static bool bulitinInit = bootstrapBuiltins();
std::shared_ptr<StrictModuleObject> strictModule =
StrictModuleObject::makeStrictModule(kModuleType, strictModName);

DictType* dict = new DictType({
{"loose_slots", StrictModuleLooseSlots(strictModule)},
{"strict_slots", StrictModuleStrictSlots(strictModule)},
{"extra_slot", StrictModuleExtraSlot(strictModule)},
{"mutable", StrictModuleMutable(strictModule)},
{"_mark_cached_property", StrictModuleMarkCachedProperty(strictModule)},
});
strictModule->setDict(std::shared_ptr<DictType>(dict));

return strictModule;
}

} // namespace strictmod::objects
4 changes: 3 additions & 1 deletion StrictModules/Objects/objects.h
View file
Expand Up @@ -30,7 +30,9 @@
namespace strictmod::objects {
//--------------------Builtin Type Declarations-----------------------
std::shared_ptr<StrictModuleObject> BuiltinsModule();
std::shared_ptr<StrictModuleObject> StrictModulesModule();
// every time this is called, a new strict module module
// is created
std::shared_ptr<StrictModuleObject> createStrictModulesModule();

std::shared_ptr<StrictType> ObjectType();
std::shared_ptr<StrictType> TypeType();
Expand Down
2 changes: 1 addition & 1 deletion StrictModules/Objects/strict_modules_builtins.cpp
View file
Expand Up @@ -21,7 +21,7 @@ std::shared_ptr<BaseStrictObject> looseSlots(
baseHasLooseSlots = true;
}
}
if (baseHasLooseSlots) {
if (!baseHasLooseSlots) {
typAttrs.setLooseSlots(true);
}
}
Expand Down

0 comments on commit 1139869

Please sign in to comment.