From ce023b8c83c095cb77d56843d08e45293a6c5319 Mon Sep 17 00:00:00 2001 From: JJ Allaire Date: Tue, 16 May 2017 09:58:24 -0400 Subject: [PATCH 1/5] Automatically generate native routine registrations during compileAttributes --- ChangeLog | 7 ++ DESCRIPTION | 2 +- R/Attributes.R | 9 ++- R/Rcpp.package.skeleton.R | 4 +- inst/NEWS.Rd | 1 + src/attributes.cpp | 163 ++++++++++++++++++++++++++++++-------- 6 files changed, 151 insertions(+), 35 deletions(-) diff --git a/ChangeLog b/ChangeLog index 709a68272..87910a8d1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2017-05-16 JJ Allaire + + * R/Attributes.R: Automatically generate native routine registrations. + * src/attributes.cpp: Automatically generate native routine registrations. + * R/Rcpp.package.skeleton.R: Don't generate native routines when creating + a package using attributes. + 2017-05-09 Dirk Eddelbuettel * R/Rcpp.package.skeleton.R (Rcpp.package.skeleton): Under R 3.4.0, run diff --git a/DESCRIPTION b/DESCRIPTION index 2826cf850..17ed754e0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: Rcpp Title: Seamless R and C++ Integration -Version: 0.12.10.4 +Version: 0.12.10.5 Date: 2017-05-07 Author: Dirk Eddelbuettel, Romain Francois, JJ Allaire, Kevin Ushey, Qiang Kou, Nathan Russell, Douglas Bates and John Chambers diff --git a/R/Attributes.R b/R/Attributes.R index 144f28466..4d7c2fb57 100644 --- a/R/Attributes.R +++ b/R/Attributes.R @@ -404,6 +404,13 @@ compileAttributes <- function(pkgdir = ".", verbose = getOption("verbose")) { depends <- unique(.splitDepends(depends)) depends <- depends[depends != "R"] + # check the NAMESPACE file to see if dynamic registration is enabled + namespaceFile <- file.path(pkgdir, "NAMESPACE") + if (!file.exists(namespaceFile)) + stop("pkgdir must refer to the directory containing an R package") + pkgNamespace <- readLines(namespaceFile, warn = FALSE) + registration <- any(grepl("^\\s*useDynLib.*\\.registration\\s*=\\s*TRUE.*$", pkgNamespace)) + # determine source directory srcDir <- file.path(pkgdir, "src") if (!file.exists(srcDir)) @@ -449,7 +456,7 @@ compileAttributes <- function(pkgdir = ".", verbose = getOption("verbose")) { # generate exports invisible(.Call("compileAttributes", PACKAGE="Rcpp", - pkgdir, pkgname, depends, cppFiles, cppFileBasenames, + pkgdir, pkgname, depends, registration, cppFiles, cppFileBasenames, includes, verbose, .Platform)) } diff --git a/R/Rcpp.package.skeleton.R b/R/Rcpp.package.skeleton.R index 95327205d..101784210 100644 --- a/R/Rcpp.package.skeleton.R +++ b/R/Rcpp.package.skeleton.R @@ -185,7 +185,9 @@ Rcpp.package.skeleton <- function(name = "anRpackage", list = character(), message(" >> copied the example module file ") } - if (getRversion() >= "3.4.0") { + # generate native routines if we aren't using attributes (which already generate + # them automatically) and we have at least R 3.4 + if (!attributes && getRversion() >= "3.4.0") { con <- file(file.path(src, "init.c"), "wt") tools::package_native_routine_registration_skeleton(root, con=con) close(con) diff --git a/inst/NEWS.Rd b/inst/NEWS.Rd index b110d5d68..66478d8f1 100644 --- a/inst/NEWS.Rd +++ b/inst/NEWS.Rd @@ -40,6 +40,7 @@ } \item Changes in Rcpp Attributes: \itemize{ + \item Automatically generate native routine registrations. \item The plugins for C++11, C++14, C++17 now set the values R 3.4.0 or later expects; a plugin for C++98 was added (Dirk in \ghpr{684} addressing \ghit{683}). diff --git a/src/attributes.cpp b/src/attributes.cpp index c4adb8f07..bfd4ddfe9 100644 --- a/src/attributes.cpp +++ b/src/attributes.cpp @@ -129,6 +129,9 @@ namespace attributes { // is the passed string quoted? bool isQuoted(const std::string& str); + // does a string end with another string? + bool endsWith(const std::string& str, const std::string& suffix); + // show a warning message void showWarning(const std::string& msg); @@ -410,6 +413,8 @@ namespace attributes { virtual const std::vector >& roxygenChunks() const = 0; virtual bool hasGeneratorOutput() const = 0; + + virtual bool hasPackageInit() const = 0; }; @@ -445,6 +450,7 @@ namespace attributes { class SourceFileAttributesParser : public SourceFileAttributes { public: explicit SourceFileAttributesParser(const std::string& sourceFile, + const std::string& packageFile, bool parseDependencies); private: @@ -491,6 +497,11 @@ namespace attributes { return false; } + // Was a package init function found? + bool hasPackageInit() const { + return hasPackageInit_; + } + // Get lines of embedded R code const std::vector& embeddedR() const { return embeddedR_; @@ -530,6 +541,7 @@ namespace attributes { CharacterVector lines_; std::vector attributes_; std::vector modules_; + bool hasPackageInit_; std::vector embeddedR_; std::vector sourceDependencies_; std::vector > roxygenChunks_; @@ -571,7 +583,7 @@ namespace attributes { virtual void writeBegin() = 0; void writeFunctions(const SourceFileAttributes& attributes, bool verbose); // see doWriteFunctions below - virtual void writeEnd() = 0; + virtual void writeEnd(bool hasPackageInit) = 0; virtual bool commit(const std::vector& includes) = 0; @@ -648,7 +660,7 @@ namespace attributes { const std::string& fileSep); virtual void writeBegin() {}; - virtual void writeEnd(); + virtual void writeEnd(bool hasPackageInit); virtual bool commit(const std::vector& includes); private: @@ -660,7 +672,11 @@ namespace attributes { const std::string& name) const; private: + // for generating C++ interfaces std::vector cppExports_; + + // for generating native routine registration + std::vector nativeRoutines_; }; // Class which manages generating PackageName_RcppExports.h header file @@ -671,7 +687,7 @@ namespace attributes { const std::string& fileSep); virtual void writeBegin(); - virtual void writeEnd(); + virtual void writeEnd(bool hasPackageInit); virtual bool commit(const std::vector& includes); private: @@ -692,7 +708,7 @@ namespace attributes { const std::string& fileSep); virtual void writeBegin() {} - virtual void writeEnd(); + virtual void writeEnd(bool hasPackageInit); virtual bool commit(const std::vector& includes); private: @@ -709,16 +725,19 @@ namespace attributes { public: RExportsGenerator(const std::string& packageDir, const std::string& package, + bool registration, const std::string& fileSep); virtual void writeBegin() {} - virtual void writeEnd(); + virtual void writeEnd(bool hasPackageInit); virtual bool commit(const std::vector& includes); private: virtual void doWriteFunctions(const SourceFileAttributes& attributes, bool verbose); + bool registration_; + }; // Class to manage and dispatch to a list of generators @@ -734,7 +753,7 @@ namespace attributes { void writeBegin(); void writeFunctions(const SourceFileAttributes& attributes, bool verbose); - void writeEnd(); + void writeEnd(bool hasPackageInit); // Commit and return a list of the files that were updated std::vector commit( @@ -1115,8 +1134,9 @@ namespace attributes { // Parse the attributes from a source file SourceFileAttributesParser::SourceFileAttributesParser( const std::string& sourceFile, + const std::string& packageName, bool parseDependencies) - : sourceFile_(sourceFile) + : sourceFile_(sourceFile), hasPackageInit_(false) { // First read the entire file into a std::stringstream so we can check // it for attributes (we don't want to do any of our more expensive @@ -1127,7 +1147,8 @@ namespace attributes { // Check for attribute signature if (contents.find("[[Rcpp::") != std::string::npos || - contents.find("RCPP_MODULE") != std::string::npos) { + contents.find("RCPP_MODULE") != std::string::npos || + contents.find("R_init_" + packageName) != std::string::npos) { // Now read into a list of strings (which we can pass to regexec) // First read into a std::deque (which will handle lots of append @@ -1203,6 +1224,27 @@ namespace attributes { } } + // Scan for package init function + hasPackageInit_ = false; + commentState.reset(); + std::string pkgInit = "R_init_" + packageName; + Rcpp::List initMatches = regexMatches(lines_, "^[^/]+" + pkgInit + ".*DllInfo.*$"); + for (int i = 0; i 0) { + hasPackageInit_ = true; + break; + } + } + // Parse embedded R embeddedR_ = parseEmbeddedR(lines_, lines); @@ -1217,7 +1259,7 @@ namespace attributes { // perform parse std::string dependency = sourceDependencies_[i].path(); - SourceFileAttributesParser parser(dependency, false); + SourceFileAttributesParser parser(dependency, packageName, false); // copy to base attributes (if it's a new attribute) for (SourceFileAttributesParser::const_iterator @@ -1824,19 +1866,28 @@ namespace attributes { attributes.hasInterface(kInterfaceCpp), packageCpp()); - // track cppExports and signatures (we use these at the end to - // generate the ValidateSignature and RegisterCCallable functions) - if (attributes.hasInterface(kInterfaceCpp)) { - for (SourceFileAttributes::const_iterator // #nocov start - it = attributes.begin(); it != attributes.end(); ++it) { - if (it->isExportedFunction()) { - // add it to the list if it's not hidden + // track cppExports, signatures, and native routines (we use these + // at the end to generate the ValidateSignature and RegisterCCallable + // functions, and to generate a package init function with native + // routine registration) + for (SourceFileAttributes::const_iterator // #nocov start + it = attributes.begin(); it != attributes.end(); ++it) { + + if (it->isExportedFunction()) { + + // add it to the cpp exports list if we are generating + // a C++ interface and it's not hidden + if (attributes.hasInterface(kInterfaceCpp)) { Function fun = it->function().renamedTo(it->exportedCppName()); if (!fun.isHidden()) - cppExports_.push_back(*it); // #nocov end + cppExports_.push_back(*it); } + + // add it to the native routines list + nativeRoutines_.push_back(*it); } - } + } // #nocov end + // verbose if requested if (verbose) { // #nocov start @@ -1851,7 +1902,7 @@ namespace attributes { } } - void CppExportsGenerator::writeEnd() + void CppExportsGenerator::writeEnd(bool hasPackageInit) { // generate a function that can be used to validate exported // functions and their signatures prior to looking up with @@ -1902,6 +1953,28 @@ namespace attributes { ostr() << " return R_NilValue;" << std::endl; ostr() << "}" << std::endl; } + + // write native routines + if (!hasPackageInit && !nativeRoutines_.empty()) { + ostr() << std::endl; + ostr() << "static const R_CallMethodDef CallEntries[] = {" << std::endl; + for (std::size_t i=0;i& arguments = function.arguments(); @@ -2266,7 +2344,7 @@ namespace attributes { } } - void RExportsGenerator::writeEnd() { + void RExportsGenerator::writeEnd(bool hasPackageInit) { if (hasCppInterface()) { // #nocov start // register all C-callable functions ostr() << "# Register entry points for exported C++ functions" @@ -2307,9 +2385,9 @@ namespace attributes { (*it)->writeFunctions(attributes, verbose); } - void ExportsGenerators::writeEnd() { + void ExportsGenerators::writeEnd(bool hasPackageInit) { for(Itr it = generators_.begin(); it != generators_.end(); ++it) - (*it)->writeEnd(); + (*it)->writeEnd(hasPackageInit); } // Commit and return a list of the files that were updated @@ -2794,6 +2872,13 @@ namespace attributes { return (quote == '\'' || quote == '\"') && (*(str.rbegin()) == quote); } + // does a string end with another string? + bool endsWith(const std::string& str, const std::string& suffix) + { + return str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; + } + // show a warning message void showWarning(const std::string& msg) { // #nocov start Rcpp::Function warning = Rcpp::Environment::base_env()["warning"]; @@ -2959,7 +3044,7 @@ namespace { filecopy(cppSourcePath_, generatedCppSourcePath(), true); // parse attributes - SourceFileAttributesParser sourceAttributes(cppSourcePath_, true); + SourceFileAttributesParser sourceAttributes(cppSourcePath_, "", true); // generate cpp for attributes and append them std::ostringstream ostr; @@ -3288,6 +3373,7 @@ END_RCPP RcppExport SEXP compileAttributes(SEXP sPackageDir, SEXP sPackageName, SEXP sDepends, + SEXP sRegistration, SEXP sCppFiles, SEXP sCppFileBasenames, SEXP sIncludes, @@ -3305,6 +3391,8 @@ BEGIN_RCPP depends.insert(std::string(*it)); } + bool registration = Rcpp::as(sRegistration); + std::vector cppFiles = Rcpp::as >(sCppFiles); std::vector cppFileBasenames = @@ -3318,7 +3406,7 @@ BEGIN_RCPP // initialize generators ExportsGenerators generators; generators.add(new CppExportsGenerator(packageDir, packageName, fileSep)); - generators.add(new RExportsGenerator(packageDir, packageName, fileSep)); + generators.add(new RExportsGenerator(packageDir, packageName, registration, fileSep)); // catch file exists exception if the include file already exists // and we are unable to overwrite it @@ -3347,13 +3435,24 @@ BEGIN_RCPP generators.writeBegin(); // Parse attributes from each file and generate code as required. + bool hasPackageInit = false; bool haveAttributes = false; std::set dependsAttribs; for (std::size_t i=0; i updated; From 3f3bf08ca1fd66b1407d757246fa5f2c09d6fd97 Mon Sep 17 00:00:00 2001 From: JJ Allaire Date: Tue, 16 May 2017 10:08:32 -0400 Subject: [PATCH 2/5] don't print init.c message if attributes=TRUE --- R/Rcpp.package.skeleton.R | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/R/Rcpp.package.skeleton.R b/R/Rcpp.package.skeleton.R index 101784210..fe73edc4c 100644 --- a/R/Rcpp.package.skeleton.R +++ b/R/Rcpp.package.skeleton.R @@ -187,13 +187,15 @@ Rcpp.package.skeleton <- function(name = "anRpackage", list = character(), # generate native routines if we aren't using attributes (which already generate # them automatically) and we have at least R 3.4 - if (!attributes && getRversion() >= "3.4.0") { - con <- file(file.path(src, "init.c"), "wt") - tools::package_native_routine_registration_skeleton(root, con=con) - close(con) - message(" >> created init.c for package registration") - } else { - message(" >> R version older than 3.4.0 detected, so NO file init.c created.") + if (!attributes) { + if (getRversion() >= "3.4.0") { + con <- file(file.path(src, "init.c"), "wt") + tools::package_native_routine_registration_skeleton(root, con=con) + close(con) + message(" >> created init.c for package registration") + } else { + message(" >> R version older than 3.4.0 detected, so NO file init.c created.") + } } lines <- readLines(package.doc <- file.path( root, "man", sprintf("%s-package.Rd", name))) From bb17924d8694aadf6f4e616a21ac8e4a15a03817 Mon Sep 17 00:00:00 2001 From: JJ Allaire Date: Wed, 17 May 2017 07:32:10 -0400 Subject: [PATCH 3/5] scan .c files during compileAttributes Necessary to discover R_init_pkgname functions defined within .c files (e.g. init.c) --- R/Attributes.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Attributes.R b/R/Attributes.R index 4d7c2fb57..61cbdc42e 100644 --- a/R/Attributes.R +++ b/R/Attributes.R @@ -422,7 +422,7 @@ compileAttributes <- function(pkgdir = ".", verbose = getOption("verbose")) { dir.create(rDir) # get a list of all source files - cppFiles <- list.files(srcDir, pattern = "\\.((c(c|pp))|(h(pp)?))$") + cppFiles <- list.files(srcDir, pattern = "\\.((c(c|pp)?)|(h(pp)?))$") # derive base names (will be used for modules) cppFileBasenames <- tools::file_path_sans_ext(cppFiles) From 30110e3ba65099ce596d686b5bdb669dd03bec13 Mon Sep 17 00:00:00 2001 From: JJ Allaire Date: Wed, 17 May 2017 19:22:01 -0400 Subject: [PATCH 4/5] register cpp interface bootstrap function --- src/attributes.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/attributes.cpp b/src/attributes.cpp index bfd4ddfe9..1476866e1 100644 --- a/src/attributes.cpp +++ b/src/attributes.cpp @@ -1958,6 +1958,11 @@ namespace attributes { if (!hasPackageInit && !nativeRoutines_.empty()) { ostr() << std::endl; ostr() << "static const R_CallMethodDef CallEntries[] = {" << std::endl; + if (hasCppInterface()) { + ostr() << " {\"" << registerCCallableExportedName() << "\", " << + "(DL_FUNC) &" << registerCCallableExportedName() << ", " << + 0 << "}," << std::endl; + } for (std::size_t i=0;i Date: Thu, 18 May 2017 09:02:38 -0400 Subject: [PATCH 5/5] use tools::package_native_routine_registration_skeleton when available to discover and register additional non-Rcpp exported native routines --- R/Attributes.R | 40 +++++++++++++++++++++++++++++++++++++++ src/attributes.cpp | 47 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/R/Attributes.R b/R/Attributes.R index 61cbdc42e..7cceaebd2 100644 --- a/R/Attributes.R +++ b/R/Attributes.R @@ -1154,3 +1154,43 @@ sourceCppFunction <- function(func, isVoid, dll, symbol) { as.character(token) } +.extraRoutineRegistrations <- function(routines) { + + declarations = character() + call_entries = character() + + # if we are running R 3.4 or higher we can use an internal utility function + # to automatically discover additional native routines that require registration + if (getRversion() >= "3.4") { + + # get the generated code from R + con <- textConnection(object = NULL, open = "w") + on.exit(close(con), add = TRUE) + tools::package_native_routine_registration_skeleton( + dir = ".", + con = con, + character_only = FALSE + ) + code <- textConnectionValue(con) + + # look for lines containing call entries + matches <- regexec('^\\s+\\{"([^"]+)",.*$', code) + matches <- regmatches(code, matches) + matches <- Filter(x = matches, function(x) { + length(x) > 0 + }) + for (match in matches) { + routine <- match[[2]] + if (!routine %in% routines) { + declaration <- grep(sprintf("^extern .* %s\\(.*$", routine), code, + value = TRUE) + declarations <- c(declarations, sub("^extern", "RcppExport", declaration)) + call_entries <- c(call_entries, match[[1]]) + } + } + } + + # return extra declaratiosn and call entries + list(declarations = declarations, + call_entries = call_entries) +} diff --git a/src/attributes.cpp b/src/attributes.cpp index 1476866e1..4f4e056f1 100644 --- a/src/attributes.cpp +++ b/src/attributes.cpp @@ -1956,26 +1956,51 @@ namespace attributes { // write native routines if (!hasPackageInit && !nativeRoutines_.empty()) { + + // build list of routines we will register + std::vector routineNames; + std::vector routineArgs; + for (std::size_t i=0;i declarations = extraRoutines["declarations"]; + std::vector callEntries = extraRoutines["call_entries"]; + + // generate declarations + if (declarations.size() > 0) { + ostr() << std::endl; + for (int i = 0; i 0) { + for (int i = 0; i