MINIFICPP-1108 - Automate the generation of processors.md

This closes #700.

Signed-off-by: Aldrin Piri <aldrin@apache.org>

main/AgentDocs.cpp

@@ -0,0 +1,175 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "AgentDocs.h"
+#include <vector>
+#include <algorithm>
+#include <iostream>
+#include <iterator>
+#include <fstream>
+#include <set>
+#include <memory>
+#include <string>
+#include "core/ClassLoader.h"
+#include "core/ConfigurableComponent.h"
+#include "core/Core.h"
+#include "core/Processor.h"
+#include "core/Property.h"
+#include "core/PropertyValidation.h"
+#include "core/Relationship.h"
+#include "io/validation.h"
+#include "utils/file/FileUtils.h"
+#include "agent/build_description.h"
+#include "agent/agent_version.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace docs {
+
+std::string AgentDocs::extractClassName(const std::string &processor) const {
+  auto lastOfIdx = processor.find_last_of(".");
+  if (lastOfIdx != std::string::npos) {
+    lastOfIdx++;  // if a value is found, increment to move beyond the .
+    int nameLength = processor.length() - lastOfIdx;
+    std::string processorName = processor.substr(lastOfIdx, nameLength);
+    return processorName;
+  }
+  return processor;
+
+}
+
+void AgentDocs::generate(const std::string &docsdir, std::ostream &genStream) {
+  std::map<std::string, ClassDescription> processorSet;
+  for (const auto &group : minifi::AgentBuild::getExtensions()) {
+
+    struct Components descriptions = BuildDescription::getClassDescriptions(group);
+    for (const auto &processorName : descriptions.processors_) {
+      processorSet.insert(std::make_pair(extractClassName(processorName.class_name_), processorName));
+    }
+  }
+  for (const auto &processor : processorSet) {
+    const std::string &filename = docsdir + utils::file::FileUtils::get_separator() + processor.first;
+    std::ofstream outfile(filename);
+
+    std::string description;
+
+    bool foundDescription = minifi::AgentDocs::getDescription(processor.first, description);
+
+    if (!foundDescription) {
+      foundDescription = minifi::AgentDocs::getDescription(processor.second.class_name_, description);
+    }
+
+    outfile << "## " << processor.first << std::endl << std::endl;
+    if (foundDescription) {
+      outfile << "### Description " << std::endl << std::endl;
+      outfile << description << std::endl;
+    }
+
+    outfile << "### Properties " << std::endl << std::endl;
+    outfile
+        << "In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language."
+        << std::endl<< std::endl;
+
+    outfile << "| Name | Default Value | Allowable Values | Description | " << std::endl << "| - | - | - | - | " << std::endl;
+    for (const auto &prop : processor.second.class_properties_) {
+      bool supportsEl = prop.second.supportsExpressionLangauge();
+      outfile << "|";
+      if (prop.second.getRequired()) {
+        outfile << "**";
+      }
+      outfile << prop.first;
+      if (prop.second.getRequired()) {
+        outfile << "**";
+      }
+      const auto allowableValues = prop.second.getAllowedValues();
+      std::stringstream s;
+      std::copy(allowableValues.begin(),allowableValues.end(), std::ostream_iterator<std::string>(s,"<br>"));
+      outfile << "|";
+      const auto defaultValue = prop.second.getDefaultValue().to_string();
+      if (defaultValue.size() == 1 && (int)defaultValue.c_str()[0] == 0x0a){
+        outfile << "\\n";
+      }
+      else{
+        outfile << defaultValue;
+      }
+      outfile << "|" << s.str() << "|" << prop.second.getDescription();
+      if (supportsEl){
+        outfile << "<br/>**Supports Expression Language: true**";
+      }
+      outfile << "|" << std::endl;
+    }
+    outfile << "### Properties " << std::endl << std::endl;
+    outfile << "| Name | Description |" << std::endl << "| - | - |" << std::endl;
+    for (const auto &rel : processor.second.class_relationships_) {
+      outfile << "|" << rel.getName() << "|" << rel.getDescription() << "|" << std::endl;
+    }
+
+    outfile << std::endl;
+
+  }
+
+  std::map<std::string,std::string> fileList;
+
+  auto fileFind = [&fileList,this](const std::string& base_path, const std::string& file) -> bool {
+    if (file.find(".extra") == std::string::npos)
+      fileList.insert( std::make_pair(file, base_path + utils::file::FileUtils::get_separator() + file) );
+    return true;
+  };
+
+  // shortened with list_dir
+  utils::file::FileUtils::list_dir(docsdir, fileFind, logging::LoggerFactory<AgentDocs>::getLogger());
+  genStream << "<!--"
+  "Licensed to the Apache Software Foundation (ASF) under one or more"
+  "contributor license agreements.  See the NOTICE file distributed with"
+  "this work for additional information regarding copyright ownership."
+  "The ASF licenses this file to You under the Apache License, Version 2.0"
+  "(the \"License\"); you may not use this file except in compliance with"
+  "the License.  You may obtain a copy of the License at"
+  "    http://www.apache.org/licenses/LICENSE-2.0"
+  "Unless required by applicable law or agreed to in writing, software"
+  "distributed under the License is distributed on an \"AS IS\" BASIS,"
+  "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied."
+  "See the License for the specific language governing permissions and"
+  "limitations under the License."
+  "-->"<<std::endl;
+  genStream << "# Processors" << std::endl << std::endl;
+  genStream << "## Table of Contents" << std::endl << std::endl;
+
+  for(const auto &file : fileList){
+    std::string lcfile=file.first;
+    std::transform(lcfile.begin(), lcfile.end(), lcfile.begin(), ::tolower);
+    genStream << "- [" << file.first << "](#" << lcfile << ")" << std::endl;
+  }
+
+  for(const auto &file : fileList){
+      std::ifstream filestream(file.second);
+      genStream << filestream.rdbuf() << std::endl;
+      std::ifstream filestreamExtra(file.second + ".extra");
+      if (filestreamExtra.good()){
+        genStream << filestreamExtra.rdbuf()<< std::endl;
+      }
+  }
+}
+
+} /* namespace docs */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */

main/AgentDocs.h

@@ -0,0 +1,44 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MAIN_AGENTDOCS_H_
+#define MAIN_AGENTDOCS_H_
+
+#include <iostream>
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace docs {
+
+class AgentDocs {
+ public:
+  AgentDocs() = default;
+  ~AgentDocs() = default;
+  void generate(const std::string &docsdir, std::ostream &genStream);
+ private:
+  inline std::string extractClassName(const std::string &processor) const;
+};
+
+} /* namespace docs */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif  // MAIN_AGENTDOCS_H_

main/CMakeLists.txt

@@ -55,7 +55,7 @@ endif()
 
 endif()
 
-add_executable(minifiexe MiNiFiMain.cpp MiNiFiWindowsService.cpp)
+add_executable(minifiexe MiNiFiMain.cpp MiNiFiWindowsService.cpp AgentDocs.cpp)
 
 if (NOT USE_SHARED_LIBS)
 	if (LIBC_STATIC)

main/MiNiFiMain.cpp

@@ -46,6 +46,7 @@
 #include <queue>
 #include <map>
 #include <iostream>
+#include <utils/file/FileUtils.h>
 #include "ResourceClaim.h"
 #include "core/Core.h"
 
@@ -54,6 +55,7 @@
 #include "core/RepositoryFactory.h"
 #include "utils/file/PathUtils.h"
 #include "FlowController.h"
+#include "AgentDocs.h"
 #include "Main.h"
 
  // Variables that allow us to avoid a timed wait.
@@ -95,6 +97,17 @@ void sigHandler(int signal) {
 	}
 }
 
+void dumpDocs(const std::shared_ptr<minifi::Configure> &configuration, const std::string &dir, std::ostream &out) {
+	auto pythoncreator = core::ClassLoader::getDefaultClassLoader().instantiate("PythonCreator", "PythonCreator");
+	if (nullptr != pythoncreator) {
+		pythoncreator->configure(configuration);
+	}
+
+	minifi::docs::AgentDocs docsCreator;
+
+	docsCreator.generate(dir, out);
+}
+
 int main(int argc, char **argv) {
 #ifdef WIN32
   CheckRunAsService();
@@ -234,6 +247,28 @@ int main(int argc, char **argv) {
 	configure->setHome(minifiHome);
 	configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE);
 
+  if (argc >= 3 && std::string("docs") == argv[1]) {
+    if (utils::file::FileUtils::create_dir(argv[2]) != 0) {
+      std::cerr << "Working directory doesn't exist and cannot be created: " << argv[2] << std::endl;
+      exit(1);
+    }
+
+    std::cerr << "Dumping docs to " << argv[2] << std::endl;
+    if (argc == 4) {
+      std::string filepath, filename;
+      utils::file::PathUtils::getFileNameAndPath(argv[3], filepath, filename);
+      if (filepath == argv[2]) {
+        std::cerr << "Target file should be out of the working directory: " << filepath << std::endl;
+        exit(1);
+      }
+      std::ofstream outref(argv[3]);
+      dumpDocs(configure, argv[2], outref);
+    } else{
+      dumpDocs(configure, argv[2], std::cout);
+    }
+    exit(0);
+  }
+
 
 	if (configure->get(minifi::Configure::nifi_graceful_shutdown_seconds, graceful_shutdown_seconds)) {
 		try {