Merge pull request #493 from nadundesilva/master

Bug Fix to Siddhi Doc Gen Plugin - Add support to update the README.md using predefined headings

modules/siddhi-doc-gen/src/main/java/org/wso2/siddhi/doc/gen/core/ExtensionsIndexGenerationMojo.java

@@ -97,7 +97,7 @@ public void execute() throws MojoExecutionException, MojoFailureException {
             docGenBasePath = rootMavenProject.getBasedir() + File.separator + Constants.DOCS_DIRECTORY;
         }
 
-        // Setting the documentation output file name if not set by user
+        // Setting the extension index output file name if not set by user
         if (indexGenFileName == null) {
             indexGenFileName = Constants.MARKDOWN_EXTENSIONS_INDEX_TEMPLATE;
         }

modules/siddhi-doc-gen/src/main/java/org/wso2/siddhi/doc/gen/core/MarkdownDocumentationGenerationMojo.java

@@ -31,6 +31,7 @@
 import org.wso2.siddhi.doc.gen.core.utils.DocumentationUtils;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.util.List;
 
 /**
@@ -84,6 +85,13 @@ public class MarkdownDocumentationGenerationMojo extends AbstractMojo {
     @Parameter(property = "home.page.file.name")
     private String homePageFileName;
 
+    /**
+     * The readme file
+     * Optional
+     */
+    @Parameter(property = "readme.file")
+    private File readmeFile;
+
     @Override
     public void execute() throws MojoExecutionException, MojoFailureException {
         // Finding the root maven project
@@ -108,7 +116,7 @@ public void execute() throws MojoExecutionException, MojoFailureException {
             docGenBasePath = rootMavenProject.getBasedir() + File.separator + Constants.DOCS_DIRECTORY;
         }
 
-        // Setting the read me file path if not set by user
+        // Setting the home page template file path if not set by user
         if (homePageTemplateFile == null) {
             homePageTemplateFile = new File(rootMavenProject.getBasedir() + File.separator
                     + Constants.README_FILE_NAME + Constants.MARKDOWN_FILE_EXTENSION);
@@ -120,9 +128,19 @@ public void execute() throws MojoExecutionException, MojoFailureException {
                     + Constants.MKDOCS_CONFIG_FILE_NAME + Constants.YAML_FILE_EXTENSION);
         }
 
-        // Setting the index file name if not set by user
+        // Setting the home page file name if not set by user
+        File homePageFile;
         if (homePageFileName == null) {
-            homePageFileName = Constants.MARKDOWN_HOME_PAGE_TEMPLATE;
+            homePageFile = new File(docGenBasePath + File.separator
+                    + Constants.HOMEPAGE_FILE_NAME + Constants.MARKDOWN_FILE_EXTENSION);
+        } else {
+            homePageFile = new File(docGenBasePath + File.separator + homePageFileName);
+        }
+
+        // Setting the readme file name if not set by user
+        if (readmeFile == null) {
+            readmeFile = new File(rootMavenProject.getBasedir() + File.separator
+                    + Constants.README_FILE_NAME + Constants.MARKDOWN_FILE_EXTENSION);
         }
 
         // Retrieving metadata
@@ -140,8 +158,19 @@ public void execute() throws MojoExecutionException, MojoFailureException {
         // Generating the documentation
         if (namespaceMetaDataList.size() > 0) {
             DocumentationUtils.generateDocumentation(namespaceMetaDataList, docGenBasePath, mavenProject.getVersion());
-            DocumentationUtils.updateHomePage(homePageTemplateFile, docGenBasePath, homePageFileName, mkdocsConfigFile,
-                    mavenProject.getVersion(), namespaceMetaDataList, getLog());
+            DocumentationUtils.updateHeadingsInMarkdownFile(homePageTemplateFile, homePageFile,
+                    rootMavenProject.getArtifactId(), mavenProject.getVersion(), namespaceMetaDataList);
+            DocumentationUtils.updateHeadingsInMarkdownFile(readmeFile, readmeFile, rootMavenProject.getArtifactId(),
+                    mavenProject.getVersion(), namespaceMetaDataList);
+
+
+            // Updating the mkdocs config to support all the API Docs pages
+            try {
+                DocumentationUtils.updateAPIPagesInMkdocsConfig(mkdocsConfigFile, docGenBasePath);
+            } catch (FileNotFoundException e) {
+                getLog().warn("Unable to find mkdocs configuration file: "
+                        + mkdocsConfigFile.getAbsolutePath() + ". Mkdocs configuration file not updated.");
+            }
         }
     }
 }

modules/siddhi-doc-gen/src/main/java/org/wso2/siddhi/doc/gen/core/MkdocsGitHubPagesDeployMojo.java

@@ -17,20 +17,17 @@
  */
 package org.wso2.siddhi.doc.gen.core;
 
-import org.apache.maven.artifact.DependencyResolutionRequiredException;
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugin.MojoFailureException;
 import org.apache.maven.plugins.annotations.LifecyclePhase;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.project.MavenProject;
-import org.wso2.siddhi.doc.gen.commons.metadata.NamespaceMetaData;
 import org.wso2.siddhi.doc.gen.core.utils.Constants;
 import org.wso2.siddhi.doc.gen.core.utils.DocumentationUtils;
 
 import java.io.File;
-import java.util.List;
 
 /**
  * Mojo for deploying mkdocs website on GitHub pages
@@ -62,25 +59,11 @@ public class MkdocsGitHubPagesDeployMojo extends AbstractMojo {
     private File docGenBaseDirectory;
 
     /**
-     * The path of the readme file in the base directory
+     * The readme file
      * Optional
      */
-    @Parameter(property = "home.page.template.file")
-    private File homePageTemplateFile;
-
-    /**
-     * The name of the index file
-     * Optional
-     */
-    @Parameter(property = "home.page.file.name")
-    private String homePageFileName;
-
-    /**
-     * The target module for which the files will be generated
-     * Optional
-     */
-    @Parameter(property = "module.target.directory")
-    private File moduleTargetDirectory;
+    @Parameter(property = "readme.file")
+    private File readmeFile;
 
     @Override
     public void execute() throws MojoExecutionException, MojoFailureException {
@@ -104,47 +87,18 @@ public void execute() throws MojoExecutionException, MojoFailureException {
             docGenBasePath = rootMavenProject.getBasedir() + File.separator + Constants.DOCS_DIRECTORY;
         }
 
-        // Setting the read me file path if not set by user
-        if (homePageTemplateFile == null) {
-            homePageTemplateFile = new File(rootMavenProject.getBasedir() + File.separator
+        // Setting the readme file name if not set by user
+        if (readmeFile == null) {
+            readmeFile = new File(rootMavenProject.getBasedir() + File.separator
                     + Constants.README_FILE_NAME + Constants.MARKDOWN_FILE_EXTENSION);
         }
 
-        // Setting the index file name if not set by user
-        if (homePageFileName == null) {
-            homePageFileName = Constants.MARKDOWN_HOME_PAGE_TEMPLATE;
-        }
-
-        // Setting the relevant modules target directory if not set by user
-        String moduleTargetPath;
-        if (moduleTargetDirectory != null) {
-            moduleTargetPath = moduleTargetDirectory.getAbsolutePath();
-        } else {
-            moduleTargetPath = mavenProject.getBuild().getDirectory();
-        }
-
-        // Retrieving metadata
-        List<NamespaceMetaData> namespaceMetaDataList;
-        try {
-            namespaceMetaDataList = DocumentationUtils.getExtensionMetaData(
-                    moduleTargetPath,
-                    mavenProject.getRuntimeClasspathElements(),
-                    getLog()
-            );
-        } catch (DependencyResolutionRequiredException e) {
-            throw new MojoFailureException("Unable to resolve dependencies of the project", e);
-        }
-
-        // Updating the documentation
+        // Updating the API Docs files
         DocumentationUtils.removeSnapshotAPIDocs(mkdocsConfigFile, docGenBasePath, getLog());
-        if (namespaceMetaDataList.size() > 0) {
-            DocumentationUtils.updateHomePage(homePageTemplateFile, docGenBasePath, homePageFileName, mkdocsConfigFile,
-                    mavenProject.getVersion(), namespaceMetaDataList, getLog());
-        }
 
         // Deploying the documentation
         DocumentationUtils.deployMkdocsOnGitHubPages(mkdocsConfigFile, mavenProject.getVersion(), getLog());
-        DocumentationUtils.updateDocumentationOnGitHub(docGenBasePath, mkdocsConfigFile,
+        DocumentationUtils.updateDocumentationOnGitHub(docGenBasePath, mkdocsConfigFile, readmeFile,
                 mavenProject.getVersion(), getLog());
     }
 }

modules/siddhi-doc-gen/src/main/java/org/wso2/siddhi/doc/gen/core/utils/Constants.java

@@ -35,6 +35,7 @@ public class Constants {
 
     public static final String README_FILE_NAME = "README";
     public static final String MKDOCS_CONFIG_FILE_NAME = "mkdocs";
+    public static final String HOMEPAGE_FILE_NAME = "index";
 
     public static final String FREEMARKER_TEMPLATE_FILE_EXTENSION = ".ftl";
     public static final String CLASS_FILE_EXTENSION = ".class";
@@ -43,10 +44,11 @@ public class Constants {
 
     public static final String MARKDOWN_DOCUMENTATION_TEMPLATE = "documentation";
     public static final String MARKDOWN_EXTENSIONS_INDEX_TEMPLATE = "extensions";
-    public static final String MARKDOWN_HOME_PAGE_TEMPLATE = "index";
+    public static final String MARKDOWN_HEADINGS_UPDATE_TEMPLATE = "headings-update";
 
     public static final String GITHUB_GPL_EXTENSION_REPOSITORY_PREFIX = "siddhi-gpl-";
     public static final String GITHUB_APACHE_EXTENSION_REPOSITORY_PREFIX = "siddhi-";
+    public static final String GITHUB_EXTENSION_REPOSITORY_PARENT_POSTFIX = "-parent";
     public static final String GITHUB_OWNER_WSO2_EXTENSIONS = "wso2-extensions";
 
     public static final String MKDOCS_CONFIG_PAGES_KEY = "pages";
@@ -69,6 +71,8 @@ public class Constants {
     public static final String GIT_COMMIT_COMMAND_MESSAGE_FORMAT = "[WSO2-Release] [Release %s] " +
             "update documentation for release %s";
 
+    public static final String URL_PREFIX_BASE = ".";
+
     public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
     public static final String CORE_NAMESPACE = "core";
     public static final String SNAPSHOT_VERSION_POSTFIX = "-SNAPSHOT";
@@ -79,4 +83,5 @@ public class Constants {
     public static final String FREEMARKER_FEATURES_HEADING = "## Features";
     public static final String FREEMARKER_LATEST_API_DOCS_HEADING = "## Latest API Docs";
     public static final String FREEMARKER_SIDDHI_HOME_PAGE = "https://wso2.github.io/siddhi";
+    public static final String FREEMARKER_SIDDHI_REPOSITORY_ARTIFACT_ID = "siddhi";
 }

modules/siddhi-doc-gen/src/main/java/org/wso2/siddhi/doc/gen/core/utils/DocumentationUtils.java

@@ -151,63 +151,38 @@ public static void generateDocumentation(List<NamespaceMetaData> namespaceMetaDa
     /**
      * Update the documentation home page
      *
-     * @param homePageTemplateFile       The path to the read me file
-     * @param documentationBaseDirectory The path of the base directory in which the documentation will be generated
-     * @param homePageFileName           The name of the documentation file that will be generated
-     * @param mkdocsConfigFile           The name of the mkdocs file
+     * @param inputFile                  The path to the input file
+     * @param outputFile                 The path to the output file
+     * @param extensionRepositoryName    The name of  the extension repository
      * @param latestDocumentationVersion The version of the latest documentation generated
      * @param namespaceMetaDataList      Metadata in this repository
-     * @param logger                     The maven plugin logger
      * @throws MojoFailureException if the Mojo fails to find template file or create new documentation file
      */
-    public static void updateHomePage(File homePageTemplateFile, String documentationBaseDirectory,
-                                      String homePageFileName, File mkdocsConfigFile,
-                                      String latestDocumentationVersion, List<NamespaceMetaData> namespaceMetaDataList,
-                                      Log logger)
+    public static void updateHeadingsInMarkdownFile(File inputFile, File outputFile,
+                                                    String extensionRepositoryName,
+                                                    String latestDocumentationVersion,
+                                                    List<NamespaceMetaData> namespaceMetaDataList)
             throws MojoFailureException {
         // Retrieving the content of the README.md file
-        List<String> homePageTemplateFileLines = new ArrayList<>();
+        List<String> inputFileLines = new ArrayList<>();
         try {
-            homePageTemplateFileLines = Files.readLines(homePageTemplateFile, Constants.DEFAULT_CHARSET);
+            inputFileLines = Files.readLines(inputFile, Constants.DEFAULT_CHARSET);
         } catch (IOException ignored) {
         }
 
-        // Retrieving the documentation file names
-        File documentationDirectory = new File(documentationBaseDirectory
-                + File.separator + Constants.API_SUB_DIRECTORY);
-        String[] documentationFiles = documentationDirectory.list(
-                (directory, fileName) -> fileName.endsWith(Constants.MARKDOWN_FILE_EXTENSION)
-        );
-
-        List<String> documentationFilesList;
-        if (documentationFiles == null) {
-            documentationFilesList = new ArrayList<>();
-        } else {
-            documentationFilesList = Arrays.asList(documentationFiles);
-        }
-
         // Generating data model
         Map<String, Object> rootDataModel = new HashMap<>();
-        rootDataModel.put("homePageTemplateFileLines", homePageTemplateFileLines);
-        rootDataModel.put("documentationFiles", documentationFilesList);
+        rootDataModel.put("inputFileLines", inputFileLines);
+        rootDataModel.put("extensionRepositoryName", extensionRepositoryName);
         rootDataModel.put("latestDocumentationVersion", latestDocumentationVersion);
         rootDataModel.put("metaData", namespaceMetaDataList);
         rootDataModel.put("formatDescription", new FormatDescriptionMethod());
 
         generateFileFromTemplate(
-                Constants.MARKDOWN_HOME_PAGE_TEMPLATE + Constants.MARKDOWN_FILE_EXTENSION
+                Constants.MARKDOWN_HEADINGS_UPDATE_TEMPLATE + Constants.MARKDOWN_FILE_EXTENSION
                         + Constants.FREEMARKER_TEMPLATE_FILE_EXTENSION,
-                rootDataModel, documentationBaseDirectory,
-                homePageFileName + Constants.MARKDOWN_FILE_EXTENSION
+                rootDataModel, outputFile.getParent(), outputFile.getName()
         );
-
-        // Updating the links in the home page to the mkdocs config
-        try {
-            updateAPIPagesInMkdocsConfig(mkdocsConfigFile, documentationFilesList);
-        } catch (FileNotFoundException e) {
-            logger.warn("Unable to find mkdocs configuration file: " + mkdocsConfigFile.getAbsolutePath()
-                    + ". Mkdocs configuration file not updated.");
-        }
     }
 
     /**
@@ -218,17 +193,14 @@ public static void updateHomePage(File homePageTemplateFile, String documentatio
      * @param logger                     The maven plugin logger
      */
     public static void removeSnapshotAPIDocs(File mkdocsConfigFile, String  documentationBaseDirectory, Log logger) {
+        // Retrieving the documentation file names
         File apiDocsDirectory = new File(documentationBaseDirectory
                 + File.separator + Constants.API_SUB_DIRECTORY);
-
-        // Retrieving the documentation file names
         String[] documentationFileNames = apiDocsDirectory.list(
                 (directory, fileName) -> fileName.endsWith(Constants.MARKDOWN_FILE_EXTENSION)
         );
 
         if (documentationFileNames != null) {
-            List<String> documentationFilesList = new ArrayList<>();
-
             // Removing snapshot files and creating a list of the files that are left out
             for (String documentationFileName : documentationFileNames) {
                 if (documentationFileName.endsWith(Constants.SNAPSHOT_VERSION_POSTFIX
@@ -240,14 +212,12 @@ public static void removeSnapshotAPIDocs(File mkdocsConfigFile, String  document
                         logger.warn("Failed to delete SNAPSHOT documentation file "
                                 + documentationFile.getAbsolutePath());
                     }
-                } else {
-                    documentationFilesList.add(documentationFileName);
                 }
             }
 
             // Updating the links in the home page to the mkdocs config
             try {
-                updateAPIPagesInMkdocsConfig(mkdocsConfigFile, documentationFilesList);
+                updateAPIPagesInMkdocsConfig(mkdocsConfigFile, documentationBaseDirectory);
             } catch (FileNotFoundException e) {
                 logger.warn("Unable to find mkdocs configuration file: "
                         + mkdocsConfigFile.getAbsolutePath() + ". Mkdocs configuration file not updated.");
@@ -258,12 +228,27 @@ public static void removeSnapshotAPIDocs(File mkdocsConfigFile, String  document
     /**
      * This add a new page to the list of pages in the mkdocs configuration
      *
-     * @param mkdocsConfigFile    The mkdocs configuration file
-     * @param apiDirectoryContent The contents of the api directory
+     * @param mkdocsConfigFile           The mkdocs configuration file
+     * @param documentationBaseDirectory The base directory of the documentation
      * @throws FileNotFoundException If mkdocs configuration file is not found
      */
-    private static void updateAPIPagesInMkdocsConfig(File mkdocsConfigFile, List<String> apiDirectoryContent)
+    public static void updateAPIPagesInMkdocsConfig(File mkdocsConfigFile, String documentationBaseDirectory)
             throws FileNotFoundException {
+        // Retrieving the documentation file names
+        File documentationDirectory = new File(documentationBaseDirectory
+                + File.separator + Constants.API_SUB_DIRECTORY);
+        String[] documentationFiles = documentationDirectory.list(
+                (directory, fileName) -> fileName.endsWith(Constants.MARKDOWN_FILE_EXTENSION)
+        );
+
+        List<String> apiDirectoryContent;
+        if (documentationFiles == null) {
+            apiDirectoryContent = new ArrayList<>();
+        } else {
+            apiDirectoryContent = Arrays.asList(documentationFiles);
+            apiDirectoryContent.sort(String::compareTo);
+        }
+
         // Creating yaml parser
         DumperOptions dumperOptions = new DumperOptions();
         dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
@@ -370,12 +355,12 @@ public static void deployMkdocsOnGitHubPages(File mkdocsConfigFile, String versi
      *
      * @param docsDirectory    The docs drectory
      * @param mkdocsConfigFile The mkdocs configuration file
+     * @param readmeFile       The read me file
      * @param version          The version of the documentation
      * @param logger           The maven logger
      */
-    public static void updateDocumentationOnGitHub(String docsDirectory, File mkdocsConfigFile, String version,
-                                                   Log logger) {
-
+    public static void updateDocumentationOnGitHub(String docsDirectory, File mkdocsConfigFile, File readmeFile,
+                                                   String version, Log logger) {
         try {
             executeCommand(new String[] {Constants.GIT_COMMAND,
                     Constants.GIT_ADD_COMMAND,
@@ -385,8 +370,7 @@ public static void updateDocumentationOnGitHub(String docsDirectory, File mkdocs
                     Constants.GIT_COMMIT_COMMAND_MESSAGE_ARGUMENT,
                     String.format(Constants.GIT_COMMIT_COMMAND_MESSAGE_FORMAT, version, version),
                     Constants.GIT_COMMIT_COMMAND_FILES_ARGUMENT,
-                    docsDirectory,
-                    mkdocsConfigFile.getAbsolutePath()}, logger);
+                    docsDirectory, mkdocsConfigFile.getAbsolutePath(), readmeFile.getAbsolutePath()}, logger);
             executeCommand(new String[] {Constants.GIT_COMMAND,
                     Constants.GIT_PUSH_COMMAND,
                     Constants.GIT_PUSH_COMMAND_REMOTE,