Skip to content
Permalink
Browse files
AMBARI-23456. Add mkdocs support & markdown skeletons (#54)
* AMBARI-23456. Add mkdocs support & markdown skeletons

* Remove version duplication

* Deleted from wrong pom.xml - fixing it

* Use ambari_server as input type example

* Some doc fix

* More doc fix + skip rat check on generated mkdocs-site

* Add sample log + additional fixes
  • Loading branch information
oleewere committed Dec 5, 2018
1 parent da2e901 commit 6ff4dc21dd428bd57afd2bc95ba7c2ab4f6a7f8c
Show file tree
Hide file tree
Showing 29 changed files with 6,075 additions and 45 deletions.
@@ -18,3 +18,4 @@ node/
.hgignore
.hgtags
.flattened-pom.xml
mkdocs-site
@@ -46,6 +46,31 @@ rpm:
deb:
$(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests -Djdk.version=$(LOGSEARCH_JAVA_VERSION)

javadoc:
$(MAVEN_BINARY) javadoc:aggregate -DskipTests -Djdk.version=$(LOGSEARCH_JAVA_VERSION)

site-only:
mkdir -p target/docs && mkdir -p target/docs/javadoc && cp mkdocs.yml target/ && cp -r docs/* target/docs
cp ambari-logsearch-docs/src/main/resources/docs.md target/docs/
cp -r ambari-logsearch-docs/target/META-INF/resources/webjars/swagger-ui/*/ target/docs/api-docs/ && rm -rf target/docs/api-docs/*.gz
sed -i '' 's#https://petstore.swagger.io/v2/swagger.json#./logsearch-swagger.yaml#g' target/docs/api-docs/index.html
sed -i '' 's#Swagger UI#Log Search REST API#g' target/docs/api-docs/index.html
cp -r target/site/apidocs/* target/docs/javadoc/

site: prop-docs javadoc site-only

serve-site: site
cd target && mkdocs serve

serve-site-only: site-only
cd target && mkdocs serve

generate-site-html: site
cd target && mkdocs build

generate-site-html-only: site-only
cd target && mkdocs build

prop-docs: install
$(MAVEN_BINARY) -pl ambari-logsearch-docs exec:java -DskipTests -Djdk.version=$(LOGSEARCH_JAVA_VERSION)

@@ -35,7 +35,8 @@ public class ConditionsImpl implements Conditions {
@ShipperConfigElementDescription(
path = "/filter/[]/conditions/fields",
type = "json object",
description = "The fields in the input element of which's value should be met."
description = "The fields in the input element of which's value should be met.",
examples = {"\"fields\"{\"type\": [\"hdfs_audit\", \"hdfs_datanode\"]}"}
)
@Expose
private FieldsImpl fields;
@@ -38,7 +38,7 @@ public class FieldsImpl implements Fields {
path = "/filter/[]/conditions/fields/type",
type = "list of strings",
description = "The acceptable values for the type field in the input element.",
examples = {"ambari_server", "\"spark_jobhistory_server\", \"spark_thriftserver\", \"livy_server\""}
examples = {"\"ambari_server\"", "\"spark_jobhistory_server\", \"spark_thriftserver\", \"livy_server\""}
)
@Expose
private Set<String> type;
@@ -74,8 +74,8 @@ public class FilterGrokDescriptorImpl extends FilterDescriptorImpl implements Fi
@ShipperConfigElementDescription(
path = "/filter/[]/deep_extract",
type = "boolean",
description = "",
examples = {""}
description = "Keep the full named regex collection for Grok filters.",
examples = {"true"}
)
@Expose
@SerializedName("deep_extract")
@@ -39,15 +39,17 @@ public class InputConfigImpl implements InputConfig {
@ShipperConfigElementDescription(
path = "/input",
type = "list of json objects",
description = "A list of input descriptions"
description = "A list of input descriptions",
examples = {"{\"input\" : [ {\"type\": \"myinput_service_type\"}] }"}
)
@Expose
private List<InputDescriptorImpl> input;

@ShipperConfigElementDescription(
path = "/filter",
type = "list of json objects",
description = "A list of filter descriptions"
description = "A list of filter descriptions",
examples = {"{\"filter\" : [ {\"filter\": \"json\", \"conditions\": {\"fields\": { \"type\": [\"logsearch_app\", \"logsearch_perf\"]} } } ]}"}
)
@Expose
private List<FilterDescriptorImpl> filter;
@@ -19,7 +19,6 @@

package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;

import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor;

@@ -31,29 +30,14 @@
description = "S3 file inputs have the following parameters in addition to the general file parameters:"
)
public class InputS3FileDescriptorImpl extends InputFileBaseDescriptorImpl implements InputS3FileDescriptor {
@ShipperConfigElementDescription(
path = "/input/[]/s3_access_key",
type = "string",
description = "The access key used for AWS credentials."
)
@Expose
@SerializedName("s3_access_key")
private String s3AccessKey;

@ShipperConfigElementDescription(
path = "/input/[]/s3_secret_key",
type = "string",
description = "The secret key used for AWS credentials."
)
@Expose
@SerializedName("s3_secret_key")
private String s3SecretKey;

@ShipperConfigElementDescription(
path = "/input/[]/s3_endpoint",
type = "string",
description = "Endpoint URL for S3."
)
@Expose
@SerializedName("s3_endpoint")
private String s3Endpoint;
@@ -24,10 +24,10 @@
<version>${revision}</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>ambari-logsearch-docs</artifactId>
<packaging>jar</packaging>
<url>http://maven.apache.org</url>
<name>Ambari Logsearch Docs</name>
<artifactId>ambari-logsearch-docs</artifactId>
<build>
<resources>
<resource>
@@ -44,6 +44,31 @@
<target>${jdk.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>unpack</id>
<phase>prepare-package</phase>
<goals>
<goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.webjars</groupId>
<artifactId>swagger-ui</artifactId>
<version>${swagger-ui.version}</version>
<outputDirectory>${project.build.directory}/</outputDirectory>
<includes>META-INF/resources/**</includes>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
@@ -56,7 +81,7 @@
</execution>
</executions>
<configuration>
<mainClass>org.apache.ambari.logsearch.doc.LogSearchMarkdownGenerator</mainClass>
<mainClass>org.apache.ambari.logsearch.doc.LogSearchDocumentationGenerator</mainClass>
<arguments>
<argument>--output-dir</argument>
<argument>${project.basedir}/../docs</argument>
@@ -83,5 +108,4 @@
</dependency>
</dependencies>


</project>
@@ -20,6 +20,11 @@

import freemarker.template.Configuration;
import freemarker.template.Template;
import io.swagger.jaxrs.config.BeanConfig;
import io.swagger.models.Swagger;
import io.swagger.models.auth.BasicAuthDefinition;
import io.swagger.util.Yaml;
import org.apache.ambari.logsearch.conf.ApiDocConfig;
import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
import org.apache.commons.cli.CommandLine;
@@ -42,7 +47,6 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@@ -51,7 +55,10 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

public class LogSearchMarkdownGenerator {
/**
* Class to generate markdown files based on property annotations + rest API docs with swagger
*/
public class LogSearchDocumentationGenerator {

private static final String SHIPPER_CONFIG_TEMPLATE_KEY = "shipperConfigs";
private static final String LOGSEARCH_PROPERTIES_TEMPLATE_KEY = "logsearchProperties";
@@ -76,6 +83,8 @@ public class LogSearchMarkdownGenerator {
private static final String SHIPPER_CONFIGURATIONS_MARKDOWN_TEMPLATE_FILE = "shipper_configurations.md.ftl";
private static final String SHIPPER_CONFIGURATIONS_MARKDOWN_OUTPUT = "shipper_configurations.md";

private static final String SWAGGER_API_DOC_FOLDER = "api-docs";
private static final String SWAGGER_YAML_FILE_NAME = "logsearch-swagger.yaml";

public static void main(String[] args) {
try {
@@ -102,13 +111,14 @@ public static void main(String[] args) {
System.out.println(String.format("Number of logsearch.properties configuration descriptors found: %d", propertyDescriptions.get(LOGSEARCH_PROPERTIES).size()));
System.out.println(String.format("Number of logfeeder.properties configuration descriptors found: %d", propertyDescriptions.get(LOGFEEDER_PROPERTIES).size()));

final List<String> shipperConfigPackagesToScan = Collections.singletonList(CONFIG_API_PACKAGE);
final List<String> shipperConfigPackagesToScan = Arrays.asList(CONFIG_API_PACKAGE, LOGFEEDER_PACKAGE);
ShipperConfigDescriptionDataHolder shipperConfigDescriptionDataHolder = createShipperConfigDescriptions(shipperConfigPackagesToScan);

System.out.println(String.format("Number of top level section shipper descriptors found: %d", shipperConfigDescriptionDataHolder.getTopLevelConfigSections().size()));
System.out.println(String.format("Number of input config section shipper descriptors found: %d", shipperConfigDescriptionDataHolder.getInputConfigSections().size()));
System.out.println(String.format("Number of filter config section shipper descriptors found: %d", shipperConfigDescriptionDataHolder.getFilterConfigSections().size()));
System.out.println(String.format("Number of mapper section shipper descriptors found: %d", shipperConfigDescriptionDataHolder.getPostMapValuesConfigSections().size()));
System.out.println(String.format("Number of output config section shipper descriptors found: %d", shipperConfigDescriptionDataHolder.getOutputConfigSections().size()));

final Configuration freemarkerConfiguration = new Configuration();
final ClassPathResource cpr = new ClassPathResource(TEMPLATES_FOLDER);
@@ -130,12 +140,25 @@ public static void main(String[] args) {
File shipperConfigsOutputFile = Paths.get(outputDir, SHIPPER_CONFIGURATIONS_MARKDOWN_OUTPUT).toFile();
writeMarkdown(freemarkerConfiguration, SHIPPER_CONFIGURATIONS_MARKDOWN_TEMPLATE_FILE, shipperConfigModels, shipperConfigsOutputFile);

String swaggerYaml = generateSwaggerYaml();
File swaggerYamlFile = Paths.get(outputDir, SWAGGER_API_DOC_FOLDER, SWAGGER_YAML_FILE_NAME).toFile();
FileUtils.writeStringToFile(swaggerYamlFile, swaggerYaml, Charset.defaultCharset());
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
}
}

private static String generateSwaggerYaml() throws Exception {
ApiDocConfig apiDocConfig = new ApiDocConfig();
BeanConfig beanConfig = apiDocConfig.swaggerConfig();
Swagger swagger = beanConfig.getSwagger();
swagger.addSecurityDefinition("basicAuth", new BasicAuthDefinition());
beanConfig.configure(swagger);
beanConfig.scanAndRead();
return Yaml.mapper().writeValueAsString(swagger);
}

private static void writeMarkdown(Configuration freemarkerConfiguration, String templateName,
Map<String, Object> models, File outputFile) throws Exception {
final StringWriter stringWriter = new StringWriter();
@@ -187,7 +210,14 @@ private static ShipperConfigDescriptionDataHolder createShipperConfigDescription
.distinct()
.collect(Collectors.toList());

return new ShipperConfigDescriptionDataHolder(topLevelConfigSections, inputConfigSection, filterConfigSection, postMapValuesConfigSection);
final List<ShipperConfigDescriptionData> outputConfigSection = shipperConfigDescription.stream()
.filter(
s -> s.getPath().startsWith("/output/[]"))
.distinct()
.collect(Collectors.toList());

return new ShipperConfigDescriptionDataHolder(topLevelConfigSections, inputConfigSection, filterConfigSection,
postMapValuesConfigSection, outputConfigSection);
}

private static List<PropertyDescriptionData> getPropertyDescriptions(List<String> packagesToScan) {
@@ -25,15 +25,18 @@ public class ShipperConfigDescriptionDataHolder {
private final List<ShipperConfigDescriptionData> inputConfigSections;
private final List<ShipperConfigDescriptionData> filterConfigSections;
private final List<ShipperConfigDescriptionData> postMapValuesConfigSections;
private final List<ShipperConfigDescriptionData> outputConfigSections;

public ShipperConfigDescriptionDataHolder(List<ShipperConfigDescriptionData> topLevelConfigSections,
List<ShipperConfigDescriptionData> inputConfigSections,
List<ShipperConfigDescriptionData> filterConfigSections,
List<ShipperConfigDescriptionData> postMapValuesConfigSections) {
List<ShipperConfigDescriptionData> postMapValuesConfigSections,
List<ShipperConfigDescriptionData> outputConfigSections) {
this.topLevelConfigSections = topLevelConfigSections;
this.inputConfigSections = inputConfigSections;
this.filterConfigSections = filterConfigSections;
this.postMapValuesConfigSections = postMapValuesConfigSections;
this.outputConfigSections = outputConfigSections;
}

public List<ShipperConfigDescriptionData> getTopLevelConfigSections() {
@@ -51,4 +54,8 @@ public List<ShipperConfigDescriptionData> getFilterConfigSections() {
public List<ShipperConfigDescriptionData> getPostMapValuesConfigSections() {
return postMapValuesConfigSections;
}

public List<ShipperConfigDescriptionData> getOutputConfigSections() {
return outputConfigSections;
}
}
@@ -0,0 +1,24 @@
<!---
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

## REST API documentation

[Swagger Doc UI](/api-docs)

## Javadoc

[Java doc link](/javadoc)

0 comments on commit 6ff4dc2

Please sign in to comment.