feat(core): move all documentation to JsonSchema

build.gradle

@@ -36,8 +36,8 @@ mainClassName = "org.kestra.cli.App"
 sourceCompatibility = 11
 
 dependencies {
-    compile project(":cli")
-    testCompile project(":cli")
+    implementation project(":cli")
+    testImplementation project(":cli")
 }
 
 /**********************************************************************************************************************\
@@ -106,6 +106,7 @@ allprojects {
         api group: 'com.google.guava', name: 'guava', version: '28.1-jre'
         api group: 'commons-io', name: 'commons-io', version: '2.6'
         api group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
+        api "io.swagger.core.v3:swagger-annotations"
     }
 }
 

cli/build.gradle

@@ -1,33 +1,33 @@
 dependencies {
     // micronaut
-    compile "info.picocli:picocli"
-    compile "io.micronaut.configuration:micronaut-picocli"
-    compile "io.micronaut:micronaut-management"
-    compile "io.micronaut:micronaut-http-server-netty"
+    implementation "info.picocli:picocli"
+    implementation "io.micronaut.configuration:micronaut-picocli"
+    implementation "io.micronaut:micronaut-management"
+    implementation "io.micronaut:micronaut-http-server-netty"
 
     // plugins
-    compile 'org.eclipse.aether:aether-api:1.1.0'
-    compile 'org.eclipse.aether:aether-spi:1.1.0'
-    compile 'org.eclipse.aether:aether-util:1.1.0'
-    compile 'org.eclipse.aether:aether-impl:1.1.0'
-    compile 'org.eclipse.aether:aether-connector-basic:1.1.0'
-    compile 'org.eclipse.aether:aether-transport-file:1.1.0'
-    compile 'org.eclipse.aether:aether-transport-http:1.1.0'
-    compile('org.apache.maven:maven-aether-provider:3.1.0') {
+    implementation 'org.eclipse.aether:aether-api:1.1.0'
+    implementation 'org.eclipse.aether:aether-spi:1.1.0'
+    implementation 'org.eclipse.aether:aether-util:1.1.0'
+    implementation 'org.eclipse.aether:aether-impl:1.1.0'
+    implementation 'org.eclipse.aether:aether-connector-basic:1.1.0'
+    implementation 'org.eclipse.aether:aether-transport-file:1.1.0'
+    implementation 'org.eclipse.aether:aether-transport-http:1.1.0'
+    implementation('org.apache.maven:maven-aether-provider:3.1.0') {
         // sisu dependency injector is not used
         exclude group: 'org.eclipse.sisu'
     }
 
     // modules
-    compile project(":core")
+    implementation project(":core")
 
-    compile project(":repository-memory")
-    compile project(":repository-elasticsearch")
+    implementation project(":repository-memory")
+    implementation project(":repository-elasticsearch")
 
-    compile project(":runner-memory")
-    compile project(":runner-kafka")
+    implementation project(":runner-memory")
+    implementation project(":runner-kafka")
 
-    compile project(":storage-local")
+    implementation project(":storage-local")
 
-    compile project(":webserver")
+    implementation project(":webserver")
 }

cli/src/test/java/org/kestra/cli/commands/plugins/PluginListCommandTest.java

@@ -36,8 +36,8 @@ void run() throws IOException, URISyntaxException {
 
         FileUtils.copyFile(
             new File(Objects.requireNonNull(PluginListCommandTest.class.getClassLoader()
-                .getResource("plugins/plugin-template-0.1.0-SNAPSHOT.jar")).toURI()),
-            new File(URI.create("file://" + pluginsPath.toAbsolutePath() + "/plugin-template-0.1.0-SNAPSHOT.jar"))
+                .getResource("plugins/plugin-template-test-0.1.0-SNAPSHOT.jar")).toURI()),
+            new File(URI.create("file://" + pluginsPath.toAbsolutePath() + "/plugin-template-test-0.1.0-SNAPSHOT.jar"))
         );
 
         ByteArrayOutputStream out = new ByteArrayOutputStream();

core/build.gradle

@@ -4,26 +4,31 @@ configurations {
 
 dependencies {
     // log
-    compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
+    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
 
     // serializers
-    compile group: "org.apache.avro", name: "avro", version: '1.10.0'
-    compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-ion', version: '2.11.3'
+    implementation group: "org.apache.avro", name: "avro", version: '1.10.0'
+    implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-ion', version: '2.11.3'
 
     // validations
-    compile group: 'org.hibernate.validator', name: 'hibernate-validator', version: '6.1.0.Final'
+    implementation group: 'org.hibernate.validator', name: 'hibernate-validator', version: '6.1.0.Final'
 
     // utils
-    compile group: 'net.jodah', name: 'failsafe', version: '2.4.0'
+    implementation group: 'net.jodah', name: 'failsafe', version: '2.4.0'
 
     // scheduler
-    compile group: 'com.cronutils', name: 'cron-utils', version: '9.0.2'
+    implementation group: 'com.cronutils', name: 'cron-utils', version: '9.0.2'
 
+    // schema
+    implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.16.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.16.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.16.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.16.0'
 
     // test
-    testCompile project(':repository-memory')
-    testCompile project(':runner-memory')
-    testCompile project(':storage-local')
+    testImplementation project(':repository-memory')
+    testImplementation project(':runner-memory')
+    testImplementation project(':storage-local')
 
     testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3'
 }

core/src/main/java/org/kestra/core/docs/Document.java

@@ -10,6 +10,6 @@
 @EqualsAndHashCode
 @ToString
 public class Document {
-    private String path;
-    private String body;
+    private final String path;
+    private final String body;
 }

core/src/main/java/org/kestra/core/docs/DocumentationGenerator.java

@@ -8,39 +8,23 @@
 import com.google.common.base.Charsets;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
-import org.kestra.core.models.annotations.Documentation;
-import org.kestra.core.models.annotations.Example;
-import org.kestra.core.models.annotations.InputProperty;
-import org.kestra.core.models.annotations.OutputProperty;
-import org.kestra.core.models.tasks.Output;
+import org.kestra.core.models.conditions.Condition;
 import org.kestra.core.models.tasks.Task;
+import org.kestra.core.models.triggers.AbstractTrigger;
 import org.kestra.core.plugins.RegisteredPlugin;
 import org.kestra.core.runners.handlebars.helpers.DateHelper;
 import org.kestra.core.runners.handlebars.helpers.JsonHelper;
 import org.kestra.core.serializers.JacksonMapper;
 
 import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 abstract public class DocumentationGenerator {
-    private static Pattern DEFAULT_PACKAGES_TO_IGNORE = Pattern.compile("^(?:"
-        + "|java"
-        + "|javax"
-        + "|org.joda.time"
-        + ")\\..*$");
-
-    private static List<String> SIMPLE_NAME = Arrays.asList(
-        "java.lang",
-        "java.util",
-        "java.net"
-    );
-
     private static final Handlebars handlebars = new Handlebars()
         .with(EscapingStrategy.NOOP)
         .registerHelpers(ConditionalHelpers.class)
@@ -55,17 +39,17 @@ abstract public class DocumentationGenerator {
     public static List<Document> generate(RegisteredPlugin registeredPlugin) {
         ArrayList<Document> result = new ArrayList<>();
 
-        result.addAll(DocumentationGenerator.generate(registeredPlugin, registeredPlugin.getTasks(), "tasks"));
-        result.addAll(DocumentationGenerator.generate(registeredPlugin, registeredPlugin.getTriggers(), "triggers"));
-        result.addAll(DocumentationGenerator.generate(registeredPlugin, registeredPlugin.getConditions(), "conditions"));
+        result.addAll(DocumentationGenerator.generate(registeredPlugin, registeredPlugin.getTasks(), Task.class, "tasks"));
+        result.addAll(DocumentationGenerator.generate(registeredPlugin, registeredPlugin.getTriggers(), AbstractTrigger.class, "triggers"));
+        result.addAll(DocumentationGenerator.generate(registeredPlugin, registeredPlugin.getConditions(), Condition.class, "conditions"));
 
         return result;
     }
 
-    private static <T> List<Document> generate(RegisteredPlugin registeredPlugin, List<Class<? extends T>> cls, String type) {
+    private static <T> List<Document> generate(RegisteredPlugin registeredPlugin, List<Class<? extends T>> cls, Class<T> baseCls, String type) {
         return cls
             .stream()
-            .map(r -> PluginDocumentation.of(registeredPlugin, r))
+            .map(r -> PluginDocumentation.of(registeredPlugin, r, baseCls))
             .map(pluginDocumentation -> {
                 try {
                     String project = ObjectUtils.firstNonNull(
@@ -94,158 +78,13 @@ public static <T> String render(PluginDocumentation<T> pluginDocumentation) thro
         );
 
         Template template = handlebars.compileInline(hbsTemplate);
-
-        String renderer = template.apply(JacksonMapper.toMap(pluginDocumentation));
+        Map<String, Object> vars = JacksonMapper.toMap(pluginDocumentation);
+        String renderer = template.apply(vars);
 
         // vuepress {{ }} evaluation
         Pattern pattern = Pattern.compile("`\\{\\{(.*?)\\}\\}`", Pattern.MULTILINE);
         renderer = pattern.matcher(renderer).replaceAll("<code v-pre>{{ $1 }}</code>");
 
         return renderer;
     }
-
-    private static List<Field> getFields(Class<?> cls) {
-        if (cls.isInterface()) {
-            return new ArrayList<>();
-        }
-
-        List<Field> fields = new ArrayList<>();
-        for (Class<?> c = cls; c != null; c = c.getSuperclass()) {
-            if (c == Task.class) {
-                break;
-            }
-
-            fields.addAll(Arrays.asList(c.getDeclaredFields()));
-        }
-
-        return fields
-            .stream()
-            .filter(f -> !Modifier.isStatic(f.getModifiers()))
-            .filter(f -> !Modifier.isTransient(f.getModifiers()))
-            .collect(Collectors.toList());
-    }
-
-    public static Documentation getClassDoc(Class<?> cls) {
-        return Arrays.stream(cls.getAnnotationsByType(Documentation.class)).findFirst().orElse(null);
-    }
-
-    public static List<Example> getClassExample(Class<?> cls) {
-        return Arrays.stream(cls.getAnnotationsByType(Example.class)).collect(Collectors.toList());
-    }
-
-    public static Map<String, InputDocumentation> getMainInputs(Class<?> cls) {
-        return new TreeMap<>(flatten(getInputs(cls)));
-    }
-
-    private static List<InputDocumentation> getInputs(Class<?> cls) {
-        return getFields(cls)
-            .stream()
-            .map(field -> new InputDocumentation(
-                cls,
-                field,
-                Arrays.stream(field.getAnnotationsByType(InputProperty.class)).findFirst().orElse(null)
-            ))
-            .collect(Collectors.toList());
-    }
-
-    private static boolean isValidChild(final Class<?> cls) {
-        return !DEFAULT_PACKAGES_TO_IGNORE.matcher(cls.getPackageName()).matches() &&
-            !cls.isEnum();
-    }
-
-    public static <T extends AbstractChildDocumentation<T>> Map<String, T> flatten(List<T> list) {
-        return flatten(list, null);
-    }
-
-    private static <T extends AbstractChildDocumentation<T>> Map<String, T> flatten(List<T> list, String parentName) {
-        Map<String, T> result = new HashMap<>();
-
-        for (T current : list) {
-            result.put(flattenKey(current.getName(), parentName), current);
-            result.putAll(flatten(current.getChilds(), current.getName()));
-        }
-
-        return result;
-    }
-
-    private static String flattenKey(String current, String parent) {
-        return (parent != null ? parent + "." : "") + current;
-    }
-
-    public static List<InputDocumentation> getChildsInputs(Field field) {
-        return isValidChild(field.getType()) ?
-            DocumentationGenerator.getInputs(field.getType()) :
-            new ArrayList<>();
-    }
-
-    public static Map<String, OutputDocumentation> getMainOutput(Class<?> cls) {
-        List<OutputDocumentation> list = Arrays.stream(cls.getGenericInterfaces())
-            .filter(type -> type instanceof ParameterizedType)
-            .map(type -> (ParameterizedType) type)
-            .flatMap(parameterizedType -> Arrays.stream(parameterizedType.getActualTypeArguments()))
-            .filter(type -> type instanceof Class)
-            .map(type -> (Class<?>) type)
-            .filter(Output.class::isAssignableFrom)
-            .flatMap(c -> getOutputs(c).stream())
-            .collect(Collectors.toList());
-
-        return new TreeMap<>(flatten(list));
-    }
-
-    private static List<OutputDocumentation> getOutputs(Class<?> cls) {
-        return getFields(cls)
-            .stream()
-            .filter(f -> !Modifier.isTransient(f.getModifiers()))
-            .map(field -> new OutputDocumentation(
-                cls,
-                field,
-                Arrays.stream(field.getAnnotationsByType(OutputProperty.class)).findFirst().orElse(null)
-            ))
-            .collect(Collectors.toList());
-    }
-
-    public static List<OutputDocumentation> getChildsOutputs(Field field) {
-        return isValidChild(field.getType()) ?
-            DocumentationGenerator.getOutputs(field.getType()) :
-            new ArrayList<>();
-    }
-
-    public static String typeName(Class<?> cls) {
-        String name = cls.getName();
-
-        if (cls.getPackage() != null && SIMPLE_NAME.contains(cls.getPackage().getName())) {
-            name = cls.getSimpleName();
-        }
-
-        if (cls.isEnum()) {
-            name = "Enum";
-        }
-
-        if (cls.isArray()) {
-            name = typeName(cls.getComponentType()) + "[]";
-        }
-
-        if (cls.isMemberClass()) {
-            name = cls.getSimpleName();
-        }
-
-        return name;
-    }
-
-    public static String typeName(Field field) {
-        return genericName(typeName(field.getType()), field);
-    }
-
-    private static String genericName(String current, Field field) {
-        String generic = Stream.of(field.getGenericType())
-            .filter(type -> type instanceof ParameterizedType)
-            .map(type -> (ParameterizedType) type)
-            .flatMap(parameterizedType -> Arrays.stream(parameterizedType.getActualTypeArguments()))
-            .filter(type -> type instanceof Class)
-            .map(type -> (Class<?>) type)
-            .map(DocumentationGenerator::typeName)
-            .collect(Collectors.joining(", "));
-
-        return generic.equals("") ? current : current + "<" + generic + ">";
-    }
 }