Skip to content

Commit

Permalink
Scripting: capture structured javadoc from stdlib (elastic#68782)
Browse files Browse the repository at this point in the history 
Clean javadoc tags and strip html.
Methods and constructors have an optional `javadoc` field.  All fields under
`javadoc` are optional but at least one will be present.

Fields also have optional `javadoc` field which, if present, is a string.
```
"javadoc": {
  "description": "...",

  // from @param <param name> <param description>
  "parameters": {
    "p1": "<p1 description>",
    "p2": "<p2 description>"
  },

  // from @return
  "return": "...",

  // from @throws <type> <description>
  "throws": [
    [
      "IndexOutOfBoundsException",
      "<description>"
    ],
    [
      "IOException",
      "<description>"
    ]
  ]
}
```

Backport: c35eebe
  • Loading branch information
@stu-elastic
stu-elastic committed Feb 10, 2021
1 parent a43e872 commit 73ace7f
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 17 deletions.
10 changes: 10 additions & 0 deletions modules/lang-painless/build.gradle
View file
Expand Up @@ -77,6 +77,16 @@ dependencies {
docImplementation project(':server')
docImplementation project(':modules:lang-painless')
docImplementation 'com.github.javaparser:javaparser-core:3.18.0'
docImplementation 'org.jsoup:jsoup:1.13.1'
if (isEclipse) {
/*
* Eclipse isn't quite "with it" enough to understand the different
* source sets. This adds the dependency to all source sets so it
* can compile the doc java files.
*/
implementation 'com.github.javaparser:javaparser-core:3.18.0'
implementation 'org.jsoup:jsoup:1.13.1'
}
}

testClusters {
Expand Down
108 changes: 103 additions & 5 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java
View file
Expand Up @@ -18,9 +18,19 @@
import com.github.javaparser.ast.comments.Comment;
import com.github.javaparser.ast.visitor.VoidVisitorAdapter;
import com.github.javaparser.javadoc.Javadoc;
import com.github.javaparser.javadoc.JavadocBlockTag;
import com.github.javaparser.javadoc.description.JavadocDescription;
import com.github.javaparser.javadoc.description.JavadocDescriptionElement;
import com.github.javaparser.javadoc.description.JavadocInlineTag;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -94,7 +104,7 @@ public void putMethod(MethodDeclaration declaration) {
methods.put(
MethodSignature.fromDeclaration(declaration),
new ParsedMethod(
declaration.getJavadoc().map(Javadoc::toText).orElse(""),
declaration.getJavadoc().map(JavadocExtractor::clean).orElse(null),
declaration.getParameters()
.stream()
.map(p -> p.getName().asString())
Expand All @@ -110,7 +120,7 @@ public void putConstructor(ConstructorDeclaration declaration) {
constructors.put(
declaration.getParameters().stream().map(p -> stripTypeParameters(p.getType().asString())).collect(Collectors.toList()),
new ParsedMethod(
declaration.getJavadoc().map(Javadoc::toText).orElse(""),
declaration.getJavadoc().map(JavadocExtractor::clean).orElse(null),
declaration.getParameters()
.stream()
.map(p -> p.getName().asString())
Expand Down Expand Up @@ -152,7 +162,7 @@ public void putField(FieldDeclaration declaration) {
return;
}
for (VariableDeclarator var : declaration.getVariables()) {
fields.put(var.getNameAsString(), declaration.getJavadoc().map(Javadoc::toText).orElse(""));
fields.put(var.getNameAsString(), declaration.getJavadoc().map(v -> JavadocExtractor.clean(v).description).orElse(""));
}
}
}
Expand Down Expand Up @@ -192,15 +202,103 @@ public int hashCode() {
}

public static class ParsedMethod {
public final String javadoc;
public final ParsedJavadoc javadoc;
public final List<String> parameterNames;

public ParsedMethod(String javadoc, List<String> parameterNames) {
public ParsedMethod(ParsedJavadoc javadoc, List<String> parameterNames) {
this.javadoc = javadoc;
this.parameterNames = parameterNames;
}
}

public static ParsedJavadoc clean(Javadoc javadoc) {
JavadocDescription description = javadoc.getDescription();
List<JavadocBlockTag> tags = javadoc.getBlockTags();
List<String> cleaned = new ArrayList<>(description.getElements().size() + tags.size());
cleaned.addAll(stripInlineTags(description));
ParsedJavadoc parsed = new ParsedJavadoc(cleaned(cleaned));
for (JavadocBlockTag tag: tags) {
String tagName = tag.getTagName();
// https://docs.oracle.com/en/java/javase/14/docs/specs/javadoc/doc-comment-spec.html#standard-tags
// ignore author, deprecated, hidden, provides, uses, see, serial*, since and version.
if ("param".equals(tagName)) {
tag.getName().ifPresent(t -> parsed.param.put(t, cleaned(stripInlineTags(tag.getContent()))));
} else if ("return".equals(tagName)) {
parsed.returns = cleaned(stripInlineTags(tag.getContent()));
} else if ("exception".equals(tagName) || "throws".equals(tagName)) {
if (tag.getName().isPresent() == false) {
throw new IllegalStateException("Missing tag " + tag.toText());
}
parsed.thrws.add(List.of(tag.getName().get(), cleaned(stripInlineTags(tag.getContent()))));
}
}
return parsed;
}

private static String cleaned(List<String> segments) {
return Jsoup.clean(String.join("", segments), Whitelist.none()).replaceAll("[\n\\s]*\n[\n\\s]*", " ");
}

private static List<String> stripInlineTags(JavadocDescription description) {
List<JavadocDescriptionElement> elements = description.getElements();
List<String> stripped = new ArrayList<>(elements.size());
for (JavadocDescriptionElement element: elements) {
if (element instanceof JavadocInlineTag) {
stripped.add(((JavadocInlineTag)element).getContent());
} else {
stripped.add(element.toText());
}
}
return stripped;
}

public static class ParsedJavadoc implements ToXContent {
public final Map<String, String> param = new HashMap<>();
public String returns;
public String description;
public List<List<String>> thrws = new ArrayList<>();

public static final ParseField PARAMETERS = new ParseField("parameters");
public static final ParseField RETURN = new ParseField("return");
public static final ParseField THROWS = new ParseField("throws");
public static final ParseField DESCRIPTION = new ParseField("description");

public ParsedJavadoc(String description) {
this.description = description;
}

public boolean isEmpty() {
return param.size() == 0 &&
(description == null || description.isEmpty()) &&
(returns == null || returns.isEmpty()) &&
thrws.size() == 0;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
if (description != null && description.isEmpty() == false) {
builder.field(DESCRIPTION.getPreferredName(), description);
}
if (param.isEmpty() == false) {
builder.field(PARAMETERS.getPreferredName(), param);
}
if (returns != null && returns.isEmpty() == false) {
builder.field(RETURN.getPreferredName(), returns);
}
if (thrws.isEmpty() == false) {
builder.field(THROWS.getPreferredName(), thrws);
}
builder.endObject();
return builder;
}

@Override
public boolean isFragment() {
return true;
}
}

private static class ClassFileVisitor extends VoidVisitorAdapter<ParsedJavaClass> {
@Override
public void visit(CompilationUnit compilationUnit, ParsedJavaClass parsed) {
Expand Down
36 changes: 24 additions & 12 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java
View file
Expand Up @@ -173,13 +173,20 @@ public static class Method implements ToXContentObject {
private final String declaring;
private final String name;
private final String rtn;
private final String javadoc;
private final JavadocExtractor.ParsedJavadoc javadoc;
private final List<String> parameters;
private final List<String> parameterNames;
public static final ParseField PARAMETER_NAMES = new ParseField("parameter_names");
public static final ParseField JAVADOC = new ParseField("javadoc");

private Method(String declaring, String name, String rtn, String javadoc, List<String> parameters, List<String> parameterNames) {
private Method(
String declaring,
String name,
String rtn,
JavadocExtractor.ParsedJavadoc javadoc,
List<String> parameters,
List<String> parameterNames
) {
this.declaring = declaring;
this.name = name;
this.rtn = rtn;
Expand Down Expand Up @@ -210,11 +217,11 @@ public static List<Method> fromInfos(
) {
List<Method> methods = new ArrayList<>(infos.size());
for (PainlessContextMethodInfo info: infos) {
String javadoc = null;
JavadocExtractor.ParsedJavadoc javadoc = null;
List<String> parameterNames = null;

String name = info.getName();
List<String> parameterTypes = info.getParameters();
List<String> parameterTypes = toDisplayParameterTypes(info.getParameters(), javaNamesToDisplayNames);

JavadocExtractor.ParsedMethod parsedMethod = parsed.getMethod(name, parameterTypes);
if (parsedMethod != null) {
Expand All @@ -227,7 +234,7 @@ public static List<Method> fromInfos(
name,
ContextGeneratorCommon.getType(javaNamesToDisplayNames, info.getRtn()),
javadoc,
toDisplayParameterTypes(parameterTypes, javaNamesToDisplayNames),
parameterTypes,
parameterNames
));
}
Expand All @@ -240,7 +247,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(PainlessContextMethodInfo.DECLARING.getPreferredName(), declaring);
builder.field(PainlessContextMethodInfo.NAME.getPreferredName(), name);
builder.field(PainlessContextMethodInfo.RTN.getPreferredName(), rtn);
if (javadoc != null && "".equals(javadoc) == false) {
if (javadoc != null && javadoc.isEmpty() == false) {
builder.field(JAVADOC.getPreferredName(), javadoc);
}
builder.field(PainlessContextMethodInfo.PARAMETERS.getPreferredName(), parameters);
Expand All @@ -257,12 +264,17 @@ public static class Constructor implements ToXContentObject {
private final String declaring;
private final List<String> parameters;
private final List<String> parameterNames;
private final String javadoc;
private final JavadocExtractor.ParsedJavadoc javadoc;

public static final ParseField JAVADOC = new ParseField("javadoc");
public static final ParseField PARAMETER_NAMES = new ParseField("parameter_names");

private Constructor(String declaring, List<String> parameters, List<String> parameterNames, String javadoc) {
private Constructor(
String declaring,
List<String> parameters,
List<String> parameterNames,
JavadocExtractor.ParsedJavadoc javadoc
) {
this.declaring = declaring;
this.parameters = parameters;
this.parameterNames = parameterNames;
Expand All @@ -289,9 +301,9 @@ private static List<Constructor> fromInfos(
) {
List<Constructor> constructors = new ArrayList<>(infos.size());
for (PainlessContextConstructorInfo info: infos) {
List<String> parameterTypes = info.getParameters();
List<String> parameterTypes = toDisplayParameterTypes(info.getParameters(), javaNamesToDisplayNames);
List<String> parameterNames = null;
String javadoc = null;
JavadocExtractor.ParsedJavadoc javadoc = null;

JavadocExtractor.ParsedMethod parsed = pj.getConstructor(parameterTypes);
if (parsed != null) {
Expand All @@ -301,7 +313,7 @@ private static List<Constructor> fromInfos(

constructors.add(new Constructor(
javaNamesToDisplayNames.get(info.getDeclaring()),
toDisplayParameterTypes(parameterTypes, javaNamesToDisplayNames),
parameterTypes,
parameterNames,
javadoc
));
Expand All @@ -317,7 +329,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (parameterNames != null && parameterNames.size() > 0) {
builder.field(PARAMETER_NAMES.getPreferredName(), parameterNames);
}
if (javadoc != null && "".equals(javadoc) == false) {
if (javadoc != null && javadoc.isEmpty() == false) {
builder.field(JAVADOC.getPreferredName(), javadoc);
}
builder.endObject();
Expand Down

0 comments on commit 73ace7f

Please sign in to comment.