From c5ef7ff6dcf80dce99c75e6d9c2ec869872b5d84 Mon Sep 17 00:00:00 2001 From: Chris Fraire Date: Thu, 9 Apr 2020 20:39:12 -0500 Subject: [PATCH] Resolve #2613 : support HCL and a Terraform derivation --- .../indexer/analysis/AnalyzerGuru.java | 43 +-- .../org/opengrok/indexer/analysis/Ctags.java | 35 ++- .../opengrok/indexer/analysis/hcl/Consts.java | 63 ++++ .../indexer/analysis/hcl/HCLAnalyzer.java | 78 +++++ .../analysis/hcl/HCLAnalyzerFactory.java | 55 ++++ .../indexer/analysis/hcl/HCLLexer.java | 268 ++++++++++++++++++ .../indexer/analysis/terraform/Consts.java | 63 ++++ .../analysis/terraform/TerraformAnalyzer.java | 78 +++++ .../terraform/TerraformAnalyzerFactory.java | 55 ++++ .../analysis/terraform/TerraformLexer.java | 32 +++ .../src/main/resources/analysis/hcl/HCL.lexh | 61 ++++ .../analysis/hcl/HCLProductions.lexh | 244 ++++++++++++++++ .../analysis/hcl/HCLSymbolTokenizer.lex | 151 ++++++++++ .../main/resources/analysis/hcl/HCLXref.lex | 129 +++++++++ .../analysis/terraform/Terraform.lexh | 25 ++ .../terraform/TerraformProductions.lexh | 91 ++++++ .../terraform/TerraformSymbolTokenizer.lex | 155 ++++++++++ .../analysis/terraform/TerraformXref.lex | 133 +++++++++ .../TerraformSymbolTokenizerTest.java | 58 ++++ .../analysis/terraform/TerraformXrefTest.java | 53 ++++ .../resources/analysis/terraform/sample.tf | 215 ++++++++++++++ .../analysis/terraform/sample_xref.html | 223 +++++++++++++++ .../analysis/terraform/samplesymbols.txt | 199 +++++++++++++ .../resources/analysis/terraform/sampletags | 17 ++ .../resources/analysis/terraform/truncated.tf | 1 + .../analysis/terraform/truncated_xref.html | 7 + 26 files changed, 2491 insertions(+), 41 deletions(-) create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/Consts.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzer.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzerFactory.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLLexer.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/Consts.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzer.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzerFactory.java create mode 100644 opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformLexer.java create mode 100644 opengrok-indexer/src/main/resources/analysis/hcl/HCL.lexh create mode 100644 opengrok-indexer/src/main/resources/analysis/hcl/HCLProductions.lexh create mode 100644 opengrok-indexer/src/main/resources/analysis/hcl/HCLSymbolTokenizer.lex create mode 100644 opengrok-indexer/src/main/resources/analysis/hcl/HCLXref.lex create mode 100644 opengrok-indexer/src/main/resources/analysis/terraform/Terraform.lexh create mode 100644 opengrok-indexer/src/main/resources/analysis/terraform/TerraformProductions.lexh create mode 100644 opengrok-indexer/src/main/resources/analysis/terraform/TerraformSymbolTokenizer.lex create mode 100644 opengrok-indexer/src/main/resources/analysis/terraform/TerraformXref.lex create mode 100644 opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformSymbolTokenizerTest.java create mode 100644 opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformXrefTest.java create mode 100644 opengrok-indexer/src/test/resources/analysis/terraform/sample.tf create mode 100644 opengrok-indexer/src/test/resources/analysis/terraform/sample_xref.html create mode 100644 opengrok-indexer/src/test/resources/analysis/terraform/samplesymbols.txt create mode 100644 opengrok-indexer/src/test/resources/analysis/terraform/sampletags create mode 100644 opengrok-indexer/src/test/resources/analysis/terraform/truncated.tf create mode 100644 opengrok-indexer/src/test/resources/analysis/terraform/truncated_xref.html diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java index 7394e7e291b..5d3b4923ec1 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java @@ -19,7 +19,7 @@ /* * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved. - * Portions Copyright (c) 2017-2019, Chris Fraire . + * Portions Copyright (c) 2017-2020, Chris Fraire . */ package org.opengrok.indexer.analysis; @@ -30,7 +30,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.io.StringReader; import java.io.Writer; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; @@ -80,6 +79,7 @@ import org.opengrok.indexer.analysis.fortran.FortranAnalyzerFactory; import org.opengrok.indexer.analysis.golang.GolangAnalyzerFactory; import org.opengrok.indexer.analysis.haskell.HaskellAnalyzerFactory; +import org.opengrok.indexer.analysis.hcl.HCLAnalyzerFactory; import org.opengrok.indexer.analysis.java.JavaAnalyzerFactory; import org.opengrok.indexer.analysis.javascript.JavaScriptAnalyzerFactory; import org.opengrok.indexer.analysis.json.JsonAnalyzerFactory; @@ -101,6 +101,7 @@ import org.opengrok.indexer.analysis.sql.SQLAnalyzerFactory; import org.opengrok.indexer.analysis.swift.SwiftAnalyzerFactory; import org.opengrok.indexer.analysis.tcl.TclAnalyzerFactory; +import org.opengrok.indexer.analysis.terraform.TerraformAnalyzerFactory; import org.opengrok.indexer.analysis.typescript.TypeScriptAnalyzerFactory; import org.opengrok.indexer.analysis.uue.UuencodeAnalyzerFactory; import org.opengrok.indexer.analysis.vb.VBAnalyzerFactory; @@ -221,8 +222,6 @@ public class AnalyzerGuru { */ private static final List analysisPkgNames = new ArrayList<>(); - public static final Reader dummyR = new StringReader(""); - public static final String dummyS = ""; public static final FieldType string_ft_stored_nanalyzed_norms = new FieldType(StringField.TYPE_STORED); public static final FieldType string_ft_nstored_nanalyzed_norms = new FieldType(StringField.TYPE_NOT_STORED); @@ -300,7 +299,9 @@ public class AnalyzerGuru { new EiffelAnalyzerFactory(), new VerilogAnalyzerFactory(), new TypeScriptAnalyzerFactory(), - new AsmAnalyzerFactory() + new AsmAnalyzerFactory(), + new HCLAnalyzerFactory(), + new TerraformAnalyzerFactory() }; for (AnalyzerFactory analyzer : analyzers) { @@ -334,7 +335,7 @@ public class AnalyzerGuru { * {@link FileAnalyzerFactory} subclasses are revised to target more or * different files. * @return a value whose lower 32-bits are a static value - * 20191120_00 + * 20200410_00 * for the current implementation and whose higher-32 bits are non-zero if * {@link #addExtension(java.lang.String, AnalyzerFactory)} * or @@ -342,7 +343,7 @@ public class AnalyzerGuru { * has been called. */ public static long getVersionNo() { - final int ver32 = 20191120_00; // Edit comment above too! + final int ver32 = 20200410_00; // Edit comment above too! long ver = ver32; if (customizationHashCode != 0) { ver |= (long) customizationHashCode << 32; @@ -633,26 +634,6 @@ public void populateDocument(Document doc, File file, String path, } } - /** - * Get the content type for a named file. - * - * @param in The input stream we want to get the content type for (if we - * cannot determine the content type by the filename) - * @param file The name of the file - * @return The contentType suitable for printing to - * response.setContentType() or null if the factory was not found - * @throws java.io.IOException If an error occurs while accessing the input - * stream. - */ - public static String getContentType(InputStream in, String file) throws IOException { - AnalyzerFactory factory = find(in, file); - String type = null; - if (factory != null) { - type = factory.getContentType(); - } - return type; - } - /** * Write a browse-able version of the file. * @@ -780,7 +761,7 @@ public static AnalyzerFactory findByFileTypeName(String fileTypeName) { public static AnalyzerFactory findFactory(String factoryClassName) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { - Class fcn = null; + Class fcn; try { fcn = Class.forName(factoryClassName); @@ -1031,9 +1012,7 @@ private static AnalyzerFactory findForStream(InputStream in, return null; } - private static AnalyzerFactory findMagicString(String opening, - String file) - throws IOException { + private static AnalyzerFactory findMagicString(String opening, String file) { // first, try to look up two words in magics String fragment = getWords(opening, 2); @@ -1207,6 +1186,6 @@ private static boolean factoriesDifferent(AnalyzerFactory a, if (a_name == null && b_name == null) { return false; } - return a_name == null || b_name == null || !a_name.equals(b_name); + return a_name == null || !a_name.equals(b_name); } } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java index 26df1f35405..c922070ebf1 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/Ctags.java @@ -19,7 +19,7 @@ /* * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved. - * Portions Copyright (c) 2017-2019, Chris Fraire . + * Portions Copyright (c) 2017-2020, Chris Fraire . */ package org.opengrok.indexer.analysis; @@ -185,22 +185,15 @@ private void initialize() { //on Solaris regexp.h used is different than on linux (gnu regexp) //http://en.wikipedia.org/wiki/Regular_expression#POSIX_basic_and_extended addScalaSupport(command); - addHaskellSupport(command); - //temporarily use our defs until ctags will fix https://github.com/universal-ctags/ctags/issues/988 addClojureSupport(command); - addKotlinSupport(command); - addSwiftSupport(command); - addRustSupport(command); - addPascalSupport(command); - addPowerShellSupport(command); - + addTerraformSupport(command); //PLEASE add new languages ONLY with POSIX syntax (see above wiki link) if (langMap == null) { @@ -401,6 +394,30 @@ private void addScalaSupport(List command) { command.add("--regex-scala=/^[[:space:]]*package[[:space:]]+([a-zA-Z0-9_.]+)/\\1/p,packages/"); } + private void addTerraformSupport(List command) { + if (!env.getCtagsLanguages().contains("Terraform")) { // Built-in would be capitalized. + command.add("--langdef=terraform"); // Lower-case if user-defined. + } + + /* + * Ignore Terraform single-line comments with short-form (only two + * separators following the pattern), exclusive matches. + */ + command.add("--regex-terraform=,^[[:space:]]*#,,{exclusive}"); + command.add("--regex-terraform=,^[[:space:]]*//,,{exclusive}"); + + /* + * Terraform "resource block declares a resource of a given type ... + * with a given local name...." Unfortunately there is no Posix + * equivalent of {Identifier} from HCL.lexh, so we must approximate with + * the possibility of leaving out some matches. + */ + command.add("--regex-terraform=" + + "/[[:<:]]resource[[:space:]]*\"([[:alpha:]][-_[:alpha:]]*)\"[[:space:]]*" + + "\"([[:alpha:]][-_[:alpha:]]*)\"[[:space:]]*\\{/" + + "\\1.\\2/s,struct,resource names/"); + } + /** * Run ctags on a file. * @param file file path to process diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/Consts.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/Consts.java new file mode 100644 index 00000000000..b9b168e6988 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/Consts.java @@ -0,0 +1,63 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.hcl; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Represents a container for HCL keywords and other string constants. + */ +public class Consts { + + private static final Set kwd = new HashSet<>(); + + public static final Set KEYWORDS = Collections.unmodifiableSet(kwd); + + static { + /* + * HCL has the irritating aspect that "there are no globally-reserved + * words, but in some contexts certain identifiers are reserved to + * function as keywords." + * + * We'll just treat the following as globally reserved and wait to see + * if that causes problems for any users. + */ + + kwd.add("false"); + kwd.add("true"); + kwd.add("null"); + kwd.add("for"); + kwd.add("endfor"); + kwd.add("in"); + kwd.add("if"); + kwd.add("else"); + kwd.add("endif"); + } + + /* private to enforce static */ + private Consts() { + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzer.java new file mode 100644 index 00000000000..5d9c62fdff0 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzer.java @@ -0,0 +1,78 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017-2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.hcl; + +import org.opengrok.indexer.analysis.AbstractAnalyzer; +import org.opengrok.indexer.analysis.AnalyzerFactory; +import org.opengrok.indexer.analysis.JFlexTokenizer; +import org.opengrok.indexer.analysis.JFlexXref; +import org.opengrok.indexer.analysis.plain.AbstractSourceCodeAnalyzer; + +import java.io.Reader; + +/** + * Represents an extension of {@link AbstractSourceCodeAnalyzer} for the HCL + * configuration language. + */ +public class HCLAnalyzer extends AbstractSourceCodeAnalyzer { + + /** + * Creates a new instance of {@link HCLAnalyzer}. + * @param factory defined instance for the analyzer + */ + protected HCLAnalyzer(AnalyzerFactory factory) { + super(factory, () -> new JFlexTokenizer(new HCLSymbolTokenizer( + AbstractAnalyzer.DUMMY_READER))); + } + + /** + * @return {@code null} as there is no aligned language + */ + @Override + public String getCtagsLang() { + return null; + } + + /** + * Gets a version number to be used to tag processed documents so that + * re-analysis can be re-done later if a stored version number is different + * from the current implementation. + * @return 20200409_05 + */ + @Override + protected int getSpecializedVersionNo() { + return 20200409_05; // Edit comment above too! + } + + /** + * Creates a wrapped instance of {@link HCLXref}. + * @param reader an instance passed to the new {@link HCLXref} + * @return a defined instance + */ + @Override + protected JFlexXref newXref(Reader reader) { + return new JFlexXref(new HCLXref(reader)); + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzerFactory.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzerFactory.java new file mode 100644 index 00000000000..b84285959cc --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLAnalyzerFactory.java @@ -0,0 +1,55 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018 Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.hcl; + +import org.opengrok.indexer.analysis.AbstractAnalyzer; +import org.opengrok.indexer.analysis.FileAnalyzerFactory; + +/** + * Represents an extension of {@link FileAnalyzerFactory} to produce + * {@link HCLAnalyzer} instances. + */ +public class HCLAnalyzerFactory extends FileAnalyzerFactory { + + private static final String name = "HCL"; + + private static final String[] SUFFIXES = {"HCL"}; + + /** + * Creates a new instance of {@link HCLAnalyzerFactory}. + */ + public HCLAnalyzerFactory() { + super(null, null, SUFFIXES, null, null, "text/plain", AbstractAnalyzer.Genre.PLAIN, name); + } + + /** + * Creates a new instance of {@link HCLAnalyzer}. + * @return the new instance + */ + @Override + protected AbstractAnalyzer newAnalyzer() { + return new HCLAnalyzer(this); + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLLexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLLexer.java new file mode 100644 index 00000000000..0388a2c860b --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/hcl/HCLLexer.java @@ -0,0 +1,268 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.hcl; + +import org.opengrok.indexer.analysis.JFlexJointLexer; +import org.opengrok.indexer.analysis.JFlexSymbolMatcher; +import org.opengrok.indexer.analysis.Resettable; +import org.opengrok.indexer.web.HtmlConsts; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.Queue; +import java.util.Stack; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Represents an abstract base class for HCL lexers. + */ +@SuppressWarnings("DuplicatedCode") +public abstract class HCLLexer extends JFlexSymbolMatcher + implements JFlexJointLexer, Resettable { + + // Defined to be the equivalent of {Identifier} from HCL.lexh + private static final Pattern HERE_TERMINATOR_MATCH = Pattern.compile( + "^\\p{javaUnicodeIdentifierStart}(\\p{javaUnicodeIdentifierPart}|-)*"); + + private HCLLexerData dataHead; + + private Stack data; + + public HCLLexer() { + dataHead = new HCLLexerData(); + } + + /** + * Resets the instance to an initial state. + */ + @Override + public void reset() { + super.reset(); + dataHead = new HCLLexerData(); + if (data != null) { + data.clear(); + } + } + + /** + * Parses a Here-document declaration, and takes the {@code capture} using + * {@link HCLLexer#offer(String)}. If the declaration is valid, + * {@code hereSettings} will have been appended. + */ + public void hereOp(String capture) throws IOException { + if (!capture.startsWith("<<")) { + throw new IllegalArgumentException("bad HERE: " + capture); + } + + offer(capture); + if (dataHead.hereSettings == null) { + dataHead.hereSettings = new LinkedList<>(); + } + + String remaining = capture; + boolean indented = false; + int i = 0; + HereDocSettings settings; + String terminator; + + String opener = remaining.substring(0, i + "<<".length()); + remaining = remaining.substring(opener.length()); + if (remaining.startsWith("-")) { + indented = true; + remaining = remaining.substring(1); + } + + // Trim any whitespace, which is allowed by HCL after the HERE op. + remaining = remaining.replaceFirst("^\\s+", ""); + + Matcher m = HERE_TERMINATOR_MATCH.matcher(remaining); + if (!m.find()) { + return; + } + terminator = m.group(0); + + int state = indented ? HEREin() : HERE(); + settings = new HereDocSettings(terminator, state); + dataHead.hereSettings.add(settings); + } + + /** + * Pushes the first Here-document state if any declarations were parsed, or + * else does nothing. + * @return true if a Here state was pushed + */ + public boolean maybeHereStart() throws IOException { + if (dataHead.hereSettings != null && dataHead.hereSettings.size() > 0) { + HereDocSettings settings = dataHead.hereSettings.peek(); + yypush(settings.state); + disjointSpan(HtmlConsts.STRING_CLASS); + return true; + } + return false; + } + + /** + * Process the {@code capture}, possibly ending the Here-document state + * just beforehand. + * @return true if the Here state ended + */ + public boolean maybeHereEnd(String capture) throws IOException { + String trimmed = capture.replaceFirst("^\\s+", ""); + HereDocSettings settings = dataHead.hereSettings.peek(); + assert settings != null; + + boolean didZspan = false; + if (trimmed.equals(settings.terminator)) { + disjointSpan(null); + didZspan = true; + dataHead.hereSettings.remove(); + } + + offer(capture); + + if (dataHead.hereSettings.size() > 0) { + settings = dataHead.hereSettings.peek(); + yybegin(settings.state); + if (didZspan) { + disjointSpan(HtmlConsts.STRING_CLASS); + } + return false; + } + yypop(); + return true; + } + + /** + * Resets the interpolation counter to 1. + */ + public void interpOp() { + dataHead.numEndBrace = 1; + } + + /** + * Determine if the interpolation should end based on the first character + * of {@code capture}, recognizing tokens that increase the nesting level + * instead. + *

+ * Calling this method has side effects to possibly modify + * {@code numEndBrace}. + * @return true if the interpolation state ended + */ + public boolean maybeInterpolationEnd(String capture) throws IOException { + if (dataHead.numEndBrace <= 0) { + return false; + } + if (capture.startsWith("}")) { + if (--dataHead.numEndBrace <= 0) { + int rem = capture.length() - 1; + String opener = capture.substring(0, 1); + popData(); + yypop(); + disjointSpan(HtmlConsts.STRING_CLASS); + offer(opener); + if (rem > 0) { + yypushback(rem); + } + return true; + } + } else if (capture.startsWith("{")) { + ++dataHead.numEndBrace; + } + return false; + } + + /** + * Calls {@link #phLOC()} if the yystate is not COMMENT or SCOMMENT. + */ + public void chkLOC() { + int yystate = yystate(); + if (yystate != COMMENT() && yystate != SCOMMENT()) { + phLOC(); + } + } + + /** + * Push the lexer state data on the stack, and initialize a new state. + */ + public void pushData() { + if (data == null) { + data = new Stack<>(); + } + data.push(dataHead); + dataHead = new HCLLexerData(); + } + + /** + * Discard the current lexer state, and pop state off the stack. + */ + public void popData() { + dataHead = data.pop(); + } + + /** + * Subclasses must override to get the constant value created by JFlex to + * represent COMMENT. + */ + public abstract int COMMENT(); + + /** + * Subclasses must override to get the constant value created by JFlex to + * represent SCOMMENT. + */ + public abstract int SCOMMENT(); + + /** + * Subclasses must override to get the constant value created by JFlex to + * represent HERE. + */ + public abstract int HERE(); + + /** + * Subclasses must override to get the constant value created by JFlex to + * represent HEREin. + */ + public abstract int HEREin(); + + private static class HereDocSettings { + private final String terminator; + private final int state; + + HereDocSettings(String terminator, int state) { + this.terminator = terminator; + this.state = state; + } + } + + private static class HCLLexerData { + private Queue hereSettings; + + /** + * When interpolating inside a quoting construct, the number of + * remaining '}' is stored. It starts at 1, and any nesting increases + * the value. + */ + private int numEndBrace; + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/Consts.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/Consts.java new file mode 100644 index 00000000000..2d220bc3c51 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/Consts.java @@ -0,0 +1,63 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.terraform; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Represents a container for Terraform keywords and other string constants. + */ +public class Consts { + + private static final Set kwd = new HashSet<>(); + private static final Set pathKwd = new HashSet<>(); + + static final Set KEYWORDS = Collections.unmodifiableSet(kwd); + static final Set PATH_KEYWORDS = Collections.unmodifiableSet(pathKwd); + + static { + /* + * HCL has the irritating aspect that "there are no globally-reserved + * words, but in some contexts certain identifiers are reserved to + * function as keywords." + * + * We'll just treat the following as globally reserved and wait to see + * if that causes problems for any users. + */ + kwd.addAll(org.opengrok.indexer.analysis.hcl.Consts.KEYWORDS); + + /* + * We will support some `path.` context-specific keywords though. + */ + pathKwd.add("module"); + pathKwd.add("root"); + pathKwd.add("cwd"); + } + + /* private to enforce static */ + private Consts() { + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzer.java new file mode 100644 index 00000000000..bee742f51b2 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzer.java @@ -0,0 +1,78 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017-2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.terraform; + +import org.opengrok.indexer.analysis.AbstractAnalyzer; +import org.opengrok.indexer.analysis.AnalyzerFactory; +import org.opengrok.indexer.analysis.JFlexTokenizer; +import org.opengrok.indexer.analysis.JFlexXref; +import org.opengrok.indexer.analysis.plain.AbstractSourceCodeAnalyzer; + +import java.io.Reader; + +/** + * Represents an extension of {@link AbstractSourceCodeAnalyzer} for the + * Terraform configuration language. + */ +public class TerraformAnalyzer extends AbstractSourceCodeAnalyzer { + + /** + * Creates a new instance of {@link TerraformAnalyzer}. + * @param factory defined instance for the analyzer + */ + protected TerraformAnalyzer(AnalyzerFactory factory) { + super(factory, () -> new JFlexTokenizer(new TerraformSymbolTokenizer( + AbstractAnalyzer.DUMMY_READER))); + } + + /** + * @return {@code "terraform"} to match the OpenGrok-customized definitions + */ + @Override + public String getCtagsLang() { + return "terraform"; + } + + /** + * Gets a version number to be used to tag processed documents so that + * re-analysis can be re-done later if a stored version number is different + * from the current implementation. + * @return 20200410_08 + */ + @Override + protected int getSpecializedVersionNo() { + return 20200410_08; // Edit comment above too! + } + + /** + * Creates a wrapped instance of {@link TerraformXref}. + * @param reader an instance passed to the new {@link TerraformXref} + * @return a defined instance + */ + @Override + protected JFlexXref newXref(Reader reader) { + return new JFlexXref(new TerraformXref(reader)); + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzerFactory.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzerFactory.java new file mode 100644 index 00000000000..88a564f7210 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformAnalyzerFactory.java @@ -0,0 +1,55 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018 Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.terraform; + +import org.opengrok.indexer.analysis.AbstractAnalyzer; +import org.opengrok.indexer.analysis.FileAnalyzerFactory; + +/** + * Represents an extension of {@link FileAnalyzerFactory} to produce + * {@link TerraformAnalyzer} instances. + */ +public class TerraformAnalyzerFactory extends FileAnalyzerFactory { + + private static final String name = "Terraform"; + + private static final String[] SUFFIXES = {"TF", "TFVARS"}; + + /** + * Creates a new instance of {@link TerraformAnalyzerFactory}. + */ + public TerraformAnalyzerFactory() { + super(null, null, SUFFIXES, null, null, "text/plain", AbstractAnalyzer.Genre.PLAIN, name); + } + + /** + * Creates a new instance of {@link TerraformAnalyzer}. + * @return the new instance + */ + @Override + protected AbstractAnalyzer newAnalyzer() { + return new TerraformAnalyzer(this); + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformLexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformLexer.java new file mode 100644 index 00000000000..cf5393f3d44 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/terraform/TerraformLexer.java @@ -0,0 +1,32 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.terraform; + +import org.opengrok.indexer.analysis.hcl.HCLLexer; + +/** + * Represents an abstract base class for Terraform lexers. + */ +abstract class TerraformLexer extends HCLLexer { +} diff --git a/opengrok-indexer/src/main/resources/analysis/hcl/HCL.lexh b/opengrok-indexer/src/main/resources/analysis/hcl/HCL.lexh new file mode 100644 index 00000000000..adf07b9d369 --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/hcl/HCL.lexh @@ -0,0 +1,61 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +MaybeWhsp = {WhspChar}* + +/* + * spec.md: Identifier = ID_Start (ID_Continue | '-')*; + */ +Identifier = \p{ID_Start} (\p{ID_Continue} | "-")* + +/* + * spec.md: Numeric Literals + * + * A numeric literal is a decimal representation of a real number. It has an + * integer part, a fractional part, and an exponent part. + * + * NumericLit = decimal+ ("." decimal+)? (expmark decimal+)?; + * decimal = '0' .. '9'; + * expmark = ('e' | 'E') ("+" | "-")?; + */ +Numeric_literal = {decimal}+ ("." {decimal}+)? ({expmark} {decimal}+)? +decimal = [0-9] +expmark = [eE] [\+\-]? + +FileExt = [Hh][Cc][Ll] +File = [a-zA-Z]{FNameChar}* "." {FileExt} + +/* + * YYINITIAL : base scanning state + * POST_IDENTIFIER : after an identifier has been matched to allow + * distinguishing a STRING from a QUOTED. E.g. after an identifier a quoted + * expression does not allow interpolation and is therefore a STRING. + * COMMENT: multi-line comment + * SCOMMENT : single-line comment + * QUOTED: quoted template + * STRING: literal string expression + * HERE : Here-docs + * HEREin : Indented Here-docs + */ +%state POST_IDENTIFIER COMMENT SCOMMENT QUOTED STRING HERE HEREin diff --git a/opengrok-indexer/src/main/resources/analysis/hcl/HCLProductions.lexh b/opengrok-indexer/src/main/resources/analysis/hcl/HCLProductions.lexh new file mode 100644 index 00000000000..b9793cc3aba --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/hcl/HCLProductions.lexh @@ -0,0 +1,244 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + + { + ^ {Identifier} / {MaybeWhsp}{EOL} { + chkLOC(); + maybeHereEnd(yytext()); + } +} + + { + ^ {MaybeWhsp} {Identifier} / {MaybeWhsp}{EOL} { + chkLOC(); + maybeHereEnd(yytext()); + } +} + + { + "<<" "-"? {MaybeWhsp} {Identifier} { + chkLOC(); + hereOp(yytext()); + } + + {Identifier} { + chkLOC(); + if (offerSymbol(yytext(), 0, true)) { + yypush(POST_IDENTIFIER); + if (returnOnSymbol()) { + return yystate(); + } + } + } + + {Numeric_literal} { + chkLOC(); + onDisjointSpanChanged(HtmlConsts.NUMBER_CLASS, yychar); + offer(yytext()); + onDisjointSpanChanged(null, yychar + yylength()); + } + + \" { + chkLOC(); + yypush(QUOTED); + onDisjointSpanChanged(HtmlConsts.STRING_CLASS, yychar); + offer(yytext()); + } + + "#" | "//" { + yypush(SCOMMENT); + onDisjointSpanChanged(HtmlConsts.COMMENT_CLASS, yychar); + offer(yytext()); + } + + "/*" { + yypush(COMMENT); + onDisjointSpanChanged(HtmlConsts.COMMENT_CLASS, yychar); + offer(yytext()); + } + + [\{\}] { + chkLOC(); + String capture = yytext(); + if (!maybeInterpolationEnd(capture)) { + offer(capture); + } + } +} + + { + \\\\ { + chkLOC(); + offer(yytext()); + } +} + + { + \\\" { + chkLOC(); + offer(yytext()); + } + + \" { + chkLOC(); + offer(yytext()); + yypop(); + onDisjointSpanChanged(null, yychar + yylength()); + } +} + + { + /* + * Handle the escaping of interpolation and directive introductions as just + * regular quoted expression. + */ + ("$$" | "%%") "{" { + chkLOC(); + offer(yytext()); + } + + // Interpolation or directive introduction + [$%] "{" { + chkLOC(); + offer(yytext()); + onDisjointSpanChanged(null, yychar); + yypush(YYINITIAL); + pushData(); + interpOp(); + } +} + + { + /* + * For HCL quoted template expressions or string literals, "literal newline + * sequences are not permitted," but we'll handle anyway to avoid a possible + * JFlex "Error: could not match input" if a user accidentally violated + * HCL syntax. + */ + + {MaybeWhsp}{EOL} { + onDisjointSpanChanged(null, yychar); + onEndOfLineMatched(yytext(), yychar); + onDisjointSpanChanged(HtmlConsts.STRING_CLASS, yychar); + } +} + + { + {MaybeWhsp}{EOL} { + onDisjointSpanChanged(null, yychar); + onEndOfLineMatched(yytext(), yychar); + onDisjointSpanChanged(HtmlConsts.COMMENT_CLASS, yychar); + } + + "*/" { + offer(yytext()); + onDisjointSpanChanged(null, yychar); + yypop(); + } +} + + { + {MaybeWhsp}{EOL} { + yypushback(yylength()); + yypop(); + onDisjointSpanChanged(null, yychar); + } +} + + { + {MaybeWhsp}{EOL} { + if (maybeHereStart()) { + yypushback(yylength()); + } else { + onEndOfLineMatched(yytext(), yychar); + } + } +} + + { + \" { + chkLOC(); + yypop(); // Pop out of POST_IDENTIFIER first. + yypush(STRING); + onDisjointSpanChanged(HtmlConsts.STRING_CLASS, yychar); + offer(yytext()); + } +} + + { + // Only one whitespace char at a time. + {WhspChar} | [[\s]--[\n\r]] { + offer(yytext()); + } +} + + { + // Any other character after handling above causes POST_IDENTIFIER to end. + [^] { + yypushback(yylength()); + yypop(); + } +} + + { + // Only one char at a time. + [^\n\r] { + chkLOC(); + offer(yytext()); + } +} + +// "string links" and "comment links" + { + {FPath} { + chkLOC(); + if (takeAllContent()) { + onPathlikeMatched(yytext(), '/', false, yychar); + } + } + + {File} { + chkLOC(); + if (takeAllContent()) { + String path = yytext(); + onFilelikeMatched(path, yychar); + } + } + + {FNameChar}+ "@" {FNameChar}+ "." {FNameChar}+ { + chkLOC(); + if (takeAllContent()) { + onEmailAddressMatched(yytext(), yychar); + } + } +} + + { + {BrowseableURI} { + chkLOC(); + if (takeAllContent()) { + onUriMatched(yytext(), yychar, null); + } + } +} diff --git a/opengrok-indexer/src/main/resources/analysis/hcl/HCLSymbolTokenizer.lex b/opengrok-indexer/src/main/resources/analysis/hcl/HCLSymbolTokenizer.lex new file mode 100644 index 00000000000..06ba41a17be --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/hcl/HCLSymbolTokenizer.lex @@ -0,0 +1,151 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018 Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +/* + * Gets HCL symbols -- ignores comments, strings, keywords + */ + +package org.opengrok.indexer.analysis.hcl; + +import java.io.IOException; +import java.util.regex.Pattern; +import org.opengrok.indexer.util.StringUtils; +import org.opengrok.indexer.web.HtmlConsts; +%% +%public +%class HCLSymbolTokenizer +%extends HCLLexer +%unicode +%int +%char +%init{ + yyline = 1; +%init} +%include CommonLexer.lexh +%{ + private String lastSymbol; + + /** + * Resets the HCL tracked state; {@inheritDoc} + */ + @Override + public void reset() { + super.reset(); + lastSymbol = null; + } + + /** Does nothing. */ + @Override + public void offer(String value) throws IOException { + } + + @Override + public boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd) + throws IOException { + if (ignoreKwd || !Consts.KEYWORDS.contains(value)) { + lastSymbol = value; + onSymbolMatched(value, yychar + captureOffset); + return true; + } + lastSymbol = null; + return false; + } + + /** Just forget any last-matched symbol. */ + @Override + public void skipSymbol() { + lastSymbol = null; + } + + /** Just forget any last-matched symbol. */ + @Override + public void offerKeyword(String value) throws IOException { + lastSymbol = null; + } + + /** Does nothing. */ + @Override + public void startNewLine() throws IOException { + } + + /** Does nothing. */ + @Override + public void disjointSpan(String className) throws IOException { + } + + /** Does nothing. */ + @Override + public void phLOC() { + } + + /** Gets the value {@code false}. */ + protected boolean takeAllContent() { + return false; + } + + /** Gets a value indicating if a symbol was just matched. */ + protected boolean returnOnSymbol() { + return lastSymbol != null; + } + + /** + * Gets the constant value created by JFlex to represent COMMENT. + */ + @Override + public int COMMENT() { + return COMMENT; + } + + /** + * Gets the constant value created by JFlex to represent SCOMMENT. + */ + @Override + public int SCOMMENT() { + return SCOMMENT; + } + + /** + * Gets the constant value created by JFlex to represent HERE. + */ + @Override + public int HERE() { + return HERE; + } + + /** + * Gets the constant value created by JFlex to represent HEREin. + */ + @Override + public int HEREin() { + return HEREin; + } +%} + +%include Common.lexh +%include CommonURI.lexh +%include CommonPath.lexh +%include HCL.lexh + +%% +%include HCLProductions.lexh diff --git a/opengrok-indexer/src/main/resources/analysis/hcl/HCLXref.lex b/opengrok-indexer/src/main/resources/analysis/hcl/HCLXref.lex new file mode 100644 index 00000000000..1a244e3c160 --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/hcl/HCLXref.lex @@ -0,0 +1,129 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +/* + * Cross references a HCL file. + */ + +package org.opengrok.indexer.analysis.hcl; + +import java.io.IOException; +import java.util.Set; +import java.util.regex.Pattern; +import org.opengrok.indexer.web.HtmlConsts; +%% +%public +%class HCLXref +%extends HCLLexer +%unicode +%int +%char +%init{ + yyline = 1; +%init} +%include CommonLexer.lexh +%include CommonXref.lexh +%{ + @Override + public void offer(String value) throws IOException { + onNonSymbolMatched(value, yychar); + } + + @Override + public boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd) + throws IOException { + Set keywords = ignoreKwd ? null : Consts.KEYWORDS; + return onFilteredSymbolMatched(value, yychar, keywords); + } + + /** Does nothing. */ + @Override + public void skipSymbol() { + } + + @Override + public void offerKeyword(String value) throws IOException { + onKeywordMatched(value, yychar); + } + + @Override + public void startNewLine() throws IOException { + onEndOfLineMatched("\n", yychar); + } + + @Override + public void disjointSpan(String className) throws IOException { + onDisjointSpanChanged(className, yychar); + } + + /** Gets the value {@code true}. */ + protected boolean takeAllContent() { + return true; + } + + /** Gets the value {@code false}. */ + protected boolean returnOnSymbol() { + return false; + } + + /** + * Gets the constant value created by JFlex to represent COMMENT. + */ + @Override + public int COMMENT() { + return COMMENT; + } + + /** + * Gets the constant value created by JFlex to represent SCOMMENT. + */ + @Override + public int SCOMMENT() { + return SCOMMENT; + } + + /** + * Gets the constant value created by JFlex to represent HERE. + */ + @Override + public int HERE() { + return HERE; + } + + /** + * Gets the constant value created by JFlex to represent HEREin. + */ + @Override + public int HEREin() { + return HEREin; + } +%} + +%include Common.lexh +%include CommonURI.lexh +%include CommonPath.lexh +%include HCL.lexh + +%% +%include HCLProductions.lexh diff --git a/opengrok-indexer/src/main/resources/analysis/terraform/Terraform.lexh b/opengrok-indexer/src/main/resources/analysis/terraform/Terraform.lexh new file mode 100644 index 00000000000..0085162b527 --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/terraform/Terraform.lexh @@ -0,0 +1,25 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +FileExt = ([Hh][Cc][Ll] | [Tf][Ff] | [Tf][Ff][Vv][Aa][Rr][Ss]) diff --git a/opengrok-indexer/src/main/resources/analysis/terraform/TerraformProductions.lexh b/opengrok-indexer/src/main/resources/analysis/terraform/TerraformProductions.lexh new file mode 100644 index 00000000000..654e8520f67 --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/terraform/TerraformProductions.lexh @@ -0,0 +1,91 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2020, Chris Fraire . + */ + + { + + "var." {Identifier} { + chkLOC(); + String capture = yytext(); + String id = capture.substring("var.".length()); + offerKeyword("var"); + offer("."); + // Push back the ID, and let the {Identifier} handling take over. + yypushback(id.length()); + } + + "local." {Identifier} { + chkLOC(); + String capture = yytext(); + String id = capture.substring("local.".length()); + offerKeyword("local"); + offer("."); + // Push back the ID, and let the {Identifier} handling take over. + yypushback(id.length()); + } + + "module." {Identifier} { + chkLOC(); + String capture = yytext(); + String id = capture.substring("module.".length()); + offerKeyword("module"); + offer("."); + // Push back the ID, and let the {Identifier} handling take over. + yypushback(id.length()); + } + + "data." {Identifier} { + chkLOC(); + String capture = yytext(); + String id = capture.substring("data.".length()); + offerKeyword("data"); + offer("."); + // Push back the ID, and let the {Identifier} handling take over. + yypushback(id.length()); + } + + "path." {Identifier} { + chkLOC(); + String capture = yytext(); + String id = capture.substring("path.".length()); + if (Consts.PATH_KEYWORDS.contains(id)) { + offerKeyword("path"); + offer("."); + offerKeyword(id); + } else { + yypushback(yylength() - "path".length()); + if (offerSymbol("path", 0, true)) { + yypush(POST_IDENTIFIER); + if (returnOnSymbol()) { + return yystate(); + } + } + } + } + + "terraform.workspace" { + chkLOC(); + offerKeyword("terraform"); + offer("."); + offerKeyword("workspace"); + } +} diff --git a/opengrok-indexer/src/main/resources/analysis/terraform/TerraformSymbolTokenizer.lex b/opengrok-indexer/src/main/resources/analysis/terraform/TerraformSymbolTokenizer.lex new file mode 100644 index 00000000000..7deb9d2e912 --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/terraform/TerraformSymbolTokenizer.lex @@ -0,0 +1,155 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018 Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +/* + * Gets Terraform symbols -- ignores comments, strings, keywords + */ + +package org.opengrok.indexer.analysis.terraform; + +import java.io.IOException; +import java.util.regex.Pattern; +import org.opengrok.indexer.util.StringUtils; +import org.opengrok.indexer.web.HtmlConsts; +%% +%public +%class TerraformSymbolTokenizer +%extends TerraformLexer +%unicode +%int +%char +%init{ + yyline = 1; +%init} +%include CommonLexer.lexh +%{ + private String lastSymbol; + + /** + * Resets the Terraform tracked state; {@inheritDoc} + */ + @Override + public void reset() { + super.reset(); + lastSymbol = null; + } + + /** Does nothing. */ + @Override + public void offer(String value) throws IOException { + } + + @Override + public boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd) + throws IOException { + if (ignoreKwd || !Consts.KEYWORDS.contains(value)) { + lastSymbol = value; + onSymbolMatched(value, yychar + captureOffset); + return true; + } + lastSymbol = null; + return false; + } + + /** Just forget any last-matched symbol. */ + @Override + public void skipSymbol() { + lastSymbol = null; + } + + /** Just forget any last-matched symbol. */ + @Override + public void offerKeyword(String value) throws IOException { + lastSymbol = null; + } + + /** Does nothing. */ + @Override + public void startNewLine() throws IOException { + } + + /** Does nothing. */ + @Override + public void disjointSpan(String className) throws IOException { + } + + /** Does nothing. */ + @Override + public void phLOC() { + } + + /** Gets the value {@code false}. */ + protected boolean takeAllContent() { + return false; + } + + /** Gets a value indicating if a symbol was just matched. */ + protected boolean returnOnSymbol() { + return lastSymbol != null; + } + + /** + * Gets the constant value created by JFlex to represent COMMENT. + */ + @Override + public int COMMENT() { + return COMMENT; + } + + /** + * Gets the constant value created by JFlex to represent SCOMMENT. + */ + @Override + public int SCOMMENT() { + return SCOMMENT; + } + + /** + * Gets the constant value created by JFlex to represent HERE. + */ + @Override + public int HERE() { + return HERE; + } + + /** + * Gets the constant value created by JFlex to represent HEREin. + */ + @Override + public int HEREin() { + return HEREin; + } +%} + +%include Common.lexh +%include CommonURI.lexh +%include CommonPath.lexh +// Terraform.lexh comes after HCL so that Terraform macros supersede. +%include HCL.lexh +%include Terraform.lexh + +%% +// TerraformProductions.lexh comes first so that its expressions are preferred. +%include TerraformProductions.lexh +%include HCLProductions.lexh diff --git a/opengrok-indexer/src/main/resources/analysis/terraform/TerraformXref.lex b/opengrok-indexer/src/main/resources/analysis/terraform/TerraformXref.lex new file mode 100644 index 00000000000..49f453419b5 --- /dev/null +++ b/opengrok-indexer/src/main/resources/analysis/terraform/TerraformXref.lex @@ -0,0 +1,133 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +/* + * Cross references a Terraform file. + */ + +package org.opengrok.indexer.analysis.terraform; + +import java.io.IOException; +import java.util.Set; +import java.util.regex.Pattern; +import org.opengrok.indexer.web.HtmlConsts; +%% +%public +%class TerraformXref +%extends TerraformLexer +%unicode +%int +%char +%init{ + yyline = 1; +%init} +%include CommonLexer.lexh +%include CommonXref.lexh +%{ + @Override + public void offer(String value) throws IOException { + onNonSymbolMatched(value, yychar); + } + + @Override + public boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd) + throws IOException { + Set keywords = ignoreKwd ? null : Consts.KEYWORDS; + return onFilteredSymbolMatched(value, yychar, keywords); + } + + /** Does nothing. */ + @Override + public void skipSymbol() { + } + + @Override + public void offerKeyword(String value) throws IOException { + onKeywordMatched(value, yychar); + } + + @Override + public void startNewLine() throws IOException { + onEndOfLineMatched("\n", yychar); + } + + @Override + public void disjointSpan(String className) throws IOException { + onDisjointSpanChanged(className, yychar); + } + + /** Gets the value {@code true}. */ + protected boolean takeAllContent() { + return true; + } + + /** Gets the value {@code false}. */ + protected boolean returnOnSymbol() { + return false; + } + + /** + * Gets the constant value created by JFlex to represent COMMENT. + */ + @Override + public int COMMENT() { + return COMMENT; + } + + /** + * Gets the constant value created by JFlex to represent SCOMMENT. + */ + @Override + public int SCOMMENT() { + return SCOMMENT; + } + + /** + * Gets the constant value created by JFlex to represent HERE. + */ + @Override + public int HERE() { + return HERE; + } + + /** + * Gets the constant value created by JFlex to represent HEREin. + */ + @Override + public int HEREin() { + return HEREin; + } +%} + +%include Common.lexh +%include CommonURI.lexh +%include CommonPath.lexh +// Terraform.lexh comes after HCL so that Terraform macros supersede. +%include HCL.lexh +%include Terraform.lexh + +%% +// TerraformProductions.lexh comes first so that its expressions are preferred. +%include TerraformProductions.lexh +%include HCLProductions.lexh diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformSymbolTokenizerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformSymbolTokenizerTest.java new file mode 100644 index 00000000000..f31a4cb521e --- /dev/null +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformSymbolTokenizerTest.java @@ -0,0 +1,58 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2010, 2019, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.terraform; + +import static org.junit.Assert.assertNotNull; +import static org.opengrok.indexer.util.CustomAssertions.assertSymbolStream; +import static org.opengrok.indexer.util.StreamUtils.readSampleSymbols; + +import org.junit.Test; + +import java.io.InputStream; +import java.util.List; + +/** + * Represents a container for tests of {@link TerraformSymbolTokenizer}. + */ +public class TerraformSymbolTokenizerTest { + + /** + * Test sample.tf v. samplesymbols.txt + */ + @Test + public void testTerraformSymbolStream() throws Exception { + testSymbols("analysis/terraform/sample.tf", "analysis/terraform/samplesymbols.txt"); + } + + private void testSymbols(String codeResource, String symbolsResource) throws Exception { + InputStream tfRes = getClass().getClassLoader().getResourceAsStream(codeResource); + assertNotNull("Should get resource " + codeResource, tfRes); + InputStream symRes = getClass().getClassLoader().getResourceAsStream(symbolsResource); + assertNotNull("Should get resource " + symbolsResource, symRes); + + List expectedSymbols = readSampleSymbols(symRes); + assertSymbolStream(TerraformSymbolTokenizer.class, tfRes, expectedSymbols); + } +} diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformXrefTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformXrefTest.java new file mode 100644 index 00000000000..2fdc2547c05 --- /dev/null +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/terraform/TerraformXrefTest.java @@ -0,0 +1,53 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. + * Portions Copyright (c) 2017, 2019-2020, Chris Fraire . + */ + +package org.opengrok.indexer.analysis.terraform; + +import static org.opengrok.indexer.util.StreamUtils.readTagsFromResource; + +import org.junit.Test; +import org.opengrok.indexer.analysis.XrefTestBase; + +import java.io.IOException; + +/** + * Represents a container for tests of {@link TerraformXref}. + */ +public class TerraformXrefTest extends XrefTestBase { + + @Test + public void sampleTest() throws IOException { + writeAndCompare(new TerraformAnalyzerFactory(), + "analysis/terraform/sample.tf", + "analysis/terraform/sample_xref.html", + readTagsFromResource("analysis/terraform/sampletags"), 153); + } + + @Test + public void shouldCloseTruncatedStringSpan() throws IOException { + writeAndCompare(new TerraformAnalyzerFactory(), + "analysis/terraform/truncated.tf", + "analysis/terraform/truncated_xref.html", null, 1); + } +} diff --git a/opengrok-indexer/src/test/resources/analysis/terraform/sample.tf b/opengrok-indexer/src/test/resources/analysis/terraform/sample.tf new file mode 100644 index 00000000000..ca9a8c8f28a --- /dev/null +++ b/opengrok-indexer/src/test/resources/analysis/terraform/sample.tf @@ -0,0 +1,215 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This is derived from Hashicat main.tf just for testing OpenGrok's Terraform + * handling and modified arbitrarily to test other Terraform or HCL syntax. + */ + +provider "azurerm" { + version = "=1.44.0" +} + +resource "azurerm_resource_group" "myresourcegroup" { + name = "${var.prefix}-workshop" + location = var.location +} + +resource "azurerm_virtual_network" "vnet" { + name = "${var.prefix}-vnet" + location = azurerm_resource_group.myresourcegroup.location + address_space = [var.address_space] + resource_group_name = azurerm_resource_group.myresourcegroup.name +} + +resource "azurerm_subnet" "subnet" { + name = "${var.prefix}-subnet" + virtual_network_name = azurerm_virtual_network.vnet.name + resource_group_name = azurerm_resource_group.myresourcegroup.name + address_prefix = var.subnet_prefix +} + +resource "azurerm_network_security_group" "catapp-sg" { + name = "${var.prefix}-sg" + location = var.location + resource_group_name = azurerm_resource_group.myresourcegroup.name + + security_rule { + name = "HTTP" + priority = 100 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "80" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + security_rule { + name = "HTTPS" + priority = 102 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "*" + destination_address_prefix = "*" + } + + security_rule { + name = "SSH" + priority = 101 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } +} + +resource "azurerm_network_interface" "catapp-nic" { + name = "${var.prefix}-catapp-nic" + location = var.location + resource_group_name = azurerm_resource_group.myresourcegroup.name + network_security_group_id = azurerm_network_security_group.catapp-sg.id + + ip_configuration { + name = "${var.prefix}ipconfig" + subnet_id = azurerm_subnet.subnet.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.catapp-pip.id + } +} + +resource "azurerm_public_ip" "catapp-pip" { + name = "${var.prefix}-ip" + location = var.location + resource_group_name = azurerm_resource_group.myresourcegroup.name + allocation_method = "Dynamic" + domain_name_label = "${var.prefix}-meow" +} + +resource "azurerm_virtual_machine" "catapp" { + name = "${var.prefix}-meow" + location = var.location + resource_group_name = azurerm_resource_group.myresourcegroup.name + vm_size = var.vm_size + + network_interface_ids = [azurerm_network_interface.catapp-nic.id] + delete_os_disk_on_termination = "true" + + storage_image_reference { + publisher = var.image_publisher + offer = var.image_offer + sku = var.image_sku + version = var.image_version + } + + storage_os_disk { + name = "${var.prefix}-osdisk" + managed_disk_type = "Standard_LRS" + caching = "ReadWrite" + create_option = "FromImage" + } + + os_profile { + computer_name = var.prefix + admin_username = var.admin_username + admin_password = var.admin_password + } + + os_profile_linux_config { + disable_password_authentication = false + } +} + +# We're using a little trick here so we can run the provisioner without +# destroying the VM. Do not do this in production. + +# If you need ongoing management (Day N) of your virtual machines a tool such +# as Chef or Puppet is a better choice. These tools track the state of +# individual files and can keep them in the correct configuration. + +# Here we do the following steps: +# Sync everything in files/ to the remote VM. +# Set up some environment variables for our script. +# Add execute permissions to our scripts. +# Run the deploy_app.sh script. +resource "null_resource" "configure-cat-app" { + depends_on = [ + azurerm_virtual_machine.catapp, + ] + + # Terraform 0.11 + # triggers { + # build_number = "${timestamp()}" + # } + + # Terraform 0.12 + triggers = { + build_number = timestamp() + } + + provisioner "file" { + source = "files/" + destination = "/home/${var.admin_username}/" + + connection { + type = "ssh" + user = var.admin_username + password = var.admin_password + host = azurerm_public_ip.catapp-pip.fqdn + } + } + + provisioner "remote-exec" { + inline = [ + "sudo apt -y update", + "sudo apt -y install apache2", + "sudo systemctl start apache2", + "sudo chown -R ${var.admin_username}:${var.admin_username} /var/www/html", + "chmod +x *.sh", + "PLACEHOLDER=${var.placeholder} WIDTH=${var.width} HEIGHT=${var.height} PREFIX=${var.prefix} ./deploy_app.sh", + ] + + connection { + type = "ssh" + user = var.admin_username + password = var.admin_password + host = azurerm_public_ip.catapp-pip.fqdn + } + } +} + +resource "no-interp-here-${var.admin_username}" { + doc1 = < + + + +sampleFile - OpenGrok cross reference for /sampleFile +1/* +2 * Licensed under the Apache License, Version 2.0 (the "License"); +3 * you may not use this file except in compliance with the License. +4 * You may obtain a copy of the License at +5 * +6 * http://www.apache.org/licenses/LICENSE-2.0 +7 * +8 * Unless required by applicable law or agreed to in writing, software +9 * distributed under the License is distributed on an "AS IS" BASIS, +10 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +11 * See the License for the specific language governing permissions and +12 * limitations under the License. +13 */ +14 +15/* +16 * This is derived from Hashicat main.tf just for testing OpenGrok's Terraform +17 * handling and modified arbitrarily to test other Terraform or HCL syntax. +18 */ +19 +20provider "azurerm" { +21 version = "=1.44.0" +22} +23 +24resource "azurerm_resource_group" "myresourcegroup" { +25 name = "${var.prefix}-workshop" +26 location = var.location +27} +28 +29resource "azurerm_virtual_network" "vnet" { +30 name = "${var.prefix}-vnet" +31 location = azurerm_resource_group.myresourcegroup.location +32 address_space = [var.address_space] +33 resource_group_name = azurerm_resource_group.myresourcegroup.name +34} +35 +36resource "azurerm_subnet" "subnet" { +37 name = "${var.prefix}-subnet" +38 virtual_network_name = azurerm_virtual_network.vnet.name +39 resource_group_name = azurerm_resource_group.myresourcegroup.name +40 address_prefix = var.subnet_prefix +41} +42 +43resource "azurerm_network_security_group" "catapp-sg" { +44 name = "${var.prefix}-sg" +45 location = var.location +46 resource_group_name = azurerm_resource_group.myresourcegroup.name +47 +48 security_rule { +49 name = "HTTP" +50 priority = 100 +51 direction = "Inbound" +52 access = "Allow" +53 protocol = "Tcp" +54 source_port_range = "*" +55 destination_port_range = "80" +56 source_address_prefix = "*" +57 destination_address_prefix = "*" +58 } +59 +60 security_rule { +61 name = "HTTPS" +62 priority = 102 +63 direction = "Inbound" +64 access = "Allow" +65 protocol = "Tcp" +66 source_port_range = "*" +67 destination_port_range = "443" +68 source_address_prefix = "*" +69 destination_address_prefix = "*" +70 } +71 +72 security_rule { +73 name = "SSH" +74 priority = 101 +75 direction = "Inbound" +76 access = "Allow" +77 protocol = "Tcp" +78 source_port_range = "*" +79 destination_port_range = "22" +80 source_address_prefix = "*" +81 destination_address_prefix = "*" +82 } +83} +84 +85resource "azurerm_network_interface" "catapp-nic" { +86 name = "${var.prefix}-catapp-nic" +87 location = var.location +88 resource_group_name = azurerm_resource_group.myresourcegroup.name +89 network_security_group_id = azurerm_network_security_group.catapp-sg.id +90 +91 ip_configuration { +92 name = "${var.prefix}ipconfig" +93 subnet_id = azurerm_subnet.subnet.id +94 private_ip_address_allocation = "Dynamic" +95 public_ip_address_id = azurerm_public_ip.catapp-pip.id +96 } +97} +98 +99resource "azurerm_public_ip" "catapp-pip" { +100 name = "${var.prefix}-ip" +101 location = var.location +102 resource_group_name = azurerm_resource_group.myresourcegroup.name +103 allocation_method = "Dynamic" +104 domain_name_label = "${var.prefix}-meow" +105} +106 +107resource "azurerm_virtual_machine" "catapp" { +108 name = "${var.prefix}-meow" +109 location = var.location +110 resource_group_name = azurerm_resource_group.myresourcegroup.name +111 vm_size = var.vm_size +112 +113 network_interface_ids = [azurerm_network_interface.catapp-nic.id] +114 delete_os_disk_on_termination = "true" +115 +116 storage_image_reference { +117 publisher = var.image_publisher +118 offer = var.image_offer +119 sku = var.image_sku +120 version = var.image_version +121 } +122 +123 storage_os_disk { +124 name = "${var.prefix}-osdisk" +125 managed_disk_type = "Standard_LRS" +126 caching = "ReadWrite" +127 create_option = "FromImage" +128 } +129 +130 os_profile { +131 computer_name = var.prefix +132 admin_username = var.admin_username +133 admin_password = var.admin_password +134 } +135 +136 os_profile_linux_config { +137 disable_password_authentication = false +138 } +139} +140 +141# We're using a little trick here so we can run the provisioner without +142# destroying the VM. Do not do this in production. +143 +144# If you need ongoing management (Day N) of your virtual machines a tool such +145# as Chef or Puppet is a better choice. These tools track the state of +146# individual files and can keep them in the correct configuration. +147 +148# Here we do the following steps: +149# Sync everything in files/ to the remote VM. +150# Set up some environment variables for our script. +151# Add execute permissions to our scripts. +152# Run the deploy_app.sh script. +153resource "null_resource" "configure-cat-app" { +154 depends_on = [ +155 azurerm_virtual_machine.catapp, +156 ] +157 +158 # Terraform 0.11 +159 # triggers { +160 # build_number = "${timestamp()}" +161 # } +162 +163 # Terraform 0.12 +164 triggers = { +165 build_number = timestamp() +166 } +167 +168 provisioner "file" { +169 source = "files/" +170 destination = "/home/${var.admin_username}/" +171 +172 connection { +173 type = "ssh" +174 user = var.admin_username +175 password = var.admin_password +176 host = azurerm_public_ip.catapp-pip.fqdn +177 } +178 } +179 +180 provisioner "remote-exec" { +181 inline = [ +182 "sudo apt -y update", +183 "sudo apt -y install apache2", +184 "sudo systemctl start apache2", +185 "sudo chown -R ${var.admin_username}:${var.admin_username} /var/www/html", +186 "chmod +x *.sh", +187 "PLACEHOLDER=${var.placeholder} WIDTH=${var.width} HEIGHT=${var.height} PREFIX=${var.prefix} ./deploy_app.sh", +188 ] +189 +190 connection { +191 type = "ssh" +192 user = var.admin_username +193 password = var.admin_password +194 host = azurerm_public_ip.catapp-pip.fqdn +195 } +196 } +197} +198 +199resource "no-interp-here-${var.admin_username}" { +200 doc1 = <<END +201 ${var.val1} +202 ${local.val2} +203 ${module.val3} +204 ${data.val4} +205 ${path.cwd} +206 END (ineligible END) +207END +208 +209 doc2 = <<- END +210 ${path.other_value} +211 Now is the winter of our discontent. +212 END +213 +214 value1 = terraform.workspace +215} +216 + diff --git a/opengrok-indexer/src/test/resources/analysis/terraform/samplesymbols.txt b/opengrok-indexer/src/test/resources/analysis/terraform/samplesymbols.txt new file mode 100644 index 00000000000..a1d0e0d50a8 --- /dev/null +++ b/opengrok-indexer/src/test/resources/analysis/terraform/samplesymbols.txt @@ -0,0 +1,199 @@ +provider # 20:provider "azurerm" { +version +resource +name +prefix +location +location +resource +name +prefix +location +azurerm_resource_group +myresourcegroup +location +address_space +address_space +resource_group_name # 33: resource_group_name = azurerm_resource_group ... +azurerm_resource_group +myresourcegroup +name +resource +name +prefix +virtual_network_name +azurerm_virtual_network +vnet +name +resource_group_name +azurerm_resource_group +myresourcegroup +name +address_prefix +subnet_prefix +resource +name # 44: name = "${var.prefix}-sg" +prefix +location +location +resource_group_name +azurerm_resource_group +myresourcegroup +name +security_rule +name +priority +direction +access +protocol +source_port_range +destination_port_range +source_address_prefix +destination_address_prefix +security_rule # 60: security_rule { +name +priority +direction +access +protocol +source_port_range +destination_port_range +source_address_prefix +destination_address_prefix +security_rule +name +priority +direction +access +protocol +source_port_range +destination_port_range +source_address_prefix +destination_address_prefix +resource +name +prefix +location +location +resource_group_name +azurerm_resource_group +myresourcegroup +name +network_security_group_id +azurerm_network_security_group +catapp-sg +id +ip_configuration # 91: ip_configuration { +name +prefix +subnet_id +azurerm_subnet +subnet +id +private_ip_address_allocation +public_ip_address_id +azurerm_public_ip +catapp-pip +id +resource +name +prefix +location +location +resource_group_name +azurerm_resource_group +myresourcegroup +name +allocation_method +domain_name_label +prefix +resource +name +prefix +location +location +resource_group_name +azurerm_resource_group +myresourcegroup +name +vm_size # 111: vm_size = var.vm_size +vm_size +network_interface_ids +azurerm_network_interface +catapp-nic +id +delete_os_disk_on_termination +storage_image_reference +publisher +image_publisher +offer +image_offer +sku +image_sku +version +image_version +storage_os_disk +name +prefix +managed_disk_type +caching +create_option +os_profile +computer_name # 131: computer_name = var.prefix +prefix +admin_username +admin_username +admin_password +admin_password +os_profile_linux_config +disable_password_authentication +false +resource +depends_on # 154: depends_on = [ +azurerm_virtual_machine +catapp +triggers +build_number +timestamp +provisioner +source +destination +admin_username +connection +type +user +admin_username +password +admin_password +host # 176: host = azurerm_public_ip.catapp-pip.fqdn +azurerm_public_ip +catapp-pip +fqdn +provisioner +inline +admin_username +admin_username +placeholder +width +height +prefix +connection +type +user +admin_username +password +admin_password +host +azurerm_public_ip +catapp-pip +fqdn +resource # 199:resource "no-interp-here-${var.admin_username}" { +doc1 +val1 +val2 +val3 +val4 +doc2 +path +other_value +value1 diff --git a/opengrok-indexer/src/test/resources/analysis/terraform/sampletags b/opengrok-indexer/src/test/resources/analysis/terraform/sampletags new file mode 100644 index 00000000000..3cd578f7d06 --- /dev/null +++ b/opengrok-indexer/src/test/resources/analysis/terraform/sampletags @@ -0,0 +1,17 @@ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 0 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_PROGRAM_AUTHOR Universal Ctags Team // +!_TAG_PROGRAM_NAME Universal Ctags /Derived from Exuberant Ctags/ +!_TAG_PROGRAM_URL https://ctags.io/ /official site/ +!_TAG_PROGRAM_VERSION 0.0.0 /30cd8e03/ +!_TAG_OUTPUT_MODE u-ctags /u-ctags or e-ctags/ +!_TAG_OUTPUT_FILESEP slash /slash or backslash/ +!_TAG_PATTERN_LENGTH_LIMIT 180 /0 for no limit/ +azurerm_resource_group.myresourcegroup grok.tf /^resource "azurerm_resource_group" "myresourcegroup" {$/;" struct line:24 +azurerm_virtual_network.vnet grok.tf /^resource "azurerm_virtual_network" "vnet" {$/;" struct line:29 +azurerm_subnet.subnet grok.tf /^resource "azurerm_subnet" "subnet" {$/;" struct line:36 +azurerm_network_security_group.catapp-sg grok.tf /^resource "azurerm_network_security_group" "catapp-sg" {$/;" struct line:43 +azurerm_network_interface.catapp-nic grok.tf /^resource "azurerm_network_interface" "catapp-nic" {$/;" struct line:85 +azurerm_public_ip.catapp-pip grok.tf /^resource "azurerm_public_ip" "catapp-pip" {$/;" struct line:99 +azurerm_virtual_machine.catapp grok.tf /^resource "azurerm_virtual_machine" "catapp" {$/;" struct line:107 +null_resource.configure-cat-app grok.tf /^resource "null_resource" "configure-cat-app" {$/;" struct line:153 diff --git a/opengrok-indexer/src/test/resources/analysis/terraform/truncated.tf b/opengrok-indexer/src/test/resources/analysis/terraform/truncated.tf new file mode 100644 index 00000000000..9c2090ee781 --- /dev/null +++ b/opengrok-indexer/src/test/resources/analysis/terraform/truncated.tf @@ -0,0 +1 @@ + "Oops this string is not ter \ No newline at end of file diff --git a/opengrok-indexer/src/test/resources/analysis/terraform/truncated_xref.html b/opengrok-indexer/src/test/resources/analysis/terraform/truncated_xref.html new file mode 100644 index 00000000000..016205ab158 --- /dev/null +++ b/opengrok-indexer/src/test/resources/analysis/terraform/truncated_xref.html @@ -0,0 +1,7 @@ + + + + +sampleFile - OpenGrok cross reference for /sampleFile +1 "Oops this string is not ter +