Skip to content

Commit

Permalink
Resolve #2613 : support HCL and a Terraform derivation
Browse files Browse the repository at this point in the history
  • Loading branch information
idodeclare authored and Vladimir Kotal committed Apr 14, 2020
1 parent 2321e18 commit c5ef7ff
Show file tree
Hide file tree
Showing 26 changed files with 2,491 additions and 41 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

/*
* Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
* Portions Copyright (c) 2017-2019, Chris Fraire <cfraire@me.com>.
* Portions Copyright (c) 2017-2020, Chris Fraire <cfraire@me.com>.
*/
package org.opengrok.indexer.analysis;

Expand All @@ -30,7 +30,6 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -80,6 +79,7 @@
import org.opengrok.indexer.analysis.fortran.FortranAnalyzerFactory;
import org.opengrok.indexer.analysis.golang.GolangAnalyzerFactory;
import org.opengrok.indexer.analysis.haskell.HaskellAnalyzerFactory;
import org.opengrok.indexer.analysis.hcl.HCLAnalyzerFactory;
import org.opengrok.indexer.analysis.java.JavaAnalyzerFactory;
import org.opengrok.indexer.analysis.javascript.JavaScriptAnalyzerFactory;
import org.opengrok.indexer.analysis.json.JsonAnalyzerFactory;
Expand All @@ -101,6 +101,7 @@
import org.opengrok.indexer.analysis.sql.SQLAnalyzerFactory;
import org.opengrok.indexer.analysis.swift.SwiftAnalyzerFactory;
import org.opengrok.indexer.analysis.tcl.TclAnalyzerFactory;
import org.opengrok.indexer.analysis.terraform.TerraformAnalyzerFactory;
import org.opengrok.indexer.analysis.typescript.TypeScriptAnalyzerFactory;
import org.opengrok.indexer.analysis.uue.UuencodeAnalyzerFactory;
import org.opengrok.indexer.analysis.vb.VBAnalyzerFactory;
Expand Down Expand Up @@ -221,8 +222,6 @@ public class AnalyzerGuru {
*/
private static final List<String> analysisPkgNames = new ArrayList<>();

public static final Reader dummyR = new StringReader("");
public static final String dummyS = "";
public static final FieldType string_ft_stored_nanalyzed_norms = new FieldType(StringField.TYPE_STORED);
public static final FieldType string_ft_nstored_nanalyzed_norms = new FieldType(StringField.TYPE_NOT_STORED);

Expand Down Expand Up @@ -300,7 +299,9 @@ public class AnalyzerGuru {
new EiffelAnalyzerFactory(),
new VerilogAnalyzerFactory(),
new TypeScriptAnalyzerFactory(),
new AsmAnalyzerFactory()
new AsmAnalyzerFactory(),
new HCLAnalyzerFactory(),
new TerraformAnalyzerFactory()
};

for (AnalyzerFactory analyzer : analyzers) {
Expand Down Expand Up @@ -334,15 +335,15 @@ public class AnalyzerGuru {
* {@link FileAnalyzerFactory} subclasses are revised to target more or
* different files.
* @return a value whose lower 32-bits are a static value
* 20191120_00
* 20200410_00
* for the current implementation and whose higher-32 bits are non-zero if
* {@link #addExtension(java.lang.String, AnalyzerFactory)}
* or
* {@link #addPrefix(java.lang.String, AnalyzerFactory)}
* has been called.
*/
public static long getVersionNo() {
final int ver32 = 20191120_00; // Edit comment above too!
final int ver32 = 20200410_00; // Edit comment above too!
long ver = ver32;
if (customizationHashCode != 0) {
ver |= (long) customizationHashCode << 32;
Expand Down Expand Up @@ -633,26 +634,6 @@ public void populateDocument(Document doc, File file, String path,
}
}

/**
* Get the content type for a named file.
*
* @param in The input stream we want to get the content type for (if we
* cannot determine the content type by the filename)
* @param file The name of the file
* @return The contentType suitable for printing to
* response.setContentType() or null if the factory was not found
* @throws java.io.IOException If an error occurs while accessing the input
* stream.
*/
public static String getContentType(InputStream in, String file) throws IOException {
AnalyzerFactory factory = find(in, file);
String type = null;
if (factory != null) {
type = factory.getContentType();
}
return type;
}

/**
* Write a browse-able version of the file.
*
Expand Down Expand Up @@ -780,7 +761,7 @@ public static AnalyzerFactory findByFileTypeName(String fileTypeName) {
public static AnalyzerFactory findFactory(String factoryClassName)
throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException,
InvocationTargetException {
Class<?> fcn = null;
Class<?> fcn;
try {
fcn = Class.forName(factoryClassName);

Expand Down Expand Up @@ -1031,9 +1012,7 @@ private static AnalyzerFactory findForStream(InputStream in,
return null;
}

private static AnalyzerFactory findMagicString(String opening,
String file)
throws IOException {
private static AnalyzerFactory findMagicString(String opening, String file) {

// first, try to look up two words in magics
String fragment = getWords(opening, 2);
Expand Down Expand Up @@ -1207,6 +1186,6 @@ private static boolean factoriesDifferent(AnalyzerFactory a,
if (a_name == null && b_name == null) {
return false;
}
return a_name == null || b_name == null || !a_name.equals(b_name);
return a_name == null || !a_name.equals(b_name);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

/*
* Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
* Portions Copyright (c) 2017-2019, Chris Fraire <cfraire@me.com>.
* Portions Copyright (c) 2017-2020, Chris Fraire <cfraire@me.com>.
*/
package org.opengrok.indexer.analysis;

Expand Down Expand Up @@ -185,22 +185,15 @@ private void initialize() {
//on Solaris regexp.h used is different than on linux (gnu regexp)
//http://en.wikipedia.org/wiki/Regular_expression#POSIX_basic_and_extended
addScalaSupport(command);

addHaskellSupport(command);

//temporarily use our defs until ctags will fix https://github.com/universal-ctags/ctags/issues/988
addClojureSupport(command);

addKotlinSupport(command);

addSwiftSupport(command);

addRustSupport(command);

addPascalSupport(command);

addPowerShellSupport(command);

addTerraformSupport(command);
//PLEASE add new languages ONLY with POSIX syntax (see above wiki link)

if (langMap == null) {
Expand Down Expand Up @@ -401,6 +394,30 @@ private void addScalaSupport(List<String> command) {
command.add("--regex-scala=/^[[:space:]]*package[[:space:]]+([a-zA-Z0-9_.]+)/\\1/p,packages/");
}

private void addTerraformSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Terraform")) { // Built-in would be capitalized.
command.add("--langdef=terraform"); // Lower-case if user-defined.
}

/*
* Ignore Terraform single-line comments with short-form (only two
* separators following the pattern), exclusive matches.
*/
command.add("--regex-terraform=,^[[:space:]]*#,,{exclusive}");
command.add("--regex-terraform=,^[[:space:]]*//,,{exclusive}");

/*
* Terraform "resource block declares a resource of a given type ...
* with a given local name...." Unfortunately there is no Posix
* equivalent of {Identifier} from HCL.lexh, so we must approximate with
* the possibility of leaving out some matches.
*/
command.add("--regex-terraform=" +
"/[[:<:]]resource[[:space:]]*\"([[:alpha:]][-_[:alpha:]]*)\"[[:space:]]*" +
"\"([[:alpha:]][-_[:alpha:]]*)\"[[:space:]]*\\{/" +
"\\1.\\2/s,struct,resource names/");
}

/**
* Run ctags on a file.
* @param file file path to process
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2020, Chris Fraire <cfraire@me.com>.
*/

package org.opengrok.indexer.analysis.hcl;

import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

/**
* Represents a container for HCL keywords and other string constants.
*/
public class Consts {

private static final Set<String> kwd = new HashSet<>();

public static final Set<String> KEYWORDS = Collections.unmodifiableSet(kwd);

static {
/*
* HCL has the irritating aspect that "there are no globally-reserved
* words, but in some contexts certain identifiers are reserved to
* function as keywords."
*
* We'll just treat the following as globally reserved and wait to see
* if that causes problems for any users.
*/

kwd.add("false");
kwd.add("true");
kwd.add("null");
kwd.add("for");
kwd.add("endfor");
kwd.add("in");
kwd.add("if");
kwd.add("else");
kwd.add("endif");
}

/* private to enforce static */
private Consts() {
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved.
* Portions Copyright (c) 2017-2020, Chris Fraire <cfraire@me.com>.
*/

package org.opengrok.indexer.analysis.hcl;

import org.opengrok.indexer.analysis.AbstractAnalyzer;
import org.opengrok.indexer.analysis.AnalyzerFactory;
import org.opengrok.indexer.analysis.JFlexTokenizer;
import org.opengrok.indexer.analysis.JFlexXref;
import org.opengrok.indexer.analysis.plain.AbstractSourceCodeAnalyzer;

import java.io.Reader;

/**
* Represents an extension of {@link AbstractSourceCodeAnalyzer} for the HCL
* configuration language.
*/
public class HCLAnalyzer extends AbstractSourceCodeAnalyzer {

/**
* Creates a new instance of {@link HCLAnalyzer}.
* @param factory defined instance for the analyzer
*/
protected HCLAnalyzer(AnalyzerFactory factory) {
super(factory, () -> new JFlexTokenizer(new HCLSymbolTokenizer(
AbstractAnalyzer.DUMMY_READER)));
}

/**
* @return {@code null} as there is no aligned language
*/
@Override
public String getCtagsLang() {
return null;
}

/**
* Gets a version number to be used to tag processed documents so that
* re-analysis can be re-done later if a stored version number is different
* from the current implementation.
* @return 20200409_05
*/
@Override
protected int getSpecializedVersionNo() {
return 20200409_05; // Edit comment above too!
}

/**
* Creates a wrapped instance of {@link HCLXref}.
* @param reader an instance passed to the new {@link HCLXref}
* @return a defined instance
*/
@Override
protected JFlexXref newXref(Reader reader) {
return new JFlexXref(new HCLXref(reader));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2010, 2018 Oracle and/or its affiliates. All rights reserved.
* Portions Copyright (c) 2017, 2020, Chris Fraire <cfraire@me.com>.
*/

package org.opengrok.indexer.analysis.hcl;

import org.opengrok.indexer.analysis.AbstractAnalyzer;
import org.opengrok.indexer.analysis.FileAnalyzerFactory;

/**
* Represents an extension of {@link FileAnalyzerFactory} to produce
* {@link HCLAnalyzer} instances.
*/
public class HCLAnalyzerFactory extends FileAnalyzerFactory {

private static final String name = "HCL";

private static final String[] SUFFIXES = {"HCL"};

/**
* Creates a new instance of {@link HCLAnalyzerFactory}.
*/
public HCLAnalyzerFactory() {
super(null, null, SUFFIXES, null, null, "text/plain", AbstractAnalyzer.Genre.PLAIN, name);
}

/**
* Creates a new instance of {@link HCLAnalyzer}.
* @return the new instance
*/
@Override
protected AbstractAnalyzer newAnalyzer() {
return new HCLAnalyzer(this);
}
}

0 comments on commit c5ef7ff

Please sign in to comment.