From 00c47b8e7081e9031edfec394bc1f42b8cbae910 Mon Sep 17 00:00:00 2001 From: Eric Seidel Date: Thu, 30 Nov 2017 15:23:21 -0500 Subject: [PATCH] normalize fortran variables like gfortran --- .../opensolaris/opengrok/analysis/Ctags.java | 9 ++++ .../opengrok/analysis/CtagsReader.java | 11 ++++- .../opengrok/analysis/JFlexNonXref.java | 2 +- .../opengrok/analysis/JFlexSymbolMatcher.java | 4 +- .../opengrok/analysis/JFlexXref.java | 6 +-- .../opengrok/analysis/JFlexXrefUtils.java | 20 +++++---- .../opengrok/analysis/SymbolMatchedEvent.java | 13 +++++- .../analysis/fortran/FortranAnalyzer.java | 13 ++++++ .../opengrok/analysis/fortran/FortranXref.lex | 3 ++ .../opengrok/search/context/Context.java | 10 +++++ .../search/context/PlainLineTokenizer.lex | 17 +++++++- .../analysis/fortran/FortranXrefTest.java | 1 + .../analysis/fortran/sample_xref.html | 42 +++++++++---------- 13 files changed, 112 insertions(+), 39 deletions(-) diff --git a/src/org/opensolaris/opengrok/analysis/Ctags.java b/src/org/opensolaris/opengrok/analysis/Ctags.java index ab438d8acac..b953bcaa4c7 100644 --- a/src/org/opensolaris/opengrok/analysis/Ctags.java +++ b/src/org/opensolaris/opengrok/analysis/Ctags.java @@ -34,6 +34,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import org.opensolaris.opengrok.configuration.RuntimeEnvironment; @@ -61,6 +62,8 @@ public class Ctags implements Resettable { private String CTagsExtraOptionsFile = null; private int tabSize; + private Function normalizeIdentifier = str -> str; + private boolean junit_testing = false; /** @@ -89,6 +92,10 @@ public void setTabSize(int tabSize) { this.tabSize = tabSize; } + public void setNormalizeIdentifier(Function normalizeIdentifier) { + this.normalizeIdentifier = normalizeIdentifier; + } + public void setCTagsExtraOptionsFile(String CTagsExtraOptionsFile) { this.CTagsExtraOptionsFile = CTagsExtraOptionsFile; } @@ -380,6 +387,7 @@ public Definitions doCtags(String file) throws IOException, CtagsReader rdr = new CtagsReader(); rdr.setSplitterSupplier(() -> { return trySplitSource(file); }); rdr.setTabSize(tabSize); + rdr.setNormalizeIdentifier(normalizeIdentifier); Definitions ret; try { ctagsIn.write(file + "\n"); @@ -443,6 +451,7 @@ public void destroy() { CtagsReader rdr = new CtagsReader(); rdr.setTabSize(tabSize); + rdr.setNormalizeIdentifier(normalizeIdentifier); try { readTags(rdr); } catch (InterruptedException ex) { diff --git a/src/org/opensolaris/opengrok/analysis/CtagsReader.java b/src/org/opensolaris/opengrok/analysis/CtagsReader.java index b04a3f13a71..13b8787d8d2 100644 --- a/src/org/opensolaris/opengrok/analysis/CtagsReader.java +++ b/src/org/opensolaris/opengrok/analysis/CtagsReader.java @@ -25,6 +25,7 @@ package org.opensolaris.opengrok.analysis; import java.util.EnumMap; +import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -93,6 +94,12 @@ public class CtagsReader { private int tabSize; + private Function normalizeIdentifier = str -> str; + + public void setNormalizeIdentifier(Function normalize) { + normalizeIdentifier = normalize; + } + /** * This should mimic * https://github.com/universal-ctags/ctags/blob/master/docs/format.rst or @@ -209,7 +216,7 @@ public void readLine(String tagLine) { //log.fine("SKIPPING LINE - NO TAB"); return; } - String def = tagLine.substring(0, p); + String def = normalizeIdentifier.apply(tagLine.substring(0, p)); int mstart = tagLine.indexOf('\t', p + 1); String kind = null; @@ -326,7 +333,7 @@ public void readLine(String tagLine) { name = arg; } if (name != null) { - addTag(defs, cidx.lineno, name, "argument", def.trim() + + addTag(defs, cidx.lineno, normalizeIdentifier.apply(name), "argument", def.trim() + signature.trim(), null, signature, cidx.lineStart, cidx.lineEnd); } else { diff --git a/src/org/opensolaris/opengrok/analysis/JFlexNonXref.java b/src/org/opensolaris/opengrok/analysis/JFlexNonXref.java index fd5fb98add9..79ab6c30a24 100644 --- a/src/org/opensolaris/opengrok/analysis/JFlexNonXref.java +++ b/src/org/opensolaris/opengrok/analysis/JFlexNonXref.java @@ -473,7 +473,7 @@ protected boolean writeSymbol(String symbol, Set keywords, int line, protected boolean writeSymbol(String symbol, Set keywords, int line, boolean caseSensitive, boolean isKeyword) throws IOException { return JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project, - symbol, keywords, line, caseSensitive, isKeyword); + symbol, symbol, keywords, line, caseSensitive, isKeyword); } /** diff --git a/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java b/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java index 41e23ce06a9..ad8de48c75c 100644 --- a/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java +++ b/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java @@ -38,6 +38,8 @@ public abstract class JFlexSymbolMatcher extends JFlexStateStacker private NonSymbolMatchedListener nonSymbolListener; private String disjointSpanClassName; + protected String normalizeIdentifier(String id) { return id; } + /** * Associates the specified listener, replacing the former one. * @param l defined instance @@ -97,7 +99,7 @@ protected String getDisjointSpanClassName() { protected void onSymbolMatched(String str, int start) { SymbolMatchedListener l = symbolListener; if (l != null) { - SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, start, + SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, normalizeIdentifier(str), start, start + str.length()); l.symbolMatched(evt); } diff --git a/src/org/opensolaris/opengrok/analysis/JFlexXref.java b/src/org/opensolaris/opengrok/analysis/JFlexXref.java index 48a2fd91bae..09180a24996 100644 --- a/src/org/opensolaris/opengrok/analysis/JFlexXref.java +++ b/src/org/opensolaris/opengrok/analysis/JFlexXref.java @@ -203,7 +203,7 @@ public void setFoldingEnabled(boolean foldingEnabled) { public void symbolMatched(SymbolMatchedEvent evt) { try { JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project, - evt.getStr(), null, matcher.getLineNumber(), false, false); + evt.getStr(), evt.getNormalizedStr(), null, matcher.getLineNumber(), false, false); } catch (IOException ex) { throw new RuntimeException(ex); } @@ -292,7 +292,7 @@ public void linkageMatched(LinkageMatchedEvent evt) { break; case LABELDEF: // Only PowerShell seems to be using this. - JFlexXrefUtils.writeSameFileLinkSymbol(out, str); + JFlexXrefUtils.writeSameFileLinkSymbol(out, str, str); break; case FILELIKE: out.write(" keywords, + String urlPrefix, Project project, String symbol, String id, Set keywords, int line, boolean caseSensitive, boolean isKeyword) throws IOException { String[] strs = new String[1]; @@ -244,7 +245,7 @@ public static boolean writeSymbol(Writer out, Definitions defs, return false; } - if (defs != null && defs.hasDefinitionAt(symbol, line, strs)) { + if (defs != null && defs.hasDefinitionAt(id, line, strs)) { // This is the definition of the symbol. String type = strs[0]; String style_class = "d"; @@ -268,7 +269,7 @@ public static boolean writeSymbol(Writer out, Definitions defs, out.append(""); } @@ -276,7 +277,7 @@ public static boolean writeSymbol(Writer out, Definitions defs, out.append(""); Util.htmlize(symbol, out); out.append(""); - } else if (defs != null && defs.occurrences(symbol) == 1) { - writeSameFileLinkSymbol(out, symbol); + } else if (defs != null && defs.occurrences(id) == 1) { + writeSameFileLinkSymbol(out, symbol, id); } else { // This is a symbol that is not defined in this file, or a symbol // that is defined more than once in this file. In either case, we @@ -295,7 +296,7 @@ public static boolean writeSymbol(Writer out, Definitions defs, out.append(""); diff --git a/src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java b/src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java index c8e962238b8..e4c5ee80be3 100644 --- a/src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java +++ b/src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java @@ -36,6 +36,7 @@ public class SymbolMatchedEvent { private final Object source; private final String str; + private final String normalizedStr; private final int start; private final int end; @@ -43,12 +44,14 @@ public class SymbolMatchedEvent { * Initializes an immutable instance of {@link SymbolMatchedEvent}. * @param source the event source * @param str the symbol string + * @param normalizedStr the symbol string, normalized according to language-specific conventions * @param start the symbol start position * @param end the symbol end position */ - public SymbolMatchedEvent(Object source, String str, int start, int end) { + public SymbolMatchedEvent(Object source, String str, String normalizedStr, int start, int end) { this.source = source; this.str = str; + this.normalizedStr = normalizedStr; this.start = start; this.end = end; } @@ -69,6 +72,14 @@ public String getStr() { return str; } + /** + * Gets the normalized symbol string. + * @return the initial value + */ + public String getNormalizedStr() { + return normalizedStr; + } + /** * Gets the symbol start position. * @return the initial value diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java b/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java index f5097086b39..7923c461ed9 100644 --- a/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java +++ b/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java @@ -24,6 +24,7 @@ package org.opensolaris.opengrok.analysis.fortran; import java.io.Reader; +import org.opensolaris.opengrok.analysis.Ctags; import org.opensolaris.opengrok.analysis.FileAnalyzer; import org.opensolaris.opengrok.analysis.JFlexTokenizer; import org.opensolaris.opengrok.analysis.JFlexXref; @@ -36,6 +37,10 @@ */ public class FortranAnalyzer extends AbstractSourceCodeAnalyzer { + public static String normalizeIdentifier(String id) { + return id.toLowerCase() + "_"; + } + FortranAnalyzer(FortranAnalyzerFactory factory) { super(factory, new JFlexTokenizer(new FortranSymbolTokenizer( FileAnalyzer.dummyReader))); @@ -50,4 +55,12 @@ public class FortranAnalyzer extends AbstractSourceCodeAnalyzer { protected JFlexXref newXref(Reader reader) { return new JFlexXref(new FortranXref(reader)); } + + @Override + public void setCtags(Ctags ctags) { + this.ctags = ctags; + if (this.ctags != null) { + this.ctags.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier); + } + } } diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex b/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex index 5041555b2b3..12ff1847cf6 100644 --- a/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex +++ b/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex @@ -60,6 +60,9 @@ import org.opensolaris.opengrok.web.HtmlConsts; break; } } + + @Override + protected String normalizeIdentifier(String id) { return FortranAnalyzer.normalizeIdentifier(id); } %} File = [a-zA-Z]{FNameChar}* ".inc" diff --git a/src/org/opensolaris/opengrok/search/context/Context.java b/src/org/opensolaris/opengrok/search/context/Context.java index a584921827d..541768fb92f 100644 --- a/src/org/opensolaris/opengrok/search/context/Context.java +++ b/src/org/opensolaris/opengrok/search/context/Context.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -44,8 +45,12 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.opensolaris.opengrok.analysis.AnalyzerGuru; import org.opensolaris.opengrok.analysis.Definitions; import org.opensolaris.opengrok.analysis.FileAnalyzer; +import org.opensolaris.opengrok.analysis.FileAnalyzerFactory; +import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzer; +import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzerFactory; import org.opensolaris.opengrok.analysis.Scopes; import org.opensolaris.opengrok.analysis.Scopes.Scope; import org.opensolaris.opengrok.analysis.plain.PlainAnalyzerFactory; @@ -416,6 +421,10 @@ public boolean getContext(Reader in, Writer out, String urlPrefix, String token; int matchState; int matchedLines = 0; + FileAnalyzerFactory factory = AnalyzerGuru.find(path); + if (factory instanceof FortranAnalyzerFactory) { + tokens.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier); + } while ((token = tokens.yylex()) != null && (!lim || matchedLines < limit_max_lines)) { for (int i = 0; i < m.length; i++) { @@ -455,6 +464,7 @@ public boolean getContext(Reader in, Writer out, String urlPrefix, } } } + tokens.resetNormalizeIdentifier(); return anything; } } diff --git a/src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex b/src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex index 6bd11c7c533..e85a70e1c3a 100644 --- a/src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex +++ b/src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex @@ -32,6 +32,7 @@ import java.io.Reader; import java.io.Writer; import java.util.List; import java.util.TreeMap; +import java.util.function.Function; import org.opensolaris.opengrok.search.Hit; import org.opensolaris.opengrok.web.Util; import org.opensolaris.opengrok.analysis.Scopes; @@ -72,6 +73,14 @@ import org.opensolaris.opengrok.analysis.Scopes.Scope; boolean alt; Scopes scopes = null; + Function normalizeIdentifier = str -> str; + public void setNormalizeIdentifier(Function normalizeIdentifier) { + this.normalizeIdentifier = normalizeIdentifier; + } + public void resetNormalizeIdentifier() { + this.normalizeIdentifier = str -> str; + } + /** * Set the writer that should receive all output * @param out The new writer to write to @@ -400,7 +409,13 @@ Printable = [\@\$\%\^\&\-+=\?\.\:] %% -{Identifier}|{Number}|{Printable} { +{Identifier} { + String text = yytext(); + markedContents.append(text); + return normalizeIdentifier.apply(text); +} + +{Number}|{Printable} { String text = yytext(); markedContents.append(text); return text; diff --git a/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java b/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java index 0b05ebb48c1..3564b027836 100644 --- a/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java +++ b/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java @@ -116,6 +116,7 @@ private Definitions getTagsDefinitions() throws IOException { res, "UTF-8")); CtagsReader rdr = new CtagsReader(); + rdr.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier); String line; while ((line = in.readLine()) != null) { rdr.readLine(line); diff --git a/test/org/opensolaris/opengrok/analysis/fortran/sample_xref.html b/test/org/opensolaris/opengrok/analysis/fortran/sample_xref.html index 80cad3eb7d7..c32d56a7c7a 100644 --- a/test/org/opensolaris/opengrok/analysis/fortran/sample_xref.html +++ b/test/org/opensolaris/opengrok/analysis/fortran/sample_xref.html @@ -6,7 +6,7 @@ sampleFile - OpenGrok cross reference for /sampleFile 1* Copyright (c) 2013 Samuel Halliday +function get_sym_list(){return [["Subroutine","xsr",[["dgesv_",173]]]];} /* ]]> */1* Copyright (c) 2013 Samuel Halliday 2* Copyright (c) 1992-2011 The University of Tennessee and The University 3* of Tennessee Research Foundation. All rights 4* reserved. @@ -178,7 +178,7 @@ 170*> \ingroup doubleGEsolve 171* 172* ===================================================================== -173 SUBROUTINE DGESV( N, NRHS, A, LDA, IPIV, B, LDB, INFO ) +173 SUBROUTINE DGESV( N, NRHS, A, LDA, IPIV, B, LDB, INFO ) 174* 175* -- LAPACK driver routine (version 3.4.0) -- 176* -- LAPACK is a software package provided by Univ. of Tennessee, -- @@ -186,49 +186,49 @@ 178* November 2011 179* 180* .. Scalar Arguments .. -181 INTEGER INFO, LDA, LDB, N, NRHS +181 INTEGER INFO, LDA, LDB, N, NRHS 182* .. 183* .. Array Arguments .. -184 INTEGER IPIV( * ) -185 DOUBLE PRECISION A( LDA, * ), B( LDB, * ) +184 INTEGER IPIV( * ) +185 DOUBLE PRECISION A( LDA, * ), B( LDB, * ) 186* .. 187* 188* ===================================================================== 189* 190* .. External Subroutines .. -191 EXTERNAL DGETRF, DGETRS, XERBLA +191 EXTERNAL DGETRF, DGETRS, XERBLA 192* .. 193* .. Intrinsic Functions .. -194 INTRINSIC MAX +194 INTRINSIC MAX 195* .. 196* .. Executable Statements .. 197* 198* Test the input parameters. 199* -200 INFO = 0 + 0xFFFF - 0XFF - 0xFF00 +200 INFO = 0 + 0xFFFF - 0XFF - 0xFF00 201 IF( N.LT.0 ) THEN -202 INFO = -1 -203 ELSE IF( NRHS.LT.0 ) THEN -204 INFO = -2 -205 ELSE IF( LDA.LT.MAX( 1, N ) ) THEN -206 INFO = -4 -207 ELSE IF( LDB.LT.MAX( 1, N ) ) THEN -208 INFO = -7 +202 INFO = -1 +203 ELSE IF( NRHS.LT.0 ) THEN +204 INFO = -2 +205 ELSE IF( LDA.LT.MAX( 1, N ) ) THEN +206 INFO = -4 +207 ELSE IF( LDB.LT.MAX( 1, N ) ) THEN +208 INFO = -7 209 END IF -210 IF( INFO.NE.0 ) THEN -211 CALL XERBLA( 'DGESV ', -INFO ) +210 IF( INFO.NE.0 ) THEN +211 CALL XERBLA( 'DGESV ', -INFO ) 212 RETURN 213 END IF 214* 215* Compute the LU factorization of A. 216* -217 CALL DGETRF( N, N, A, LDA, IPIV, INFO ) -218 IF( INFO.EQ.0 ) THEN +217 CALL DGETRF( N, N, A, LDA, IPIV, INFO ) +218 IF( INFO.EQ.0 ) THEN 219* 220* Solve the system A*X = B, overwriting B with X. 221* -222 CALL DGETRS( 'No transpose', N, NRHS, A, LDA, IPIV, B, LDB, -223 $ INFO ) +222 CALL DGETRS( 'No transpose', N, NRHS, A, LDA, IPIV, B, LDB, +223 $ INFO ) 224 END IF 225 RETURN 226*