normalize fortran variables like gfortran

oracle · Apr 17, 2018 · 00c47b8 · 00c47b8
1 parent acee724
commit 00c47b8
Show file tree

Hide file tree

Showing 13 changed files with 112 additions and 39 deletions.
diff --git a/src/org/opensolaris/opengrok/analysis/Ctags.java b/src/org/opensolaris/opengrok/analysis/Ctags.java
@@ -34,6 +34,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
@@ -61,6 +62,8 @@ public class Ctags implements Resettable {
     private String CTagsExtraOptionsFile = null;
     private int tabSize;
 
+    private Function<String, String> normalizeIdentifier = str -> str;
+
     private boolean junit_testing = false;
 
     /**
@@ -89,6 +92,10 @@ public void setTabSize(int tabSize) {
         this.tabSize = tabSize;
     }
 
+    public void setNormalizeIdentifier(Function<String, String> normalizeIdentifier) {
+        this.normalizeIdentifier = normalizeIdentifier;
+    }
+
     public void setCTagsExtraOptionsFile(String CTagsExtraOptionsFile) {
         this.CTagsExtraOptionsFile = CTagsExtraOptionsFile;
     }
@@ -380,6 +387,7 @@ public Definitions doCtags(String file) throws IOException,
         CtagsReader rdr = new CtagsReader();
         rdr.setSplitterSupplier(() -> { return trySplitSource(file); });
         rdr.setTabSize(tabSize);
+        rdr.setNormalizeIdentifier(normalizeIdentifier);
         Definitions ret;
         try {
             ctagsIn.write(file + "\n");
@@ -443,6 +451,7 @@ public void destroy() {
 
         CtagsReader rdr = new CtagsReader();
         rdr.setTabSize(tabSize);
+        rdr.setNormalizeIdentifier(normalizeIdentifier);
         try {
             readTags(rdr);
         } catch (InterruptedException ex) {

diff --git a/src/org/opensolaris/opengrok/analysis/CtagsReader.java b/src/org/opensolaris/opengrok/analysis/CtagsReader.java
@@ -25,6 +25,7 @@
 package org.opensolaris.opengrok.analysis;
 
 import java.util.EnumMap;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -93,6 +94,12 @@ public class CtagsReader {
 
     private int tabSize;
 
+    private Function<String, String> normalizeIdentifier = str -> str;
+
+    public void setNormalizeIdentifier(Function<String, String> normalize) {
+        normalizeIdentifier = normalize;
+    }
+
     /**
      * This should mimic
      * https://github.com/universal-ctags/ctags/blob/master/docs/format.rst or
@@ -209,7 +216,7 @@ public void readLine(String tagLine) {
             //log.fine("SKIPPING LINE - NO TAB");
             return;
         }
-        String def = tagLine.substring(0, p);
+        String def = normalizeIdentifier.apply(tagLine.substring(0, p));
         int mstart = tagLine.indexOf('\t', p + 1);
 
         String kind = null;
@@ -326,7 +333,7 @@ public void readLine(String tagLine) {
                     name = arg;
                 }
                 if (name != null) {
-                    addTag(defs, cidx.lineno, name, "argument", def.trim() +
+                    addTag(defs, cidx.lineno, normalizeIdentifier.apply(name), "argument", def.trim() +
                         signature.trim(), null, signature, cidx.lineStart,
                         cidx.lineEnd);
                 } else {

diff --git a/src/org/opensolaris/opengrok/analysis/JFlexNonXref.java b/src/org/opensolaris/opengrok/analysis/JFlexNonXref.java
@@ -473,7 +473,7 @@ protected boolean writeSymbol(String symbol, Set<String> keywords, int line,
     protected boolean writeSymbol(String symbol, Set<String> keywords, int line,
         boolean caseSensitive, boolean isKeyword) throws IOException {
         return JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-            symbol, keywords, line, caseSensitive, isKeyword);
+            symbol, symbol, keywords, line, caseSensitive, isKeyword);
     }
 
     /**

diff --git a/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java b/src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java
@@ -38,6 +38,8 @@ public abstract class JFlexSymbolMatcher extends JFlexStateStacker
     private NonSymbolMatchedListener nonSymbolListener;
     private String disjointSpanClassName;
 
+    protected String normalizeIdentifier(String id) { return id; }
+
     /**
      * Associates the specified listener, replacing the former one.
      * @param l defined instance
@@ -97,7 +99,7 @@ protected String getDisjointSpanClassName() {
     protected void onSymbolMatched(String str, int start) {
         SymbolMatchedListener l = symbolListener;
         if (l != null) {
-            SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, start,
+            SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, normalizeIdentifier(str), start,
                 start + str.length());
             l.symbolMatched(evt);
         }

diff --git a/src/org/opensolaris/opengrok/analysis/JFlexXref.java b/src/org/opensolaris/opengrok/analysis/JFlexXref.java
@@ -203,7 +203,7 @@ public void setFoldingEnabled(boolean foldingEnabled) {
     public void symbolMatched(SymbolMatchedEvent evt) {
         try {
             JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-                evt.getStr(), null, matcher.getLineNumber(), false, false);
+                evt.getStr(), evt.getNormalizedStr(), null, matcher.getLineNumber(), false, false);
         } catch (IOException ex) {
             throw new RuntimeException(ex);
         }
@@ -292,7 +292,7 @@ public void linkageMatched(LinkageMatchedEvent evt) {
                     break;
                 case LABELDEF:
                     // Only PowerShell seems to be using this.
-                    JFlexXrefUtils.writeSameFileLinkSymbol(out, str);
+                    JFlexXrefUtils.writeSameFileLinkSymbol(out, str, str);
                     break;
                 case FILELIKE:
                     out.write("<a href=\"");
@@ -585,7 +585,7 @@ public void startNewLine() throws IOException {
      */
     protected void writeKeyword(String symbol, int line) throws IOException {
         JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-            symbol, null, line, false, true);
+            symbol, symbol, null, line, false, true);
     }
 
     /**

diff --git a/src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java b/src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java
@@ -218,6 +218,7 @@ public static void writeEMailAddress(Writer out, String address)
      * @param urlPrefix a defined instance
      * @param project a possibly defined instance or null
      * @param symbol the symbol to write
+     * @param id the symbol to write, normalized according to language-specific conventions
      * @param keywords a set of keywords recognized by this analyzer (no links
      * will be generated if the symbol is a keyword)
      * @param line the line number on which the symbol appears
@@ -229,7 +230,7 @@ public static void writeEMailAddress(Writer out, String address)
      * @throws IOException if an error occurs while writing to the stream
      */
     public static boolean writeSymbol(Writer out, Definitions defs,
-        String urlPrefix, Project project, String symbol, Set<String> keywords,
+        String urlPrefix, Project project, String symbol, String id, Set<String> keywords,
         int line, boolean caseSensitive, boolean isKeyword)
             throws IOException {
         String[] strs = new String[1];
@@ -244,7 +245,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             return false;
         }
 
-        if (defs != null && defs.hasDefinitionAt(symbol, line, strs)) {
+        if (defs != null && defs.hasDefinitionAt(id, line, strs)) {
             // This is the definition of the symbol.
             String type = strs[0];
             String style_class = "d";
@@ -268,15 +269,15 @@ public static boolean writeSymbol(Writer out, Definitions defs,
                 out.append("<a class=\"");
                 out.append(style_class);
                 out.append("\" name=\"");
-                Util.htmlize(symbol, out);
+                Util.htmlize(id, out);
                 out.append("\"/>");
             }
 
             // 2) Create a link that searches for all references to this symbol.
             out.append("<a href=\"");
             out.append(urlPrefix);
             out.append("refs=");
-            Util.qurlencode(symbol, out);
+            Util.qurlencode(id, out);
             appendProject(out, project);
             out.append("\" class=\"");
             out.append(style_class);
@@ -285,8 +286,8 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             out.append(">");
             Util.htmlize(symbol, out);
             out.append("</a>");
-        } else if (defs != null && defs.occurrences(symbol) == 1) {
-            writeSameFileLinkSymbol(out, symbol);
+        } else if (defs != null && defs.occurrences(id) == 1) {
+            writeSameFileLinkSymbol(out, symbol, id);
         } else {
             // This is a symbol that is not defined in this file, or a symbol
             // that is defined more than once in this file. In either case, we
@@ -295,7 +296,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             out.append("<a href=\"");
             out.append(urlPrefix);
             out.append("defs=");
-            Util.qurlencode(symbol, out);
+            Util.qurlencode(id, out);
             appendProject(out, project);
             out.append("\"");
             out.append(" class=\"intelliWindow-symbol\"");
@@ -312,10 +313,11 @@ public static boolean writeSymbol(Writer out, Definitions defs,
      * exactly one location in the same file.
      * @param out a defined, target instance
      * @param symbol the symbol to write
+     * @param id the symbol to write, normalized according to language-specific conventions
      * @throws IOException if {@link Writer#append(java.lang.CharSequence)}
      * fails
      */
-    public static void writeSameFileLinkSymbol(Writer out, String symbol)
+    public static void writeSameFileLinkSymbol(Writer out, String symbol, String id)
             throws IOException {
         // This is a reference to a symbol defined exactly once in this file.
         String style_class = "d";
@@ -324,7 +326,7 @@ public static void writeSameFileLinkSymbol(Writer out, String symbol)
         out.append("<a class=\"");
         out.append(style_class);
         out.append(" intelliWindow-symbol\" href=\"#");
-        Util.URIEncode(symbol, out);
+        Util.URIEncode(id, out);
         out.append("\"");
         out.append(" data-definition-place=\"defined-in-file\"");
         out.append(">");

diff --git a/src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java b/src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java
@@ -36,19 +36,22 @@ public class SymbolMatchedEvent {
 
     private final Object source;
     private final String str;
+    private final String normalizedStr;
     private final int start;
     private final int end;
 
     /**
      * Initializes an immutable instance of {@link SymbolMatchedEvent}.
      * @param source the event source
      * @param str the symbol string
+     * @param normalizedStr the symbol string, normalized according to language-specific conventions
      * @param start the symbol start position
      * @param end the symbol end position
      */
-    public SymbolMatchedEvent(Object source, String str, int start, int end) {
+    public SymbolMatchedEvent(Object source, String str, String normalizedStr, int start, int end) {
         this.source = source;
         this.str = str;
+        this.normalizedStr = normalizedStr;
         this.start = start;
         this.end = end;
     }
@@ -69,6 +72,14 @@ public String getStr() {
         return str;
     }
 
+    /**
+     * Gets the normalized symbol string.
+     * @return the initial value
+     */
+    public String getNormalizedStr() {
+        return normalizedStr;
+    }
+
     /**
      * Gets the symbol start position.
      * @return the initial value

diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java b/src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java
@@ -24,6 +24,7 @@
 package org.opensolaris.opengrok.analysis.fortran;
 
 import java.io.Reader;
+import org.opensolaris.opengrok.analysis.Ctags;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 import org.opensolaris.opengrok.analysis.JFlexXref;
@@ -36,6 +37,10 @@
  */
 public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
 
+    public static String normalizeIdentifier(String id) {
+        return id.toLowerCase() + "_";
+    }
+
     FortranAnalyzer(FortranAnalyzerFactory factory) {
         super(factory, new JFlexTokenizer(new FortranSymbolTokenizer(
             FileAnalyzer.dummyReader)));
@@ -50,4 +55,12 @@ public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
     protected JFlexXref newXref(Reader reader) {
         return new JFlexXref(new FortranXref(reader));
     }
+
+    @Override
+    public void setCtags(Ctags ctags) {
+        this.ctags = ctags;
+        if (this.ctags != null) {
+            this.ctags.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
+        }
+    }
 }
diff --git a/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex b/src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex
@@ -60,6 +60,9 @@ import org.opensolaris.opengrok.web.HtmlConsts;
                 break;
         }
     }
+
+    @Override
+    protected String normalizeIdentifier(String id) { return FortranAnalyzer.normalizeIdentifier(id); }
 %}
 
 File = [a-zA-Z]{FNameChar}* ".inc"

diff --git a/src/org/opensolaris/opengrok/search/context/Context.java b/src/org/opensolaris/opengrok/search/context/Context.java
@@ -37,15 +37,20 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.opensolaris.opengrok.analysis.AnalyzerGuru;
 import org.opensolaris.opengrok.analysis.Definitions;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
+import org.opensolaris.opengrok.analysis.FileAnalyzerFactory;
+import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzer;
+import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.Scopes;
 import org.opensolaris.opengrok.analysis.Scopes.Scope;
 import org.opensolaris.opengrok.analysis.plain.PlainAnalyzerFactory;
@@ -416,6 +421,10 @@ public boolean getContext(Reader in, Writer out, String urlPrefix,
             String token;
             int matchState;
             int matchedLines = 0;
+            FileAnalyzerFactory factory = AnalyzerGuru.find(path);
+            if (factory instanceof FortranAnalyzerFactory) {
+                tokens.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
+            }
             while ((token = tokens.yylex()) != null && (!lim ||
                     matchedLines < limit_max_lines)) {
                 for (int i = 0; i < m.length; i++) {
@@ -455,6 +464,7 @@ public boolean getContext(Reader in, Writer out, String urlPrefix,
                 }
             }
         }
+        tokens.resetNormalizeIdentifier();
         return anything;
     }
 }
diff --git a/src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex b/src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex
@@ -32,6 +32,7 @@ import java.io.Reader;
 import java.io.Writer;
 import java.util.List;
 import java.util.TreeMap;
+import java.util.function.Function;
 import org.opensolaris.opengrok.search.Hit;
 import org.opensolaris.opengrok.web.Util;
 import org.opensolaris.opengrok.analysis.Scopes;
@@ -72,6 +73,14 @@ import org.opensolaris.opengrok.analysis.Scopes.Scope;
   boolean alt;
   Scopes scopes = null;
 
+  Function<String, String> normalizeIdentifier = str -> str;
+  public void setNormalizeIdentifier(Function<String, String> normalizeIdentifier) {
+    this.normalizeIdentifier = normalizeIdentifier;
+  }
+  public void resetNormalizeIdentifier() {
+    this.normalizeIdentifier = str -> str;
+  }
+
   /**
    * Set the writer that should receive all output
    * @param out The new writer to write to
@@ -400,7 +409,13 @@ Printable = [\@\$\%\^\&\-+=\?\.\:]
 
 
 %%
-{Identifier}|{Number}|{Printable}    {
+{Identifier} {
+    String text = yytext();
+    markedContents.append(text);
+    return normalizeIdentifier.apply(text);
+}
+
+{Number}|{Printable}    {
     String text = yytext();
     markedContents.append(text);
     return text;

diff --git a/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java b/test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java
@@ -116,6 +116,7 @@ private Definitions getTagsDefinitions() throws IOException {
             res, "UTF-8"));
 
         CtagsReader rdr = new CtagsReader();
+        rdr.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
         String line;
         while ((line = in.readLine()) != null) {
             rdr.readLine(line);