normalize fortran identifiers like gfortran

src/org/opensolaris/opengrok/analysis/Ctags.java

@@ -36,6 +36,10 @@
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.opensolaris.opengrok.analysis.AnalyzerGuru;
+import org.opensolaris.opengrok.analysis.FileAnalyzerFactory;
+import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzer;
+import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzerFactory;
 import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
 import org.opensolaris.opengrok.logger.LoggerFactory;
 import org.opensolaris.opengrok.util.IOUtils;
@@ -380,6 +384,10 @@ public Definitions doCtags(String file) throws IOException,
         CtagsReader rdr = new CtagsReader();
         rdr.setSplitterSupplier(() -> { return trySplitSource(file); });
         rdr.setTabSize(tabSize);
+        FileAnalyzerFactory factory = AnalyzerGuru.find(file);
+        if (factory instanceof FortranAnalyzerFactory) {
+            rdr.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
+        }
         Definitions ret;
         try {
             ctagsIn.write(file + "\n");

src/org/opensolaris/opengrok/analysis/CtagsReader.java

@@ -25,6 +25,7 @@
 package org.opensolaris.opengrok.analysis;
 
 import java.util.EnumMap;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -93,6 +94,12 @@ public class CtagsReader {
 
     private int tabSize;
 
+    private Function<String, String> normalizeIdentifier = str -> str;
+
+    public void setNormalizeIdentifier(Function<String, String> normalize) {
+        this.normalizeIdentifier = normalize;
+    }
+
     /**
      * This should mimic
      * https://github.com/universal-ctags/ctags/blob/master/docs/format.rst or
@@ -141,7 +148,7 @@ public enum tagFields {
          * VALIDATION happens of input - but then we gain LOTS of speed, due to
          * not comparing the same field names again and again fully.
          */
-        public static int charCmpEndOffset = 0;
+        public static int charCmpEndOffset = 1; // Need to distinguish FORTRAN's subroutine from signature
 
         /**
          * Quickly get if the field name matches allowed/consumed ones
@@ -279,8 +286,10 @@ public void readLine(String tagLine) {
                 " I will continue with line # 0) for symbol {0}", def);
         }
 
+        // NOTE: bestIndexOfTag searches the source, so it needs the
+        // precise, non-normalized symbol.
         CpatIndex cidx = bestIndexOfTag(lineno, whole, def);
-        addTag(defs, cidx.lineno, def, type, match, classInher, signature,
+        addTag(defs, cidx.lineno, normalizeIdentifier.apply(def), type, match, classInher, signature,
             cidx.lineStart, cidx.lineEnd);
 
         String[] args;
@@ -326,7 +335,7 @@ public void readLine(String tagLine) {
                     name = arg;
                 }
                 if (name != null) {
-                    addTag(defs, cidx.lineno, name, "argument", def.trim() +
+                    addTag(defs, cidx.lineno, normalizeIdentifier.apply(name), "argument", def.trim() +
                         signature.trim(), null, signature, cidx.lineStart,
                         cidx.lineEnd);
                 } else {

src/org/opensolaris/opengrok/analysis/JFlexNonXref.java

@@ -473,7 +473,7 @@ protected boolean writeSymbol(String symbol, Set<String> keywords, int line,
     protected boolean writeSymbol(String symbol, Set<String> keywords, int line,
         boolean caseSensitive, boolean isKeyword) throws IOException {
         return JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-            symbol, keywords, line, caseSensitive, isKeyword);
+            symbol, symbol, keywords, line, caseSensitive, isKeyword);
     }
 
     /**

src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java

@@ -38,6 +38,8 @@ public abstract class JFlexSymbolMatcher extends JFlexStateStacker
     private NonSymbolMatchedListener nonSymbolListener;
     private String disjointSpanClassName;
 
+    public String normalizeIdentifier(String id) { return id; }
+
     /**
      * Associates the specified listener, replacing the former one.
      * @param l defined instance
@@ -97,7 +99,7 @@ protected String getDisjointSpanClassName() {
     protected void onSymbolMatched(String str, int start) {
         SymbolMatchedListener l = symbolListener;
         if (l != null) {
-            SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, start,
+            SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, normalizeIdentifier(str), start,
                 start + str.length());
             l.symbolMatched(evt);
         }

src/org/opensolaris/opengrok/analysis/JFlexTokenizer.java

@@ -105,7 +105,7 @@ public final boolean incrementToken() throws IOException {
      */
     @Override
     public void symbolMatched(SymbolMatchedEvent evt) {
-        setAttribs(evt.getStr(), evt.getStart(), evt.getEnd());
+        setAttribs(evt.getNormalizedStr(), evt.getStart(), evt.getEnd());
     }
 
     /**

src/org/opensolaris/opengrok/analysis/JFlexXref.java

@@ -203,7 +203,7 @@ public void setFoldingEnabled(boolean foldingEnabled) {
     public void symbolMatched(SymbolMatchedEvent evt) {
         try {
             JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-                evt.getStr(), null, matcher.getLineNumber(), false, false);
+                evt.getStr(), evt.getNormalizedStr(), null, matcher.getLineNumber(), false, false);
         } catch (IOException ex) {
             throw new RuntimeException(ex);
         }
@@ -292,7 +292,7 @@ public void linkageMatched(LinkageMatchedEvent evt) {
                     break;
                 case LABELDEF:
                     // Only PowerShell seems to be using this.
-                    JFlexXrefUtils.writeSameFileLinkSymbol(out, str);
+                    JFlexXrefUtils.writeSameFileLinkSymbol(out, str, str);
                     break;
                 case FILELIKE:
                     out.write("<a href=\"");
@@ -585,7 +585,7 @@ public void startNewLine() throws IOException {
      */
     protected void writeKeyword(String symbol, int line) throws IOException {
         JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
-            symbol, null, line, false, true);
+            symbol, symbol, null, line, false, true);
     }
 
     /**

src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java

@@ -229,6 +229,7 @@ public static void writeEMailAddress(Writer out, String address)
      * @param urlPrefix a defined instance
      * @param project a possibly defined instance or null
      * @param symbol the symbol to write
+     * @param id the symbol to write, normalized according to language-specific conventions
      * @param keywords a set of keywords recognized by this analyzer (no links
      * will be generated if the symbol is a keyword)
      * @param line the line number on which the symbol appears
@@ -240,7 +241,7 @@ public static void writeEMailAddress(Writer out, String address)
      * @throws IOException if an error occurs while writing to the stream
      */
     public static boolean writeSymbol(Writer out, Definitions defs,
-        String urlPrefix, Project project, String symbol, Set<String> keywords,
+        String urlPrefix, Project project, String symbol, String id, Set<String> keywords,
         int line, boolean caseSensitive, boolean isKeyword)
             throws IOException {
         String[] strs = new String[1];
@@ -255,7 +256,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             return false;
         }
 
-        if (defs != null && defs.hasDefinitionAt(symbol, line, strs)) {
+        if (defs != null && defs.hasDefinitionAt(id, line, strs)) {
             // This is the definition of the symbol.
             String type = strs[0];
             String style_class = "d";
@@ -281,15 +282,15 @@ public static boolean writeSymbol(Writer out, Definitions defs,
                 out.append("<a class=\"");
                 out.append(style_class);
                 out.append("\" name=\"");
-                Util.htmlize(symbol, out);
+                Util.htmlize(id, out);
                 out.append("\"/>");
             }
 
             // 2) Create a link that searches for all references to this symbol.
             out.append("<a href=\"");
             out.append(urlPrefix);
             out.append("refs=");
-            Util.qurlencode(symbol, out);
+            Util.qurlencode(id, out);
             appendProject(out, project);
             out.append("\" class=\"");
             out.append(style_class);
@@ -298,8 +299,8 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             out.append(">");
             Util.htmlize(symbol, out);
             out.append("</a>");
-        } else if (defs != null && defs.occurrences(symbol) == 1) {
-            writeSameFileLinkSymbol(out, symbol);
+        } else if (defs != null && defs.occurrences(id) == 1) {
+            writeSameFileLinkSymbol(out, symbol, id);
         } else {
             // This is a symbol that is not defined in this file, or a symbol
             // that is defined more than once in this file. In either case, we
@@ -308,7 +309,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
             out.append("<a href=\"");
             out.append(urlPrefix);
             out.append("defs=");
-            Util.qurlencode(symbol, out);
+            Util.qurlencode(id, out);
             appendProject(out, project);
             out.append("\"");
             out.append(" class=\"intelliWindow-symbol\"");
@@ -325,10 +326,11 @@ public static boolean writeSymbol(Writer out, Definitions defs,
      * exactly one location in the same file.
      * @param out a defined, target instance
      * @param symbol the symbol to write
+     * @param id the symbol to write, normalized according to language-specific conventions
      * @throws IOException if {@link Writer#append(java.lang.CharSequence)}
      * fails
      */
-    public static void writeSameFileLinkSymbol(Writer out, String symbol)
+    public static void writeSameFileLinkSymbol(Writer out, String symbol, String id)
             throws IOException {
         // This is a reference to a symbol defined exactly once in this file.
         String style_class = "d";
@@ -337,7 +339,7 @@ public static void writeSameFileLinkSymbol(Writer out, String symbol)
         out.append("<a class=\"");
         out.append(style_class);
         out.append(" intelliWindow-symbol\" href=\"#");
-        Util.URIEncode(symbol, out);
+        Util.URIEncode(id, out);
         out.append("\"");
         out.append(" data-definition-place=\"defined-in-file\"");
         out.append(">");

src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java

@@ -36,19 +36,22 @@ public class SymbolMatchedEvent {
 
     private final Object source;
     private final String str;
+    private final String normalizedStr;
     private final int start;
     private final int end;
 
     /**
      * Initializes an immutable instance of {@link SymbolMatchedEvent}.
      * @param source the event source
      * @param str the symbol string
+     * @param normalizedStr the symbol string, normalized according to language-specific conventions
      * @param start the symbol start position
      * @param end the symbol end position
      */
-    public SymbolMatchedEvent(Object source, String str, int start, int end) {
+    public SymbolMatchedEvent(Object source, String str, String normalizedStr, int start, int end) {
         this.source = source;
         this.str = str;
+        this.normalizedStr = normalizedStr;
         this.start = start;
         this.end = end;
     }
@@ -69,6 +72,14 @@ public String getStr() {
         return str;
     }
 
+    /**
+     * Gets the normalized symbol string.
+     * @return the initial value
+     */
+    public String getNormalizedStr() {
+        return normalizedStr;
+    }
+
     /**
      * Gets the symbol start position.
      * @return the initial value

src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java

@@ -24,6 +24,7 @@
 package org.opensolaris.opengrok.analysis.fortran;
 
 import java.io.Reader;
+import org.opensolaris.opengrok.analysis.Ctags;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
 import org.opensolaris.opengrok.analysis.JFlexTokenizer;
 import org.opensolaris.opengrok.analysis.JFlexXref;
@@ -36,6 +37,10 @@
  */
 public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
 
+    public static String normalizeIdentifier(String id) {
+        return id.toLowerCase() + "_";
+    }
+
     FortranAnalyzer(FortranAnalyzerFactory factory) {
         super(factory, new JFlexTokenizer(new FortranSymbolTokenizer(
             FileAnalyzer.dummyReader)));

src/org/opensolaris/opengrok/analysis/fortran/FortranSymbolTokenizer.lex

@@ -38,6 +38,11 @@ import org.opensolaris.opengrok.analysis.JFlexSymbolMatcher;
 %include CommonLexer.lexh
 %char
 
+%{
+  @Override
+  public String normalizeIdentifier(String id) { return FortranAnalyzer.normalizeIdentifier(id); }
+%}
+
 // (OK to exclude LCOMMENT state used in FortranXref.)
 %state STRING SCOMMENT QSTRING
 

src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex

@@ -60,6 +60,9 @@ import org.opensolaris.opengrok.web.HtmlConsts;
                 break;
         }
     }
+
+    @Override
+    public String normalizeIdentifier(String id) { return FortranAnalyzer.normalizeIdentifier(id); }
 %}
 
 File = [a-zA-Z]{FNameChar}* ".inc"

src/org/opensolaris/opengrok/search/context/Context.java

@@ -37,15 +37,20 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.opensolaris.opengrok.analysis.AnalyzerGuru;
 import org.opensolaris.opengrok.analysis.Definitions;
 import org.opensolaris.opengrok.analysis.FileAnalyzer;
+import org.opensolaris.opengrok.analysis.FileAnalyzerFactory;
+import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzer;
+import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzerFactory;
 import org.opensolaris.opengrok.analysis.Scopes;
 import org.opensolaris.opengrok.analysis.Scopes.Scope;
 import org.opensolaris.opengrok.analysis.plain.PlainAnalyzerFactory;
@@ -416,6 +421,10 @@ public boolean getContext(Reader in, Writer out, String urlPrefix,
             String token;
             int matchState;
             int matchedLines = 0;
+            FileAnalyzerFactory factory = AnalyzerGuru.find(path);
+            if (factory instanceof FortranAnalyzerFactory) {
+                tokens.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
+            }
             while ((token = tokens.yylex()) != null && (!lim ||
                     matchedLines < limit_max_lines)) {
                 for (int i = 0; i < m.length; i++) {
@@ -455,6 +464,7 @@ public boolean getContext(Reader in, Writer out, String urlPrefix,
                 }
             }
         }
+        tokens.resetNormalizeIdentifier();
         return anything;
     }
 }

src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex

@@ -32,6 +32,7 @@ import java.io.Reader;
 import java.io.Writer;
 import java.util.List;
 import java.util.TreeMap;
+import java.util.function.Function;
 import org.opensolaris.opengrok.search.Hit;
 import org.opensolaris.opengrok.web.Util;
 import org.opensolaris.opengrok.analysis.Scopes;
@@ -72,6 +73,14 @@ import org.opensolaris.opengrok.analysis.Scopes.Scope;
   boolean alt;
   Scopes scopes = null;
 
+  Function<String, String> normalizeIdentifier = str -> str;
+  public void setNormalizeIdentifier(Function<String, String> normalizeIdentifier) {
+    this.normalizeIdentifier = normalizeIdentifier;
+  }
+  public void resetNormalizeIdentifier() {
+    this.normalizeIdentifier = str -> str;
+  }
+
   /**
    * Set the writer that should receive all output
    * @param out The new writer to write to
@@ -400,7 +409,13 @@ Printable = [\@\$\%\^\&\-+=\?\.\:]
 
 
 %%
-{Identifier}|{Number}|{Printable}    {
+{Identifier} {
+    String text = yytext();
+    markedContents.append(text);
+    return normalizeIdentifier.apply(text);
+}
+
+{Number}|{Printable}    {
     String text = yytext();
     markedContents.append(text);
     return text;