From 1f9aac496a5802fc3286c02aa0980c765e93df02 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Wed, 22 Jun 2022 11:40:16 -0400 Subject: [PATCH 1/2] [Java.Interop.Tools.JavaSource] Fix tag parsing errors The number of "## Unable to translate remarks for ..." errors have been cut nearly in half, from 740 to 391. The rules for `@param`, `@return`, `@throws`, and `@unknown` have been updated to handle more variations of these tags. In some cases, these tags are only followed by a single word or no content at all. New line character filters have been added to `nonSpaceTerm` to fix instances where an "empty" tag would be merged with a tag on a new line. See [`Android.Database.DatabaseUtils.GetCollationKey(String)`][0]. Before `nonSpaceTerm` change: To be added. the collation key return the collation key To be added. After `nonSpaceTerm` change: name return the collation key the collation key New (mostly empty) rules for `@hide`, `@inheritDoc`, and `@{see}` tags have also been added. [0]: https://github.com/xamarin/android-api-docs/blob/a7e914965a0e80c9454149399551d87a18997b2e/docs/Mono.Android/en/Android.Database/DatabaseUtils.xml#L1505-L1508 --- ...avadocToXmldocGrammar.BlockTagsBnfTerms.cs | 47 +++++++++++++++---- ...vadocToXmldocGrammar.InlineTagsBnfTerms.cs | 15 +++++- .../SourceJavadocToXmldocParser.cs | 2 + ...cToXmldocGrammar.BlockTagsBnfTermsTests.cs | 44 +++++++++++++++-- ...ToXmldocGrammar.InlineTagsBnfTermsTests.cs | 10 ++++ .../SourceJavadocToXmldocParserTests.cs | 12 +++++ 6 files changed, 115 insertions(+), 15 deletions(-) diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs index ea43046e8..9246b18b2 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs @@ -25,6 +25,8 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) | DeprecatedDeclaration | DeprecatedSinceDeclaration | ExceptionDeclaration + | InheritDocDeclaration + | HideDeclaration | ParamDeclaration | ReturnDeclaration | SeeDeclaration @@ -78,7 +80,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = p; }; - var nonSpaceTerm = new RegexBasedTerminal ("[^ ]", "[^ ]+") { + var nonSpaceTerm = new RegexBasedTerminal ("[^ \n\r]", "[^ \n\r]+") { AstConfig = new AstNodeConfig { NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value, }, @@ -99,19 +101,42 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) FinishParse (context, parseNode); }; - ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValues; + // Ignore @hide tags + HideDeclaration.Rule = "@hide"; + HideDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + FinishParse (context, parseNode); + }; + + InheritDocDeclaration.Rule = "@inheritDoc"; + InheritDocDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.InheritDocTag)) { + return; + } + // TODO: Iterate through parents for corresponding javadoc element. + FinishParse (context, parseNode); + }; + + ParamDeclaration.Rule = "@param" + nonSpaceTerm + | "@param" + nonSpaceTerm + BlockValues + ; ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => { if (!grammar.ShouldImport (ImportJavadoc.ParamTag)) { return; } - var p = new XElement ("param", - new XAttribute ("name", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))), - AstNodeToXmlContent (parseNode.ChildNodes [2])); + var paramName = string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1])); + var p = new XElement ("param", new XAttribute ("name", paramName)); + if (parseNode.ChildNodes.Count >= 3) { + p.Add (AstNodeToXmlContent (parseNode.ChildNodes [2])); + } else { + p.Add (paramName); + } FinishParse (context, parseNode).Parameters.Add (p); parseNode.AstNode = p; }; - ReturnDeclaration.Rule = "@return" + BlockValues; + ReturnDeclaration.Rule = "@return" + | "@return" + BlockValues + ; ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => { if (!grammar.ShouldImport (ImportJavadoc.ReturnTag)) { return; @@ -158,7 +183,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = p; }; - ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValues; + ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + | "@throws" + nonSpaceTerm + BlockValues + ; ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => { if (!grammar.ShouldImport (ImportJavadoc.ExceptionTag)) { return; @@ -205,7 +232,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = parseNode.Token.Value.ToString (); - UnknownTagDeclaration.Rule = unknownTagTerminal + BlockValues; + UnknownTagDeclaration.Rule = unknownTagTerminal + | unknownTagTerminal + BlockValues + ; UnknownTagDeclaration.AstConfig.NodeCreator = (context, parseNode) => { if (!grammar.ShouldImport (ImportJavadoc.Remarks)) { return; @@ -242,6 +271,8 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) public readonly NonTerminal DeprecatedDeclaration = new NonTerminal (nameof (DeprecatedDeclaration)); public readonly NonTerminal DeprecatedSinceDeclaration = new NonTerminal (nameof (DeprecatedSinceDeclaration)); public readonly NonTerminal ExceptionDeclaration = new NonTerminal (nameof (ExceptionDeclaration)); + public readonly NonTerminal HideDeclaration = new NonTerminal (nameof (HideDeclaration)); + public readonly NonTerminal InheritDocDeclaration = new NonTerminal (nameof (InheritDocDeclaration)); public readonly NonTerminal ParamDeclaration = new NonTerminal (nameof (ParamDeclaration)); public readonly NonTerminal ReturnDeclaration = new NonTerminal (nameof (ReturnDeclaration)); public readonly NonTerminal SeeDeclaration = new NonTerminal (nameof (SeeDeclaration)); diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs index 9815661ab..3685ea7b6 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs @@ -26,6 +26,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) | LinkDeclaration | LinkplainDeclaration | LiteralDeclaration + | SeeDeclaration | ValueDeclaration ; @@ -75,6 +76,14 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = new XText (content); }; + SeeDeclaration.Rule = grammar.ToTerm ("{@see") + InlineValue + "}"; + SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + // TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see + // Also need to convert to appropriate CREF value, ignore for now + var target = parseNode.ChildNodes [1].AstNode; + parseNode.AstNode = new XElement ("c", target); + }; + ValueDeclaration.Rule = grammar.ToTerm ("{@value}") | grammar.ToTerm ("{@value") + InlineValue + "}"; ValueDeclaration.AstConfig.NodeCreator = (context, parseNode) => { @@ -114,10 +123,12 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) public readonly NonTerminal LinkplainDeclaration = new NonTerminal (nameof (LinkplainDeclaration)); // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#literal - public readonly NonTerminal LiteralDeclaration = new NonTerminal (nameof (LinkplainDeclaration)); + public readonly NonTerminal LiteralDeclaration = new NonTerminal (nameof (LiteralDeclaration)); + + public readonly NonTerminal SeeDeclaration = new NonTerminal (nameof (SeeDeclaration)); // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#value - public readonly NonTerminal ValueDeclaration = new NonTerminal (nameof (ValueDeclaration)); + public readonly NonTerminal ValueDeclaration = new NonTerminal (nameof (ValueDeclaration)); } } } diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs index 4f29894cb..af671967d 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs @@ -26,6 +26,7 @@ internal enum ImportJavadoc { SinceTag = 1 << 9, VersionTag = 1 << 10, ExtraRemarks = 1 << 11, + InheritDocTag = 1 << 12, } [Flags] @@ -43,6 +44,7 @@ public enum XmldocStyle { | ImportJavadoc.SinceTag | ImportJavadoc.VersionTag | ImportJavadoc.ExtraRemarks + | ImportJavadoc.InheritDocTag , IntelliSense = ImportJavadoc.Summary | ImportJavadoc.ExceptionTag diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs index c695de2be..323fd8b34 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs @@ -56,14 +56,39 @@ public void ExceptionDeclaration () //Assert.AreEqual ("Just Because.", r.Root.AstNode.ToString ()); } + [Test] + public void InheritDocDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.InheritDocDeclaration); + + var r = p.Parse ("@inheritDoc"); + Assert.IsFalse (r.HasErrors (), "@inheritDoc: " + DumpMessages (r, p)); + // TODO: Enable after adding support for @inheritDoc + Assert.IsNull (r.Root.AstNode, "@inheritDoc should be ignored, but node was not null."); + } + + [Test] + public void HideDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.HideDeclaration); + + var r = p.Parse ("@hide"); + Assert.IsFalse (r.HasErrors (), "@hide: " + DumpMessages (r, p)); + Assert.IsNull (r.Root.AstNode, "@hide should be ignored, but node was not null."); + } + [Test] public void ParamDeclaration () { var p = CreateParser (g => g.BlockTagsTerms.ParamDeclaration); - var r = p.Parse ("@param a Insert description here\n"); + var r = p.Parse ("@param a Insert description here\nand here."); Assert.IsFalse (r.HasErrors (), "@param: " + DumpMessages (r, p)); - Assert.AreEqual ("Insert description here", r.Root.AstNode.ToString ()); + Assert.AreEqual ($"Insert description here{Environment.NewLine}and here.", r.Root.AstNode.ToString ()); + + r = p.Parse ("@param b"); + Assert.IsFalse (r.HasErrors (), "name only @param: " + DumpMessages (r, p)); + Assert.AreEqual ("b", r.Root.AstNode.ToString ()); } [Test] @@ -120,16 +145,21 @@ public void ThrowsDeclaration () var p = CreateParser (g => g.BlockTagsTerms.ThrowsDeclaration); var r = p.Parse ("@throws Throwable the {@code Exception} raised by this method"); - Assert.IsFalse (r.HasErrors (), "@throws: " + DumpMessages (r, p)); + Assert.IsFalse (r.HasErrors (), "{@code} @throws: " + DumpMessages (r, p)); Assert.IsNull (r.Root.AstNode, "@throws should be ignored, but node with code block was not null."); // TODO: Re-enable when we can generate valid crefs //Assert.AreEqual ("the Exception raised by this method", r.Root.AstNode.ToString ()); r = p.Parse ("@throws Throwable something or other!"); - Assert.IsFalse (r.HasErrors (), "@throws: " + DumpMessages (r, p)); - Assert.IsNull (r.Root.AstNode, "@throws should be ignored, but node was not null."); + Assert.IsFalse (r.HasErrors (), " @throws: " + DumpMessages (r, p)); + Assert.IsNull (r.Root.AstNode, "@throws should be ignored, but node with was not null."); // TODO: Re-enable when we can generate valid crefs //Assert.AreEqual ("something or other!", r.Root.AstNode.ToString ()); + + r = p.Parse ("@throws android.content.ActivityNotFoundException"); + Assert.IsFalse (r.HasErrors (), "name only @throws: " + DumpMessages (r, p)); + // TODO: Re-enable when we can generate valid crefs + Assert.IsNull (r.Root.AstNode, "@throws should be ignored, but name only node was not null."); } [Test] @@ -140,6 +170,10 @@ public void UnknownTagDeclaration () var r = p.Parse ("@this-is-not-supported something {@code foo} else."); Assert.IsFalse (r.HasErrors (), "@this-is-not-supported: " + DumpMessages (r, p)); Assert.AreEqual (null, r.Root.AstNode); + + r = p.Parse ("@standalonetag"); + Assert.IsFalse (r.HasErrors (), "@standalonetag: " + DumpMessages (r, p)); + Assert.AreEqual (null, r.Root.AstNode); } } } diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs index 869a90d6c..a1d36378e 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs @@ -75,6 +75,16 @@ public void LiteralDeclaration () Assert.AreEqual ("A<B>C", r.Root.AstNode.ToString ()); } + [Test] + public void SeeDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.SeeDeclaration); + + var r = p.Parse ("{@see #cancelNotification(String, String, int)}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("#cancelNotification(String, String, int)", r.Root.AstNode.ToString ()); + } + [Test] public void ValueDeclaration () { diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs index fa951fc4d..14255216e 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs @@ -104,11 +104,15 @@ What about soft paragraphs?

What about hard paragraphs? @param a something +@param b +@param c @see #method() @apiSince 1 ", FullXml = @" something + b + c

This is the summary sentence. This is the summary sentence. Insert @@ -120,6 +124,8 @@ more description here. ", IntelliSenseXml = @" something + b + c This is the summary sentence. ", }, @@ -183,19 +189,25 @@ more description here. @param manifest The value of the {@code android:versionCode} manifest attribute. +@param empty +@return the return value ", FullXml = $@" The value of the android:versionCode manifest attribute. + empty See accept(2). See accept(2). Insert more description here. How about another link accept(2) + the return value ", IntelliSenseXml = $@" The value of the android:versionCode manifest attribute. + empty See accept(2). + the return value ", }, }; From eed53cd30c4f2bd522fbc3bf6adc8c9693f9ca06 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 30 Jun 2022 13:56:32 -0400 Subject: [PATCH 2/2] Add tab to nonSpaceTerm regex --- .../SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs index 9246b18b2..54a0d1be0 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs @@ -80,7 +80,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = p; }; - var nonSpaceTerm = new RegexBasedTerminal ("[^ \n\r]", "[^ \n\r]+") { + var nonSpaceTerm = new RegexBasedTerminal ("[^ \n\r\t]", "[^ \n\r\t]+") { AstConfig = new AstNodeConfig { NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value, },