Harden Spoon Javadoc parsing and provenance

analyzer-core/pom.xml

@@ -49,6 +49,10 @@
             <groupId>fr.inria.gforge.spoon</groupId>
             <artifactId>spoon-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>fr.inria.gforge.spoon</groupId>
+            <artifactId>spoon-javadoc</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

analyzer-core/src/main/java/org/assertlab/cocomut/ContextExtractor.java

@@ -21,7 +21,7 @@
  * - Javadoc documentation (method-level and class-level)
  * - Class hierarchy (via Spoon source model, with SootUp when available)
  * - Related class methods
- * - Metrics (LOC, cyclomatic complexity)
+ * - Lexical metrics (LOC and branch-keyword complexity estimates)
  * - Combines with call graph from Phase 3
  *
  * Input: ProjectMetadata from Phase 1, MethodInfo from Phase 2, CallGraphGenerator from Phase 3
@@ -154,7 +154,9 @@ private int calculateCyclomaticComplexity(String methodBody) {
             return 1;
         }
 
-        // McCabe complexity: 1 + one decision point per branch keyword.
+        // Lexical branch-keyword estimate: 1 + one decision point per matched token.
+        // This is intentionally exposed as a coarse estimate, not an AST-level
+        // McCabe/control-flow graph measurement.
         // \bif\b already matches the `if` inside `else if`, so do NOT add a
         // separate \belse\s+if\b term — that would count each else-if twice.
         int complexity = 1;

analyzer-core/src/main/java/org/assertlab/cocomut/MethodContext.java

@@ -28,7 +28,7 @@ public class MethodContext {
     private final Map<String, String> classMethods;  // Other methods in class
     private final CallGraphResult callGraph;  // Call graph from Phase 3
     private final int linesOfCode;
-    private final int cyclomatic;             // McCabe complexity
+    private final int cyclomatic;             // lexical branch-keyword estimate
     private final List<String> annotations;
     private final List<String> thrownExceptions;
     private final List<String> fieldReads;

docs/javadoc-reference-policy.md

@@ -1,14 +1,16 @@
 # Javadoc Reference Policy
 
-CoCoMUT parses common Javadoc tags using the official Oracle/JDK doc-comment
-syntax and standard doclet model. The implementation should not add
-repository-specific parsing rules for a single project such as Apache Commons
-Lang.
+CoCoMUT parses common Javadoc tags using Spoon's official `spoon-javadoc`
+module, which follows the Oracle/JDK documentation-comment syntax and standard
+doclet model. The dependency is resolved from normal Maven providers as
+`fr.inria.gforge.spoon:spoon-javadoc`; it is not read from a local Spoon
+checkout. The implementation should not add repository-specific parsing rules
+for a single project such as Apache Commons Lang.
 
 ## Supported Tags
 
-CoCoMUT extracts structured metadata for common block and inline tags used in
-method documentation:
+CoCoMUT extracts structured metadata for common block tags used in method
+documentation:
 
 - `@param`
 - `@return`
@@ -19,6 +21,11 @@ method documentation:
 - `@implSpec`
 - `@implNote`
 - `@see`
+
+CoCoMUT also parses common inline tags as part of rendered text, reference
+metadata, and documentation metrics. They are not emitted as first-class
+`structured_tags` objects unless the schema names them explicitly:
+
 - `{@link ...}`
 - `{@linkplain ...}`
 - `{@code ...}`
@@ -42,6 +49,7 @@ Type
 Type#member
 #member
 package.Type#member(parameter.Types)
+Type#member(Type, int)
 module/package.Type#member(parameter.Types) label text
 ```
 
@@ -59,6 +67,12 @@ Javadoc text.
 Each resolved reference also gets derived taxonomy fields for empirical
 analysis:
 
+- `parser`: the parser path used for the reference, normally `spoon-javadoc`;
+- `parse_confidence`: `high` for typed Spoon references, `medium` for Spoon
+  text fallback, and `low` for CoCoMUT fallback text parsing;
+- `spoon_reference`: Spoon's typed reference rendering when available;
+- `target`: the source Javadoc spelling, preserved for auditability;
+- `canonical_target`: Spoon's normalized target when it differs from `target`;
 - `reference_target_kind`: `method`, `field`, `type`, `url`, `text`,
   `method_or_field`, or `unknown`;
 - `reference_domain`: `project`, `external_jdk`, `external_library`,
@@ -70,6 +84,30 @@ These taxonomy fields summarize CoCoMUT's resolution result. They are not
 additional Javadoc syntax and should not replace canonical method/type/field
 URIs when a project-local target is resolved.
 
+CoCoMUT keeps a fallback text parser only for cases where `spoon-javadoc` cannot
+produce reference or structured-tag elements. If Spoon parses some references
+but misses a raw reference that CoCoMUT's compatibility scanner can recognize,
+the fallback object is still emitted for coverage and auditability, marked with
+`parser=cocomut-fallback`, `parse_confidence=low`, and a fallback reason.
+
+When `spoon-javadoc` produces a typed `CtReference`, CoCoMUT resolves semantics
+from that typed Spoon reference rather than reparsing `target`. The raw
+`target` field remains the source spelling for audit and compatibility; it is
+not the authoritative semantic identity when `spoon_reference` is present.
+
+Auxiliary documentation files such as `doc-files/...`, `{@docRoot}/...`,
+`@filename ...`, and `{@snippet file="..."}` are recorded under
+`file_references`. These are path references, not program-element references.
+They use conservative project-root containment checks and include
+`parser=cocomut-file-regex`, `parse_confidence=low`, and a `source_form` marker
+describing the recognized textual form.
+
+`{@inheritDoc}` is represented as an inheritance candidate relation. CoCoMUT
+reports whether an inherited candidate exists and includes bounded inherited
+Javadoc snippets, but it does not silently expand inherited `@param`, `@return`,
+or `@throws` text into the child method's `structured_tags`. Rows that use
+inheritDoc therefore carry `inheritdoc_policy=candidate_only`.
+
 ## Resolution Policy
 
 CoCoMUT resolves project-local references in this order:

docs/symbol-model.md

@@ -198,10 +198,12 @@ Primary references:
 - JDK documentation-comment specification for the standard doclet.
 - Javadoc tool reference.
 
-These documents are versioned with the JDK. CoCoMUT implements the stable
-traditional Javadoc reference forms used by `@see`, `{@link ...}`, and
-`{@linkplain ...}`; it does not currently implement newer Markdown
-documentation-comment syntax as a separate parser mode.
+These documents are versioned with the JDK. CoCoMUT delegates program-element
+reference parsing to Spoon's official `spoon-javadoc` module from Maven
+providers, then maps the typed Spoon reference to CoCoMUT's URI and taxonomy
+model. It implements the stable traditional Javadoc reference forms used by
+`@see`, `{@link ...}`, and `{@linkplain ...}`; it does not currently implement
+newer Markdown documentation-comment syntax as a separate parser mode.
 
 Important fields:
 
@@ -210,9 +212,15 @@ kind                 type_reference|member_reference|field_reference|external_ur
 resolution           resolved_method|resolved_type|resolved_field|
                      resolved_inherited_method|resolved_inherited_field|
                      overload_ambiguous|external_symbol|external|text|unresolved
+parser               parser path, normally spoon-javadoc
+parse_confidence     high|medium|low parser confidence marker
+target               source Javadoc spelling preserved for auditability
+canonical_target     Spoon-normalized target when it differs from target
+spoon_reference      typed Spoon reference rendering used for semantic resolution
 reference_target_kind method|field|type|url|text|method_or_field|unknown
 reference_domain    project|external_jdk|external_library|external_web|text|unresolved
 reference_scope     same_type|same_package|same_module|external|text|unknown
+raw_pairing_confidence low|none when raw spelling could not be paired confidently
 method_uri           present for resolved project methods
 field_uri            present for resolved project fields
 type_uri             present for resolved project types
@@ -231,6 +239,13 @@ different project package, an external JDK/library symbol, a web URL, or a text
 reference. They do not replace canonical CoCoMUT URIs for resolved project
 symbols.
 
+Auxiliary documentation file paths are reported separately under
+`javadoc_metadata.file_references`. They are not program-element references.
+Each file-reference entry includes `parser=cocomut-file-regex`,
+`parse_confidence=low`, a `source_form` marker such as `doc_root`,
+`doc_files`, `filename_tag`, `snippet_file_attribute`, or `regex_text`, and a
+project-root-contained `resolved_path` when one can be safely formed.
+
 For project-local method references, `referenced_method` intentionally embeds a
 compact method context rather than only an excerpt. It includes the referenced
 method URI, signature, source code without leading Javadoc, method Javadoc,

pom.xml

@@ -90,6 +90,11 @@
                 <artifactId>spoon-core</artifactId>
                 <version>${spoon.version}</version>
             </dependency>
+            <dependency>
+                <groupId>fr.inria.gforge.spoon</groupId>
+                <artifactId>spoon-javadoc</artifactId>
+                <version>${spoon.version}</version>
+            </dependency>
             <dependency>
                 <groupId>junit</groupId>
                 <artifactId>junit</artifactId>

schemas/README.md

@@ -26,7 +26,7 @@ callers                   Optional caller context from SootUp call graph
 callees                   Optional callee context from SootUp call graph
 metadata                  Schema, backend, method identity, and call graph metadata
 provenance                Extraction source and confidence information
-documentation_metrics     Computed Javadoc quality flags
+documentation_metrics     Parser-backed Javadoc quality flags and parser provenance
 javadoc_metadata          Parsed @see, @since, inline links, deprecation, inheritDoc hints
 dynamic_features          Static hints for reflection, proxies, service loaders, DI, native code
 selection                 Project/method/type/package target provenance
@@ -63,11 +63,14 @@ selection. See
 Important `javadoc_metadata` fields:
 
 ```text
-see                       Raw @see targets
-inline_links              Raw inline {@link ...}/{@linkplain ...} targets
+see                       Targets from final merged javadoc_references entries tagged @see
+inline_links              Targets from final merged javadoc_references entries tagged link/linkplain
 javadoc_references        Resolved reference objects for @see/link/linkplain targets
 file_references           Referenced doc-files/images/html/text/sample-source paths when present
+                          plus parser, parse_confidence, and source_form
 structured_tags           Parsed param/return/throws/since/apiNote/implSpec/implNote/deprecated text
+                          plus parser and parse_confidence
+inheritdoc_policy         not_applicable|candidate_only
 inheritdoc_resolution     not_used|resolved_candidate|unresolved
 inherited_javadoc_candidates
                           Candidate inherited Javadoc snippets for {@inheritDoc}
@@ -79,7 +82,13 @@ inherited_javadoc_candidates
 tag                       see|link|linkplain
 raw                       Raw Javadoc reference text
 target                    Parsed reference target
+canonical_target          Spoon-normalized target when it differs from target
 label                     Optional rendered-label text
+parser                    spoon-javadoc|spoon-javadoc-text-fallback|cocomut-fallback
+parse_confidence          high|medium|low
+spoon_reference           typed Spoon reference rendering when available
+raw_pairing_confidence    low/none when raw spelling was not trusted for typed resolution
+fallback_reason           why a low-confidence fallback reference was emitted
 kind                      type_reference|member_reference|field_reference|external_url|text_reference
 resolution                resolved_type|resolved_method|resolved_field|
                           resolved_inherited_method|resolved_inherited_field|
@@ -115,6 +124,16 @@ HTML anchor links, and program-element references such as
 `module/package.Type#member label`. Module prefixes are normalized before
 symbol lookup.
 
+Javadoc reference and structured-tag parsing uses Spoon's official
+`spoon-javadoc` parser first. CoCoMUT keeps a text fallback for raw references
+that Spoon cannot represent; fallback-derived objects are marked with
+`parser=cocomut-fallback`, `parse_confidence=low`, and a fallback reason.
+
+`{@inheritDoc}` is reported as a candidate relation rather than silently
+expanded into child structured tags. When present, `inheritdoc_policy` is
+`candidate_only`, and inherited source Javadocs are exposed through
+`inherited_javadoc_candidates` for downstream inspection.
+
 External references are intentionally symbol-level only in the current schema.
 CoCoMUT does not fetch JDK/dependency source jars or generated Javadoc pages for
 external `@see` / `{@link ...}` targets, because that behavior depends heavily

schemas/method-context.schema.json

@@ -36,8 +36,14 @@
         "parameters": {"type": "array"},
         "annotations": {"type": "array"},
         "throws": {"type": "array"},
-        "lines_of_code": {"type": "integer"},
-        "cyclomatic_complexity": {"type": "integer"},
+        "lines_of_code": {
+          "type": "integer",
+          "description": "Lexical estimate derived from emitted method source; not an AST structural metric."
+        },
+        "cyclomatic_complexity": {
+          "type": "integer",
+          "description": "Lexical branch-token estimate derived from emitted method source; not an AST/control-flow McCabe metric."
+        },
         "code": {
           "type": "string",
           "description": "Method or constructor source without leading Javadoc. Method annotations are kept."
@@ -91,7 +97,20 @@
       },
       "additionalProperties": true
     },
-    "documentation_metrics": {"type": "object"},
+    "documentation_metrics": {
+      "type": "object",
+      "properties": {
+        "parser": {
+          "type": "string",
+          "description": "Parser path used for documentation metrics, normally spoon-javadoc."
+        },
+        "parse_confidence": {
+          "type": "string",
+          "description": "high for spoon-javadoc parsed metrics, low for fallback text parsing."
+        }
+      },
+      "additionalProperties": true
+    },
     "javadoc_metadata": {
       "type": "object",
       "properties": {
@@ -101,9 +120,19 @@
           "type": "array",
           "items": {"$ref": "#/$defs/javadocReference"}
         },
-        "file_references": {"type": "array"},
-        "structured_tags": {"type": "object"},
+        "file_references": {
+          "type": "array",
+          "description": "Auxiliary documentation file/path references. Entries include parser, parse_confidence, source_form, containment-checked resolved_path, and exists."
+        },
+        "structured_tags": {
+          "type": "object",
+          "description": "Parser-backed param/return/throws/since/apiNote/implSpec/implNote/deprecated tags. Includes parser and parse_confidence."
+        },
         "uses_inheritdoc": {"type": "boolean"},
+        "inheritdoc_policy": {
+          "type": "string",
+          "description": "not_applicable when inheritDoc is absent; candidate_only when inherited documentation candidates are reported without silently expanding child structured tags."
+        },
         "inheritdoc_resolution": {"type": "string"},
         "inherited_javadoc_candidates": {"type": "array"}
       },
@@ -169,8 +198,32 @@
         "tag": {"type": "string"},
         "raw": {"type": "string"},
         "target": {"type": "string"},
+        "canonical_target": {
+          "type": "string",
+          "description": "Parser-normalized target when it differs from the source Javadoc spelling preserved in target."
+        },
         "label": {"type": "string"},
         "text": {"type": "string"},
+        "parser": {
+          "type": "string",
+          "description": "Parser path used for the reference, for example spoon-javadoc or cocomut-fallback."
+        },
+        "parse_confidence": {
+          "type": "string",
+          "description": "high for typed spoon-javadoc references, medium for Spoon text fallback, low for CoCoMUT fallback text parsing."
+        },
+        "spoon_reference": {
+          "type": "string",
+          "description": "Spoon-javadoc typed reference rendering when available."
+        },
+        "raw_pairing_confidence": {
+          "type": "string",
+          "description": "Confidence for pairing legacy raw target spelling with Spoon's typed reference; low/none pairings do not drive semantic resolution."
+        },
+        "fallback_reason": {
+          "type": "string",
+          "description": "Reason a low-confidence fallback reference was emitted, for example spoon_no_references or not_represented_by_spoon."
+        },
         "kind": {"type": "string"},
         "resolution": {"type": "string"},
         "reference_target_kind": {"type": "string"},