Allow production lists to refer to tokens from other production groups.

doc/usage/restructuredtext/directives.rst

@@ -1206,20 +1206,29 @@ the definition of the symbol.  There is this directive:
    the following definition.  If the definition spans multiple lines, each
    continuation line must begin with a colon placed at the same column as in
    the first line.
+   Blank lines are not allowed within ``productionlist`` directive arguments.
+
+   The definition can contain token names which are marked as interpreted text
+   (e.g., "``sum ::= `integer` "+" `integer```") -- this generates
+   cross-references to the productions of these tokens.  Outside of the
+   production list, you can reference to token productions using
+   :rst:role:`token`.
 
    The *productionGroup* argument to :rst:dir:`productionlist` serves to
    distinguish different sets of production lists that belong to different
    grammars.  Multiple production lists with the same *productionGroup* thus
    define rules in the same scope.
 
-   Blank lines are not allowed within ``productionlist`` directive arguments.
+   Inside of the production list, tokens implicitly refer to productions
+   from the current group. You can refer to the production of another
+   grammar by prefixing the token with its group name and a colon, e.g,
+   "``otherGroup:sum``". If the group of the token should not be shown in
+   the production, it can be prefixed by a tilde, e.g.,
+   "``~otherGroup:sum``". To refer to a production from an unnamed
+   grammar, the token should be prefixed by a colon, e.g., "``:sum``".
 
-   The definition can contain token names which are marked as interpreted text
-   (e.g. "``sum ::= `integer` "+" `integer```") -- this generates
-   cross-references to the productions of these tokens.  Outside of the
-   production list, you can reference to token productions using
-   :rst:role:`token`.
-   However, if you have given a *productionGroup* argument you must prefix the
+   Outside of the production list,
+   if you have given a *productionGroup* argument you must prefix the
    token name in the cross-reference with the group name and a colon,
    e.g., "``myGroup:sum``" instead of just "``sum``".
    If the group should not be shown in the title of the link either

sphinx/domains/std.py

@@ -45,7 +45,7 @@
 # RE for option descriptions
 option_desc_re = re.compile(r'((?:/|--|-|\+)?[^\s=]+)(=?\s*.*)')
 # RE for grammar tokens
-token_re = re.compile(r'`(\w+)`', re.U)
+token_re = re.compile(r'`((~?\w*:)?\w+)`', re.U)
 
 
 class GenericObject(ObjectDescription[str]):
@@ -464,9 +464,23 @@ def token_xrefs(text: str, productionGroup: str = '') -> List[Node]:
         if m.start() > pos:
             txt = text[pos:m.start()]
             retnodes.append(nodes.Text(txt, txt))
-        refnode = pending_xref(m.group(1), reftype='token', refdomain='std',
-                               reftarget=productionGroup + m.group(1))
-        refnode += nodes.literal(m.group(1), m.group(1), classes=['xref'])
+        token = m.group(1)
+        if ':' in token:
+            if token[0] == '~':
+                _, title = token.split(':')
+                target = token[1:]
+            elif token[0] == ':':
+                title = token[1:]
+                target = title
+            else:
+                title = token
+                target = token
+        else:
+            title = token
+            target = productionGroup + token
+        refnode = pending_xref(title, reftype='token', refdomain='std',
+                               reftarget=target)
+        refnode += nodes.literal(token, title, classes=['xref'])
         retnodes.append(refnode)
         pos = m.end()
     if pos < len(text):

tests/test_domain_std.py

@@ -405,6 +405,22 @@ def test_productionlist(app, status, warning):
     assert "A</strong> ::=  B C D    E F G" in text
 
 
+def test_productionlist2(app):
+    text = (".. productionlist:: P2\n"
+            "   A: `:A` `A`\n"
+            "   B: `P1:B` `~P1:B`\n")
+    doctree = restructuredtext.parse(app, text)
+    refnodes = list(doctree.traverse(pending_xref))
+    assert_node(refnodes[0], pending_xref, reftarget="A")
+    assert_node(refnodes[1], pending_xref, reftarget="P2:A")
+    assert_node(refnodes[2], pending_xref, reftarget="P1:B")
+    assert_node(refnodes[3], pending_xref, reftarget="P1:B")
+    assert_node(refnodes[0], [pending_xref, nodes.literal, "A"])
+    assert_node(refnodes[1], [pending_xref, nodes.literal, "A"])
+    assert_node(refnodes[2], [pending_xref, nodes.literal, "P1:B"])
+    assert_node(refnodes[3], [pending_xref, nodes.literal, "B"])
+
+
 def test_disabled_docref(app):
     text = (":doc:`index`\n"
             ":doc:`!index`\n")