From 63b4163c6f2641766c6b865afdfbec94070de97f Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 27 Jun 2021 23:50:45 +0200 Subject: [PATCH 01/92] Begin revamp for proc-macros --- src/SUMMARY.md | 56 ++++---- src/macros/syntax.md | 5 - src/macros/syntax/ast.md | 123 ----------------- src/macros/syntax/expansion.md | 127 ------------------ src/proc-macros.md | 1 + src/syntax-extensions.md | 9 ++ src/syntax-extensions/ast.md | 123 +++++++++++++++++ src/syntax-extensions/expansion.md | 115 ++++++++++++++++ .../source-analysis.md} | 0 9 files changed, 277 insertions(+), 282 deletions(-) delete mode 100644 src/macros/syntax.md delete mode 100644 src/macros/syntax/ast.md delete mode 100644 src/macros/syntax/expansion.md create mode 100644 src/proc-macros.md create mode 100644 src/syntax-extensions.md create mode 100644 src/syntax-extensions/ast.md create mode 100644 src/syntax-extensions/expansion.md rename src/{macros/syntax/source-analysys.md => syntax-extensions/source-analysis.md} (100%) diff --git a/src/SUMMARY.md b/src/SUMMARY.md index e19bb3c..e5c55a8 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -2,30 +2,32 @@ [Introduction](./introduction.md) -- [Macros, A Methodical Introduction](./macros.md) - - [Syntax Extensions](./macros/syntax.md) - - [Source Analysis](./macros/syntax/source-analysys.md) - - [Macros in the Ast](./macros/syntax/ast.md) - - [Expansion](./macros/syntax/expansion.md) - - [macro_rules!](./macros/macro_rules.md) - - [Minutiae](./macros/minutiae.md) - - [Fragment Specifiers](./macros/minutiae/fragment-specifiers.md) - - [Metavariables and Expansion Redux](./macros/minutiae/metavar-and-expansion.md) - - [Hygiene](./macros/minutiae/hygiene.md) - - [Non-Identifier Identifiers](./macros/minutiae/identifiers.md) - - [Debugging](./macros/minutiae/debugging.md) - - [Scoping](./macros/minutiae/scoping.md) - - [Import and Export](./macros/minutiae/import-export.md) -- [Macros, A Practical Introduction](./macros-practical.md) -- [Patterns](./patterns.md) - - [Callbacks](./patterns/callbacks.md) - - [Incremental TT Munchers](./patterns/tt-muncher.md) - - [Internal Rules](./patterns/internal-rules.md) - - [Push-down Accumulation](./patterns/push-down-acc.md) - - [Repetition Replacement](./patterns/repetition-replacement.md) - - [TT Bundling](./patterns/tt-bundling.md) -- [Building Blocks](./building-blocks.md) - - [AST Coercion](./building-blocks/ast-coercion.md) - - [Counting](./building-blocks/counting.md) - - [Abacus Counting](./building-blocks/abacus-counting.md) - - [Parsing Rust](./building-blocks/parsing.md) +- [Syntax Extensions](./syntax-extensions.md) + - [Source Analysis](./syntax-extensions/source-analysis.md) + - [Macros in the Ast](./syntax-extensions/ast.md) + - [Expansion](./syntax-extensions/expansion.md) +- [Macros](./macros.md) + - [A Methodical Introduction](./macros.md) + - [macro_rules!](./macros/macro_rules.md) + - [Minutiae](./macros/minutiae.md) + - [Fragment Specifiers](./macros/minutiae/fragment-specifiers.md) + - [Metavariables and Expansion Redux](./macros/minutiae/metavar-and-expansion.md) + - [Hygiene](./macros/minutiae/hygiene.md) + - [Non-Identifier Identifiers](./macros/minutiae/identifiers.md) + - [Debugging](./macros/minutiae/debugging.md) + - [Scoping](./macros/minutiae/scoping.md) + - [Import and Export](./macros/minutiae/import-export.md) + - [A Practical Introduction](./macros-practical.md) + - [Patterns](./patterns.md) + - [Callbacks](./patterns/callbacks.md) + - [Incremental TT Munchers](./patterns/tt-muncher.md) + - [Internal Rules](./patterns/internal-rules.md) + - [Push-down Accumulation](./patterns/push-down-acc.md) + - [Repetition Replacement](./patterns/repetition-replacement.md) + - [TT Bundling](./patterns/tt-bundling.md) + - [Building Blocks](./building-blocks.md) + - [AST Coercion](./building-blocks/ast-coercion.md) + - [Counting](./building-blocks/counting.md) + - [Abacus Counting](./building-blocks/abacus-counting.md) + - [Parsing Rust](./building-blocks/parsing.md) +- [Proc-Macros](./proc-macros.md) diff --git a/src/macros/syntax.md b/src/macros/syntax.md deleted file mode 100644 index 9611d7a..0000000 --- a/src/macros/syntax.md +++ /dev/null @@ -1,5 +0,0 @@ -# Syntax Extensions - -Before talking about `macro_rules!`, it is worthwhile to discuss the general mechanism they are built on: -syntax extensions. To do that, we must discuss how Rust source is processed by the compiler, and the -general mechanisms on which user-defined `macro_rules!` macros are built. \ No newline at end of file diff --git a/src/macros/syntax/ast.md b/src/macros/syntax/ast.md deleted file mode 100644 index 2709200..0000000 --- a/src/macros/syntax/ast.md +++ /dev/null @@ -1,123 +0,0 @@ -# Macros in the AST - -As previously mentioned, macro processing in Rust happens *after* the construction of the AST. As -such, the syntax used to invoke a macro *must* be a proper part of the language's syntax. In fact, -there are several "syntax extension" forms which are part of Rust's syntax. Specifically, the -following 4 forms (by way of examples): - -* `# [ $arg ]`; *e.g.* `#[derive(Clone)]`, `#[no_mangle]`, … -* `# ! [ $arg ]`; *e.g.* `#![allow(dead_code)]`, `#![crate_name="blang"]`, … -* `$name ! $arg`; *e.g.* `println!("Hi!")`, `concat!("a", "b")`, … -* `$name ! $arg0 $arg1`; *e.g.* `macro_rules! dummy { () => {}; }`. - -The first two are [attributes] which annotate items, expressions and statements. They can be -classified into different kinds, [built-in attributes], [macro attributes] and [derive attributes]. -[macro attributes] and [derive attributes] can be implemented with the second macro system that Rust -offers, [procedural macros]. [built-in attributes] on the other hand are attributes implemented by -the compiler. - -The third form, `$name ! $arg`, is the one of interest to us: function-like macros. It is the form -available for use with `macro_rules!` macros. Note that this form is not *limited* to `macro_rules!` -macros: it is a generic syntax extension form. For example, whilst [`format!`] is a macro, -[`format_args!`] (which is used to *implement* [`format!`]) is *not*. - -The fourth form is essentially a variation which is *not* available to macros. In fact, the only case -where this form is used *at all* is with `macro_rules!` itself which, again we will come back to. - -Disregarding all but the third form (`$name ! $arg`), the question becomes: how does the Rust parser -know what `$arg` looks like for every possible syntax extension? The answer is that it doesn't -*have to*. Instead, the argument of a syntax extension invocation is a *single* token tree. More -specifically, it is a single, *non-leaf* token tree; `(...)`, `[...]`, or `{...}`. With that -knowledge, it should become apparent how the parser can understand all of the following invocation -forms: - -```rust,ignore -bitflags! { - struct Color: u8 { - const RED = 0b0001, - const GREEN = 0b0010, - const BLUE = 0b0100, - const BRIGHT = 0b1000, - } -} - -lazy_static! { - static ref FIB_100: u32 = { - fn fib(a: u32) -> u32 { - match a { - 0 => 0, - 1 => 1, - a => fib(a-1) + fib(a-2) - } - } - - fib(100) - }; -} - -fn main() { - use Color::*; - let colors = vec![RED, GREEN, BLUE]; - println!("Hello, World!"); -} -``` - -Although the above invocations may *look* like they contain various kinds of Rust code, the parser -simply sees a collection of meaningless token trees. To make this clearer, we can replace all these -syntactic "black boxes" with ⬚, leaving us with: - -```text -bitflags! ⬚ - -lazy_static! ⬚ - -fn main() { - let colors = vec! ⬚; - println! ⬚; -} -``` - -Just to reiterate: the parser does not assume *anything* about ⬚; it remembers the tokens it -contains, but doesn't try to *understand* them. This means ⬚ can be anything, even invalid Rust! As -to why this is a good thing, we will come back to that at a later point. - -The important takeaways are: - -* There are multiple kinds of syntax extension in Rust. We will *only* be talking about macros - defined by the `macro_rules!` construct. -* Just because you see something of the form `$name! $arg`, doesn't mean it's actually a macro; it - might be another kind of syntax extension, a procedural macro for example. -* The input to every macro is a single non-leaf token tree. -* Macros (really, syntax extensions in general) are parsed as *part* of the abstract syntax tree. - -> **Aside**: due to the first point, some of what will be said below (including the next paragraph) -> will apply to syntax extensions *in general*.[^writer-is-lazy] - -[^writer-is-lazy]: This is rather convenient as "macro" is much quicker and easier to type than -"syntax extension". - -The last point is the most important, as it has *significant* implications. Because macros are -parsed into the AST, they can **only** appear in positions where they are explicitly supported. -Specifically macros can appear in place of the following: - -* Patterns -* Statements -* Expressions -* Items(this includes `impl` Items) -* Types - -Some things *not* on this list: - -* Identifiers -* Match arms -* Struct fields - -There is absolutely, definitely *no way* to use macros in any position *not* on the first list. - -[attributes]: https://doc.rust-lang.org/reference/attributes.html -[built-in attributes]: https://doc.rust-lang.org/reference/attributes.html#built-in-attributes-index -[macro attributes]: https://doc.rust-lang.org/reference/procedural-macros.html#attribute-macros -[derive attributes]: https://doc.rust-lang.org/reference/procedural-macros.html#derive-macro-helper-attributes -[procedural macros]: https://doc.rust-lang.org/reference/procedural-macros.html -[`format!`]: https://doc.rust-lang.org/std/macro.format.html -[`format_args!`]: https://doc.rust-lang.org/std/macro.format_args.html diff --git a/src/macros/syntax/expansion.md b/src/macros/syntax/expansion.md deleted file mode 100644 index 07077eb..0000000 --- a/src/macros/syntax/expansion.md +++ /dev/null @@ -1,127 +0,0 @@ -# Expansion - -Expansion is a relatively simple affair. At some point *after* the construction of the AST, but -before the compiler begins constructing its semantic understanding of the program, it will expand -all macros. - -This involves traversing the AST, locating macro invocations and replacing them with their -expansion. In the case of non-macro syntax extensions, *how* this happens is up to the particular -syntax extension. That said, syntax extensions go through *exactly* the same process that macros do -once their expansion is complete. - -Once the compiler has run a syntax extension, it expects the result to be parsable as one of a -limited set of syntax elements, based on context. For example, if you invoke a macro at module scope, -the compiler will parse the result into an AST node that represents an item. If you invoke a macro -in expression position, the compiler will parse the result into an expression AST node. - -In fact, it can turn a syntax extension result into any of the following: - -* an expression, -* a pattern, -* a type, -* zero or more items, or -* zero or more statements. - -In other words, *where* you can invoke a macro determines what its result will be interpreted as. - -The compiler will take this AST node and completely replace the macro's invocation node with the -output node. *This is a structural operation*, not a textural one! - -For example, consider the following: - -```rust,ignore -let eight = 2 * four!(); -``` - -We can visualize this partial AST as follows: - -```text -┌─────────────┐ -│ Let │ -│ name: eight │ ┌─────────┐ -│ init: ◌ │╶─╴│ BinOp │ -└─────────────┘ │ op: Mul │ - ┌╴│ lhs: ◌ │ - ┌────────┐ │ │ rhs: ◌ │╶┐ ┌────────────┐ - │ LitInt │╶┘ └─────────┘ └╴│ Macro │ - │ val: 2 │ │ name: four │ - └────────┘ │ body: () │ - └────────────┘ -``` - -From context, `four!()` *must* expand to an expression (the initializer can *only* be an expression). -Thus, whatever the actual expansion is, it will be interpreted as a complete expression. In this -case, we will assume `four!` is defined such that it expands to the expression `1 + 3`. As a result, -expanding this invocation will result in the AST changing to: - -```text -┌─────────────┐ -│ Let │ -│ name: eight │ ┌─────────┐ -│ init: ◌ │╶─╴│ BinOp │ -└─────────────┘ │ op: Mul │ - ┌╴│ lhs: ◌ │ - ┌────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ - │ LitInt │╶┘ └─────────┘ └╴│ BinOp │ - │ val: 2 │ │ op: Add │ - └────────┘ ┌╴│ lhs: ◌ │ - ┌────────┐ │ │ rhs: ◌ │╶┐ ┌────────┐ - │ LitInt │╶┘ └─────────┘ └╴│ LitInt │ - │ val: 1 │ │ val: 3 │ - └────────┘ └────────┘ -``` - -This can be written out like so: - -```rust,ignore -let eight = 2 * (1 + 3); -``` - -Note that we added parentheses *despite* them not being in the expansion. Remember that the -compiler always treats the expansion of a macro as a complete AST node, **not** as a mere sequence -of tokens. To put it another way, even if you don't explicitly wrap a complex expression in -parentheses, there is no way for the compiler to "misinterpret" the result, or change the order of -evaluation. - -It is important to understand that macro expansions are treated as AST nodes, as this design has two -further implications: - -* In addition to there being a limited number of invocation *positions*, macros can *only* expand to -the kind of AST node the parser *expects* at that position. -* As a consequence of the above, macros *absolutely cannot* expand to incomplete or syntactically -invalid constructs. - -There is one further thing to note about expansion: what happens when a syntax extension expands to -something that contains *another* syntax extension invocation. For example, consider an alternative -definition of `four!`; what happens if it expands to `1 + three!()`? - -```rust,ignore -let x = four!(); -``` - -Expands to: - -```rust,ignore -let x = 1 + three!(); -``` - -This is resolved by the compiler checking the result of expansions for additional macro invocations, -and expanding them. Thus, a second expansion step turns the above into: - -```rust,ignore -let x = 1 + 3; -``` - -The takeaway here is that expansion happens in "passes"; as many as is needed to completely expand -all invocations. - -Well, not *quite*. In fact, the compiler imposes an upper limit on the number of such recursive -passes it is willing to run before giving up. This is known as the macro recursion limit and -defaults to 32. If the 32nd expansion contains a macro invocation, the compiler will abort with an -error indicating that the recursion limit was exceeded. - -This limit can be raised using the `#![recursion_limit="…"]` [attribute][recursion_limit], though it *must* be done -crate-wide. Generally, it is recommended to try and keep macros below this limit wherever possible -as it may impact compilation times. - -[recursion_limit]: https://doc.rust-lang.org/reference/attributes/limits.html#the-recursion_limit-attribute \ No newline at end of file diff --git a/src/proc-macros.md b/src/proc-macros.md new file mode 100644 index 0000000..d782b95 --- /dev/null +++ b/src/proc-macros.md @@ -0,0 +1 @@ +# Proc-Macros diff --git a/src/syntax-extensions.md b/src/syntax-extensions.md new file mode 100644 index 0000000..e1cab4b --- /dev/null +++ b/src/syntax-extensions.md @@ -0,0 +1,9 @@ +# Syntax Extensions + + + +Before talking about Rust's many macros it is worthwhile to discuss the general mechanism they are built on: syntax extensions. +To do that, we must discuss how Rust source is processed by the compiler, and the general mechanisms on which user-defined macros and proc-macros are built upon. diff --git a/src/syntax-extensions/ast.md b/src/syntax-extensions/ast.md new file mode 100644 index 0000000..57839bf --- /dev/null +++ b/src/syntax-extensions/ast.md @@ -0,0 +1,123 @@ +# Macros in the AST + +As previously mentioned, macro processing in Rust happens *after* the construction of the AST. +As such, the syntax used to invoke a macro *must* be a proper part of the language's syntax. +In fact, there are several "syntax extension" forms which are part of Rust's syntax. +Specifically, the following 4 forms (by way of examples): + +1. `# [ $arg ]`; *e.g.* `#[derive(Clone)]`, `#[no_mangle]`, … +2. `# ! [ $arg ]`; *e.g.* `#![allow(dead_code)]`, `#![crate_name="blang"]`, … +3. `$name ! $arg`; *e.g.* `println!("Hi!")`, `concat!("a", "b")`, … +4. `$name ! $arg0 $arg1`; *e.g.* `macro_rules! dummy { () => {}; }`. + +The first two are [attributes] which annotate items, expressions and statements. They can be +classified into different kinds, [built-in attributes], [proc-macro attributes] and [derive attributes]. +[macro attributes] and [derive attributes] can be implemented with the second macro system that Rust +offers, [procedural macros]. [built-in attributes] on the other hand are attributes implemented by +the compiler. + +The third form `$name ! $arg` are function-like macros. It is the form available for use with `macro_rules!`, `macro` and also procedural macros. +Note that this form is not *limited* to `macro_rules!` macros: it is a generic syntax extension form. +For example, whilst [`format!`] is a `macro_rules!` macro, [`format_args!`] (which is used to *implement* [`format!`]) is *not* as it is a compiler builtin. + + +The fourth form is essentially a variation which is *not* available to macros. +In fact, the only case where this form is used *at all* is with the `macro_rules!` construct itself which. + +So, starting with the third form, how does the Rust parser know what the `$arg` in (`$name ! $arg`) looks like for every possible syntax extension? +The answer is that it doesn't *have to*. +Instead, the argument of a syntax extension invocation is a *single* token tree. +More specifically, it is a single, *non-leaf* token tree; `(...)`, `[...]`, or `{...}`. With that +knowledge, it should become apparent how the parser can understand all of the following invocation +forms: + +```rust,ignore +bitflags! { + struct Color: u8 { + const RED = 0b0001, + const GREEN = 0b0010, + const BLUE = 0b0100, + const BRIGHT = 0b1000, + } +} + +lazy_static! { + static ref FIB_100: u32 = { + fn fib(a: u32) -> u32 { + match a { + 0 => 0, + 1 => 1, + a => fib(a-1) + fib(a-2) + } + } + + fib(100) + }; +} + +fn main() { + use Color::*; + let colors = vec![RED, GREEN, BLUE]; + println!("Hello, World!"); +} +``` + +Although the above invocations may *look* like they contain various kinds of Rust code, the parser simply sees a collection of meaningless token trees. +To make this clearer, we can replace all these syntactic "black boxes" with ⬚, leaving us with: + +```text +bitflags! ⬚ + +lazy_static! ⬚ + +fn main() { + let colors = vec! ⬚; + println! ⬚; +} +``` + +Just to reiterate: the parser does not assume *anything* about ⬚; +it remembers the tokens it contains, but doesn't try to *understand* them. +This means ⬚ can be anything, even invalid Rust! +As to why this is a good thing, we will come back to that at a later point. + +So, does this also apply to `$arg` in form 1 and 2, and to ther two args in form 4? Kind of. +The `$arg$` for form 1 and 2 is a bit different in that it is not directly a token tree, but a *simple path* that is either followed by an `=` token and a literal expression, or a token tree. +We will explore this more in-depth in the appropriate proc-macro chapter. +The important part here is that this form as well, makes use of token trees to describe the input. +The 4th form in general is more special and accepts a very specific grammar that also makes use of token trees though. +The specifics of this form do not matter at this point so we will skip them until they become relevant. + +The important takeaways from this are: + +* There are multiple kinds of syntax extensions in Rust. +* Just seeing something of the form `$name! $arg`, doesn't tell you what kind of syntax extension it might be. + It could be a `macro_rules!` macro, a `proc-macro` or maybe even a builtin. +* The input to every `!` macro invocation, that is form 3, is a single non-leaf token tree. +* Syntax extensions are parsed as *part* of the abstract syntax tree. + +The last point is the most important, as it has *significant* implications. +Because syntax extensions are parsed into the AST, they can **only** appear in positions where they are explicitly supported. +Specifically syntax extensions can appear in place of the following: + +* Patterns +* Statements +* Expressions +* Items(this includes `impl` items) +* Types + +Some things *not* on this list: + +* Identifiers +* Match arms +* Struct fields + +There is absolutely, definitely *no way* to use syntax extensions in any position *not* on the first list. + +[attributes]: https://doc.rust-lang.org/reference/attributes.html +[built-in attributes]: https://doc.rust-lang.org/reference/attributes.html#built-in-attributes-index +[proc-macro attributes]: https://doc.rust-lang.org/reference/procedural-macros.html#attribute-macros +[derive attributes]: https://doc.rust-lang.org/reference/procedural-macros.html#derive-macro-helper-attributes +[procedural macros]: https://doc.rust-lang.org/reference/procedural-macros.html +[`format!`]: https://doc.rust-lang.org/std/macro.format.html +[`format_args!`]: https://doc.rust-lang.org/std/macro.format_args.html diff --git a/src/syntax-extensions/expansion.md b/src/syntax-extensions/expansion.md new file mode 100644 index 0000000..135c543 --- /dev/null +++ b/src/syntax-extensions/expansion.md @@ -0,0 +1,115 @@ +# Expansion + +Expansion is a relatively simple affair. +At some point *after* the construction of the AST, but before the compiler begins constructing its semantic understanding of the program, it will expand all syntax extensions. + +This involves traversing the AST, locating syntax extension invocations and replacing them with their expansion. + +Once the compiler has run a syntax extension, it expects the result to be parsable as one of a limited set of syntax elements, based on context. +For example, if you invoke a syntax extension at module scope, the compiler will parse the result into an AST node that represents an item. +If you invoke a syntax extension in expression position, the compiler will parse the result into an expression AST node. + +In fact, it can turn a syntax extension result into any of the following: + +* an expression, +* a pattern, +* a type, +* zero or more items, or +* zero or more statements. + +In other words, *where* you can invoke a syntax extension determines what its result will be interpreted as. + +The compiler will take this AST node and completely replace the syntax extension's invocation node with the output node. +*This is a structural operation*, not a textural one! + +For example, consider the following: + +```rust,ignore +let eight = 2 * four!(); +``` + +We can visualize this partial AST as follows: + +```text +┌─────────────┐ +│ Let │ +│ name: eight │ ┌─────────┐ +│ init: ◌ │╶─╴│ BinOp │ +└─────────────┘ │ op: Mul │ + ┌╴│ lhs: ◌ │ + ┌────────┐ │ │ rhs: ◌ │╶┐ ┌────────────┐ + │ LitInt │╶┘ └─────────┘ └╴│ Macro │ + │ val: 2 │ │ name: four │ + └────────┘ │ body: () │ + └────────────┘ +``` + +From context, `four!()` *must* expand to an expression (the initializer can *only* be an expression). +Thus, whatever the actual expansion is, it will be interpreted as a complete expression. +In this case, we will assume `four!` is defined such that it expands to the expression `1 + 3`. +As a result, expanding this invocation will result in the AST changing to: + +```text +┌─────────────┐ +│ Let │ +│ name: eight │ ┌─────────┐ +│ init: ◌ │╶─╴│ BinOp │ +└─────────────┘ │ op: Mul │ + ┌╴│ lhs: ◌ │ + ┌────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ + │ LitInt │╶┘ └─────────┘ └╴│ BinOp │ + │ val: 2 │ │ op: Add │ + └────────┘ ┌╴│ lhs: ◌ │ + ┌────────┐ │ │ rhs: ◌ │╶┐ ┌────────┐ + │ LitInt │╶┘ └─────────┘ └╴│ LitInt │ + │ val: 1 │ │ val: 3 │ + └────────┘ └────────┘ +``` + +This can be written out like so: + +```rust,ignore +let eight = 2 * (1 + 3); +``` + +Note that we added parentheses *despite* them not being in the expansion. +Remember that the compiler always treats the expansion of a syntax extension as a complete AST node, **not** as a mere sequence of tokens. +To put it another way, even if you don't explicitly wrap a complex expression in parentheses, there is no way for the compiler to "misinterpret" the result, or change the order of evaluation. + +It is important to understand that syntax extension expansions are treated as AST nodes, as this design has two further implications: + +* In addition to there being a limited number of invocation *positions*, syntax extension can *only* expand to the kind of AST node the parser *expects* at that position. +* As a consequence of the above, syntax extension *absolutely cannot* expand to incomplete or syntactically invalid constructs. + +There is one further thing to note about expansion: what happens when a syntax extension expands to something that contains *another* syntax extension invocation. +For example, consider an alternative definition of `four!`; what happens if it expands to `1 + three!()`? + +```rust,ignore +let x = four!(); +``` + +Expands to: + +```rust,ignore +let x = 1 + three!(); +``` + +This is resolved by the compiler checking the result of expansions for additional syntax extension invocations, and expanding them. +Thus, a second expansion step turns the above into: + +```rust,ignore +let x = 1 + 3; +``` + +The takeaway here is that expansion happens in "passes"; +as many as is needed to completely expand all invocations. + +Well, not *quite*. +In fact, the compiler imposes an upper limit on the number of such recursive passes it is willing to run before giving up. +This is known as the syntax extension recursion limit and defaults to 32. +If the 32nd expansion contains a syntax extension invocation, the compiler will abort with an error indicating that the recursion limit was exceeded. + +This limit can be raised using the `#![recursion_limit="…"]` [attribute][recursion_limit], though it *must* be done crate-wide. +Generally, it is recommended to try and keep syntax extension below this limit wherever possible as it may impact compilation times. + +[recursion_limit]: https://doc.rust-lang.org/reference/attributes/limits.html#the-recursion_limit-attribute diff --git a/src/macros/syntax/source-analysys.md b/src/syntax-extensions/source-analysis.md similarity index 100% rename from src/macros/syntax/source-analysys.md rename to src/syntax-extensions/source-analysis.md From 7962cd2306e0b13d3b4b2a034a8ca7f1c7ba13e8 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 29 Jun 2021 01:21:51 +0200 Subject: [PATCH 02/92] Move some files around --- book.toml | 5 ++-- src/SUMMARY.md | 26 +++++++++---------- src/{ => macros}/building-blocks.md | 0 .../building-blocks/abacus-counting.md | 0 .../building-blocks/ast-coercion.md | 0 src/{ => macros}/building-blocks/counting.md | 0 src/{ => macros}/building-blocks/parsing.md | 0 src/{ => macros}/macros-practical-table.html | 0 src/{ => macros}/macros-practical.md | 0 src/{ => macros}/patterns.md | 0 src/{ => macros}/patterns/callbacks.md | 0 src/{ => macros}/patterns/internal-rules.md | 0 src/{ => macros}/patterns/push-down-acc.md | 0 .../patterns/repetition-replacement.md | 0 src/{ => macros}/patterns/tt-bundling.md | 0 src/{ => macros}/patterns/tt-muncher.md | 0 16 files changed, 16 insertions(+), 15 deletions(-) rename src/{ => macros}/building-blocks.md (100%) rename src/{ => macros}/building-blocks/abacus-counting.md (100%) rename src/{ => macros}/building-blocks/ast-coercion.md (100%) rename src/{ => macros}/building-blocks/counting.md (100%) rename src/{ => macros}/building-blocks/parsing.md (100%) rename src/{ => macros}/macros-practical-table.html (100%) rename src/{ => macros}/macros-practical.md (100%) rename src/{ => macros}/patterns.md (100%) rename src/{ => macros}/patterns/callbacks.md (100%) rename src/{ => macros}/patterns/internal-rules.md (100%) rename src/{ => macros}/patterns/push-down-acc.md (100%) rename src/{ => macros}/patterns/repetition-replacement.md (100%) rename src/{ => macros}/patterns/tt-bundling.md (100%) rename src/{ => macros}/patterns/tt-muncher.md (100%) diff --git a/book.toml b/book.toml index cd6f649..1e926a5 100644 --- a/book.toml +++ b/book.toml @@ -4,9 +4,10 @@ language = "en" multilingual = false src = "src" title = "The Little Book of Rust Macros" +edition = "2018" [build] -create-missing = true +create-missing = false build-dir = "book" [output.html] @@ -14,4 +15,4 @@ default-theme = "ayu" site-url = "/tlborm/" mathjax-support = true git-repository-url = "https://github.com/veykril/tlborm/" -additional-css = ["res/rust-syntax-bg-highlight.css"] \ No newline at end of file +additional-css = ["res/rust-syntax-bg-highlight.css"] diff --git a/src/SUMMARY.md b/src/SUMMARY.md index e5c55a8..bfee408 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -17,17 +17,17 @@ - [Debugging](./macros/minutiae/debugging.md) - [Scoping](./macros/minutiae/scoping.md) - [Import and Export](./macros/minutiae/import-export.md) - - [A Practical Introduction](./macros-practical.md) - - [Patterns](./patterns.md) - - [Callbacks](./patterns/callbacks.md) - - [Incremental TT Munchers](./patterns/tt-muncher.md) - - [Internal Rules](./patterns/internal-rules.md) - - [Push-down Accumulation](./patterns/push-down-acc.md) - - [Repetition Replacement](./patterns/repetition-replacement.md) - - [TT Bundling](./patterns/tt-bundling.md) - - [Building Blocks](./building-blocks.md) - - [AST Coercion](./building-blocks/ast-coercion.md) - - [Counting](./building-blocks/counting.md) - - [Abacus Counting](./building-blocks/abacus-counting.md) - - [Parsing Rust](./building-blocks/parsing.md) + - [A Practical Introduction](./macros/macros-practical.md) + - [Patterns](./macros/patterns.md) + - [Callbacks](./macros/patterns/callbacks.md) + - [Incremental TT Munchers](./macros/patterns/tt-muncher.md) + - [Internal Rules](./macros/patterns/internal-rules.md) + - [Push-down Accumulation](./macros/patterns/push-down-acc.md) + - [Repetition Replacement](./macros/patterns/repetition-replacement.md) + - [TT Bundling](./macros/patterns/tt-bundling.md) + - [Building Blocks](./macros/building-blocks.md) + - [AST Coercion](./macros/building-blocks/ast-coercion.md) + - [Counting](./macros/building-blocks/counting.md) + - [Abacus Counting](./macros/building-blocks/abacus-counting.md) + - [Parsing Rust](./macros/building-blocks/parsing.md) - [Proc-Macros](./proc-macros.md) diff --git a/src/building-blocks.md b/src/macros/building-blocks.md similarity index 100% rename from src/building-blocks.md rename to src/macros/building-blocks.md diff --git a/src/building-blocks/abacus-counting.md b/src/macros/building-blocks/abacus-counting.md similarity index 100% rename from src/building-blocks/abacus-counting.md rename to src/macros/building-blocks/abacus-counting.md diff --git a/src/building-blocks/ast-coercion.md b/src/macros/building-blocks/ast-coercion.md similarity index 100% rename from src/building-blocks/ast-coercion.md rename to src/macros/building-blocks/ast-coercion.md diff --git a/src/building-blocks/counting.md b/src/macros/building-blocks/counting.md similarity index 100% rename from src/building-blocks/counting.md rename to src/macros/building-blocks/counting.md diff --git a/src/building-blocks/parsing.md b/src/macros/building-blocks/parsing.md similarity index 100% rename from src/building-blocks/parsing.md rename to src/macros/building-blocks/parsing.md diff --git a/src/macros-practical-table.html b/src/macros/macros-practical-table.html similarity index 100% rename from src/macros-practical-table.html rename to src/macros/macros-practical-table.html diff --git a/src/macros-practical.md b/src/macros/macros-practical.md similarity index 100% rename from src/macros-practical.md rename to src/macros/macros-practical.md diff --git a/src/patterns.md b/src/macros/patterns.md similarity index 100% rename from src/patterns.md rename to src/macros/patterns.md diff --git a/src/patterns/callbacks.md b/src/macros/patterns/callbacks.md similarity index 100% rename from src/patterns/callbacks.md rename to src/macros/patterns/callbacks.md diff --git a/src/patterns/internal-rules.md b/src/macros/patterns/internal-rules.md similarity index 100% rename from src/patterns/internal-rules.md rename to src/macros/patterns/internal-rules.md diff --git a/src/patterns/push-down-acc.md b/src/macros/patterns/push-down-acc.md similarity index 100% rename from src/patterns/push-down-acc.md rename to src/macros/patterns/push-down-acc.md diff --git a/src/patterns/repetition-replacement.md b/src/macros/patterns/repetition-replacement.md similarity index 100% rename from src/patterns/repetition-replacement.md rename to src/macros/patterns/repetition-replacement.md diff --git a/src/patterns/tt-bundling.md b/src/macros/patterns/tt-bundling.md similarity index 100% rename from src/patterns/tt-bundling.md rename to src/macros/patterns/tt-bundling.md diff --git a/src/patterns/tt-muncher.md b/src/macros/patterns/tt-muncher.md similarity index 100% rename from src/patterns/tt-muncher.md rename to src/macros/patterns/tt-muncher.md From c922ca3e2b0213709aa14f8d3880e00dad5b352c Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 29 Jun 2021 02:50:07 +0200 Subject: [PATCH 03/92] Fix links --- CONTRIBUTING.md | 3 + book.toml | 6 +- src/SUMMARY.md | 5 +- src/introduction.md | 3 +- src/macros.md | 24 ++++--- src/macros/building-blocks/counting.md | 28 ++++---- src/macros/building-blocks/parsing.md | 14 ++-- src/macros/macro_rules.md | 21 +++--- src/macros/macros-methodical.md | 7 ++ src/macros/macros-practical.md | 81 ++++++++++------------ src/macros/minutiae/fragment-specifiers.md | 8 ++- src/macros/patterns/internal-rules.md | 14 ++-- src/macros/patterns/tt-bundling.md | 2 +- src/macros/patterns/tt-muncher.md | 4 +- src/proc-macros.md | 1 - src/syntax-extensions.md | 6 +- src/syntax-extensions/ast.md | 2 +- src/syntax-extensions/hygiene.md | 3 + 18 files changed, 122 insertions(+), 110 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 src/macros/macros-methodical.md delete mode 100644 src/proc-macros.md create mode 100644 src/syntax-extensions/hygiene.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..dab71d9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Style + +Prefer a sentence-per-line format. diff --git a/book.toml b/book.toml index 1e926a5..8e93cae 100644 --- a/book.toml +++ b/book.toml @@ -7,9 +7,13 @@ title = "The Little Book of Rust Macros" edition = "2018" [build] -create-missing = false +create-missing = true build-dir = "book" +[output.linkcheck] +traverse-parent-directories = false +exclude = ["mathjax"] + [output.html] default-theme = "ayu" site-url = "/tlborm/" diff --git a/src/SUMMARY.md b/src/SUMMARY.md index bfee408..da0d8f0 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -6,8 +6,9 @@ - [Source Analysis](./syntax-extensions/source-analysis.md) - [Macros in the Ast](./syntax-extensions/ast.md) - [Expansion](./syntax-extensions/expansion.md) + - [Hygiene]() - [Macros](./macros.md) - - [A Methodical Introduction](./macros.md) + - [A Methodical Introduction](./macros/macros-methodical.md) - [macro_rules!](./macros/macro_rules.md) - [Minutiae](./macros/minutiae.md) - [Fragment Specifiers](./macros/minutiae/fragment-specifiers.md) @@ -30,4 +31,4 @@ - [Counting](./macros/building-blocks/counting.md) - [Abacus Counting](./macros/building-blocks/abacus-counting.md) - [Parsing Rust](./macros/building-blocks/parsing.md) -- [Proc-Macros](./proc-macros.md) + - [Proc-Macros]() diff --git a/src/introduction.md b/src/introduction.md index e12d454..e8a524f 100644 --- a/src/introduction.md +++ b/src/introduction.md @@ -1 +1,2 @@ -{{#include ../README.md}} \ No newline at end of file + +{{#include ../README.md}} diff --git a/src/macros.md b/src/macros.md index 3add87c..80bcea0 100644 --- a/src/macros.md +++ b/src/macros.md @@ -1,15 +1,19 @@ -# Macros, A Methodical Introduction +# Macros -This chapter will introduce Rust's [Macro-By-Example][mbe] system: [`macro_rules!`][mbe]. Rather -than trying to cover it based on practical examples, it will instead attempt to give you a complete -and thorough explanation of *how* the system works. As such, this is intended for people who just -want the system as a whole explained, rather than be guided through it. +This chapter will introduce Rust's [Macro-By-Example][mbe] system: [`macro_rules!`][mbe]. +There are two different introductions in this chapter, a [methodical] and a [practical]. -There is also the [Macros chapter of the Rust Book] which is a more approachable, high-level -explanation, the reference [chapter](https://doc.rust-lang.org/reference/macros-by-example.html) -and the [practical introduction](./macros-practical.html) chapter of this book, which is a guided -implementation of a single macro. +The former will attempt to give you a complete and thorough explanation of *how* the system works, while the latter one will cover more practical examples. +As such, the [methodical introduction][methodical] is intended for people who just want the system as a whole explained, while the [practical introduction][practical] guides one through the implementation of a single macro. + +Following up the two introductions it offers some generally very useful [patterns] and [building blocks] for creating feature rich macros. + +Should the information presented here not suffice, then there is also the [Macros chapter of the Rust Book] which is a more approachable, high-level explanation as well as the reference [chapter](https://doc.rust-lang.org/reference/macros-by-example.html) which goes more into the precise details of things. [mbe]: https://doc.rust-lang.org/reference/macros-by-example.html -[Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html \ No newline at end of file +[Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html +[practical]: ./macros/macros-practical.html +[methodical]: ./macros/macros-methodical.html +[patterns]: ./macros/patterns.md +[building blocks]: ./macros/building-blocks.md diff --git a/src/macros/building-blocks/counting.md b/src/macros/building-blocks/counting.md index 185a5ba..3c993a3 100644 --- a/src/macros/building-blocks/counting.md +++ b/src/macros/building-blocks/counting.md @@ -13,7 +13,7 @@ macro_rules! replace_expr { macro_rules! count_tts { ($($tts:tt)*) => {0usize $(+ replace_expr!($tts 1usize))*}; } -# +# # fn main() { # assert_eq!(count_tts!(0 1 2), 3); # } @@ -38,7 +38,7 @@ macro_rules! count_tts { () => {0usize}; ($_head:tt $($tail:tt)*) => {1usize + count_tts!($($tail)*)}; } -# +# # fn main() { # assert_eq!(count_tts!(0 1 2), 3); # } @@ -79,19 +79,19 @@ fn main() { assert_eq!(700, count_tts!( ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, - + ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, - + ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, - + ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, - + ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, - + // Repetition breaks somewhere after this ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, ,,,,,,,,,, @@ -117,7 +117,7 @@ macro_rules! replace_expr { macro_rules! count_tts { ($($tts:tt)*) => {<[()]>::len(&[$(replace_expr!($tts ())),*])}; } -# +# # fn main() { # assert_eq!(count_tts!(0 1 2), 3); # } @@ -140,7 +140,7 @@ macro_rules! count_idents { } }; } -# +# # fn main() { # const COUNT: u32 = count_idents!(A, B, C); # assert_eq!(COUNT, 3); @@ -156,7 +156,7 @@ Secondly, this approach is *not* hygienic, meaning that if whatever identifier y ## Bit twiddling -Another recursive approach using bit operations: +Another recursive approach using bit operations: ```rust macro_rules! count_tts { @@ -164,7 +164,7 @@ macro_rules! count_tts { ($odd:tt $($a:tt $b:tt)*) => { (count_tts!($($a)*) << 1) | 1 }; ($($a:tt $even:tt)*) => { count_tts!($($a)*) << 1 }; } -# +# # fn main() { # assert_eq!(count_tts!(0 1 2), 3); # } @@ -211,10 +211,10 @@ can count 1 for the uneven discard and multiply by 2 again since we also halved. ((((0 << 1) | 1) << 1 << 1) | 1) << 1; ``` -Now to check if we expanded correctly manually we can use a one of the tools we introduced for -[`debugging`](/macros/minutiae/debugging.html). When expanding the macro there we should get: +Now to check if we expanded correctly manually we can use a one of the tools we introduced for [`debugging`](../minutiae/debugging.html). +When expanding the macro there we should get: ```rust,ignore ((((0 << 1) | 1) << 1 << 1) | 1) << 1; ``` -That's the same so we didn't make any mistakes, great! \ No newline at end of file +That's the same so we didn't make any mistakes, great! diff --git a/src/macros/building-blocks/parsing.md b/src/macros/building-blocks/parsing.md index 1b0617d..1ecc461 100644 --- a/src/macros/building-blocks/parsing.md +++ b/src/macros/building-blocks/parsing.md @@ -123,7 +123,7 @@ macro_rules! struct_item_matcher { #struct_item_matcher!( # #[derive(Copy, Clone)] -# pub(crate) struct Foo { +# pub(crate) struct Foo { # pub bar: i32, # baz: &'static str, # qux: f32 @@ -164,7 +164,7 @@ macro_rules! enum_item_matcher { ),* $(,)? //∨~~rest of input~~∨ ) $(, $($tt:tt)* )? ) => { - + // process rest of the enum $( enum_item_matcher!(@variant $( $tt )*) )? }; @@ -203,7 +203,7 @@ macro_rules! enum_item_matcher { #enum_item_matcher!( # #[derive(Copy, Clone)] -# pub(crate) enum Foo { +# pub(crate) enum Foo { # Bar, # Baz, # } @@ -222,7 +222,7 @@ macro_rules! enum_item_matcher { #); ``` -[patterns]:/patterns.html -[Push Down Accumulator]:/patterns/push-down-acc.html -[Internal Rules]:/patterns/internal-rules.html -[Incremental TT Muncher]:/patterns/tt-muncher.html +[patterns]: ../patterns.md +[Push Down Accumulator]: ../patterns/push-down-acc.md +[Internal Rules]: ../patterns/internal-rules.md +[Incremental TT Muncher]: ../patterns/tt-muncher.md diff --git a/src/macros/macro_rules.md b/src/macros/macro_rules.md index fc197ce..36c8613 100644 --- a/src/macros/macro_rules.md +++ b/src/macros/macro_rules.md @@ -93,8 +93,7 @@ kind of capture which is also called the fragment-specifier, which must be one o * `ty`: a type * `vis`: a possible empty visibility qualifier (e.g. `pub`, `pub(in crate)`, ...) -For more in-depth description of the fragement specifiers, check out the -[Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. +For more in-depth description of the fragement specifiers, check out the [Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. For example, here is a `macro_rules!` macro which captures its input as an expression under the metavariable `$e`: @@ -105,14 +104,12 @@ macro_rules! one_expression { } ``` -These metavariables leverage the Rust compiler's parser, ensuring that they are always "correct". An -`expr` metavariables will *always* capture a complete, valid expression for the version of Rust being -compiled. +These metavariables leverage the Rust compiler's parser, ensuring that they are always "correct". +An `expr` metavariables will *always* capture a complete, valid expression for the version of Rust being compiled. -You can mix literal token trees and metavariables, within limits ([explained below]). +You can mix literal token trees and metavariables, within limits (explained in [Metavariables and Expansion Redux]). -To refer to a metavariable you simply write `$name`, as the type of the variable is already -specified in the matcher. For example: +To refer to a metavariable you simply write `$name`, as the type of the variable is already specified in the matcher. For example: ```rust,ignore macro_rules! times_five { @@ -143,10 +140,10 @@ macro_rules! repeat { } ``` -There is also a special metavariable called [`$crate`] which can be used to refer to the current -crate. +There is also a special metavariable called [`$crate`] which can be used to refer to the current crate. -[`$crate`]:./minutiae/hygiene.html#crate +[Metavariables and Expansion Redux]: ./minutiae/metavar-and-expansion.md +[`$crate`]: ./minutiae/hygiene.md#crate ## Repetitions @@ -246,6 +243,6 @@ error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times   -For the complete grammar definition you may want to consult the +For the complete grammar definition you may want to consult the [Macros By Example](https://doc.rust-lang.org/reference/macros-by-example.html#macros-by-example) chapter of the Rust reference. diff --git a/src/macros/macros-methodical.md b/src/macros/macros-methodical.md new file mode 100644 index 0000000..3db1f6d --- /dev/null +++ b/src/macros/macros-methodical.md @@ -0,0 +1,7 @@ +# Macros, A Methodical Introduction + +This chapter will introduce Rust's [Macro-By-Example][mbe] system by explaining the system as a whole. +It will do so by first going into the construct's syntax and its key parts and then following it up with more general information that one should at least be aware of. + +[mbe]: https://doc.rust-lang.org/reference/macros-by-example.html +[Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html diff --git a/src/macros/macros-practical.md b/src/macros/macros-practical.md index 231ecc0..c8e09c7 100644 --- a/src/macros/macros-practical.md +++ b/src/macros/macros-practical.md @@ -5,7 +5,7 @@ example. It does *not* attempt to explain all of the intricacies of the system; you comfortable with how and why macros are written. There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-06-macros.html) -which is another high-level explanation, and the [methodical introduction](/macros.html) chapter of +which is another high-level explanation, and the [methodical introduction](../macros.md) chapter of this book, which explains the macro system in detail. ## A Little Context @@ -18,21 +18,17 @@ one or more *previous* values, with one or more initial values to get the whole example, the [Fibonacci sequence](https://en.wikipedia.org/wiki/Fibonacci_number) can be defined by the relation: -\\[ F_{n} = 0, 1, ..., F_{n-1} + F_{n-2}\\] +\\[ F_{n} = 0, 1, ..., F_{n-1} + F_{n-2}\\](mathjax) -Thus, the first two numbers in the sequence are 0 and 1, with the third being -\\( F_{0} + F_{1} = 0 + 1 = 1\\), the fourth \\( F_{1} + F_{2} = 1 + 1 = 2\\), and so on forever. +Thus, the first two numbers in the sequence are 0 and 1, with the third being \\( F_{0} + F_{1} = 0 + 1 = 1\\), the fourth \\( F_{1} + F_{2} = 1 + 1 = 2\\), and so on forever. -Now, *because* such a sequence can go on forever, that makes defining a `fibonacci` function a -little tricky, since you obviously don't want to try returning a complete vector. What you *want* is -to return something which will lazily compute elements of the sequence as needed. +Now, *because* such a sequence can go on forever, that makes defining a `fibonacci` function a little tricky, since you obviously don't want to try returning a complete vector. +What you *want* is to return something which will lazily compute elements of the sequence as needed. -In Rust, that means producing an [`Iterator`]. This is not especially *hard*, but there is a fair -amount of boilerplate involved: you need to define a custom type, work out what state needs to be -stored in it, then implement the [`Iterator`] trait for it. +In Rust, that means producing an [`Iterator`]. +This is not especially *hard*, but there is a fair amount of boilerplate involved: you need to define a custom type, work out what state needs to be stored in it, then implement the [`Iterator`] trait for it. -However, recurrence relations are simple enough that almost all of these details can be abstracted -out with a little macro-based code generation. +However, recurrence relations are simple enough that almost all of these details can be abstracted out with a little macro-based code generation. So, with all that having been said, let's get started. @@ -40,8 +36,7 @@ So, with all that having been said, let's get started. ## Construction -Usually, when working on a new macro, the first thing I do is decide what the macro invocation -should look like. In this specific case, my first attempt looked like this: +Usually, when working on a new macro, the first thing I do is decide what the macro invocation should look like. In this specific case, my first attempt looked like this: ```rust,ignore let fib = recurrence![a[n] = 0, 1, ..., a[n-1] + a[n-2]]; @@ -70,8 +65,8 @@ says the input to the macro must match: - the literal token sequence `,` `...` `,`, - a valid *expression* captured into the [metavariable] `recur` (`$recur:expr`). -[repeating]:/macros/macro_rules.html#repetitions -[metavariable]:/macros/macro_rules.html#metavariables +[repeating]: ./macro_rules.html#repetitions +[metavariable]: ./macro_rules.html#metavariables Finally, the rule says that *if* the input matches this rule, then the macro invocation should be replaced by the token sequence `/* ... */`. @@ -215,7 +210,7 @@ Here, I've added a new metavariable: `sty` which should be a type. > **Aside**: if you're wondering, the bit after the colon in a metavariable can be one of several > kinds of syntax matchers. The most common ones are `item`, `expr`, and `ty`. A complete -> explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](/macros/macro_rules.html#Metavariables). +> explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macro_rules.html#Metavariables). > > There's one other thing to be aware of: in the interests of future-proofing the language, the > compiler restricts what tokens you're allowed to put *after* a matcher, depending on what kind @@ -223,7 +218,7 @@ Here, I've added a new metavariable: `sty` which should be a type. > *only* be followed by one of `=>`, `,`, and `;`. > > A complete list can be found in -> [Macros, A Methodical Introduction; Minutiae; Metavariables and Expansion Redux](/macros/minutiae/metavar-and-expansion.html). +> [Macros, A Methodical Introduction; Minutiae; Metavariables and Expansion Redux](./minutiae/metavar-and-expansion.html). ## Indexing and Shuffling @@ -490,7 +485,7 @@ error: local ambiguity: multiple parsing options: built-in NTs expr ('inits') or --> src/main.rs:75:45 | 75 | let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-1] + a[n-2]]; - | + | ``` Here, we've run into a limitation of `macro_rules`. The problem is that second comma. When it sees @@ -625,16 +620,16 @@ macro_rules! recurrence { # let a = IndexOffset { slice: &self.mem, offset: n }; # (a[n-1] + a[n-2]) # }; -# +# # { # use std::mem::swap; -# +# # let mut swap_tmp = next_val; # for i in (0..2).rev() { # swap(&mut swap_tmp, &mut self.mem[i]); # } # } -# +# # self.pos += 1; # Some(next_val) # } @@ -804,7 +799,7 @@ macro_rules! count_exprs { ``` > **JFTE**: this is not the *only*, or even the *best* -> way of counting things. You may wish to peruse the [Counting](/blocks/counting.html) section later. +> way of counting things. You may wish to peruse the [Counting](./building-blocks/counting.md) section later. With this, we can now modify `recurrence` to determine the necessary size of `mem`. @@ -891,10 +886,10 @@ macro_rules! recurrence { }; } /* ... */ -# +# # fn main() { # let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-1] + a[n-2]]; -# +# # for e in fib.take(10) { println!("{}", e) } # } ``` @@ -926,11 +921,11 @@ With that done, we can now substitute the last thing: the `recur` expression. # #[inline(always)] # fn index<'b>(&'b self, index: usize) -> &'b $sty { # use std::num::Wrapping; -# +# # let index = Wrapping(index); # let offset = Wrapping(self.offset); # let window = Wrapping(MEM_SIZE); -# +# # let real_index = index - offset + window; # &self.slice[real_index.0] # } @@ -1261,43 +1256,43 @@ And it compiles! Now, let's try with a different sequence. # ($head:expr) => (1); # ($head:expr, $($tail:expr),*) => (1 + count_exprs!($($tail),*)); # } -# +# # macro_rules! recurrence { # ( $seq:ident [ $ind:ident ]: $sty:ty = $($inits:expr),+ ; ... ; $recur:expr ) => { # { # use std::ops::Index; -# +# # const MEM_SIZE: usize = count_exprs!($($inits),+); -# +# # struct Recurrence { # mem: [$sty; MEM_SIZE], # pos: usize, # } -# +# # struct IndexOffset<'a> { # slice: &'a [$sty; MEM_SIZE], # offset: usize, # } -# +# # impl<'a> Index for IndexOffset<'a> { # type Output = $sty; -# +# # #[inline(always)] # fn index<'b>(&'b self, index: usize) -> &'b $sty { # use std::num::Wrapping; -# +# # let index = Wrapping(index); # let offset = Wrapping(self.offset); # let window = Wrapping(MEM_SIZE); -# +# # let real_index = index - offset + window; # &self.slice[real_index.0] # } # } -# +# # impl Iterator for Recurrence { # type Item = $sty; -# +# # #[inline] # fn next(&mut self) -> Option<$sty> { # if self.pos < MEM_SIZE { @@ -1310,27 +1305,27 @@ And it compiles! Now, let's try with a different sequence. # let $seq = IndexOffset { slice: &self.mem, offset: $ind }; # $recur # }; -# +# # { # use std::mem::swap; -# +# # let mut swap_tmp = next_val; # for i in (0..MEM_SIZE).rev() { # swap(&mut swap_tmp, &mut self.mem[i]); # } # } -# +# # self.pos += 1; # Some(next_val) # } # } # } -# +# # Recurrence { mem: [$($inits),+], pos: 0 } # } # }; # } -# +# # fn main() { for e in recurrence!(f[i]: f64 = 1.0; ...; f[i-1] * i as f64).take(10) { println!("{}", e) @@ -1355,4 +1350,4 @@ Which gives us: Success! -[`macro_rules!`]:/macros/macro_rules.html +[`macro_rules!`]: ./macro_rules.html diff --git a/src/macros/minutiae/fragment-specifiers.md b/src/macros/minutiae/fragment-specifiers.md index 8149f17..5e5c8d6 100644 --- a/src/macros/minutiae/fragment-specifiers.md +++ b/src/macros/minutiae/fragment-specifiers.md @@ -69,7 +69,7 @@ blocks! { The `statement` fragment solely matches a [statement](https://doc.rust-lang.org/reference/statements.html) without its trailing semicolon, unless its an item statement that requires one. What would be an item statement that requires one? A Unit-Struct would be a simple one, as defining one requires a -trailing semicolon. +trailing semicolon. Let's use a simple example to show exactly what is meant with this. We use a macro that merely emits what it captures: @@ -230,7 +230,7 @@ paths! { ## `tt` The `tt` fragment matches a TokenTree. If you need a refresher on what exactly a TokenTree was you -may want to revisit the [TokenTree chapter](../syntax/source-analysys.html#token-trees) of this +may want to revisit the [TokenTree chapter](../../syntax-extensions/source-analysis.html#token-trees) of this book. The `tt` fragment is one of the most powerful fragments, as it can match nearly anything while still allowing you to inspect the contents of it at a later state in the macro. @@ -283,7 +283,7 @@ repetition to it, meaning you don't, and in fact cannot, wrap it in a direct rep ```rust macro_rules! visibilities { - // ∨~~Note this comma, since we cannot repeat a `vis` fragment on its own + // ∨~~Note this comma, since we cannot repeat a `vis` fragment on its own ($($vis:vis,)*) => (); } @@ -315,3 +315,5 @@ literals! { } # fn main() {} ``` + +[`macro_rules`]: ../macro_rules.md diff --git a/src/macros/patterns/internal-rules.md b/src/macros/patterns/internal-rules.md index 774a32c..a0134be 100644 --- a/src/macros/patterns/internal-rules.md +++ b/src/macros/patterns/internal-rules.md @@ -9,7 +9,7 @@ macro_rules! foo { foo!(@as_expr $($tts)*) }; } -# +# # fn main() { # assert_eq!(foo!(42), 42); # } @@ -23,12 +23,12 @@ handled in the 2015 Edition of Rust due to macros not being namespaced in said e one the troubles of having to re-export all the internal macros as well polluting the global macro namespace or even worse, macro name collisions with other crates. In short, it's quite a hassle. This fortunately isn't really a problem anymore nowadays with a rustc version >= 1.30, for more -information consult the [Import and Export chapter](/macros/minutiae/import-export.html). +information consult the [Import and Export chapter](../minutiae/import-export.html). Nevertheless, let's talk about how we can unify multiple macros into one with this technique and what exactly this technique even is. -We have two macros, the common [`as_expr!` macro](/building-blocks/ast-coercion.html) and a `foo` +We have two macros, the common [`as_expr!` macro](../building-blocks/ast-coercion.html) and a `foo` macro that makes use of the first macro: ```rust @@ -41,7 +41,7 @@ macro_rules! foo { as_expr!($($tts)*) }; } -# +# # fn main() { # assert_eq!(foo!(42), 42); # } @@ -65,7 +65,7 @@ macro_rules! foo { foo!(@as_expr $($tts)*) }; } -# +# # fn main() { # assert_eq!(foo!(42), 42); # } @@ -83,7 +83,7 @@ to follow certain fragments[^ambiguity-restrictions], which in Rust 1.12 became other symbols or unique prefixes may be used as desired, but use of `@` has started to become widespread, so using it may aid readers in understanding your macro. -[^ambiguity-restrictions]:[ambiguity-restrictions](/macros/minutiae/metavar-and-expansion.html) +[^ambiguity-restrictions]:[ambiguity-restrictions](../minutiae/metavar-and-expansion.html) > **Note**: in the early days of Rust the `@` token was previously used in prefix position to denote > a garbage-collected pointer, back when the language used sigils to denote pointer types. Its @@ -94,4 +94,4 @@ Additionally, internal rules will often come *before* any "bare" rules, to avoid `macro_rules!` incorrectly attempting to parse an internal invocation as something it cannot possibly be, such as an expression. -[TT Munchers]:./tt-muncher.html \ No newline at end of file +[TT Munchers]:./tt-muncher.html diff --git a/src/macros/patterns/tt-bundling.md b/src/macros/patterns/tt-bundling.md index 6d797f1..dc9a895 100644 --- a/src/macros/patterns/tt-bundling.md +++ b/src/macros/patterns/tt-bundling.md @@ -61,4 +61,4 @@ The example above bundles the `$a` and `$b` expressions into a group which can t a single [`tt`] by the recursive rule. This group is then destructured by the terminal rules to access the expressions. -[`tt`]:./fragment-specifiers.html#tt +[`tt`]: ../minutiae/fragment-specifiers.html#tt diff --git a/src/macros/patterns/tt-muncher.md b/src/macros/patterns/tt-muncher.md index 21e33d4..27ab307 100644 --- a/src/macros/patterns/tt-muncher.md +++ b/src/macros/patterns/tt-muncher.md @@ -17,7 +17,7 @@ macro_rules! mixed_rules { } }; } -# +# # fn main() { # let a = 42; # let b = "Ho-dee-oh-di-oh-di-oh!"; @@ -55,4 +55,4 @@ adding additional rules to account for variation in the input (as opposed to rec intermediate layer), or by making compromises on the input syntax to make using standard repetitions more tractable. -[`tt`]:./fragment-specifiers.html#tt +[`tt`]: ../minutiae/fragment-specifiers.html#tt diff --git a/src/proc-macros.md b/src/proc-macros.md deleted file mode 100644 index d782b95..0000000 --- a/src/proc-macros.md +++ /dev/null @@ -1 +0,0 @@ -# Proc-Macros diff --git a/src/syntax-extensions.md b/src/syntax-extensions.md index e1cab4b..7133ee2 100644 --- a/src/syntax-extensions.md +++ b/src/syntax-extensions.md @@ -1,9 +1,5 @@ # Syntax Extensions - - Before talking about Rust's many macros it is worthwhile to discuss the general mechanism they are built on: syntax extensions. + To do that, we must discuss how Rust source is processed by the compiler, and the general mechanisms on which user-defined macros and proc-macros are built upon. diff --git a/src/syntax-extensions/ast.md b/src/syntax-extensions/ast.md index 57839bf..1f8ef95 100644 --- a/src/syntax-extensions/ast.md +++ b/src/syntax-extensions/ast.md @@ -12,7 +12,7 @@ Specifically, the following 4 forms (by way of examples): The first two are [attributes] which annotate items, expressions and statements. They can be classified into different kinds, [built-in attributes], [proc-macro attributes] and [derive attributes]. -[macro attributes] and [derive attributes] can be implemented with the second macro system that Rust +[proc-macro attributes] and [derive attributes] can be implemented with the second macro system that Rust offers, [procedural macros]. [built-in attributes] on the other hand are attributes implemented by the compiler. diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md new file mode 100644 index 0000000..503131b --- /dev/null +++ b/src/syntax-extensions/hygiene.md @@ -0,0 +1,3 @@ +# Hygiene + +TODO: Talk about hygiene in general, move most of macro/hygiene.rs, if not all, to here From aa56ed152092e9064549f607f893bc72a4c756ef Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 21:39:58 +0200 Subject: [PATCH 04/92] General hygiene chapter --- src/SUMMARY.md | 2 +- src/macros/macro_rules.md | 4 +--- src/syntax-extensions/hygiene.md | 27 ++++++++++++++++++++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/SUMMARY.md b/src/SUMMARY.md index da0d8f0..33fdf4b 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -6,7 +6,7 @@ - [Source Analysis](./syntax-extensions/source-analysis.md) - [Macros in the Ast](./syntax-extensions/ast.md) - [Expansion](./syntax-extensions/expansion.md) - - [Hygiene]() + - [Hygiene](./syntax-extensions/hygiene.md) - [Macros](./macros.md) - [A Methodical Introduction](./macros/macros-methodical.md) - [macro_rules!](./macros/macro_rules.md) diff --git a/src/macros/macro_rules.md b/src/macros/macro_rules.md index 36c8613..f1eeb6e 100644 --- a/src/macros/macro_rules.md +++ b/src/macros/macro_rules.md @@ -243,6 +243,4 @@ error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times   -For the complete grammar definition you may want to consult the -[Macros By Example](https://doc.rust-lang.org/reference/macros-by-example.html#macros-by-example) -chapter of the Rust reference. +For the complete grammar definition you may want to consult the [Macros By Example](https://doc.rust-lang.org/reference/macros-by-example.html#macros-by-example) chapter of the Rust reference. diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index 503131b..dc0b55e 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -1,3 +1,28 @@ # Hygiene -TODO: Talk about hygiene in general, move most of macro/hygiene.rs, if not all, to here +Hygiene is an important concept for macros, it describes the ability for a macro to work in any context, not affecting nor being affected by its surroundings. +In other words this means that a syntax extension should be invocable anywhere without interfering with its surrounding context which is best shown with an example. + +Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, then given the following snippet: +```rust,ignore +make_local!(); +assert_eq!(local, 0); +``` +For `make_local` to be considered fully hygienic this snippet should not compile. +On the other hand if this example was to compile, the macro couldn't be considered hygienic, as it impacted its surrounding context by introducing a new name into it. + +Now let's assume we have some syntax extension `use_local` that expands to `local = 42;`, then given the following snippet: +```rust,ignore +let mut local = 0; +use_local!(); +``` + +In this case for `use_local` to be considered fully hygienic, this snippet again should not compile as otherwise it would be affected by its surrounding context and also affect its surrounding context as well. + +This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and [proc-macro `hygiene`] chapters as there are specifics to each. + +> **Aside**: Rust's syntax extension aren't necessarily fully hygienic, as it depends on the kind of syntax extension being used, for which in some cases one can create fully hygienic ones, but not so for others. + +[`macro_rules!` `hygiene`]: ../macros/minutiae/hygiene.md + +[proc-macro `hygiene`]: ../macros/minutiae/hygiene.md From 34242838663e46e67b59129365f4d06ef9118e29 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 22:02:33 +0200 Subject: [PATCH 05/92] Fix up some book links pointing to the html files instead of the markdown files --- src/SUMMARY.md | 4 +-- src/macros.md | 4 +-- src/macros/building-blocks/abacus-counting.md | 2 +- src/macros/building-blocks/ast-coercion.md | 2 +- src/macros/building-blocks/counting.md | 2 +- src/macros/macros-practical.md | 10 +++---- src/macros/minutiae/fragment-specifiers.md | 4 +-- src/macros/minutiae/identifiers.md | 12 ++++----- src/macros/minutiae/import-export.md | 6 ++--- src/macros/minutiae/metavar-and-expansion.md | 26 +++++++++---------- src/syntax-extensions.md | 6 +++-- src/syntax-extensions/hygiene.md | 11 ++++---- 12 files changed, 46 insertions(+), 43 deletions(-) diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 33fdf4b..076c774 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -7,7 +7,7 @@ - [Macros in the Ast](./syntax-extensions/ast.md) - [Expansion](./syntax-extensions/expansion.md) - [Hygiene](./syntax-extensions/hygiene.md) -- [Macros](./macros.md) +- [Macros-By-Example](./macros.md) - [A Methodical Introduction](./macros/macros-methodical.md) - [macro_rules!](./macros/macro_rules.md) - [Minutiae](./macros/minutiae.md) @@ -31,4 +31,4 @@ - [Counting](./macros/building-blocks/counting.md) - [Abacus Counting](./macros/building-blocks/abacus-counting.md) - [Parsing Rust](./macros/building-blocks/parsing.md) - - [Proc-Macros]() + - [Procedural Macros]() diff --git a/src/macros.md b/src/macros.md index 80bcea0..744665b 100644 --- a/src/macros.md +++ b/src/macros.md @@ -13,7 +13,7 @@ Should the information presented here not suffice, then there is also the [Macro [mbe]: https://doc.rust-lang.org/reference/macros-by-example.html [Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html -[practical]: ./macros/macros-practical.html -[methodical]: ./macros/macros-methodical.html +[practical]: ./macros/macros-practical.md +[methodical]: ./macros/macros-methodical.md [patterns]: ./macros/patterns.md [building blocks]: ./macros/building-blocks.md diff --git a/src/macros/building-blocks/abacus-counting.md b/src/macros/building-blocks/abacus-counting.md index 34b8409..ff97f98 100644 --- a/src/macros/building-blocks/abacus-counting.md +++ b/src/macros/building-blocks/abacus-counting.md @@ -84,7 +84,7 @@ Note that the example at the top combines some of the rules together (for exampl increment on `()` and `($($count:tt)+)` into an increment on `($($count:tt)*)`). If you want to extract the actual *value* of the counter, this can be done using a regular -[counter macro](./counting.html). For the example above, the terminal rules can be replaced with the following: +[counter macro](./counting.md). For the example above, the terminal rules can be replaced with the following: ```rust,ignore macro_rules! abacus { diff --git a/src/macros/building-blocks/ast-coercion.md b/src/macros/building-blocks/ast-coercion.md index d5a8def..9dd25d5 100644 --- a/src/macros/building-blocks/ast-coercion.md +++ b/src/macros/building-blocks/ast-coercion.md @@ -26,4 +26,4 @@ treat the final `tt` sequence as a particular kind of grammar construct. Note that this specific set of macros is determined by what macros are allowed to expand to, *not* what they are able to capture. -[push-down accumulation]: ../patterns/push-down-acc.html +[push-down accumulation]: ../patterns/push-down-acc.md diff --git a/src/macros/building-blocks/counting.md b/src/macros/building-blocks/counting.md index 3c993a3..91e57bc 100644 --- a/src/macros/building-blocks/counting.md +++ b/src/macros/building-blocks/counting.md @@ -211,7 +211,7 @@ can count 1 for the uneven discard and multiply by 2 again since we also halved. ((((0 << 1) | 1) << 1 << 1) | 1) << 1; ``` -Now to check if we expanded correctly manually we can use a one of the tools we introduced for [`debugging`](../minutiae/debugging.html). +Now to check if we expanded correctly manually we can use a one of the tools we introduced for [`debugging`](../minutiae/debugging.md). When expanding the macro there we should get: ```rust,ignore ((((0 << 1) | 1) << 1 << 1) | 1) << 1; diff --git a/src/macros/macros-practical.md b/src/macros/macros-practical.md index c8e09c7..1d53d76 100644 --- a/src/macros/macros-practical.md +++ b/src/macros/macros-practical.md @@ -65,8 +65,8 @@ says the input to the macro must match: - the literal token sequence `,` `...` `,`, - a valid *expression* captured into the [metavariable] `recur` (`$recur:expr`). -[repeating]: ./macro_rules.html#repetitions -[metavariable]: ./macro_rules.html#metavariables +[repeating]: ./macro_rules.md#repetitions +[metavariable]: ./macro_rules.md#metavariables Finally, the rule says that *if* the input matches this rule, then the macro invocation should be replaced by the token sequence `/* ... */`. @@ -210,7 +210,7 @@ Here, I've added a new metavariable: `sty` which should be a type. > **Aside**: if you're wondering, the bit after the colon in a metavariable can be one of several > kinds of syntax matchers. The most common ones are `item`, `expr`, and `ty`. A complete -> explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macro_rules.html#Metavariables). +> explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macro_rules.md#Metavariables). > > There's one other thing to be aware of: in the interests of future-proofing the language, the > compiler restricts what tokens you're allowed to put *after* a matcher, depending on what kind @@ -218,7 +218,7 @@ Here, I've added a new metavariable: `sty` which should be a type. > *only* be followed by one of `=>`, `,`, and `;`. > > A complete list can be found in -> [Macros, A Methodical Introduction; Minutiae; Metavariables and Expansion Redux](./minutiae/metavar-and-expansion.html). +> [Macros, A Methodical Introduction; Minutiae; Metavariables and Expansion Redux](./minutiae/metavar-and-expansion.md). ## Indexing and Shuffling @@ -1350,4 +1350,4 @@ Which gives us: Success! -[`macro_rules!`]: ./macro_rules.html +[`macro_rules!`]: ./macro_rules.md diff --git a/src/macros/minutiae/fragment-specifiers.md b/src/macros/minutiae/fragment-specifiers.md index 5e5c8d6..3f578ab 100644 --- a/src/macros/minutiae/fragment-specifiers.md +++ b/src/macros/minutiae/fragment-specifiers.md @@ -132,7 +132,7 @@ expression solely consists of only a block expression or control flow expression The fine details of what was just mentioned here can be looked up in the [reference](https://doc.rust-lang.org/reference/statements.html). -[^debugging]:See the [debugging chapter](./debugging.html) for tips on how to do this. +[^debugging]:See the [debugging chapter](./debugging.md) for tips on how to do this. ## `pat` @@ -230,7 +230,7 @@ paths! { ## `tt` The `tt` fragment matches a TokenTree. If you need a refresher on what exactly a TokenTree was you -may want to revisit the [TokenTree chapter](../../syntax-extensions/source-analysis.html#token-trees) of this +may want to revisit the [TokenTree chapter](../../syntax-extensions/source-analysis.md#token-trees) of this book. The `tt` fragment is one of the most powerful fragments, as it can match nearly anything while still allowing you to inspect the contents of it at a later state in the macro. diff --git a/src/macros/minutiae/identifiers.md b/src/macros/minutiae/identifiers.md index ff0493c..2bd8f59 100644 --- a/src/macros/minutiae/identifiers.md +++ b/src/macros/minutiae/identifiers.md @@ -48,7 +48,7 @@ impl Dummy { self } } -# +# # fn main() { # println!("{:?}", Dummy(4).double().0); # } @@ -87,7 +87,7 @@ impl Dummy { mut_self } } -# +# # fn main() { # println!("{:?}", Dummy(4).double().0); # } @@ -134,7 +134,7 @@ impl Dummy { self }} } -# +# # fn main() { # println!("{:?}", Dummy(4).double().0); # } @@ -159,7 +159,7 @@ impl Dummy { self }} } -# +# # fn main() { # println!("{:?}", Dummy(4).double().0); # } @@ -182,7 +182,7 @@ struct Dummy(i32); impl Dummy { double_method! {_, 0} } -# +# # fn main() { # println!("{:?}", Dummy(4).double().0); # } @@ -209,4 +209,4 @@ Except, no, because `self` isn't a pattern. Joy. The only work around for this (in cases where you want to accept some combination of these tokens) is to use a [`tt`] matcher instead. -[`tt`]:./fragment-specifiers.html#tt +[`tt`]:./fragment-specifiers.md#tt diff --git a/src/macros/minutiae/import-export.md b/src/macros/minutiae/import-export.md index 8ebc3d0..fb845ae 100644 --- a/src/macros/minutiae/import-export.md +++ b/src/macros/minutiae/import-export.md @@ -43,7 +43,7 @@ The following code will work as expected: X!(); // X is defined #[macro_use] extern crate macs; X!(); -# +# # fn main() {} ``` @@ -83,7 +83,7 @@ to guarantee any given macro will be available when imported by another crate. It is recommended that you *always* use absolute paths to non-macro names, to avoid conflicts, *including* names in the standard library. -[`$crate`]:./hygiene.html#crate +[`$crate`]:./hygiene.md#crate ## Edition 2018 @@ -111,4 +111,4 @@ So scoping applies there the same way as before as well. > The `$crate` prefix works in this version for everything, macros and items alike since this Edition > came out with Rust 1.31. -[scoping chapter]:./scoping.html \ No newline at end of file +[scoping chapter]:./scoping.md diff --git a/src/macros/minutiae/metavar-and-expansion.md b/src/macros/minutiae/metavar-and-expansion.md index 55c66d2..93f2f77 100644 --- a/src/macros/minutiae/metavar-and-expansion.md +++ b/src/macros/minutiae/metavar-and-expansion.md @@ -134,16 +134,16 @@ capture with anything else, the only thing you can do with the result from then directly into the output. -[`item`]:./fragment-specifiers.html#item -[`block`]:./fragment-specifiers.html#block -[`stmt`]:./fragment-specifiers.html#stmt -[`pat`]:./fragment-specifiers.html#pat -[`expr`]:./fragment-specifiers.html#expr -[`ty`]:./fragment-specifiers.html#ty -[`ident`]:./fragment-specifiers.html#ident -[`path`]:./fragment-specifiers.html#path -[`tt`]:./fragment-specifiers.html#tt -[`meta`]:./fragment-specifiers.html#meta -[`lifetime`]:./fragment-specifiers.html#lifetime -[`vis`]:./fragment-specifiers.html#vis -[`literal`]:./fragment-specifiers.html#literal +[`item`]:./fragment-specifiers.md#item +[`block`]:./fragment-specifiers.md#block +[`stmt`]:./fragment-specifiers.md#stmt +[`pat`]:./fragment-specifiers.md#pat +[`expr`]:./fragment-specifiers.md#expr +[`ty`]:./fragment-specifiers.md#ty +[`ident`]:./fragment-specifiers.md#ident +[`path`]:./fragment-specifiers.md#path +[`tt`]:./fragment-specifiers.md#tt +[`meta`]:./fragment-specifiers.md#meta +[`lifetime`]:./fragment-specifiers.md#lifetime +[`vis`]:./fragment-specifiers.md#vis +[`literal`]:./fragment-specifiers.md#literal diff --git a/src/syntax-extensions.md b/src/syntax-extensions.md index 7133ee2..b274d0a 100644 --- a/src/syntax-extensions.md +++ b/src/syntax-extensions.md @@ -1,5 +1,7 @@ # Syntax Extensions -Before talking about Rust's many macros it is worthwhile to discuss the general mechanism they are built on: syntax extensions. +Before talking about Rust's different macro systems it is worthwhile to discuss the general mechanism they are built on: *syntax extensions*. -To do that, we must discuss how Rust source is processed by the compiler, and the general mechanisms on which user-defined macros and proc-macros are built upon. +To do that, we must first discuss how Rust source is processed by the compiler, and the general mechanisms on which user-defined macros and proc-macros are built upon. + +> **Note**: This book will use the term *syntax extension* from now on when talking about all of rust's different macro kinds in general to reduce potential confusion with the upcoming [declarative macro 2.0](https://github.com/rust-lang/rust/issues/39412) proposal which uses the `macro` keyword. diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index dc0b55e..1abece6 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -1,13 +1,16 @@ # Hygiene Hygiene is an important concept for macros, it describes the ability for a macro to work in any context, not affecting nor being affected by its surroundings. -In other words this means that a syntax extension should be invocable anywhere without interfering with its surrounding context which is best shown with an example. +In other words this means that a syntax extension should be invocable anywhere without interfering with its surrounding context. + +This is best shown by example: Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, then given the following snippet: ```rust,ignore make_local!(); assert_eq!(local, 0); ``` + For `make_local` to be considered fully hygienic this snippet should not compile. On the other hand if this example was to compile, the macro couldn't be considered hygienic, as it impacted its surrounding context by introducing a new name into it. @@ -19,10 +22,8 @@ use_local!(); In this case for `use_local` to be considered fully hygienic, this snippet again should not compile as otherwise it would be affected by its surrounding context and also affect its surrounding context as well. -This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and [proc-macro `hygiene`] chapters as there are specifics to each. +This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and proc-macro `hygiene` chapters as there are specifics to each. -> **Aside**: Rust's syntax extension aren't necessarily fully hygienic, as it depends on the kind of syntax extension being used, for which in some cases one can create fully hygienic ones, but not so for others. +> **Aside**: A Rust syntax extension might or might not be fully hygienic, depending on its kind and definition. [`macro_rules!` `hygiene`]: ../macros/minutiae/hygiene.md - -[proc-macro `hygiene`]: ../macros/minutiae/hygiene.md From f9e46e779b4bbfdfc72e9c13fd69fd24496d0395 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 22:29:56 +0200 Subject: [PATCH 06/92] Sort the fragment specifier listing --- .github/workflows/gh-pages.yml | 2 +- src/macros/minutiae/fragment-specifiers.md | 346 ++++++++++----------- 2 files changed, 165 insertions(+), 183 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 8a0b822..c9f73c5 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,4 +25,4 @@ jobs: publish_dir: ./book force_orphan: true user_name: 'github-actions[bot]' - user_email: 'github-actions[bot]@users.noreply.github.com' \ No newline at end of file + user_email: 'github-actions[bot]@users.noreply.github.com' diff --git a/src/macros/minutiae/fragment-specifiers.md b/src/macros/minutiae/fragment-specifiers.md index 3f578ab..eaded98 100644 --- a/src/macros/minutiae/fragment-specifiers.md +++ b/src/macros/minutiae/fragment-specifiers.md @@ -1,53 +1,27 @@ # Fragment Specifiers -As shown in the [`macro_rules`] chapter, Rust, as of 1.46, has 13 fragment specifiers. This section -will go a bit more into detail for some of them and tries to always show a few examples of what a -matcher can match with. +As shown in the [`macro_rules`] chapter, Rust, as of 1.46, has 13 fragment specifiers. +This section will go a bit more into detail for some of them and tries to always show a few examples of what a matcher can match with. -> Note that capturing with anything but the `ident`, `lifetime` and `tt` fragments will render the -> captured AST opaque, making it impossible to further inspect it in future macro invocations. +> Note that capturing with anything but the `ident`, `lifetime` and `tt` fragments will render the captured AST opaque, making it impossible to further inspect it in future macro invocations. -* [`item`](#item) * [`block`](#block) -* [`stmt`](#stmt) -* [`pat`](#pat) * [`expr`](#expr) -* [`ty`](#ty) * [`ident`](#ident) +* [`item`](#item) +* [`lifetime`](#lifetime) +* [`literal`](#literal) +* [`meta`](#meta) +* [`pat`](#pat) * [`path`](#path) +* [`stmt`](#stmt) * [`tt`](#tt) -* [`meta`](#meta) -* [`lifetime`](#lifetime) +* [`ty`](#ty) * [`vis`](#vis) -* [`literal`](#literal) - -## `item` - -The `item` fragment simply matches any of Rust's -[item](https://doc.rust-lang.org/reference/items.html) *definitions*, not identifiers that refer to -items. Item examples: - -```rust -macro_rules! items { - ($($item:item)*) => (); -} - -items! { - struct Foo; - enum Bar { - Baz - } - impl Foo {} - /*...*/ -} -# fn main() {} -``` ## `block` -The `block` fragment solely matches a [block expression](https://doc.rust-lang.org/reference/expressions/block-expr.html), -which consists of an opening `{` brace, followed by any amount of statements and finally followed -by a closing `}` brace. +The `block` fragment solely matches a [block expression](https://doc.rust-lang.org/reference/expressions/block-expr.html), which consists of an opening `{` brace, followed by any amount of statements and finally followed by a closing `}` brace. ```rust macro_rules! blocks { @@ -64,155 +38,143 @@ blocks! { # fn main() {} ``` -## `stmt` - -The `statement` fragment solely matches a [statement](https://doc.rust-lang.org/reference/statements.html) -without its trailing semicolon, unless its an item statement that requires one. What would be an -item statement that requires one? A Unit-Struct would be a simple one, as defining one requires a -trailing semicolon. +## `expr` -Let's use a simple example to show exactly what is meant with this. We use a macro that merely emits -what it captures: +The `expr` fragment matches any kind of [expression](https://doc.rust-lang.org/reference/expressions.html)(Rust has a lot of them, given it *is* an expression orientated language). -```rust,ignore -macro_rules! statements { - ($($stmt:stmt)*) => ($($stmt)*); +```rust +macro_rules! expressions { + ($($expr:expr)*) => (); } -fn main() { - statements! { - struct Foo; - fn foo() {} - let zig = 3 - let zig = 3; - 3 - 3; - if true {} else {} - {} - } +expressions! { + "literal" + funcall() + future.await + break 'foo bar } - +# fn main() {} ``` -Expanding this, via the [playground](https://play.rust-lang.org/) for example[^debugging], gives us roughly the -following: +## `ident` -```rust,ignore -/* snip */ +The `ident` fragment matches an [identifier](https://doc.rust-lang.org/reference/identifiers.html) or *keyword*. -fn main() { - struct Foo; - fn foo() { } - let zig = 3; - let zig = 3; - ; - 3; - 3; - ; - if true { } else { } - { } +```rust +macro_rules! idents { + ($($ident:ident)*) => (); } -``` - -From this we can tell a few things: -The first you should be able to see immediately is that while the `stmt` fragment doesn't capture -trailing semicolons, it still emits them when required, even if the statement is already followed by -one. The simple reason for that is that semicolons on their own are already valid statements. So we -are actually invoking our macro here with not 8 statements, but 11! +idents! { + // _ <- This is not an ident, it is a pattern + foo + async + O_________O + _____O_____ +} +# fn main() {} +``` -Another thing you should be able to notice here is that the trailing semicolon of the `struct Foo;` -item statement is being matched, otherwise we would've seen an extra one like in the other cases. -This makes sense as we already said, that for item statements that require one, the trailing -semicolon will be matched with. +## `item` -A last observation is that expressions get emitted back with a trailing semicolon, unless the -expression solely consists of only a block expression or control flow expression. +The `item` fragment simply matches any of Rust's [item](https://doc.rust-lang.org/reference/items.html) *definitions*, not identifiers that refer to items. +Item examples: -The fine details of what was just mentioned here can be looked up in the -[reference](https://doc.rust-lang.org/reference/statements.html). +```rust +macro_rules! items { + ($($item:item)*) => (); +} -[^debugging]:See the [debugging chapter](./debugging.md) for tips on how to do this. +items! { + struct Foo; + enum Bar { + Baz + } + impl Foo {} + /*...*/ +} +# fn main() {} +``` -## `pat` +## `lifetime` -The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html). +The `lifetime` fragment matches a [lifetime or label](https://doc.rust-lang.org/reference/tokens.html#lifetimes-and-loop-labels). +It's quite similar to [`ident`](#ident) but with a prepended `'`. ```rust -macro_rules! patterns { - ($($pat:pat)*) => (); +macro_rules! lifetimes { + ($($lifetime:lifetime)*) => (); } -patterns! { - "literal" - _ - 0..5 - ref mut PatternsAreNice +lifetimes! { + 'static + 'shiv + '_ } # fn main() {} ``` -## `expr` +## `literal` -The `expr` fragment matches any kind of [expression](https://doc.rust-lang.org/reference/expressions.html) -(Rust has a lot of them, given it *is* an expression orientated language). +The `literal` fragment matches any [literal expression](https://doc.rust-lang.org/reference/expressions/literal-expr.html). ```rust -macro_rules! expressions { - ($($expr:expr)*) => (); +macro_rules! literals { + ($($literal:literal)*) => (); } -expressions! { - "literal" - funcall() - future.await - break 'foo bar +literals! { + -1 + "hello world" + 2.3 + b'b' + true } # fn main() {} ``` -## `ty` +## `meta` -The `ty` fragment matches any kind of [type expression](https://doc.rust-lang.org/reference/types.html#type-expressions). -A type expression is the syntax with which one refers to a type in the language. +The `meta` fragment matches an [attribute](https://doc.rust-lang.org/reference/attributes.html), to be more precise, the contents of an attribute. +You will usually see this fragment being used in a matcher like `#[$meta:meta]` or `#![$meta:meta]`. ```rust -macro_rules! types { - ($($type:ty)*) => (); +macro_rules! metas { + ($($meta:meta)*) => (); } -types! { - foo::bar - bool - [u8] +metas! { + ASimplePath + super::man + path = "home" + foo(bar) } # fn main() {} ``` -## `ident` +> A neat thing about doc comments: They are actually attributes in the form of `#[doc="…"]` where the `...` is the actual comment string, meaning you can act on doc comments in macros! -The `ident` fragment matches an [identifier](https://doc.rust-lang.org/reference/identifiers.html) -or *keyword*. +## `pat` + +The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html). ```rust -macro_rules! idents { - ($($ident:ident)*) => (); +macro_rules! patterns { + ($($pat:pat)*) => (); } -idents! { - // _ <- This is not an ident, it is a pattern - foo - async - O_________O - _____O_____ +patterns! { + "literal" + _ + 0..5 + ref mut PatternsAreNice } # fn main() {} ``` ## `path` -The `path` fragment matches a so called [TypePath](https://doc.rust-lang.org/reference/paths.html#paths-in-types) -style path. +The `path` fragment matches a so called [TypePath](https://doc.rust-lang.org/reference/paths.html#paths-in-types) style path. ```rust macro_rules! paths { @@ -227,50 +189,89 @@ paths! { # fn main() {} ``` -## `tt` +## `stmt` -The `tt` fragment matches a TokenTree. If you need a refresher on what exactly a TokenTree was you -may want to revisit the [TokenTree chapter](../../syntax-extensions/source-analysis.md#token-trees) of this -book. The `tt` fragment is one of the most powerful fragments, as it can match nearly anything while -still allowing you to inspect the contents of it at a later state in the macro. +The `statement` fragment solely matches a [statement](https://doc.rust-lang.org/reference/statements.html) without its trailing semicolon, unless its an item statement that requires one. +What would be an item statement that requires one? +A Unit-Struct would be a simple one, as defining one requires a trailing semicolon. -## `meta` +Let's use a simple example to show exactly what is meant with this. +We use a macro that merely emits what it captures: -The `meta` fragment matches an [attribute](https://doc.rust-lang.org/reference/attributes.html), to -be more precise, the contents of an attribute. You will usually see this fragment being used in a -matcher like `#[$meta:meta]` or `#![$meta:meta]`. +```rust,ignore +macro_rules! statements { + ($($stmt:stmt)*) => ($($stmt)*); +} -```rust -macro_rules! metas { - ($($meta:meta)*) => (); +fn main() { + statements! { + struct Foo; + fn foo() {} + let zig = 3 + let zig = 3; + 3 + 3; + if true {} else {} + {} + } } -metas! { - ASimplePath - super::man - path = "home" - foo(bar) +``` + +Expanding this, via the [playground](https://play.rust-lang.org/) for example[^debugging], gives us roughly the following: + +```rust,ignore +/* snip */ + +fn main() { + struct Foo; + fn foo() { } + let zig = 3; + let zig = 3; + ; + 3; + 3; + ; + if true { } else { } + { } } -# fn main() {} ``` -> A neat thing about doc comments: They are actually attributes in the form of `#[doc="…"]` where -> the `...` is the actual comment string, meaning you can act on doc comments in macros! +From this we can tell a few things: -## `lifetime` +The first you should be able to see immediately is that while the `stmt` fragment doesn't capture trailing semicolons, it still emits them when required, even if the statement is already followed by one. +The simple reason for that is that semicolons on their own are already valid statements. +So we are actually invoking our macro here with not 8 statements, but 11! -The `lifetime` fragment matches a [lifetime or label](https://doc.rust-lang.org/reference/tokens.html#lifetimes-and-loop-labels). -It's quite similar to [`ident`](#ident) but with a prepended `'`. +Another thing you should be able to notice here is that the trailing semicolon of the `struct Foo;` item statement is being matched, otherwise we would've seen an extra one like in the other cases. +This makes sense as we already said, that for item statements that require one, the trailing semicolon will be matched with. + +A last observation is that expressions get emitted back with a trailing semicolon, unless the expression solely consists of only a block expression or control flow expression. + +The fine details of what was just mentioned here can be looked up in the [reference](https://doc.rust-lang.org/reference/statements.html). + +[^debugging]:See the [debugging chapter](./debugging.md) for tips on how to do this. + +## `tt` + +The `tt` fragment matches a TokenTree. +If you need a refresher on what exactly a TokenTree was you may want to revisit the [TokenTree chapter](../../syntax-extensions/source-analysis.md#token-trees) of this book. +The `tt` fragment is one of the most powerful fragments, as it can match nearly anything while still allowing you to inspect the contents of it at a later state in the macro. + +## `ty` + +The `ty` fragment matches any kind of [type expression](https://doc.rust-lang.org/reference/types.html#type-expressions). +A type expression is the syntax with which one refers to a type in the language. ```rust -macro_rules! lifetimes { - ($($lifetime:lifetime)*) => (); +macro_rules! types { + ($($type:ty)*) => (); } -lifetimes! { - 'static - 'shiv - '_ +types! { + foo::bar + bool + [u8] } # fn main() {} ``` @@ -278,8 +279,8 @@ lifetimes! { ## `vis` The `vis` fragment matches a *possibly empty* [Visibility qualifier](https://doc.rust-lang.org/reference/visibility-and-privacy.html). -Emphasis lies on the *possibly empty* part. You can think of this fragment having an implicit `?` -repetition to it, meaning you don't, and in fact cannot, wrap it in a direct repetition. +Emphasis lies on the *possibly empty* part. +You can think of this fragment having an implicit `?` repetition to it, meaning you don't, and in fact cannot, wrap it in a direct repetition. ```rust macro_rules! visibilities { @@ -297,23 +298,4 @@ visibilities! { # fn main() {} ``` -## `literal` - -The `literal` fragment matches any [literal expression](https://doc.rust-lang.org/reference/expressions/literal-expr.html). - -```rust -macro_rules! literals { - ($($literal:literal)*) => (); -} - -literals! { - -1 - "hello world" - 2.3 - b'b' - true -} -# fn main() {} -``` - [`macro_rules`]: ../macro_rules.md From 9f988bfc6248791284058c3c12c9ae1fdb6a94fa Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 22:40:06 +0200 Subject: [PATCH 07/92] Change line styles in minutiae chapters --- src/macros/minutiae/debugging.md | 40 +++++++------- src/macros/minutiae/hygiene.md | 36 ++++++------- src/macros/minutiae/identifiers.md | 42 +++++++-------- src/macros/minutiae/import-export.md | 57 +++++++++----------- src/macros/minutiae/metavar-and-expansion.md | 46 ++++++++-------- src/macros/minutiae/scoping.md | 56 ++++++++----------- 6 files changed, 120 insertions(+), 157 deletions(-) diff --git a/src/macros/minutiae/debugging.md b/src/macros/minutiae/debugging.md index e73bcc2..cf6fc1c 100644 --- a/src/macros/minutiae/debugging.md +++ b/src/macros/minutiae/debugging.md @@ -1,8 +1,8 @@ # Debugging -`rustc` provides a number of tools to debug `macro_rules!` macros. One of the most useful is -[`trace_macros!`], which is a directive to the compiler instructing it to dump every `macro_rules!` -macro invocation prior to expansion. For example, given the following: +`rustc` provides a number of tools to debug `macro_rules!` macros. +One of the most useful is [`trace_macros!`], which is a directive to the compiler instructing it to dump every `macro_rules!` macro invocation prior to expansion. +For example, given the following: ```rust,ignore # // Note: make sure to use a nightly channel compiler. @@ -18,7 +18,7 @@ trace_macros!(true); each_tt!(spim wak plee whum); trace_macros!(false); each_tt!(trom qlip winp xod); -# +# # fn main() {} ``` @@ -43,11 +43,11 @@ note: trace_macro = note: to `` ``` -This is *particularly* invaluable when debugging deeply recursive `macro_rules!` macros. You can -also enable this from the command-line by adding `-Z trace-macros` to the compiler command line. +This is *particularly* invaluable when debugging deeply recursive `macro_rules!` macros. +You can also enable this from the command-line by adding `-Z trace-macros` to the compiler command line. -Secondly, there is [`log_syntax!`] which causes the compiler to output all tokens passed to it. For -example, this makes the compiler sing a song: +Secondly, there is [`log_syntax!`] which causes the compiler to output all tokens passed to it. +For example, this makes the compiler sing a song: ```rust # // Note: make sure to use a nightly channel compiler. @@ -66,14 +66,15 @@ sing! { ; '\x08' '\'' + '$' ? '\x7f' , # '"' ~ | ) '\x07' } -# +# # fn main() {} ``` This can be used to do slightly more targeted debugging than [`trace_macros!`]. -Sometimes, it is what the macro *expands to* that proves problematic. For this, the `--pretty` -argument to the compiler can be used. Given the following code: +Sometimes, it is what the macro *expands to* that proves problematic. +For this, the `--pretty` argument to the compiler can be used. +Given the following code: ```rust,ignore // Shorthand for initialising a `String`. @@ -123,18 +124,13 @@ fn main() { Other options to `--pretty` can be listed using `rustc -Z unstable-options --help -v`; a full list is not provided since, as implied by the name, any such list would be subject to change at any time. -But not just `rustc` exposes means to aid in debugging macros. For the aforementioned -`--pretty=expanded` option, there exists a nice `cargo` addon called -[`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) -which is basically just a wrapper around it. +But not just `rustc` exposes means to aid in debugging macros. +For the aforementioned `--pretty=expanded` option, there exists a nice `cargo` addon called [`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) which is basically just a wrapper around it. -You can also use the [playground](https://play.rust-lang.org/), clicking on its `TOOLS` button in -the top right gives you the option to expand macros right there! +You can also use the [playground](https://play.rust-lang.org/), clicking on its `TOOLS` button in the top right gives you the option to expand macros right there! -Another amazing tool is [`lukaslueg`'s](https://github.com/lukaslueg) -[`macro_railroad`](https://github.com/lukaslueg/macro_railroad), a tool that allows you visualize -and generate syntax diagrams for Rust's `macro_rules!` macros. It visualizes the accepted -macro's grammar as an automata. +Another amazing tool is [`lukaslueg`'s](https://github.com/lukaslueg) [`macro_railroad`](https://github.com/lukaslueg/macro_railroad), a tool that allows you visualize and generate syntax diagrams for Rust's `macro_rules!` macros. +It visualizes the accepted macro's grammar as an automata. [`trace_macros!`]:https://doc.rust-lang.org/std/macro.trace_macros.html -[`log_syntax!`]:https://doc.rust-lang.org/std/macro.log_syntax.html \ No newline at end of file +[`log_syntax!`]:https://doc.rust-lang.org/std/macro.log_syntax.html diff --git a/src/macros/minutiae/hygiene.md b/src/macros/minutiae/hygiene.md index b347e3d..fb1bd9c 100644 --- a/src/macros/minutiae/hygiene.md +++ b/src/macros/minutiae/hygiene.md @@ -1,25 +1,23 @@ # Hygiene -`macro_rules!` macros in Rust are *partially* hygienic. Specifically, they are hygienic when it -comes to most identifiers, but *not* when it comes to generic type parameters or lifetimes. +`macro_rules!` macros in Rust are *partially* hygienic. +Specifically, they are hygienic when it comes to most identifiers, but *not* when it comes to generic type parameters or lifetimes. -Hygiene works by attaching an invisible "syntax context" value to all identifiers. When two -identifiers are compared, *both* the identifiers' textual names *and* syntax contexts must be -identical for the two to be considered equal. +Hygiene works by attaching an invisible "syntax context" value to all identifiers. +When two identifiers are compared, *both* the identifiers' textual names *and* syntax contexts must be identical for the two to be considered equal. To illustrate this, consider the following code: 
macro_rules! using_a {
    ($e:expr) => {
        {
            let a = 42;
            $e
        }
    }
}

let four = using_a!(a / 10);
-We will use the background colour to denote the syntax context. Now, let's expand the macro -invocation: +We will use the background colour to denote the syntax context. +Now, let's expand the macro invocation: 
let four = {
    let a = 42;
    a / 10
};
First, recall that `macro_rules!` invocations effectively *disappear* during expansion. -Second, if you attempt to compile this code, the compiler will respond with something along the -following lines: +Second, if you attempt to compile this code, the compiler will respond with something along the following lines: ```text error[E0425]: cannot find value `a` in this scope @@ -29,14 +27,12 @@ error[E0425]: cannot find value `a` in this scope | ^ not found in this scope ``` -Note that the background colour (*i.e.* syntax context) for the expanded macro *changes* as part of -expansion. Each `macro_rules!` macro expansion is given a new, unique syntax context for its -contents. As a result, there are *two different `a`s* in the expanded code: one in the first syntax -context, the second in the other. In other words, a is not -the same identifier as a, however similar they may appear. +Note that the background colour (*i.e.* syntax context) for the expanded macro *changes* as part of expansion. +Each `macro_rules!` macro expansion is given a new, unique syntax context for its contents. +As a result, there are *two different `a`s* in the expanded code: one in the first syntax context, the second in the other. +In other words, a is not the same identifier as a, however similar they may appear. -That said, tokens that were substituted *into* the expanded output *retain* their original syntax -context (by virtue of having been provided to the macro as opposed to being part of the macro itself). +That said, tokens that were substituted *into* the expanded output *retain* their original syntax context (by virtue of having been provided to the macro as opposed to being part of the macro itself). Thus, the solution is to modify the macro as follows: 
macro_rules! using_a {
    ($a:ident, $e:expr) => {
        {
            let $a = 42;
            $e
        }
    }
}

let four = using_a!(a, a / 10);
@@ -49,9 +45,8 @@ The compiler will accept this code because there is only one `a` being used. ### `$crate` -Hygiene is also the reason that we need the `$crate` metavariable when our macro needs access to -other items in the defining crate. What this special metavariable does is that it expands to an -absolute path to the defining crate. +Hygiene is also the reason that we need the `$crate` metavariable when our macro needs access to other items in the defining crate. +What this special metavariable does is that it expands to an absolute path to the defining crate. ```rust,ignore //// Definitions in the `helper_macro` crate. @@ -76,8 +71,7 @@ fn unit() { } ``` -Note that, because `$crate` refers to the current crate, it must be used with a fully qualified -module path when referring to non-macro items: +Note that, because `$crate` refers to the current crate, it must be used with a fully qualified module path when referring to non-macro items: ```rust pub mod inner { diff --git a/src/macros/minutiae/identifiers.md b/src/macros/minutiae/identifiers.md index 2bd8f59..ca2d05b 100644 --- a/src/macros/minutiae/identifiers.md +++ b/src/macros/minutiae/identifiers.md @@ -1,11 +1,11 @@ # Non-Identifier Identifiers -There are two tokens which you are likely to run into eventually that *look* like identifiers, -but aren't. Except when they are. +There are two tokens which you are likely to run into eventually that *look* like identifiers, but aren't. +Except when they are. -First is `self`. This is *very definitely* a keyword. However, it also happens to fit the definition -of an identifier. In regular Rust code, there's no way for `self` to be interpreted as an identifier, -but it *can* happen with `macro_rules!` macros: +First is `self`. +This is *very definitely* a keyword. However, it also happens to fit the definition of an identifier. +In regular Rust code, there's no way for `self` to be interpreted as an identifier, but it *can* happen with `macro_rules!` macros: ```rust macro_rules! what_is { @@ -31,8 +31,9 @@ the keyword `self` ``` But that makes no sense; `call_with_ident!` required an identifier, matched one, and substituted it! -So `self` is both a keyword and not a keyword at the same time. You might wonder how this is in any -way important. Take this example: +So `self` is both a keyword and not a keyword at the same time. +You might wonder how this is in any way important. +Take this example: ```rust macro_rules! make_mutable { @@ -69,9 +70,8 @@ error: `mut` must be followed by a named binding = note: `mut` may be followed by `variable` and `variable @ pattern` ``` -So the macro will happily match `self` as an identifier, allowing you to use it in cases where you -can't actually use it. But, fine; it somehow remembers that `self` is a keyword even when it's an -identifier, so you *should* be able to do this, right? +So the macro will happily match `self` as an identifier, allowing you to use it in cases where you can't actually use it. +But, fine; it somehow remembers that `self` is a keyword even when it's an identifier, so you *should* be able to do this, right? ```rust macro_rules! make_self_mutable { @@ -112,10 +112,9 @@ error[E0424]: expected value, found module `self` | ``` -Now the compiler thinks we refer to our module with `self`, but that doesn't make sense. We already -have a `self` right there, in the function signature which is definitely not a module. It's almost -like it's complaining that the `self` it's trying to use isn't the *same* `self`... as though the -`self` keyword has hygiene, like an... identifier. +Now the compiler thinks we refer to our module with `self`, but that doesn't make sense. +We already have a `self` right there, in the function signature which is definitely not a module. +It's almost like it's complaining that the `self` it's trying to use isn't the *same* `self`... as though the `self` keyword has hygiene, like an... identifier. ```rust macro_rules! double_method { @@ -165,7 +164,8 @@ impl Dummy { # } ``` -At last, *this works*. So `self` is both a keyword *and* an identifier when it feels like it. +At last, *this works*. +So `self` is both a keyword *and* an identifier when it feels like it. Surely this works for other, similar constructs, right? ```rust @@ -199,14 +199,12 @@ error: no rules expected the token `_` | ^ no rules expected this token in macro call ``` -No, of course not. `_` is a keyword that is valid in patterns and expressions, but somehow *isn't* -an identifier like the keyword `self` is, despite matching the definition of an identifier just the -same. +No, of course not. +`_` is a keyword that is valid in patterns and expressions, but somehow *isn't* an identifier like the keyword `self` is, despite matching the definition of an identifier just the same. -You might think you can get around this by using `$self_:pat` instead; that way, `_` will match! -Except, no, because `self` isn't a pattern. Joy. +You might think you can get around this by using `$self_:pat` instead; that way, `_` will match! Except, no, because `self` isn't a pattern. +Joy. -The only work around for this (in cases where you want to accept some combination of these tokens) -is to use a [`tt`] matcher instead. +The only work around for this (in cases where you want to accept some combination of these tokens) is to use a [`tt`] matcher instead. [`tt`]:./fragment-specifiers.md#tt diff --git a/src/macros/minutiae/import-export.md b/src/macros/minutiae/import-export.md index fb845ae..014f605 100644 --- a/src/macros/minutiae/import-export.md +++ b/src/macros/minutiae/import-export.md @@ -1,14 +1,13 @@ # Import and Export -Importing `macro_rules!` macros differs between the two Rust Editions, 2015 and 2018. It is -recommended to read both parts nevertheless, as the 2018 Edition can still use the constructs that -are explained in the 2015 Edition. +Importing `macro_rules!` macros differs between the two Rust Editions, 2015 and 2018. +It is recommended to read both parts nevertheless, as the 2018 Edition can still use the constructs that are explained in the 2015 Edition. ## Edition 2015 -In Edition 2015 you have to use the `#[macro_use]` attribute that has already been introduced in the -[scoping chapter]. This can be applied to *either* modules or external crates. For -example: +In Edition 2015 you have to use the `#[macro_use]` attribute that has already been introduced in the [scoping chapter]. +This can be applied to *either* modules or external crates. +For example: ```rust #[macro_use] @@ -22,8 +21,8 @@ X!(); # fn main() {} ``` -`macro_rules!` macros can be exported from the current crate using `#[macro_export]`. Note that this -*ignores* all visibility. +`macro_rules!` macros can be exported from the current crate using `#[macro_export]`. +Note that this *ignores* all visibility. Given the following definition for a library package `macs`: @@ -47,14 +46,12 @@ X!(); # fn main() {} ``` -This works, as said in the [scoping chapter], because `#[macro_use]` works slightly different on -extern crates, as it basically *hoists* the exported macros out of the crate to the top of the -module. +This works, as said in the [scoping chapter], because `#[macro_use]` works slightly different on extern crates, as it basically *hoists* the exported macros out of the crate to the top of the module. > Note: you can *only* `#[macro_use]` an external crate from the root module. -Finally, when importing `macro_rules!` macros from an external crate, you can control *which* macros -you import. You can use this to limit namespace pollution, or to override specific macros, like so: +Finally, when importing `macro_rules!` macros from an external crate, you can control *which* macros you import. +You can use this to limit namespace pollution, or to override specific macros, like so: ```rust,ignore // Import *only* the `X!` macro. @@ -69,30 +66,26 @@ X!(); // X is defined, and so is Y! fn main() {} ``` -When exporting `macro_rules!` macros, it is often useful to refer to non-macro symbols in the -defining crate. Because crates can be renamed, there is a special substitution variable available: -[`$crate`]. This will *always* expand to an absolute path prefix to the containing crate -(*e.g.* `:: macs`). +When exporting `macro_rules!` macros, it is often useful to refer to non-macro symbols in the defining crate. +Because crates can be renamed, there is a special substitution variable available: [`$crate`]. +This will *always* expand to an absolute path prefix to the containing crate(*e.g.* `:: macs`). -Note that unless your compiler version is >= 1.30, this does *not* work for `macro_rules!` macros, -since `macro_rules!` macros do not interact with regular name resolution in any way. Otherwise, you -cannot use something like `$crate::Y!` to refer to a particular macro within your crate. The -implication, combined with selective imports via `#[macro_use]` is that there is currently *no way* -to guarantee any given macro will be available when imported by another crate. +Note that unless your compiler version is >= 1.30, this does *not* work for `macro_rules!` macros, since `macro_rules!` macros do not interact with regular name resolution in any way. +Otherwise, you cannot use something like `$crate::Y!` to refer to a particular macro within your crate. +The implication, combined with selective imports via `#[macro_use]` is that there is currently *no way* to guarantee any given macro will be available when imported by another crate. -It is recommended that you *always* use absolute paths to non-macro names, to avoid conflicts, -*including* names in the standard library. +It is recommended that you *always* use absolute paths to non-macro names, to avoid conflicts, *including* names in the standard library. [`$crate`]:./hygiene.md#crate ## Edition 2018 -The 2018 Edition made our lives a lot easier when it comes to `macro_rules!` macros. Why you ask? -Quite simply because it managed to make them feel more like proper items than some special thing in -the language. What this means is that we can properly import and use them in a namespaced fashion! +The 2018 Edition made our lives a lot easier when it comes to `macro_rules!` macros. +Why you ask? +Quite simply because it managed to make them feel more like proper items than some special thing in the language. +What this means is that we can properly import and use them in a namespaced fashion! -So instead of using `#[macro_use]` to import every exported macro from a crate into the global -namespace we can now do the following: +So instead of using `#[macro_use]` to import every exported macro from a crate into the global namespace we can now do the following: ```rs use some_crate::some_macro; @@ -104,11 +97,9 @@ fn main() { } ``` -Unfortunately, this only applies for external crates, if you use `macro_rules!` macros that you have -defined in your own crate you are still required to go with `#[macro_use]` on the defining modules. +Unfortunately, this only applies for external crates, if you use `macro_rules!` macros that you have defined in your own crate you are still required to go with `#[macro_use]` on the defining modules. So scoping applies there the same way as before as well. -> The `$crate` prefix works in this version for everything, macros and items alike since this Edition -> came out with Rust 1.31. +> The `$crate` prefix works in this version for everything, macros and items alike since this Edition came out with Rust 1.31. [scoping chapter]:./scoping.md diff --git a/src/macros/minutiae/metavar-and-expansion.md b/src/macros/minutiae/metavar-and-expansion.md index 93f2f77..568e582 100644 --- a/src/macros/minutiae/metavar-and-expansion.md +++ b/src/macros/minutiae/metavar-and-expansion.md @@ -1,7 +1,7 @@ # Metavariables and Expansion Redux -Once the parser begins consuming tokens for a metavariable, *it cannot stop or backtrack*. This means -that the second rule of the following macro *cannot ever match*, no matter what input is provided: +Once the parser begins consuming tokens for a metavariable, *it cannot stop or backtrack*. +This means that the second rule of the following macro *cannot ever match*, no matter what input is provided: ```ignore macro_rules! dead_rule { @@ -10,19 +10,19 @@ macro_rules! dead_rule { } ``` -Consider what happens if this macro is invoked as `dead_rule!(x+)`. The interpreter will start at -the first rule, and attempt to parse the input as an expression. The first token `x` is valid as -an expression. The second token is *also* valid in an expression, forming a binary addition node. +Consider what happens if this macro is invoked as `dead_rule!(x+)`. +The interpreter will start at the first rule, and attempt to parse the input as an expression. +The first token `x` is valid as an expression. +The second token is *also* valid in an expression, forming a binary addition node. -At this point, given that there is no right-hand side of the addition, you might expect the parser -to give up and try the next rule. Instead, the parser will panic and abort the entire compilation, -citing a syntax error. +At this point, given that there is no right-hand side of the addition, you might expect the parser to give up and try the next rule. +Instead, the parser will panic and abort the entire compilation, citing a syntax error. As such, it is important in general that you write macro rules from most-specific to least-specific. To defend against future syntax changes altering the interpretation of macro input, `macro_rules!` -restricts what can follow various metavariables. The complete list, showing what may follow what -fragment specifier, as of Rust 1.46 is as follows: +restricts what can follow various metavariables. +The complete list, showing what may follow what fragment specifier, as of Rust 1.46 is as follows: * [`stmt`] and [`expr`]: `=>`, `,`, or `;` * [`pat`]: `=>`, `,`, `=`, `|`, `if`, `in` @@ -32,13 +32,12 @@ fragment specifier, as of Rust 1.46 is as follows: metavariable with an [`ident`], [`ty`], or [`path`] fragment specifier. * All other fragment specifiers have no restrictions. -Repetitions also adhere to these restrictions, meaning if a repetition can repeat multiple times -(`*` or `+`), then the contents must be able to follow themselves. If a repetition can repeat zero -times (`?` or `*`) then what comes after the repetition must be able to follow what comes before. +Repetitions also adhere to these restrictions, meaning if a repetition can repeat multiple times(`*` or `+`), then the contents must be able to follow themselves. +If a repetition can repeat zero times (`?` or `*`) then what comes after the repetition must be able to follow what comes before. -The parser also does not perform any kind of lookahead. That means if the compiler cannot -unambiguously determine how to parse the macro invocation one token at a time, it will abort with an -ambiguity error. A simple example that triggers this: +The parser also does not perform any kind of lookahead. +That means if the compiler cannot unambiguously determine how to parse the macro invocation one token at a time, it will abort with an ambiguity error. +A simple example that triggers this: ```rust macro_rules! ambiguity { @@ -50,11 +49,9 @@ macro_rules! ambiguity { ambiguity!(an_identifier); ``` -The parser does not look ahead past the identifier to see if the following token is a `)`, which -would allow it to parse properly. +The parser does not look ahead past the identifier to see if the following token is a `)`, which would allow it to parse properly. -One aspect of substitution that often surprises people is that substitution is *not* token-based, -despite very much *looking* like it. +One aspect of substitution that often surprises people is that substitution is *not* token-based, despite very much *looking* like it. Consider the following: @@ -93,8 +90,8 @@ got something else got something else ``` -By parsing the input into an AST node, the substituted result becomes *un-destructible*; *i.e.* you -cannot examine the contents or match against it ever again. +By parsing the input into an AST node, the substituted result becomes *un-destructible*; +*i.e.* you cannot examine the contents or match against it ever again. Here is *another* example which can be particularly confusing: @@ -129,9 +126,8 @@ something else (#[no_mangle]) something else (#[inline]) ``` -The only way to avoid this is to capture using the [`tt`], [`ident`] or [`lifetime`] kinds. Once you -capture with anything else, the only thing you can do with the result from then on is substitute it -directly into the output. +The only way to avoid this is to capture using the [`tt`], [`ident`] or [`lifetime`] kinds. +Once you capture with anything else, the only thing you can do with the result from then on is substitute it directly into the output. [`item`]:./fragment-specifiers.md#item diff --git a/src/macros/minutiae/scoping.md b/src/macros/minutiae/scoping.md index 095b141..c147155 100644 --- a/src/macros/minutiae/scoping.md +++ b/src/macros/minutiae/scoping.md @@ -1,17 +1,14 @@ # Scoping -The way in which function-like macros are scoped can be somewhat unintuitive. They use two forms -of scopes: textual scope, and path-based scope. +The way in which function-like macros are scoped can be somewhat unintuitive. +They use two forms of scopes: textual scope, and path-based scope. -When such a macro is invoked by an unqualified identifier(an identifier that isn't part of a -mulit-part path), it is first looked up in textual scoping and then in path-based scoping should -the first lookup not yield any results. If it is invoked by a qualified identifier it will skip the -textual scoping lookup and instead only do a look up in the path-based scoping. +When such a macro is invoked by an unqualified identifier(an identifier that isn't part of a mulit-part path), it is first looked up in textual scoping and then in path-based scoping should the first lookup not yield any results. +If it is invoked by a qualified identifier it will skip the textual scoping lookup and instead only do a look up in the path-based scoping. ## Textual Scope -Firstly, unlike everything else in the language, function-like macros will remain visible in -sub-modules. +Firstly, unlike everything else in the language, function-like macros will remain visible in sub-modules. ```rust macro_rules! X { () => {}; } @@ -27,12 +24,10 @@ mod c { # fn main() {} ``` -> **Note**: In these examples, remember that all of them have the *same behavior* when the module -> contents are in separate files. +> **Note**: In these examples, remember that all of them have the *same behavior* when the module contents are in separate files. -Secondly, *also* unlike everything else in the language, `macro_rules!` macros are only accessible -*after* their definition. Also note that this example demonstrates how `macro_rules!` macros do not -"leak" out of their defining scope: +Secondly, *also* unlike everything else in the language, `macro_rules!` macros are only accessible *after* their definition. +Also note that this example demonstrates how `macro_rules!` macros do not "leak" out of their defining scope: ```rust mod a { @@ -82,8 +77,7 @@ mod c { # fn main() {} ``` -Defining `macro_rules!` macros multiple times is allowed and the most recent declaration will simply -shadow previous ones unless it has gone out of scope. +Defining `macro_rules!` macros multiple times is allowed and the most recent declaration will simply shadow previous ones unless it has gone out of scope. ```rust macro_rules! X { (1) => {}; } @@ -102,9 +96,8 @@ X!(2); ``` -`macro_rules!` macros can be exported from a module using the `#[macro_use]` attribute. Using this -on a module is similar to saying that you do not want to have the module's macro's scope end with -the module. +`macro_rules!` macros can be exported from a module using the `#[macro_use]` attribute. +Using this on a module is similar to saying that you do not want to have the module's macro's scope end with the module. ```rust mod a { @@ -121,8 +114,7 @@ mod c { # fn main() {} ``` -Note that this can interact in somewhat bizarre ways due to the fact that identifiers in a -`macro_rules!` macro (including other macros) are only resolved upon expansion: +Note that this can interact in somewhat bizarre ways due to the fact that identifiers in a `macro_rules!` macro (including other macros) are only resolved upon expansion: ```rust mod a { @@ -140,9 +132,7 @@ mod c { # fn main() {} ``` -Another complication is that `#[macro_use]` applied to an `extern crate` *does not* behave this way: -such declarations are effectively *hoisted* to the top of the module. Thus, assuming `X!` is defined -in an external crate called `mac`, the following holds: +Another complication is that `#[macro_use]` applied to an `extern crate` *does not* behave this way: such declarations are effectively *hoisted* to the top of the module. Thus, assuming `X!` is defined in an external crate called `mac`, the following holds: ```rust,ignore mod a { @@ -159,8 +149,7 @@ mod c { # fn main() {} ``` -Finally, note that these scoping behaviors apply to *functions* as well, with the exception of -`#[macro_use]` (which isn't applicable): +Finally, note that these scoping behaviors apply to *functions* as well, with the exception of `#[macro_use]` (which isn't applicable): ```rust macro_rules! X { @@ -182,17 +171,16 @@ fn b() { macro_rules! Y { () => {"One more"} } assert_eq!(X!(), "One more"); } -# +# # fn main() { # a(); # b(); # } ``` -These scoping rules are why a common piece of advice is to place all `macro_rules!` macros which -should be accessible "crate wide" at the very top of your root module, before any other modules. -This ensures they are available *consistently*. This also applies to `mod` definitions for files, as -in: +These scoping rules are why a common piece of advice is to place all `macro_rules!` macros which should be accessible "crate wide" at the very top of your root module, before any other modules. +This ensures they are available *consistently*. +This also applies to `mod` definitions for files, as in: ```rs #[macro_use] @@ -204,8 +192,8 @@ The order here is important, swap the declaration order and it won't compile. ## Path-Based Scope -By default, a `macro_rules!` macro has no path-based scope. However, if it has the `#[macro_export]` -attribute, then it is declared in the crate root scope and can be referred to similar to how you -refer to any other item. The [Import and Export] chapter goes more in-depth into said attribute. +By default, a `macro_rules!` macro has no path-based scope. +However, if it has the `#[macro_export]` attribute, then it is declared in the crate root scope and can be referred to similar to how you refer to any other item. +The [Import and Export] chapter goes more in-depth into said attribute. -[Import and Export]: ./import-export.html \ No newline at end of file +[Import and Export]: ./import-export.html From fac0a2563bb51666c51bd3b2675d0b3494ce0288 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 22:50:05 +0200 Subject: [PATCH 08/92] Update README --- README.md | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 1d8ce40..8a69e74 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,27 @@ # The Little Book of Rust Macros -> **Note**: This is a continuation of [Daniel Keep's Book](https://github.com/DanielKeep/tlborm) -> which has not been updated since the early summer of 2016, adapted to make use of -> [mdBook](https://github.com/rust-lang/mdBook). - -View the [rendered version here](https://veykril.github.io/tlborm/) and the -[repository here](https://github.com/veykril/tlborm). - -This book is an attempt to distill the Rust community's collective knowledge of Rust macros, -`Macros by Example` to be precise. As such, both additions (in the form of pull requests) and -requests (in the form of issues) are welcome. -The [original Little Book of Rust Macros](https://github.com/DanielKeep/tlborm) has helped me -immensely with understanding ***Macros by Example*** style macros while I was still learning the -language. Unfortunately, the author of the book has since left the project untouched, while the Rust -language as well as it's macro-system keeps evolving. Which is why I wanted to revive the project -and keep it up to date with current Rust helping other newcomers understand macros, a part of the -language a lot of people seem to have trouble with. - -> This book expects you to have basic knowledge of Rust, it will not explain language features or -> constructs that are irrelevant to macros. No prior knowledge of macros is assumed. Having read and -> understood the first seven chapters of the [Rust Book](https://doc.rust-lang.org/stable/book/) is -> a must, though having read the majority of the book is recommended. +> **Note**: This is a continuation of [Daniel Keep's Book](https://github.com/DanielKeep/tlborm) which has not been updated since the early summer of 2016, adapted to make use of [mdBook](https://github.com/rust-lang/mdBook). + +View the [rendered version here](https://veykril.github.io/tlborm/) and the [repository here](https://github.com/veykril/tlborm). + +A chinese version of this book can be found [here](https://zjp-cn.github.io/tlborm/). + +This book is an attempt to distill the Rust community's collective knowledge of Rust macros, the `Macros by Example` ones as well as procedural macros. +As such, both additions (in the form of pull requests) and requests (in the form of issues) are very much welcome. + +The [original Little Book of Rust Macros](https://github.com/DanielKeep/tlborm) has helped me immensely with understanding ***Macros by Example*** style macros while I was still learning the language. +Unfortunately, the original book hasn't been updated since april of 2016, while the Rust language as well as it's macro-system keeps evolving. +Which is why I took up the task to update the book and keep it updated as well as I can while also adding new found things to it. +In hopes that it will help out all the fresh faces coming to Rust understanding its macro systems, a part of the language a people tend to have trouble with. + +> This book expects you to have basic knowledge of Rust, it will not explain language features or constructs that are irrelevant to macros. +> No prior knowledge of macros is assumed. +> Having read and understood the first seven chapters of the [Rust Book](https://doc.rust-lang.org/stable/book/) is > a must, though having read the majority of the book is recommended. ## Thanks -Thanks to Daniel Keep for the original work and thanks to the following people for suggestions and -corrections to the original: IcyFoxy, Rym, TheMicroWorm, Yurume, akavel, cmr, eddyb, ogham, and -snake_case. +A big thank you to Daniel Keep for the original work as well as all the contributors that added to the original. ## License -This work inherits the licenses of the original, hence it is licensed under both the -[Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/) -and the [MIT license](http://opensource.org/licenses/MIT). \ No newline at end of file +This work inherits the licenses of the original, hence it is licensed under both the [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/) and the [MIT license](http://opensource.org/licenses/MIT). From 460b6344672ac4ecb684eb89b7c386d6a237bbf8 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 23:30:57 +0200 Subject: [PATCH 09/92] Fixup more line styles in src/macros --- src/macros.md | 1 + src/macros/building-blocks.md | 2 +- src/macros/building-blocks/abacus-counting.md | 37 +- src/macros/building-blocks/ast-coercion.md | 13 +- src/macros/building-blocks/counting.md | 60 ++-- src/macros/building-blocks/parsing.md | 38 +-- src/macros/macro_rules.md | 85 +++-- src/macros/macros-practical.md | 320 ++++++++---------- src/macros/minutiae.md | 4 +- src/macros/minutiae/scoping.md | 2 +- src/macros/patterns/callbacks.md | 8 +- src/macros/patterns/internal-rules.md | 61 ++-- src/macros/patterns/push-down-acc.md | 28 +- src/macros/patterns/repetition-replacement.md | 19 +- src/macros/patterns/tt-bundling.md | 17 +- src/macros/patterns/tt-muncher.md | 25 +- src/syntax-extensions/hygiene.md | 4 +- 17 files changed, 313 insertions(+), 411 deletions(-) diff --git a/src/macros.md b/src/macros.md index 744665b..557f13c 100644 --- a/src/macros.md +++ b/src/macros.md @@ -10,6 +10,7 @@ Following up the two introductions it offers some generally very useful [pattern Should the information presented here not suffice, then there is also the [Macros chapter of the Rust Book] which is a more approachable, high-level explanation as well as the reference [chapter](https://doc.rust-lang.org/reference/macros-by-example.html) which goes more into the precise details of things. +> **Note**: This book will usually use the term *mbe*(**M**acro-**B**y-**E**xample), *mbe macro* or `macro_rules!` macro when talking about `macro_rules!` macros. [mbe]: https://doc.rust-lang.org/reference/macros-by-example.html [Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html diff --git a/src/macros/building-blocks.md b/src/macros/building-blocks.md index 58b9280..23906f4 100644 --- a/src/macros/building-blocks.md +++ b/src/macros/building-blocks.md @@ -1,3 +1,3 @@ # Building Blocks -Reusable snippets of macro code. \ No newline at end of file +Reusable snippets of `macro_rules!` macro code. diff --git a/src/macros/building-blocks/abacus-counting.md b/src/macros/building-blocks/abacus-counting.md index ff97f98..96737ba 100644 --- a/src/macros/building-blocks/abacus-counting.md +++ b/src/macros/building-blocks/abacus-counting.md @@ -1,10 +1,9 @@ # Abacus Counters -> **Provisional**: needs a more compelling example. Matching nested groups that are *not* denoted by -> Rust groups is sufficiently unusual that it may not merit inclusion. +> **Provisional**: needs a more compelling example. +> Matching nested groups that are *not* denoted by Rust groups is sufficiently unusual that it may not merit inclusion. -> **Note**: this section assumes understanding of [push-down accumulation](#push-down-accumulation) -> and [incremental TT munchers](#incremental-tt-munchers). +> **Note**: this section assumes understanding of [push-down accumulation](#push-down-accumulation) and [incremental TT munchers](#incremental-tt-munchers). ```rust macro_rules! abacus { @@ -32,16 +31,15 @@ fn main() { } ``` -This technique can be used in cases where you need to keep track of a varying counter that starts at - or near zero, and must support the following operations: +This technique can be used in cases where you need to keep track of a varying counter that starts at or near zero, and must support the following operations: * Increment by one. * Decrement by one. * Compare to zero (or any other fixed, finite value). -A value of *n* is represented by *n* instances of a specific token stored in a group. Modifications -are done using recursion and [push-down accumulation](#push-down-accumulation). Assuming the token -used is `x`, the operations above are implemented as follows: +A value of *n* is represented by *n* instances of a specific token stored in a group. +Modifications are done using recursion and [push-down accumulation](#push-down-accumulation). +Assuming the token used is `x`, the operations above are implemented as follows: * Increment by one: match `($($count:tt)*)`, substitute `(x $($count)*)`. * Decrement by one: match `(x $($count:tt)*)`, substitute `($($count)*)`. @@ -58,12 +56,11 @@ In this way, operations on the counter are like flicking tokens back and forth l In fairness, it could *also* have been called ["unary counting"](https://en.wikipedia.org/wiki/Unary_numeral_system). In cases where you want to represent negative values, *-n* can be represented as *n* instances of a -*different* token. In the example given above, *+n* is stored as *n* `+` tokens, and *-m* is stored -as *m* `-` tokens. +*different* token. +In the example given above, *+n* is stored as *n* `+` tokens, and *-m* is stored as *m* `-` tokens. -In this case, the operations become slightly more complicated; increment and decrement effectively -reverse their usual meanings when the counter is negative. To which given `+` and `-` for the -positive and negative tokens respectively, the operations change to: +In this case, the operations become slightly more complicated; increment and decrement effectively reverse their usual meanings when the counter is negative. +To which given `+` and `-` for the positive and negative tokens respectively, the operations change to: * Increment by one: * match `()`, substitute `(+)`. @@ -80,11 +77,10 @@ positive and negative tokens respectively, the operations change to: * Compare to -2: match `(--)`. * *(and so on...)* -Note that the example at the top combines some of the rules together (for example, it combines -increment on `()` and `($($count:tt)+)` into an increment on `($($count:tt)*)`). +Note that the example at the top combines some of the rules together (for example, it combines increment on `()` and `($($count:tt)+)` into an increment on `($($count:tt)*)`). -If you want to extract the actual *value* of the counter, this can be done using a regular -[counter macro](./counting.md). For the example above, the terminal rules can be replaced with the following: +If you want to extract the actual *value* of the counter, this can be done using a regular [counter macro](./counting.md). +For the example above, the terminal rules can be replaced with the following: ```rust,ignore macro_rules! abacus { @@ -106,9 +102,8 @@ macro_rules! count_tts { } ``` -> **JFTE**: strictly speaking, the above formulation of -> `abacus!` is needlessly complex. It can be implemented much more efficiently using repetition, -> provided you *do not* need to match against the counter's value in a macro: +> **JFTE**: strictly speaking, the above formulation of `abacus!` is needlessly complex. +> It can be implemented much more efficiently using repetition, provided you *do not* need to match against the counter's value in a macro: > > ```ignore > macro_rules! abacus { diff --git a/src/macros/building-blocks/ast-coercion.md b/src/macros/building-blocks/ast-coercion.md index 9dd25d5..75e4d44 100644 --- a/src/macros/building-blocks/ast-coercion.md +++ b/src/macros/building-blocks/ast-coercion.md @@ -1,9 +1,8 @@ # AST Coercion -The Rust parser is not very robust in the face of `tt` substitutions. Problems can arise when the -parser is expecting a particular grammar construct and *instead* finds a lump of substituted `tt` -tokens. Rather than attempt to parse them, it will often just *give up*. In these cases, it is -necessary to employ an AST coercion. +The Rust parser is not very robust in the face of `tt` substitutions. +Problems can arise when the parser is expecting a particular grammar construct and *instead* finds a lump of substituted `tt` tokens. +Rather than attempt to parse them, it will often just *give up*. In these cases, it is necessary to employ an AST coercion. ```rust # #![allow(dead_code)] @@ -20,10 +19,8 @@ fn main() { } ``` -These coercions are often used with [push-down accumulation] macros in order to get the parser to -treat the final `tt` sequence as a particular kind of grammar construct. +These coercions are often used with [push-down accumulation] macros in order to get the parser to treat the final `tt` sequence as a particular kind of grammar construct. -Note that this specific set of macros is determined by what macros are allowed to expand to, *not* -what they are able to capture. +Note that this specific set of macros is determined by what macros are allowed to expand to, *not* what they are able to capture. [push-down accumulation]: ../patterns/push-down-acc.md diff --git a/src/macros/building-blocks/counting.md b/src/macros/building-blocks/counting.md index 91e57bc..df0287f 100644 --- a/src/macros/building-blocks/counting.md +++ b/src/macros/building-blocks/counting.md @@ -2,8 +2,8 @@ ## Repetition with replacement -Counting things in a macro is a surprisingly tricky task. The simplest way is to use replacement -with a repetition match. +Counting things in a macro is a surprisingly tricky task. +The simplest way is to use replacement with a repetition match. ```rust macro_rules! replace_expr { @@ -19,15 +19,14 @@ macro_rules! count_tts { # } ``` -This is a fine approach for smallish numbers, but will likely *crash the compiler* with inputs of -around 500 or so tokens. Consider that the output will look something like this: +This is a fine approach for smallish numbers, but will likely *crash the compiler* with inputs of around 500 or so tokens. +Consider that the output will look something like this: ```rust,ignore 0usize + 1usize + /* ~500 `+ 1usize`s */ + 1usize ``` -The compiler must parse this into an AST, which will produce what is effectively a perfectly -unbalanced binary tree 500+ levels deep. +The compiler must parse this into an AST, which will produce what is effectively a perfectly unbalanced binary tree 500+ levels deep. ## Recursion @@ -44,15 +43,13 @@ macro_rules! count_tts { # } ``` -> **Note**: As of `rustc` 1.2, the compiler has *grievous* performance problems when large numbers -> of integer literals of unknown type must undergo inference. We are using explicitly -> `usize`-typed literals here to avoid that. +> **Note**: As of `rustc` 1.2, the compiler has *grievous* performance problems when large numbers of integer literals of unknown type must undergo inference. +> We are using explicitly `usize`-typed literals here to avoid that. > -> If this is not suitable (such as when the type must be substitutable), you can help matters by -> using `as` (*e.g.* `0 as $ty`, `1 as $ty`, *etc.*). +> If this is not suitable (such as when the type must be substitutable), you can help matters by using `as` (*e.g.* `0 as $ty`, `1 as $ty`, *etc.*). -This *works*, but will trivially exceed the recursion limit. Unlike the repetition approach, you can -extend the input size by matching multiple tokens at once. +This *works*, but will trivially exceed the recursion limit. +Unlike the repetition approach, you can extend the input size by matching multiple tokens at once. ```rust macro_rules! count_tts { @@ -147,12 +144,10 @@ macro_rules! count_idents { # } ``` -This method does have two drawbacks. First, as implied above, it can *only* count valid identifiers -(which are also not keywords), and it does not allow those identifiers to repeat. +This method does have two drawbacks. +First, as implied above, it can *only* count valid identifiers(which are also not keywords), and it does not allow those identifiers to repeat. -Secondly, this approach is *not* hygienic, meaning that if whatever identifier you use in place of -`__CountIdentsLast` is provided as input, the macro will fail due to the duplicate variants in the -`enum`. +Secondly, this approach is *not* hygienic, meaning that if whatever identifier you use in place of `__CountIdentsLast` is provided as input, the macro will fail due to the duplicate variants in the `enum`. ## Bit twiddling @@ -170,16 +165,13 @@ macro_rules! count_tts { # } ``` -This approach is pretty smart as it effectively halves its input whenever its even and then -multiplying the counter by 2 (or in this case shifting 1 bit to the left which is equivalent). If -the input is uneven it simply takes one token tree from the input `or`s the token tree to the -previous counter which is equivalent to adding 1 as the lowest bit has to be a 0 at this point due -to the previous shifting. Rinse and repeat until we hit the base rule `() => 0`. +This approach is pretty smart as it effectively halves its input whenever its even and then multiplying the counter by 2 (or in this case shifting 1 bit to the left which is equivalent). +If the input is uneven it simply takes one token tree from the input `or`s the token tree to the previous counter which is equivalent to adding 1 as the lowest bit has to be a 0 at this point due to the previous shifting. +Rinse and repeat until we hit the base rule `() => 0`. -The benefit of this is that the constructed AST expression that makes up the counter value will grow -with a complexity of `O(log(n))` instead of `O(n)` like the other approaches. Be aware that you can -still hit the recursion limit with this if you try hard enough. Credits for this method go to Reddit -user [`YatoRust`](https://www.reddit.com/r/rust/comments/d3yag8/the_little_book_of_rust_macros/). +The benefit of this is that the constructed AST expression that makes up the counter value will grow with a complexity of `O(log(n))` instead of `O(n)` like the other approaches. +Be aware that you can still hit the recursion limit with this if you try hard enough. +Credits for this method go to Reddit user [`YatoRust`](https://www.reddit.com/r/rust/comments/d3yag8/the_little_book_of_rust_macros/). Let's go through the procedure by hand once: @@ -187,17 +179,15 @@ Let's go through the procedure by hand once: ```rust,ignore count_tts!(0 0 0 0 0 0 0 0 0 0); ``` -This invocation will match the third rule due to the fact that we have an even number of token trees -(10). The matcher names the odd token trees in the sequence `$a` and the even ones `$even` but the -expansion only makes use of `$a`, which means it effectively discards all the even elements cutting -the input in half. So the invocation now becomes: +This invocation will match the third rule due to the fact that we have an even number of token trees(10). +The matcher names the odd token trees in the sequence `$a` and the even ones `$even` but the expansion only makes use of `$a`, which means it effectively discards all the even elements cutting the input in half. +So the invocation now becomes: ```rust,ignore count_tts!(0 0 0 0 0) << 1; ``` -This invocation will now match the second rule as its input is an uneven amount of token trees. In -this case the first token tree is discarded to make the input even again, then we also do the -halving step in this invocation again since we know the input would be even now anyways. Therefor we -can count 1 for the uneven discard and multiply by 2 again since we also halved. +This invocation will now match the second rule as its input is an uneven amount of token trees. +In this case the first token tree is discarded to make the input even again, then we also do the halving step in this invocation again since we know the input would be even now anyways. +Therefor we can count 1 for the uneven discard and multiply by 2 again since we also halved. ```rust,ignore ((count_tts!(0 0) << 1) | 1) << 1; ``` diff --git a/src/macros/building-blocks/parsing.md b/src/macros/building-blocks/parsing.md index 1ecc461..8d200f6 100644 --- a/src/macros/building-blocks/parsing.md +++ b/src/macros/building-blocks/parsing.md @@ -1,13 +1,11 @@ # Parsing Rust -Parsing some of Rust's items can be useful in certain situations. This section will show a few -macros that can parse some of Rust's more complex items like structs and functions to a certain extent. -The goal of these macros is not to be able to parse the entire grammar of the items but to parse -parts that are in general quite useful without being too complex to parse. This means we ignore -things like generics and such. +Parsing some of Rust's items can be useful in certain situations. +This section will show a few macros that can parse some of Rust's more complex items like structs and functions to a certain extent. +The goal of these macros is not to be able to parse the entire grammar of the items but to parse parts that are in general quite useful without being too complex to parse. This means we ignore things like generics and such. -The main points of interest of these macros are their `matchers`. The transcribers are only there -for example purposes and are usually not that impressive. +The main points of interest of these macros are their `matchers`. +The transcribers are only there for example purposes and are usually not that impressive. ## Function @@ -42,23 +40,18 @@ macro_rules! function_item_matcher { # } ``` -A simple function matcher that ignores qualifiers like `unsafe`, `async`, ... as well a generics and -where clauses. If parsing those is required it is likely that you are better off using a proc-macro -instead. +A simple function matcher that ignores qualifiers like `unsafe`, `async`, ... as well a generics and where clauses. +If parsing those is required it is likely that you are better off using a proc-macro instead. -This lets you for example, inspect the function signature, generate some extra things from it and -then re-emit the entire function again. Kind of like a `Derive` proc-macro but weaker and for -functions. +This lets you for example, inspect the function signature, generate some extra things from it and then re-emit the entire function again. +Kind of like a `Derive` proc-macro but weaker and for functions. -> Ideally we would like to use a pattern fragment specifier instead of an ident for the arguments -> but this is currently not allowed. Fortunately people don't use patterns in function signatures -> that often so this is okay. +> Ideally we would like to use a pattern fragment specifier instead of an ident for the arguments but this is currently not allowed. +> Fortunately people don't use non-identifier patterns in function signatures that often so this is okay(a shame, really). ### Method -The macro for parsing basic functions is nice and all, but sometimes we would like to also parse -methods, functions that refer to their object via some form of `self` usage. This makes things a bit -trickier: +The macro for parsing basic functions is nice and all, but sometimes we would like to also parse methods, functions that refer to their object via some form of `self` usage. This makes things a bit trickier: > WIP @@ -146,11 +139,8 @@ macro_rules! struct_item_matcher { # Enum -Parsing enums is a bit more complex than structs so we will finally make use of some of the -[patterns] we have discussed, [Incremental TT Muncher] and [Internal Rules]. Instead of just -building the parsed enum again we will merely visit all the tokens of the enum, as rebuilding the -enum would require us to collect all the parsed tokens temporarily again via a -[Push Down Accumulator]. +Parsing enums is a bit more complex than structs so we will finally make use of some of the [patterns] we have discussed, [Incremental TT Muncher] and [Internal Rules]. +Instead of just building the parsed enum again we will merely visit all the tokens of the enum, as rebuilding the enum would require us to collect all the parsed tokens temporarily again via a [Push Down Accumulator]. ```rust macro_rules! enum_item_matcher { diff --git a/src/macros/macro_rules.md b/src/macros/macro_rules.md index f1eeb6e..69ac1d4 100644 --- a/src/macros/macro_rules.md +++ b/src/macros/macro_rules.md @@ -1,8 +1,8 @@ # `macro_rules!` -With all that in mind, we can introduce `macro_rules!` itself. As noted previously, `macro_rules!` -is *itself* a syntax extension, meaning it is *technically* not part of the Rust syntax. It uses the -following forms: +With all that in mind, we can introduce `macro_rules!` itself. +As noted previously, `macro_rules!` is *itself* a syntax extension, meaning it is *technically* not part of the Rust syntax. +It uses the following forms: ```rust,ignore macro_rules! $name { @@ -13,8 +13,8 @@ macro_rules! $name { } ``` -There must be *at least* one rule, and you can omit the semicolon after the last rule. You can use -brackets(`[]`), parentheses(`()`) or braces(`{}`). +There must be *at least* one rule, and you can omit the semicolon after the last rule. +You can use brackets(`[]`), parentheses(`()`) or braces(`{}`). Each *"rule"* looks like the following: @@ -22,28 +22,24 @@ Each *"rule"* looks like the following: ($matcher) => {$expansion} ``` -Like before, the types of parentheses used can be any kind, but parentheses around the matcher and -braces around the expansion are somewhat conventional. The expansion part of a rule is also called -its *transcriber*. +Like before, the types of parentheses used can be any kind, but parentheses around the matcher and braces around the expansion are somewhat conventional. +The expansion part of a rule is also called its *transcriber*. -Note that the choice of the parentheses does not matter in regards to how the macro may be invoked. -In fact, macros can be invoked with any kind of parentheses as well, but invocations with `{ .. }` -and `( ... );`, notice the trailing semicolon, are special in that their expansion will *always* be -parsed as an *item*. +Note that the choice of the parentheses does not matter in regards to how the mbe macro may be invoked. +In fact, function-like macros can be invoked with any kind of parentheses as well, but invocations with `{ .. }` and `( ... );`, notice the trailing semicolon, are special in that their expansion will *always* be parsed as an *item*. -If you are wondering, the `macro_rules!` invocation expands to... *nothing*. At least, nothing that -appears in the AST; rather, it manipulates compiler-internal structures to register the macro. As -such, you can *technically* use `macro_rules!` in any position where an empty expansion is valid. +If you are wondering, the `macro_rules!` invocation expands to... *nothing*. +At least, nothing that appears in the AST; rather, it manipulates compiler-internal structures to register the mbe macro. +As such, you can *technically* use `macro_rules!` in any position where an empty expansion is valid. ## Matching -When a `macro_rules!` macro is invoked, the `macro_rules!` interpreter goes through the rules one by -one, in declaration order. For each rule, it tries to match the contents of the input token tree -against that rule's `matcher`. A matcher must match the *entirety* of the input to be considered a -match. +When a `macro_rules!` macro is invoked, the `macro_rules!` interpreter goes through the rules one by one, in declaration order. +For each rule, it tries to match the contents of the input token tree against that rule's `matcher`. +A matcher must match the *entirety* of the input to be considered a match. -If the input matches the matcher, the invocation is replaced by the `expansion`; otherwise, the next -rule is tried. If all rules fail to match, macro expansion fails with an error. +If the input matches the matcher, the invocation is replaced by the `expansion`; otherwise, the next rule is tried. +If all rules fail to match, the expansion fails with an error. The simplest example is of an empty matcher: @@ -55,13 +51,13 @@ macro_rules! four { This matches if and only if the input is also empty (*i.e.* `four!()`, `four![]` or `four!{}`). -Note that the specific grouping tokens you use when you invoke the macro *are not* matched. That is, -you can invoke the above macro as `four![]` and it will still match. Only the *contents* of the -input token tree are considered. +Note that the specific grouping tokens you use when you invoke the function-like macro *are not* matched, they are in fact not passed to the invocation at all. +That is, you can invoke the above macro as `four![]` and it will still match. +Only the *contents* of the input token tree are considered. -Matchers can also contain literal token trees, which must be matched exactly. This is done by simply -writing the token trees normally. For example, to match the sequence `4 fn ['spang "whammo"] @_@`, -you would write: +Matchers can also contain literal token trees, which must be matched exactly. +This is done by simply writing the token trees normally. +For example, to match the sequence `4 fn ['spang "whammo"] @_@`, you would write: ```rust,ignore macro_rules! gibberish { @@ -73,11 +69,10 @@ You can use any token tree that you can write. ## Metavariables -Matchers can also contain captures. These allow input to be matched based on some general grammar -category, with the result captured to a metavariable which can then be substituted into the output. +Matchers can also contain captures. +These allow input to be matched based on some general grammar category, with the result captured to a metavariable which can then be substituted into the output. -Captures are written as a dollar (`$`) followed by an identifier, a colon (`:`), and finally the -kind of capture which is also called the fragment-specifier, which must be one of the following: +Captures are written as a dollar (`$`) followed by an identifier, a colon (`:`), and finally the kind of capture which is also called the fragment-specifier, which must be one of the following: * `block`: a block (i.e. a block of statements and/or an expression, surrounded by braces) * `expr`: an expression @@ -93,10 +88,9 @@ kind of capture which is also called the fragment-specifier, which must be one o * `ty`: a type * `vis`: a possible empty visibility qualifier (e.g. `pub`, `pub(in crate)`, ...) -For more in-depth description of the fragement specifiers, check out the [Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. +For more in-depth description of the fragment specifiers, check out the [Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. -For example, here is a `macro_rules!` macro which captures its input as an expression under the -metavariable `$e`: +For example, here is a `macro_rules!` macro which captures its input as an expression under the metavariable `$e`: ```rust,ignore macro_rules! one_expression { @@ -117,9 +111,8 @@ macro_rules! times_five { } ``` -Much like macro expansion, metavariables are substituted as complete AST nodes. This means that no -matter what sequence of tokens is captured by `$e`, it will be interpreted as a single, complete -expression. +Much like macro expansion, metavariables are substituted as complete AST nodes. +This means that no matter what sequence of tokens is captured by `$e`, it will be interpreted as a single, complete expression. You can also have multiple metavariables in a single matcher: @@ -147,8 +140,8 @@ There is also a special metavariable called [`$crate`] which can be used to refe ## Repetitions -Matchers can contain repetitions. These allow a sequence of tokens to be matched. These have the -general form `$ ( ... ) sep rep`. +Matchers can contain repetitions. These allow a sequence of tokens to be matched. +These have the general form `$ ( ... ) sep rep`. * `$` is a literal dollar token. * `( ... )` is the paren-grouped matcher being repeated. @@ -161,14 +154,12 @@ general form `$ ( ... ) sep rep`. Since `?` represents at most one occurrence, it cannot be used with a separator. -Repetitions can contain any other valid matcher, including literal token trees, metavariables, and -other repetitions allowing arbitrary nesting. +Repetitions can contain any other valid matcher, including literal token trees, metavariables, and other repetitions allowing arbitrary nesting. -Repetitions use the same syntax in the expansion and repeated metavariables can only be accessed -inside of repetitions in the expansion. +Repetitions use the same syntax in the expansion and repeated metavariables can only be accessed inside of repetitions in the expansion. -For example, below is a macro which formats each element as a string. It matches zero or more -comma-separated expressions and expands to an expression that constructs a vector. +For example, below is a mbe macro which formats each element as a string. +It matches zero or more comma-separated expressions and expands to an expression that constructs a vector. ```rust macro_rules! vec_strs { @@ -206,8 +197,8 @@ fn main() { } ``` -You can repeat multiple metavariables in a single repetition as long as all metavariables repeat -equally often. So this invocation of the following macro works: +You can repeat multiple metavariables in a single repetition as long as all metavariables repeat equally often. +So this invocation of the following macro works: ```rust macro_rules! repeat_two { diff --git a/src/macros/macros-practical.md b/src/macros/macros-practical.md index 1d53d76..c390ddd 100644 --- a/src/macros/macros-practical.md +++ b/src/macros/macros-practical.md @@ -1,22 +1,17 @@ # Macros, A Practical Introduction -This chapter will introduce the Rust macro-by-example system using a relatively simple, practical -example. It does *not* attempt to explain all of the intricacies of the system; its goal is to get -you comfortable with how and why macros are written. +This chapter will introduce the Rust macro-by-example system using a relatively simple, practical example. +It does *not* attempt to explain all of the intricacies of the system; its goal is to get you comfortable with how and why macros are written. -There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-06-macros.html) -which is another high-level explanation, and the [methodical introduction](../macros.md) chapter of -this book, which explains the macro system in detail. +There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-06-macros.html) which is another high-level explanation, and the [methodical introduction](../macros.md) chapter of this book, which explains the macro system in detail. ## A Little Context -> **Note**: don't panic! What follows is the only math that will be talked about. You can quite -> safely skip this section if you just want to get to the meat of the article. +> **Note**: don't panic! What follows is the only math that will be talked about. +> You can quite safely skip this section if you just want to get to the meat of the article. -If you aren't familiar, a recurrence relation is a sequence where each value is defined in terms of -one or more *previous* values, with one or more initial values to get the whole thing started. For -example, the [Fibonacci sequence](https://en.wikipedia.org/wiki/Fibonacci_number) can be defined by -the relation: +If you aren't familiar, a recurrence relation is a sequence where each value is defined in terms of one or more *previous* values, with one or more initial values to get the whole thing started. +For example, the [Fibonacci sequence](https://en.wikipedia.org/wiki/Fibonacci_number) can be defined by the relation: \\[ F_{n} = 0, 1, ..., F_{n-1} + F_{n-2}\\](mathjax) @@ -28,7 +23,7 @@ What you *want* is to return something which will lazily compute elements of the In Rust, that means producing an [`Iterator`]. This is not especially *hard*, but there is a fair amount of boilerplate involved: you need to define a custom type, work out what state needs to be stored in it, then implement the [`Iterator`] trait for it. -However, recurrence relations are simple enough that almost all of these details can be abstracted out with a little macro-based code generation. +However, recurrence relations are simple enough that almost all of these details can be abstracted out with a little `macro_rules!` macro-based code generation. So, with all that having been said, let's get started. @@ -36,7 +31,8 @@ So, with all that having been said, let's get started. ## Construction -Usually, when working on a new macro, the first thing I do is decide what the macro invocation should look like. In this specific case, my first attempt looked like this: +Usually, when working on a new `macro_rules!` macro, the first thing I do is decide what the invocation should look like. +In this specific case, my first attempt looked like this: ```rust,ignore let fib = recurrence![a[n] = 0, 1, ..., a[n-1] + a[n-2]]; @@ -44,9 +40,8 @@ let fib = recurrence![a[n] = 0, 1, ..., a[n-1] + a[n-2]]; for e in fib.take(10) { println!("{}", e) } ``` -From that, we can take a stab at how the macro should be defined, even if we aren't sure of the -actual expansion. This is useful because if you can't figure out how to parse the input syntax, then -*maybe* you need to change it. +From that, we can take a stab at how the `macro_rules!` macro should be defined, even if we aren't sure of the actual expansion. +This is useful because if you can't figure out how to parse the input syntax, then *maybe* you need to change it. ```rust,ignore macro_rules! recurrence { @@ -55,9 +50,10 @@ macro_rules! recurrence { # fn main() {} ``` -Assuming you aren't familiar with the syntax, allow me to elucidate. This is defining a macro, using -the [`macro_rules!`] system, called `recurrence!`. This macro has a single parsing rule. That rule -says the input to the macro must match: +Assuming you aren't familiar with the syntax, allow me to elucidate. +This is defining a syntax extension, using the [`macro_rules!`] system, called `recurrence!`. +This `macro_rules!` macro has a single parsing rule. +That rule says the input to the invocation must match: - the literal token sequence `a` `[` `n` `]` `=`, - a [repeating] (the `$( ... )`) sequence, using `,` as a separator, and one or more (`+`) repeats of: @@ -68,28 +64,26 @@ says the input to the macro must match: [repeating]: ./macro_rules.md#repetitions [metavariable]: ./macro_rules.md#metavariables -Finally, the rule says that *if* the input matches this rule, then the macro invocation should be -replaced by the token sequence `/* ... */`. +Finally, the rule says that *if* the input matches this rule, then the invocation should be replaced by the token sequence `/* ... */`. -It's worth noting that `inits`, as implied by the name, actually contains *all* the expressions that -match in this position, not just the first or last. What's more, it captures them *as a sequence* as -opposed to, say, irreversibly pasting them all together. Also note that you can do "zero or more" -with a repetition by using `*` instead of `+` and even optional, "zero or one" with `?`. There is no -support for "zero or one" or more specific numbers of repetitions. +It's worth noting that `inits`, as implied by the name, actually contains *all* the expressions that match in this position, not just the first or last. +What's more, it captures them *as a sequence* as opposed to, say, irreversibly pasting them all together. +Also note that you can do "zero or more" with a repetition by using `*` instead of `+` and even optional, "zero or one" with `?`. +There is no support for "zero or one" or more specific numbers of repetitions. -As an exercise, let's take the proposed input and feed it through the rule, to see how it is -processed. The "Position" column will show which part of the syntax pattern needs to be matched -against next, denoted by a "⌂". Note that in some cases, there might be more than one possible -"next" element to match against. "Input" will contain all of the tokens that have *not* been -consumed yet. `inits` and `recur` will contain the contents of those bindings. +As an exercise, let's take the proposed input and feed it through the rule, to see how it is processed. +The "Position" column will show which part of the syntax pattern needs to be matched against next, denoted by a "⌂". +Note that in some cases, there might be more than one possible "next" element to match against. +"Input" will contain all of the tokens that have *not* been consumed yet. +`inits` and `recur` will contain the contents of those bindings. {{#include macros-practical-table.html}} -The key take-away from this is that the macro system will *try* to incrementally match the tokens -provided as input to the macro against the provided rules. We'll come back to the "try" part. +The key take-away from this is that the macro system will *try* to incrementally match the tokens provided as input to the macro against the provided rules. +We'll come back to the "try" part. -Now, let's begin writing the final, fully expanded form. For this expansion, I was looking for -something like: +Now, let's begin writing the final, fully expanded form. +For this expansion, I was looking for something like: ```rust,ignore let fib = { @@ -99,8 +93,9 @@ let fib = { } ``` -This will be the actual iterator type. `mem` will be the memo buffer to hold the last few values so -the recurrence can be computed. `pos` is to keep track of the value of `n`. +This will be the actual iterator type. +`mem` will be the memo buffer to hold the last few values so the recurrence can be computed. +`pos` is to keep track of the value of `n`. > **Aside**: I've chosen `u64` as a "sufficiently large" type for the elements of this sequence. > Don't worry about how this will work out for *other* sequences; we'll come to it. @@ -109,7 +104,6 @@ the recurrence can be computed. `pos` is to keep track of the value of `n`. impl Iterator for Recurrence { type Item = u64; - #[inline] fn next(&mut self) -> Option { if self.pos < 2 { let next_val = self.mem[self.pos]; @@ -134,9 +128,9 @@ We need a branch to yield the initial values of the sequence; nothing tricky. } ``` -This is a bit harder; we'll come back and look at *how* exactly to define `a`. Also, -`TODO_shuffle_down_and_append` is another placeholder; I want something that places `next_val` on -the end of the array, shuffling the rest down by one space, dropping the 0th element. +This is a bit harder; we'll come back and look at *how* exactly to define `a`. +Also, `TODO_shuffle_down_and_append` is another placeholder; +I want something that places `next_val` on the end of the array, shuffling the rest down by one space, dropping the 0th element. ```rust,ignore @@ -146,8 +140,8 @@ the end of the array, shuffling the rest down by one space, dropping the 0th ele for e in fib.take(10) { println!("{}", e) } ``` -Lastly, return an instance of our new structure, which can then be iterated over. To summarise, the -complete expansion is: +Lastly, return an instance of our new structure, which can then be iterated over. +To summarise, the complete expansion is: ```rust,ignore let fib = { @@ -159,7 +153,6 @@ let fib = { impl Iterator for Recurrence { type Item = u64; - #[inline] fn next(&mut self) -> Option { if self.pos < 2 { let next_val = self.mem[self.pos]; @@ -184,14 +177,12 @@ let fib = { for e in fib.take(10) { println!("{}", e) } ``` -> **Aside**: Yes, this *does* mean we're defining a different `Recurrence` struct and its -> implementation for each macro invocation. Most of this will optimise away in the final binary, -> with some judicious use of `#[inline]` attributes. +> **Aside**: Yes, this *does* mean we're defining a different `Recurrence` struct and its implementation for each invocation. +> Most of this will optimise away in the final binary. -It's also useful to check your expansion as you're writing it. If you see anything in the expansion -that needs to vary with the invocation, but *isn't* in the actual macro syntax, you should work out -where to introduce it. In this case, we've added `u64`, but that's not neccesarily what the user -wants, nor is it in the macro syntax. So let's fix that. +It's also useful to check your expansion as you're writing it. +If you see anything in the expansion that needs to vary with the invocation, but *isn't* in the actual accepted syntax of our macro, you should work out where to introduce it. +In this case, we've added `u64`, but that's not necessarily what the user wants, nor is it in the macro syntax. So let's fix that. ```rust macro_rules! recurrence { @@ -208,23 +199,21 @@ for e in fib.take(10) { println!("{}", e) } Here, I've added a new metavariable: `sty` which should be a type. -> **Aside**: if you're wondering, the bit after the colon in a metavariable can be one of several -> kinds of syntax matchers. The most common ones are `item`, `expr`, and `ty`. A complete -> explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macro_rules.md#Metavariables). +> **Aside**: if you're wondering, the bit after the colon in a metavariable can be one of several kinds of syntax matchers. +> The most common ones are `item`, `expr`, and `ty`. +> A complete explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macro_rules.md#Metavariables). > -> There's one other thing to be aware of: in the interests of future-proofing the language, the -> compiler restricts what tokens you're allowed to put *after* a matcher, depending on what kind -> it is. Typically, this comes up when trying to match expressions or statements; those can -> *only* be followed by one of `=>`, `,`, and `;`. +> There's one other thing to be aware of: in the interests of future-proofing the language, the compiler restricts what tokens you're allowed to put *after* a matcher, depending on what kind it is. +> Typically, this comes up when trying to match expressions or statements; +> those can *only* be followed by one of `=>`, `,`, and `;`. > -> A complete list can be found in -> [Macros, A Methodical Introduction; Minutiae; Metavariables and Expansion Redux](./minutiae/metavar-and-expansion.md). +> A complete list can be found in [Macros, A Methodical Introduction; Minutiae; Metavariables and Expansion Redux](./minutiae/metavar-and-expansion.md). ## Indexing and Shuffling -I will skim a bit over this part, since it's effectively tangential to the macro stuff. We want to -make it so that the user can access previous values in the sequence by indexing `a`; we want it to -act as a sliding window keeping the last few (in this case, 2) elements of the sequence. +I will skim a bit over this part, since it's effectively tangential to the macro-related stuff. +We want to make it so that the user can access previous values in the sequence by indexing `a`; +we want it to act as a sliding window keeping the last few (in this case, 2) elements of the sequence. We can do this pretty easily with a wrapper type: @@ -237,7 +226,6 @@ struct IndexOffset<'a> { impl<'a> Index for IndexOffset<'a> { type Output = u64; - #[inline(always)] fn index<'b>(&'b self, index: usize) -> &'b u64 { use std::num::Wrapping; @@ -251,17 +239,12 @@ impl<'a> Index for IndexOffset<'a> { } ``` -> **Aside**: since lifetimes come up *a lot* with people new to Rust, a quick explanation: `'a` and -> `'b` are lifetime parameters that are used to track where a reference -> (*i.e.* a borrowed pointer to some data) is valid. In this case, `IndexOffset` borrows a -> reference to our iterator's data, so it needs to keep track of how long it's allowed to hold -> that reference for, using `'a`. +> **Aside**: since lifetimes come up *a lot* with people new to Rust, a quick explanation: `'a` and `'b` are lifetime parameters that are used to track where a reference (*i.e.* a borrowed pointer to some data) is valid. +> In this case, `IndexOffset` borrows a reference to our iterator's data, so it needs to keep track of how long it's allowed to hold that reference for, using `'a`. > -> `'b` is used because the `Index::index` function (which is how subscript syntax is actually -> implemented) is *also* parameterised on a lifetime, on account of returning a borrowed reference. -> `'a` and `'b` are not necessarily the same thing in all cases. The borrow checker will make sure -> that even though we don't explicitly relate `'a` and `'b` to one another, we don't accidentally -> violate memory safety. +> `'b` is used because the `Index::index` function (which is how subscript syntax is actually implemented) is *also* parameterised on a lifetime, on account of returning a borrowed reference. +> `'a` and `'b` are not necessarily the same thing in all cases. +> The borrow checker will make sure that even though we don't explicitly relate `'a` and `'b` to one another, we don't accidentally violate memory safety. This changes the definition of `a` to: @@ -269,9 +252,8 @@ This changes the definition of `a` to: let a = IndexOffset { slice: &self.mem, offset: n }; ``` -The only remaining question is what to do about `TODO_shuffle_down_and_append`. I wasn't able to -find a method in the standard library with exactly the semantics I wanted, but it isn't hard to do -by hand. +The only remaining question is what to do about `TODO_shuffle_down_and_append`. +I wasn't able to find a method in the standard library with exactly the semantics I wanted, but it isn't hard to do by hand. ```rust,ignore { @@ -368,15 +350,11 @@ fn main() { } ``` -Note that I've changed the order of the declarations of `n` and `a`, as well as wrapped them -(along with the recurrence expression) in a block. The reason for the first should be obvious -(`n` needs to be defined first so I can use it for `a`). The reason for the second is that the -borrowed reference `&self.mem` will prevent the swaps later on from happening (you cannot mutate -something that is aliased elsewhere). The block ensures that the `&self.mem` borrow expires before -then. +Note that I've changed the order of the declarations of `n` and `a`, as well as wrapped them(along with the recurrence expression) in a block. +The reason for the first should be obvious(`n` needs to be defined first so I can use it for `a`). +The reason for the second is that the borrowed reference `&self.mem` will prevent the swaps later on from happening (you cannot mutate something that is aliased elsewhere). The block ensures that the `&self.mem` borrow expires before then. -Incidentally, the only reason the code that does the `mem` swaps is in a block is to narrow the -scope in which `std::mem::swap` is available, for the sake of being tidy. +Incidentally, the only reason the code that does the `mem` swaps is in a block is to narrow the scope in which `std::mem::swap` is available, for the sake of being tidy. If we take this code and run it, we get: @@ -393,8 +371,9 @@ If we take this code and run it, we get: 34 ``` -Success! Now, let's copy & paste this into the macro expansion, and replace the expanded code with -an invocation. This gives us: +Success! +Now, let's copy & paste this into the macro expansion, and replace the expanded code with an invocation. +This gives us: ```rust macro_rules! recurrence { @@ -421,7 +400,6 @@ macro_rules! recurrence { impl<'a> Index for IndexOffset<'a> { type Output = u64; - #[inline(always)] fn index<'b>(&'b self, index: usize) -> &'b u64 { use std::num::Wrapping; @@ -437,7 +415,6 @@ macro_rules! recurrence { impl Iterator for Recurrence { type Item = u64; - #[inline] fn next(&mut self) -> Option { if self.pos < 2 { let next_val = self.mem[self.pos]; @@ -477,8 +454,8 @@ fn main() { } ``` -Obviously, we aren't *using* the metavariables yet, but we can change that fairly easily. However, -if we try to compile this, `rustc` aborts, telling us: +Obviously, we aren't *using* the metavariables yet, but we can change that fairly easily. +However, if we try to compile this, `rustc` aborts, telling us: ```text error: local ambiguity: multiple parsing options: built-in NTs expr ('inits') or 1 other option. @@ -488,32 +465,30 @@ error: local ambiguity: multiple parsing options: built-in NTs expr ('inits') or | ``` -Here, we've run into a limitation of `macro_rules`. The problem is that second comma. When it sees -it during expansion, `macro_rules` can't decide if it's supposed to parse *another* expression for -`inits`, or `...`. Sadly, it isn't quite clever enough to realise that `...` isn't a valid -expression, so it gives up. Theoretically, this *should* work as desired, but currently doesn't. +Here, we've run into a limitation of the `macro_rules` system. +The problem is that second comma. +When it sees it during expansion, `macro_rules` can't decide if it's supposed to parse *another* expression for `inits`, or `...`. +Sadly, it isn't quite clever enough to realise that `...` isn't a valid expression, so it gives up. +Theoretically, this *should* work as desired, but currently doesn't. -> **Aside**: I *did* fib a little about how our rule would be interpreted by the macro system. In -> general, it *should* work as described, but doesn't in this case. The `macro_rules` machinery, -> as it stands, has its foibles, and its worthwhile remembering that on occasion, you'll need to -> contort a little to get it to work. +> **Aside**: I *did* fib a little about how our rule would be interpreted by the macro system. +> In general, it *should* work as described, but doesn't in this case. +> The `macro_rules` machinery, as it stands, has its foibles, and its worthwhile remembering that on occasion, you'll need to contort a little to get it to work. > -> In this *particular* case, there are two issues. First, the macro system doesn't know what does -> and does not constitute the various grammar elements (*e.g.* an expression); that's the parser's -> job. As such, it doesn't know that `...` isn't an expression. Secondly, it has no way of trying -> to capture a compound grammar element (like an expression) without 100% committing to that -> capture. +> In this *particular* case, there are two issues. +> First, the macro system doesn't know what does and does not constitute the various grammar elements (*e.g.* an expression); that's the parser's job. +> As such, it doesn't know that `...` isn't an expression. +> Secondly, it has no way of trying to capture a compound grammar element (like an expression) without 100% committing to that capture. > -> In other words, it can ask the parser to try and parse some input as an expression, but the parser -> will respond to any problems by aborting. The only way the macro system can currently deal with -> this is to just try to forbid situations where this could be a problem. +> In other words, it can ask the parser to try and parse some input as an expression, but the parser will respond to any problems by aborting. +> The only way the macro system can currently deal with this is to just try to forbid situations where this could be a problem. > -> On the bright side, this is a state of affairs that exactly *no one* is enthusiastic about. The -> `macro` keyword has already been reserved for a more rigorously-defined future macro system. +> On the bright side, this is a state of affairs that exactly *no one* is enthusiastic about. +> The `macro` keyword has already been reserved for a more rigorously-defined future [macro system](https://github.com/rust-lang/rust/issues/39412). > Until then, needs must. -Thankfully, the fix is relatively simple: we remove the comma from the syntax. To keep things -balanced, we'll remove *both* commas around `...`: +Thankfully, the fix is relatively simple: we remove the comma from the syntax. +To keep things balanced, we'll remove *both* commas around `...`: ```rust macro_rules! recurrence { @@ -533,12 +508,10 @@ fn main() { } ``` -Success! ... or so I thought. Turns out this is being rejected by the compiler nowadays, while it -was fine back when this was written. The reason for this is that the compiler now recognizes the -`...` as a token, and as we know we may only use `=>`, `,` or `;` after an expression fragment. So -unfortunately we are now out of luck as our dreamed up syntax will not work out this way, so let us -just choose one that looks the most befitting that we are allowed to use instead, -I'd say replacing `,` with `;` works. +Success! ... or so we thought. +Turns out this is being rejected by the compiler nowadays, while it was fine back when this was written. +The reason for this is that the compiler now recognizes the `...` as a token, and as we know we may only use `=>`, `,` or `;` after an expression fragment. +So unfortunately we are now out of luck as our dreamed up syntax will not work out this way, so let us just choose one that looks the most befitting that we are allowed to use instead, I'd say replacing `,` with `;` works. ```rust macro_rules! recurrence { @@ -563,8 +536,8 @@ Success! But for real this time. ### Substitution -Substituting something you've captured in a macro is quite simple; you can insert the contents of a -metavariable `$sty:ty` by using `$sty`. So, let's go through and fix the `u64`s: +Substituting something you've captured in a macro is quite simple; you can insert the contents of a metavariable `$sty:ty` by using `$sty`. +So, let's go through and fix the `u64`s: ```rust macro_rules! recurrence { @@ -648,19 +621,20 @@ fn main() { } ``` -Let's tackle a harder one: how to turn `inits` into both the array literal `[0, 1]` *and* the array -type, `[$sty; 2]`. The first one we can do like so: +Let's tackle a harder one: how to turn `inits` into both the array literal `[0, 1]` *and* the array type, `[$sty; 2]`. +The first one we can do like so: ```rust,ignore Recurrence { mem: [$($inits),+], pos: 0 } // ^~~~~~~~~~~ changed ``` -This effectively does the opposite of the capture: repeat `inits` one or more times, separating each - with a comma. This expands to the expected sequence of tokens: `0, 1`. +This effectively does the opposite of the capture: repeat `inits` one or more times, separating each with a comma. +This expands to the expected sequence of tokens: `0, 1`. -Somehow turning `inits` into a literal `2` is a little trickier. It turns out that there's no direct -way to do this, but we *can* do it by using a second macro. Let's take this one step at a time. +Somehow turning `inits` into a literal `2` is a little trickier. +It turns out that there's no direct way to do this, but we *can* do it by using a second `macro_rules!` macro. +Let's take this one step at a time. ```rust macro_rules! count_exprs { @@ -685,17 +659,14 @@ macro_rules! count_exprs { ``` > **Aside**: You may have noticed I used parentheses here instead of curly braces for the expansion. -> `macro_rules` really doesn't care *what* you use, so long as it's one of the "matcher" pairs: -> `( )`, `{ }` or `[ ]`. In fact, you can switch out the matchers on the macro itself -> (*i.e.* the matchers right after the macro name), the matchers around the syntax rule, and the -> matchers around the corresponding expansion. +> `macro_rules` really doesn't care *what* you use, so long as it's one of the "matcher" pairs: `( )`, `{ }` or `[ ]`. +> In fact, you can switch out the matchers on the macro itself(*i.e.* the matchers right after the macro name), the matchers around the syntax rule, and the matchers around the corresponding expansion. > -> You can also switch out the matchers used when you *invoke* a macro, but in a more limited fashion: -> a macro invoked as `{ ... }` or `( ... );` will *always* be parsed as an *item* (*i.e.* like a -> `struct` or `fn` declaration). This is important when using macros in a function body; it helps -> disambiguate between "parse like an expression" and "parse like a statement". +> You can also switch out the matchers used when you *invoke* a macro, but in a more limited fashion: a macro invoked as `{ ... }` or `( ... );` will *always* be parsed as an *item* (*i.e.* like a `struct` or `fn` declaration). +> This is important when using macros in a function body; it helps disambiguate between "parse like an expression" and "parse like a statement". -What if you have *one* expression? That should be a literal `1`. +What if you have *one* expression? +That should be a literal `1`. ```rust macro_rules! count_exprs { @@ -749,7 +720,8 @@ macro_rules! count_exprs { # } ``` -This is fine since Rust can fold `1 + 1` into a constant value. What if we have three expressions? +This is fine since Rust can fold `1 + 1` into a constant value. +What if we have three expressions? ```rust macro_rules! count_exprs { @@ -771,13 +743,12 @@ macro_rules! count_exprs { # } ``` -> **Aside**: You might be wondering if we could reverse the order of these rules. In this particular -> case, *yes*, but the macro system can sometimes be picky about what it is and is not willing to -> recover from. If you ever find yourself with a multi-rule macro that you *swear* should work, -> but gives you errors about unexpected tokens, try changing the order of the rules. +> **Aside**: You might be wondering if we could reverse the order of these rules. +> In this particular case, *yes*, but the macro system can sometimes be picky about what it is and is not willing to recover from. +> If you ever find yourself with a multi-rule macro that you *swear* should work, but gives you errors about unexpected tokens, try changing the order of the rules. -Hopefully, you can see the pattern here. We can always reduce the list of expressions by matching -one expression, followed by zero or more expressions, expanding that into 1 + a count. +Hopefully, you can see the pattern here. +We can always reduce the list of expressions by matching one expression, followed by zero or more expressions, expanding that into 1 + a count. ```rust macro_rules! count_exprs { @@ -798,8 +769,8 @@ macro_rules! count_exprs { # } ``` -> **JFTE**: this is not the *only*, or even the *best* -> way of counting things. You may wish to peruse the [Counting](./building-blocks/counting.md) section later. +> **JFTE**: this is not the *only*, or even the *best* way of counting things. +> You may wish to peruse the [Counting](./building-blocks/counting.md) section later for a more efficient way. With this, we can now modify `recurrence` to determine the necessary size of `mem`. @@ -969,7 +940,7 @@ With that done, we can now substitute the last thing: the `recur` expression. # } ``` -And, when we compile our finished macro... +And, when we compile our finished `macro_rules!` macro... ```text error[E0425]: cannot find value `a` in this scope @@ -997,16 +968,17 @@ error[E0425]: cannot find value `n` in this scope | ^ not found in this scope ``` -... wait, what? That can't be right... let's check what the macro is expanding to. +... wait, what? +That can't be right... let's check what the macro is expanding to. ```shell $ rustc -Z unstable-options --pretty expanded recurrence.rs ``` -The `--pretty expanded` argument tells `rustc` to perform macro expansion, then turn the resulting -AST back into source code. Because this option isn't considered stable yet, we also need -`-Z unstable-options`. The output (after cleaning up some formatting) is shown below; in particular, -note the place in the code where `$recur` was substituted: +The `--pretty expanded` argument tells `rustc` to perform macro expansion, then turn the resulting AST back into source code. +Because this option isn't considered stable yet, we also need `-Z unstable-options`. +The output (after cleaning up some formatting) is shown below; +in particular, note the place in the code where `$recur` was substituted: ```rust,ignore #![feature(no_std)] @@ -1103,17 +1075,17 @@ fn main() { } ``` -But that looks fine! If we add a few missing `#![feature(...)]` attributes and feed it to a nightly -build of `rustc`, it even compiles! ... *what?!* +But that looks fine! +If we add a few missing `#![feature(...)]` attributes and feed it to a nightly build of `rustc`, it even compiles! ... *what?!* -> **Aside**: You can't compile the above with a non-nightly build of `rustc`. This is because the -> expansion of the `println!` macro depends on internal compiler details which are *not* publicly -> stabilised. +> **Aside**: You can't compile the above with a non-nightly build of `rustc`. +> This is because the expansion of the `println!` macro depends on internal compiler details which are *not* publicly stabilized. ### Being Hygienic -The issue here is that identifiers in Rust macros are *hygienic*. That is, identifiers from two -different contexts *cannot* collide. To show the difference, let's take a simpler example. +The issue here is that identifiers in Rust syntax extensions are *hygienic*. +That is, identifiers from two different contexts *cannot* collide. +To show the difference, let's take a simpler example. ```rust,ignore macro_rules! using_a { @@ -1129,10 +1101,11 @@ let four = using_a!(a / 10); # fn main() {} ``` -This macro simply takes an expression, then wraps it in a block with a variable `a` defined. We then -use this as a round-about way of computing `4`. There are actually *two* syntax contexts involved in -this example, but they're invisible. So, to help with this, let's give each context a different -colour. Let's start with the unexpanded code, where there is only a single context: +This macro simply takes an expression, then wraps it in a block with a variable `a` defined. +We then use this as a round-about way of computing `4`. +There are actually *two* syntax contexts involved in this example, but they're invisible. +So, to help with this, let's give each context a different colour. +Let's start with the unexpanded code, where there is only a single context: 
macro_rules! using_a {
    ($e:expr) => {
        {
            let a = 42;
            $e
        }
    }
}

let four = using_a!(a / 10);
@@ -1140,17 +1113,15 @@ Now, let's expand the invocation. 
let four = {
    let a = 42;
    a / 10
};
-As you can see, the a that's defined by the macro is in a -different context to the a we provided in our invocation. -As such, the compiler treats them as completely different identifiers, -*even though they have the same lexical appearance*. +As you can see, the a that's defined by the macro invocation is in a different context to the a we provided in our invocation. +As such, the compiler treats them as completely different identifiers, *even though they have the same lexical appearance*. -This is something to be *really* careful of when working on macros: macros can produce ASTs which -will not compile, but which *will* compile if written out by hand, or dumped using -`--pretty expanded`. +This is something to be *really* careful of when working on `macro_rules!` macros, syntax extensions in general even: they can produce ASTs which +will not compile, but which *will* compile if written out by hand, or dumped using `--pretty expanded`. -The solution to this is to capture the identifier *with the appropriate syntax context*. To do that, -we need to again adjust our macro syntax. To continue with our simpler example: +The solution to this is to capture the identifier *with the appropriate syntax context*. +To do that, we need to again adjust our macro syntax. +To continue with our simpler example: 
macro_rules! using_a {
    ($a:ident, $e:expr) => {
        {
            let $a = 42;
            $e
        }
    }
}

let four = using_a!(a, a / 10);
@@ -1158,9 +1129,9 @@ This now expands to: 
let four = {
    let a = 42;
    a / 10
};
-Now, the contexts match, and the code will compile. We can make this adjustment to our -`recurrence!` macro by explicitly capturing `a` and `n`. After making the necessary changes, we -have: +Now, the contexts match, and the code will compile. +We can make this adjustment to our `recurrence!` macro by explicitly capturing `a` and `n`. +After making the necessary changes, we have: ```rust macro_rules! count_exprs { @@ -1248,7 +1219,8 @@ fn main() { } ``` -And it compiles! Now, let's try with a different sequence. +And it compiles! +Now, let's try with a different sequence. ```rust # macro_rules! count_exprs { diff --git a/src/macros/minutiae.md b/src/macros/minutiae.md index 36fd354..be7a25e 100644 --- a/src/macros/minutiae.md +++ b/src/macros/minutiae.md @@ -1,4 +1,4 @@ # Minutiae -This section goes through some of the finer details of the `macro_rules!` system. At a minimum, you should -try to be at least *aware* of these details and issues. \ No newline at end of file +This section goes through some of the finer details of the `macro_rules!` system. +At a minimum, you should try to be at least *aware* of these details and issues. diff --git a/src/macros/minutiae/scoping.md b/src/macros/minutiae/scoping.md index c147155..9466f3c 100644 --- a/src/macros/minutiae/scoping.md +++ b/src/macros/minutiae/scoping.md @@ -1,6 +1,6 @@ # Scoping -The way in which function-like macros are scoped can be somewhat unintuitive. +The way in which mbe macros are scoped can be somewhat unintuitive. They use two forms of scopes: textual scope, and path-based scope. When such a macro is invoked by an unqualified identifier(an identifier that isn't part of a mulit-part path), it is first looked up in textual scoping and then in path-based scoping should the first lookup not yield any results. diff --git a/src/macros/patterns/callbacks.md b/src/macros/patterns/callbacks.md index acaac1e..2ee59fd 100644 --- a/src/macros/patterns/callbacks.md +++ b/src/macros/patterns/callbacks.md @@ -24,11 +24,11 @@ fn main() { } ``` -Due to the order that macros are expanded in, it is (as of Rust 1.2) impossible to pass information -to a macro from the expansion of *another* macro. This can make modularizing macros very difficult. +Due to the order that macros are expanded in, it is (as of Rust 1.2) impossible to pass information to a macro from the expansion of *another* macro. +This can make modularizing macros very difficult. -An alternative is to use recursion and pass a callback. Here is a trace of the above example to -demonstrate how this takes place: +An alternative is to use recursion and pass a callback. +Here is a trace of the above example to demonstrate how this takes place: ```rust,ignore recognize_tree! { expand_to_larch ! ( ) } diff --git a/src/macros/patterns/internal-rules.md b/src/macros/patterns/internal-rules.md index a0134be..7d1b3ee 100644 --- a/src/macros/patterns/internal-rules.md +++ b/src/macros/patterns/internal-rules.md @@ -15,21 +15,17 @@ macro_rules! foo { # } ``` -Internal rules can be used to unify multiple macros into one, or to make it easier to read and write -[TT Munchers] by explicitly naming what rule you wish to call in a macro. +Internal rules can be used to unify multiple `macro_rules!` macros into one, or to make it easier to read and write [TT Munchers] by explicitly naming what rule you wish to call in a macro. -So why is it useful to unify multiple macros into one? The main reasoning for this is how macros are -handled in the 2015 Edition of Rust due to macros not being namespaced in said edition. This gives -one the troubles of having to re-export all the internal macros as well polluting the global macro -namespace or even worse, macro name collisions with other crates. In short, it's quite a hassle. -This fortunately isn't really a problem anymore nowadays with a rustc version >= 1.30, for more -information consult the [Import and Export chapter](../minutiae/import-export.html). +So why is it useful to unify multiple mbe macros into one? +The main reasoning for this is how they are handled in the 2015 Edition of Rust due to `macro_rules!` macros not being namespaced in said edition. +This gives one the troubles of having to re-export all the internal `macro_rules!` macros as well polluting the global macro namespace or even worse, macro name collisions with other crates. +In short, it's quite a hassle. +This fortunately isn't really a problem anymore nowadays with a rustc version >= 1.30, for more information consult the [Import and Export chapter](../minutiae/import-export.html). -Nevertheless, let's talk about how we can unify multiple macros into one with this technique and -what exactly this technique even is. +Nevertheless, let's talk about how we can unify multiple `macro_rules!` macros into one with this technique and what exactly this technique even is. -We have two macros, the common [`as_expr!` macro](../building-blocks/ast-coercion.html) and a `foo` -macro that makes use of the first macro: +We have two `macro_rules!` macros, the common [`as_expr!` macro](../building-blocks/ast-coercion.html) and a `foo` macro that makes use of the first one: ```rust #[macro_export] @@ -47,14 +43,12 @@ macro_rules! foo { # } ``` -This is definitely not the nicest solution we could have for this macro, as it pollutes the global -macro namespace as mentioned earlier. In this specific case `as_expr` is also a very simple macro -that we only used once, so let's "embed" this macro in our `foo` macro with internal rules! To do so -we simply prepend a new matcher for our macro consists of the matcher used in the `as_expr` macro, -but with a small addition. We prepend a tokentree that makes it match only when specifically asked -to. In this case we can for example use `@as_expr`, so our matcher becomes -`(@as_expr $e:expr) => {$e};`. With this we get the macro that was defined at the very top of this -page: +This is definitely not the nicest solution we could have for this macro, as it pollutes the global macro namespace as mentioned earlier. +In this specific case `as_expr` is also a very simple macro that we only used once, so let's "embed" this macro in our `foo` macro with internal rules! +To do so we simply prepend a new matcher for our macro consists of the matcher used in the `as_expr` macro, but with a small addition. +We prepend a tokentree that makes it match only when specifically asked to. +In this case we can for example use `@as_expr`, so our matcher becomes `(@as_expr $e:expr) => {$e};`. +With this we get the macro that was defined at the very top of this page: ```rust #[macro_export] @@ -71,27 +65,20 @@ macro_rules! foo { # } ``` -You see how we embedded the `as_expr` macro in the `foo` one? All that changed is that instead of -invoking the `as_expr` macro, we now invoke `foo` recursively but with a special token tree -prepended to the arguments, `foo!(@as_expr $($tts)*)`. If you look closely you might even see that -this pattern can be combined quite nicely with [TT Munchers]! +You see how we embedded the `as_expr` macro in the `foo` one? +All that changed is that instead of invoking the `as_expr` macro, we now invoke `foo` recursively but with a special token tree prepended to the arguments, `foo!(@as_expr $($tts)*)`. +If you look closely you might even see that this pattern can be combined quite nicely with [TT Munchers]! -The reason for using `@` was that, as of Rust 1.2, the `@` token is *not* used in prefix position; as -such, it cannot conflict with anything. This reasoning became obsolete later on when in Rust 1.7 -macro matchers got future proofed by emitting a warning to prevent certain tokens from being allowed -to follow certain fragments[^ambiguity-restrictions], which in Rust 1.12 became a hard-error. There -other symbols or unique prefixes may be used as desired, but use of `@` has started to become -widespread, so using it may aid readers in understanding your macro. +The reason for using `@` was that, as of Rust 1.2, the `@` token is *not* used in prefix position; as such, it cannot conflict with anything. +This reasoning became obsolete later on when in Rust 1.7 macro matchers got future proofed by emitting a warning to prevent certain tokens from being allowed to follow certain fragments[^ambiguity-restrictions], which in Rust 1.12 became a hard-error. +There other symbols or unique prefixes may be used as desired, but use of `@` has started to become widespread, so using it may aid readers in understanding your macro. [^ambiguity-restrictions]:[ambiguity-restrictions](../minutiae/metavar-and-expansion.html) -> **Note**: in the early days of Rust the `@` token was previously used in prefix position to denote -> a garbage-collected pointer, back when the language used sigils to denote pointer types. Its -> only *current* purpose is for binding names to patterns. For this, however, it is used as an -> *infix* operator, and thus does not conflict with its use here. +> **Note**: in the early days of Rust the `@` token was previously used in prefix position to denote a garbage-collected pointer, back when the language used sigils to denote pointer types. +> Its only *current* purpose is for binding names to patterns. +> For this, however, it is used as an *infix* operator, and thus does not conflict with its use here. -Additionally, internal rules will often come *before* any "bare" rules, to avoid issues with -`macro_rules!` incorrectly attempting to parse an internal invocation as something it cannot -possibly be, such as an expression. +Additionally, internal rules will often come *before* any "bare" rules, to avoid issues with `macro_rules!` incorrectly attempting to parse an internal invocation as something it cannot possibly be, such as an expression. [TT Munchers]:./tt-muncher.html diff --git a/src/macros/patterns/push-down-acc.md b/src/macros/patterns/push-down-acc.md index 3a1e7c0..2bab372 100644 --- a/src/macros/patterns/push-down-acc.md +++ b/src/macros/patterns/push-down-acc.md @@ -23,8 +23,8 @@ let strings: [String; 3] = init_array![String::from("hi!"); 3]; # assert_eq!(format!("{:?}", strings), "[\"hi!\", \"hi!\", \"hi!\"]"); ``` -All macros in Rust **must** result in a complete, supported syntax element (such as an expression, -item, *etc.*). This means that it is impossible to have a macro expand to a partial construct. +All syntax extensions in Rust **must** result in a complete, supported syntax element (such as an expression, item, *etc.*). +This means that it is impossible to have a syntax extension expand to a partial construct. One might hope that the above example could be more directly expressed like so: @@ -52,13 +52,11 @@ The expectation is that the expansion of the array literal would proceed as foll [e, e, e] ``` -However, this would require each intermediate step to expand to an incomplete expression. Even -though the intermediate results will never be used *outside* of a macro context, it is still -forbidden. +However, this would require each intermediate step to expand to an incomplete expression. +Even though the intermediate results will never be used *outside* of a macro context, it is still forbidden. -Push-down, however, allows us to incrementally build up a sequence of tokens without needing to -actually have a complete construct at any point prior to completion. In the example given at the -top, the sequence of macro invocations proceeds as follows: +Push-down, however, allows us to incrementally build up a sequence of tokens without needing to actually have a complete construct at any point prior to completion. +In the example given at the top, the sequence of invocations proceeds as follows: ```rust,ignore init_array! { String:: from ( "hi!" ) ; 3 } @@ -69,14 +67,10 @@ init_array! { @ accum ( 0 , e.clone() ) -> ( e.clone() , e.clone() , e.clone() , init_array! { @ as_expr [ e.clone() , e.clone() , e.clone() , ] } ``` -As you can see, each layer adds to the accumulated output until the terminating rule finally emits -it as a complete construct. +As you can see, each layer adds to the accumulated output until the terminating rule finally emits it as a complete construct. -The only critical part of the above formulation is the use of `$($body:tt)*` to preserve the output -without triggering parsing. The use of `($input) -> ($output)` is simply a convention adopted to -help clarify the behavior of such macros. +The only critical part of the above formulation is the use of `$($body:tt)*` to preserve the output without triggering parsing. +The use of `($input) -> ($output)` is simply a convention adopted to help clarify the behavior of such macros. -Push-down accumulation is frequently used as part of -[incremental TT munchers](./tt-muncher.md), as it allows arbitrarily complex intermediate -results to be constructed. [Internal Rules](./internal-rules.md) were of use -here as well, as they simplify creating such macros. +Push-down accumulation is frequently used as part of [incremental TT munchers](./tt-muncher.md), as it allows arbitrarily complex intermediate results to be constructed. +[Internal Rules](./internal-rules.md) were of use here as well, as they simplify creating such macros. diff --git a/src/macros/patterns/repetition-replacement.md b/src/macros/patterns/repetition-replacement.md index afaa479..394e09a 100644 --- a/src/macros/patterns/repetition-replacement.md +++ b/src/macros/patterns/repetition-replacement.md @@ -6,11 +6,9 @@ macro_rules! replace_expr { } ``` -This pattern is where a matched repetition sequence is simply discarded, with the variable being -used to instead drive some repeated pattern that is related to the input only in terms of length. +This pattern is where a matched repetition sequence is simply discarded, with the variable being used to instead drive some repeated pattern that is related to the input only in terms of length. -For example, consider constructing a default instance of a tuple with more than 12 elements (the -limit as of Rust 1.2). +For example, consider constructing a default instance of a tuple with more than 12 elements (the limit as of Rust 1.2). ```rust macro_rules! tuple_default { @@ -25,17 +23,16 @@ macro_rules! tuple_default { ) }; } -# +# # macro_rules! replace_expr { # ($_t:tt $sub:expr) => {$sub}; # } -# +# # assert_eq!(tuple_default!(i32, bool, String), (i32::default(), bool::default(), String::default())); ``` -> **JFTE**: we *could* have simply used -> `$tup_tys::default()`. +> **JFTE**: we *could* have simply used `$tup_tys::default()`. -Here, we are not actually *using* the matched types. Instead, we throw them away and replace them -with a single, repeated expression. To put it another way, we don't care *what* the types are, only -*how many* there are. +Here, we are not actually *using* the matched types. +Instead, we throw them away and replace them with a single, repeated expression. +To put it another way, we don't care *what* the types are, only *how many* there are. diff --git a/src/macros/patterns/tt-bundling.md b/src/macros/patterns/tt-bundling.md index dc9a895..631bfb5 100644 --- a/src/macros/patterns/tt-bundling.md +++ b/src/macros/patterns/tt-bundling.md @@ -47,18 +47,13 @@ fn main() { } ``` -In particularly complex recursive macros, a large number of arguments may be needed in order to -carry identifiers and expressions to successive layers. However, depending on the implementation -there may be many intermediate layers which need to forward these arguments, but do not need to -*use* them. +In particularly complex recursive macros, a large number of arguments may be needed in order to carry identifiers and expressions to successive layers. +However, depending on the implementation there may be many intermediate layers which need to forward these arguments, but do not need to *use* them. -As such, it can be very useful to bundle all such arguments together into a single TT by placing -them in a group. This allows layers which do not need to use the arguments to simply capture and -substitute a single [`tt`], rather than having to exactly capture and substitute the entire argument -group. +As such, it can be very useful to bundle all such arguments together into a single TT by placing them in a group. +This allows layers which do not need to use the arguments to simply capture and substitute a single [`tt`], rather than having to exactly capture and substitute the entire argument group. -The example above bundles the `$a` and `$b` expressions into a group which can then be forwarded as -a single [`tt`] by the recursive rule. This group is then destructured by the terminal rules to -access the expressions. +The example above bundles the `$a` and `$b` expressions into a group which can then be forwarded as a single [`tt`] by the recursive rule. +This group is then destructured by the terminal rules to access the expressions. [`tt`]: ../minutiae/fragment-specifiers.html#tt diff --git a/src/macros/patterns/tt-muncher.md b/src/macros/patterns/tt-muncher.md index 27ab307..4198439 100644 --- a/src/macros/patterns/tt-muncher.md +++ b/src/macros/patterns/tt-muncher.md @@ -32,27 +32,22 @@ macro_rules! mixed_rules { # } ``` -This pattern is perhaps the *most powerful* macro parsing technique available, allowing one to parse -grammars of significant complexity. +This pattern is perhaps the *most powerful* macro parsing technique available, allowing one to parse grammars of significant complexity. -A `TT muncher` is a recursive macro that works by incrementally processing its input one step at a -time. At each step, it matches and removes (munches) some sequence of tokens from the start of its -input, generates some intermediate output, then recurses on the input tail. +A `TT muncher` is a recursive `macro_rules!` macro that works by incrementally processing its input one step at a time. +At each step, it matches and removes (munches) some sequence of tokens from the start of its input, generates some intermediate output, then recurses on the input tail. -The reason for "TT" in the name specifically is that the unprocessed part of the input is *always* -captured as `$($tail:tt)*`. This is done as a [`tt`] repetition is the only way to *losslessly* -capture part of a macro's input. +The reason for "TT" in the name specifically is that the unprocessed part of the input is *always* captured as `$($tail:tt)*`. +This is done as a [`tt`] repetition is the only way to *losslessly* capture part of a macro's input. -The only hard restrictions on TT munchers are those imposed on the macro system as a whole: +The only hard restrictions on TT munchers are those imposed on the `macro_rules!` macro system as a whole: * You can only match against literals and grammar constructs which can be captured by `macro_rules!`. * You cannot match unbalanced groups. -It is important, however, to keep the macro recursion limit in mind. `macro_rules!` does not have -*any* form of tail recursion elimination or optimization. It is recommended that, when writing a TT -muncher, you make reasonable efforts to keep recursion as limited as possible. This can be done by -adding additional rules to account for variation in the input (as opposed to recursion into an -intermediate layer), or by making compromises on the input syntax to make using standard repetitions -more tractable. +It is important, however, to keep the macro recursion limit in mind. +`macro_rules!` does not have *any* form of tail recursion elimination or optimization. +It is recommended that, when writing a TT muncher, you make reasonable efforts to keep recursion as limited as possible. +This can be done by adding additional rules to account for variation in the input (as opposed to recursion into an intermediate layer), or by making compromises on the input syntax to make using standard repetitions more tractable. [`tt`]: ../minutiae/fragment-specifiers.html#tt diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index 1abece6..0c28346 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -22,8 +22,6 @@ use_local!(); In this case for `use_local` to be considered fully hygienic, this snippet again should not compile as otherwise it would be affected by its surrounding context and also affect its surrounding context as well. -This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and proc-macro `hygiene` chapters as there are specifics to each. - -> **Aside**: A Rust syntax extension might or might not be fully hygienic, depending on its kind and definition. +This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and proc-macro `hygiene` chapters, mainly explaining how hygienic these syntax extensions can be, be it fully or only partially. [`macro_rules!` `hygiene`]: ../macros/minutiae/hygiene.md From 0cf145cfbde42ab8755a0d9cb99be85073bea8d6 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 10 Jul 2021 23:44:39 +0200 Subject: [PATCH 10/92] Add very sparse glossary base --- src/SUMMARY.md | 1 + src/glossary.md | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 src/glossary.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 076c774..627d5e6 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -32,3 +32,4 @@ - [Abacus Counting](./macros/building-blocks/abacus-counting.md) - [Parsing Rust](./macros/building-blocks/parsing.md) - [Procedural Macros]() + - [Glossary](./glossary.md) diff --git a/src/glossary.md b/src/glossary.md new file mode 100644 index 0000000..9cc9ebe --- /dev/null +++ b/src/glossary.md @@ -0,0 +1,11 @@ +# Glossary + +A place for obscure words and their descriptions. +If you feel like there is an important word missing here, please open an [issue](https://github.com/Veykril/tlborm/issues/new) or a pull request. + +## Function-like macro +A function like macro describes a syntax extension that can be invoked via the form `identifier!(...)`. +It is called this way due to its resemblance of a function call. + +## Syntax Extension +The mechanism rust's `macro_rules!` and procedural macros are built on. From 4c4145b119b508a094c9a39ffc366a2594a01b60 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 11 Jul 2021 00:10:26 +0200 Subject: [PATCH 11/92] Minor fixes --- README.md | 4 +++- book.toml | 1 - src/SUMMARY.md | 3 ++- src/macros.md | 2 +- src/macros/macros-practical.md | 6 +++--- src/macros/minutiae/debugging.md | 8 ++++---- src/macros/minutiae/identifiers.md | 3 ++- src/macros/minutiae/scoping.md | 2 +- src/syntax-extensions/hygiene.md | 8 ++++++++ 9 files changed, 24 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8a69e74..76ceb1b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ A chinese version of this book can be found [here](https://zjp-cn.github.io/tlbo This book is an attempt to distill the Rust community's collective knowledge of Rust macros, the `Macros by Example` ones as well as procedural macros. As such, both additions (in the form of pull requests) and requests (in the form of issues) are very much welcome. +If something's unclear, opens up questions or is not understandable as written down, fear not to make an issue asking for clarification. +The goal is for this book to become the best learning resource possible. The [original Little Book of Rust Macros](https://github.com/DanielKeep/tlborm) has helped me immensely with understanding ***Macros by Example*** style macros while I was still learning the language. Unfortunately, the original book hasn't been updated since april of 2016, while the Rust language as well as it's macro-system keeps evolving. @@ -16,7 +18,7 @@ In hopes that it will help out all the fresh faces coming to Rust understanding > This book expects you to have basic knowledge of Rust, it will not explain language features or constructs that are irrelevant to macros. > No prior knowledge of macros is assumed. -> Having read and understood the first seven chapters of the [Rust Book](https://doc.rust-lang.org/stable/book/) is > a must, though having read the majority of the book is recommended. +> Having read and understood the first seven chapters of the [Rust Book](https://doc.rust-lang.org/stable/book/) is a must, though having read the majority of the book is recommended. ## Thanks diff --git a/book.toml b/book.toml index 8e93cae..d180c05 100644 --- a/book.toml +++ b/book.toml @@ -12,7 +12,6 @@ build-dir = "book" [output.linkcheck] traverse-parent-directories = false -exclude = ["mathjax"] [output.html] default-theme = "ayu" diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 627d5e6..6ab599d 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -32,4 +32,5 @@ - [Abacus Counting](./macros/building-blocks/abacus-counting.md) - [Parsing Rust](./macros/building-blocks/parsing.md) - [Procedural Macros]() - - [Glossary](./glossary.md) + + [Glossary](./glossary.md) diff --git a/src/macros.md b/src/macros.md index 557f13c..dd86617 100644 --- a/src/macros.md +++ b/src/macros.md @@ -1,4 +1,4 @@ -# Macros +# Macros By Example This chapter will introduce Rust's [Macro-By-Example][mbe] system: [`macro_rules!`][mbe]. There are two different introductions in this chapter, a [methodical] and a [practical]. diff --git a/src/macros/macros-practical.md b/src/macros/macros-practical.md index c390ddd..4943e90 100644 --- a/src/macros/macros-practical.md +++ b/src/macros/macros-practical.md @@ -13,7 +13,7 @@ There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/bo If you aren't familiar, a recurrence relation is a sequence where each value is defined in terms of one or more *previous* values, with one or more initial values to get the whole thing started. For example, the [Fibonacci sequence](https://en.wikipedia.org/wiki/Fibonacci_number) can be defined by the relation: -\\[ F_{n} = 0, 1, ..., F_{n-1} + F_{n-2}\\](mathjax) +\\[F_{n} = 0, 1, ..., F_{n-1} + F_{n-2}\\] Thus, the first two numbers in the sequence are 0 and 1, with the third being \\( F_{0} + F_{1} = 0 + 1 = 1\\), the fourth \\( F_{1} + F_{2} = 1 + 1 = 2\\), and so on forever. @@ -141,7 +141,7 @@ for e in fib.take(10) { println!("{}", e) } ``` Lastly, return an instance of our new structure, which can then be iterated over. -To summarise, the complete expansion is: +To summarize, the complete expansion is: ```rust,ignore let fib = { @@ -242,7 +242,7 @@ impl<'a> Index for IndexOffset<'a> { > **Aside**: since lifetimes come up *a lot* with people new to Rust, a quick explanation: `'a` and `'b` are lifetime parameters that are used to track where a reference (*i.e.* a borrowed pointer to some data) is valid. > In this case, `IndexOffset` borrows a reference to our iterator's data, so it needs to keep track of how long it's allowed to hold that reference for, using `'a`. > -> `'b` is used because the `Index::index` function (which is how subscript syntax is actually implemented) is *also* parameterised on a lifetime, on account of returning a borrowed reference. +> `'b` is used because the `Index::index` function (which is how subscript syntax is actually implemented) is *also* parameterized on a lifetime, on account of returning a borrowed reference. > `'a` and `'b` are not necessarily the same thing in all cases. > The borrow checker will make sure that even though we don't explicitly relate `'a` and `'b` to one another, we don't accidentally violate memory safety. diff --git a/src/macros/minutiae/debugging.md b/src/macros/minutiae/debugging.md index cf6fc1c..5f201ea 100644 --- a/src/macros/minutiae/debugging.md +++ b/src/macros/minutiae/debugging.md @@ -77,7 +77,7 @@ For this, the `--pretty` argument to the compiler can be used. Given the following code: ```rust,ignore -// Shorthand for initialising a `String`. +// Shorthand for initializing a `String`. macro_rules! S { ($e:expr) => {String::from($e)}; } @@ -103,7 +103,7 @@ produces the following output (modified for formatting): use std::prelude::v1::*; #[macro_use] extern crate std as std; -// Shorthand for initialising a `String`. +// Shorthand for initializing a `String`. fn main() { let world = String::from("World"); ::std::io::_print(::std::fmt::Arguments::new_v1( @@ -121,8 +121,8 @@ fn main() { } ``` -Other options to `--pretty` can be listed using `rustc -Z unstable-options --help -v`; a full list -is not provided since, as implied by the name, any such list would be subject to change at any time. +Other options to `--pretty` can be listed using `rustc -Z unstable-options --help -v`; +a full list is not provided since, as implied by the name, any such list would be subject to change at any time. But not just `rustc` exposes means to aid in debugging macros. For the aforementioned `--pretty=expanded` option, there exists a nice `cargo` addon called [`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) which is basically just a wrapper around it. diff --git a/src/macros/minutiae/identifiers.md b/src/macros/minutiae/identifiers.md index ca2d05b..f05b536 100644 --- a/src/macros/minutiae/identifiers.md +++ b/src/macros/minutiae/identifiers.md @@ -4,7 +4,8 @@ There are two tokens which you are likely to run into eventually that *look* lik Except when they are. First is `self`. -This is *very definitely* a keyword. However, it also happens to fit the definition of an identifier. +This is *very definitely* a keyword. +However, it also happens to fit the definition of an identifier. In regular Rust code, there's no way for `self` to be interpreted as an identifier, but it *can* happen with `macro_rules!` macros: ```rust diff --git a/src/macros/minutiae/scoping.md b/src/macros/minutiae/scoping.md index 9466f3c..e886206 100644 --- a/src/macros/minutiae/scoping.md +++ b/src/macros/minutiae/scoping.md @@ -3,7 +3,7 @@ The way in which mbe macros are scoped can be somewhat unintuitive. They use two forms of scopes: textual scope, and path-based scope. -When such a macro is invoked by an unqualified identifier(an identifier that isn't part of a mulit-part path), it is first looked up in textual scoping and then in path-based scoping should the first lookup not yield any results. +When such a macro is invoked by an unqualified identifier(an identifier that isn't part of a multi-segment path), it is first looked up in textual scoping and then in path-based scoping should the first lookup not yield any results. If it is invoked by a qualified identifier it will skip the textual scoping lookup and instead only do a look up in the path-based scoping. ## Textual Scope diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index 0c28346..22440e3 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -3,6 +3,13 @@ Hygiene is an important concept for macros, it describes the ability for a macro to work in any context, not affecting nor being affected by its surroundings. In other words this means that a syntax extension should be invocable anywhere without interfering with its surrounding context. +In a perfect world all syntax extensions in rust would be fully hygienic, unfortunately this isn't the case, so care should be taken to avoid writing syntax extensions that aren't fully hygienic. +We will go into general hygiene concepts here which will be touched upon in the corresponding hygiene chapters for the different syntax extensions rust has to offer. + +  + +Hygiene mainly affects identifiers and paths emitted by syntax extensions. + This is best shown by example: Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, then given the following snippet: @@ -23,5 +30,6 @@ use_local!(); In this case for `use_local` to be considered fully hygienic, this snippet again should not compile as otherwise it would be affected by its surrounding context and also affect its surrounding context as well. This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and proc-macro `hygiene` chapters, mainly explaining how hygienic these syntax extensions can be, be it fully or only partially. +There also exists this [github gist](https://gist.github.com/Kestrer/8c05ebd4e0e9347eb05f265dfb7252e1) that explains how to write hygienic syntax extensions while going into a hygiene a bit overall. [`macro_rules!` `hygiene`]: ../macros/minutiae/hygiene.md From 6426fb856d05c5d0cba3a7b4a757af9082145662 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 11 Jul 2021 13:17:45 +0200 Subject: [PATCH 12/92] Make linkcheck optional --- book.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/book.toml b/book.toml index d180c05..6fa7438 100644 --- a/book.toml +++ b/book.toml @@ -12,6 +12,7 @@ build-dir = "book" [output.linkcheck] traverse-parent-directories = false +optional = true [output.html] default-theme = "ayu" From 415561b696e159bdc832e9e1fd59935b194426f6 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 11 Jul 2021 13:24:52 +0200 Subject: [PATCH 13/92] Fix workflow `publish_dir` --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index c9f73c5..eaac01f 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -22,7 +22,7 @@ jobs: uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./book + publish_dir: ./book/html force_orphan: true user_name: 'github-actions[bot]' user_email: 'github-actions[bot]@users.noreply.github.com' From 717e1f19baed257a896631a52768ff07869fa8ee Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 14 Jul 2021 01:04:53 +0200 Subject: [PATCH 14/92] Improve the fragment specifiers chapter --- src/macros/minutiae/fragment-specifiers.md | 38 ++++++++++++++-------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/macros/minutiae/fragment-specifiers.md b/src/macros/minutiae/fragment-specifiers.md index eaded98..49d657c 100644 --- a/src/macros/minutiae/fragment-specifiers.md +++ b/src/macros/minutiae/fragment-specifiers.md @@ -1,9 +1,9 @@ # Fragment Specifiers -As shown in the [`macro_rules`] chapter, Rust, as of 1.46, has 13 fragment specifiers. -This section will go a bit more into detail for some of them and tries to always show a few examples of what a matcher can match with. +As shown in the [`macro_rules`] chapter, Rust, as of 1.53, has 13 fragment specifiers. +This section will go a bit more into detail for some of them and shows a few example inputs of what each matcher matches. -> Note that capturing with anything but the `ident`, `lifetime` and `tt` fragments will render the captured AST opaque, making it impossible to further inspect it in future macro invocations. +> **Note**: Capturing with anything but the `ident`, `lifetime` and `tt` fragments will render the captured AST opaque, making it impossible to further match it with other fragment specifiers in future macro invocations. * [`block`](#block) * [`expr`](#expr) @@ -21,7 +21,7 @@ This section will go a bit more into detail for some of them and tries to always ## `block` -The `block` fragment solely matches a [block expression](https://doc.rust-lang.org/reference/expressions/block-expr.html), which consists of an opening `{` brace, followed by any amount of statements and finally followed by a closing `}` brace. +The `block` fragment solely matches a [block expression](https://doc.rust-lang.org/reference/expressions/block-expr.html), which consists of an opening `{` brace, followed by any number of statements and finally followed by a closing `}` brace. ```rust macro_rules! blocks { @@ -78,7 +78,7 @@ idents! { ## `item` The `item` fragment simply matches any of Rust's [item](https://doc.rust-lang.org/reference/items.html) *definitions*, not identifiers that refer to items. -Item examples: +This includes visibility modifiers. ```rust macro_rules! items { @@ -91,6 +91,7 @@ items! { Baz } impl Foo {} + pub use crate::foo; /*...*/ } # fn main() {} @@ -136,7 +137,9 @@ literals! { ## `meta` The `meta` fragment matches an [attribute](https://doc.rust-lang.org/reference/attributes.html), to be more precise, the contents of an attribute. -You will usually see this fragment being used in a matcher like `#[$meta:meta]` or `#![$meta:meta]`. +That is, it will match a simple path, one without generic arguments followed by a delimited token tree or an `=` followed by a literal expression. + +> **Note**: You will usually see this fragment being used in a matcher like `#[$meta:meta]` or `#![$meta:meta]` to actually capture an attribute. ```rust macro_rules! metas { @@ -152,11 +155,11 @@ metas! { # fn main() {} ``` -> A neat thing about doc comments: They are actually attributes in the form of `#[doc="…"]` where the `...` is the actual comment string, meaning you can act on doc comments in macros! +> **Doc-Comment Fact**: Doc-Comments like `/// ...` and `!// ...` are actually syntax sugar for attributes! They desugar to `#[doc="..."]` and `#![doc="..."]` respectively, meaning you can match on them like with attributes! ## `pat` -The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html). +The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html), except or-patterns. ```rust macro_rules! patterns { @@ -175,6 +178,7 @@ patterns! { ## `path` The `path` fragment matches a so called [TypePath](https://doc.rust-lang.org/reference/paths.html#paths-in-types) style path. +This includes the function style trait forms, `Fn() -> ()`. ```rust macro_rules! paths { @@ -185,6 +189,7 @@ paths! { ASimplePath ::A::B::C::D G::::C + FnMut(u32) -> () } # fn main() {} ``` @@ -192,6 +197,7 @@ paths! { ## `stmt` The `statement` fragment solely matches a [statement](https://doc.rust-lang.org/reference/statements.html) without its trailing semicolon, unless its an item statement that requires one. + What would be an item statement that requires one? A Unit-Struct would be a simple one, as defining one requires a trailing semicolon. @@ -240,8 +246,9 @@ fn main() { From this we can tell a few things: The first you should be able to see immediately is that while the `stmt` fragment doesn't capture trailing semicolons, it still emits them when required, even if the statement is already followed by one. -The simple reason for that is that semicolons on their own are already valid statements. -So we are actually invoking our macro here with not 8 statements, but 11! +The simple reason for that is that semicolons on their own are already valid statements which the fragment captures eagerly. +So our macro isn't capturing 8 times, but 11! +This can be important when doing multiples repetitions and expanding these in one repetition expansion, as the repetition numbers have to match in those cases. Another thing you should be able to notice here is that the trailing semicolon of the `struct Foo;` item statement is being matched, otherwise we would've seen an extra one like in the other cases. This makes sense as we already said, that for item statements that require one, the trailing semicolon will be matched with. @@ -250,6 +257,8 @@ A last observation is that expressions get emitted back with a trailing semicolo The fine details of what was just mentioned here can be looked up in the [reference](https://doc.rust-lang.org/reference/statements.html). +Fortunately, these fine details here are usually not of importance whatsoever, with the small exception that was mentioned earlier in regards to repetitions which by itself shouldn't be a common problem to run into. + [^debugging]:See the [debugging chapter](./debugging.md) for tips on how to do this. ## `tt` @@ -258,10 +267,11 @@ The `tt` fragment matches a TokenTree. If you need a refresher on what exactly a TokenTree was you may want to revisit the [TokenTree chapter](../../syntax-extensions/source-analysis.md#token-trees) of this book. The `tt` fragment is one of the most powerful fragments, as it can match nearly anything while still allowing you to inspect the contents of it at a later state in the macro. +This allows one to make use of very powerful patterns like the [tt-muncher](../patterns/tt-muncher.md) or the [push-down-accumulator](../patterns/push-down-acc.md). + ## `ty` The `ty` fragment matches any kind of [type expression](https://doc.rust-lang.org/reference/types.html#type-expressions). -A type expression is the syntax with which one refers to a type in the language. ```rust macro_rules! types { @@ -272,6 +282,7 @@ types! { foo::bar bool [u8] + impl IntoIterator } # fn main() {} ``` @@ -280,7 +291,8 @@ types! { The `vis` fragment matches a *possibly empty* [Visibility qualifier](https://doc.rust-lang.org/reference/visibility-and-privacy.html). Emphasis lies on the *possibly empty* part. -You can think of this fragment having an implicit `?` repetition to it, meaning you don't, and in fact cannot, wrap it in a direct repetition. +You can kind of think of this fragment as having an implicit `?` repetition to it, meaning you don't, and in fact cannot, wrap it in a direct repetition, though this only applies for matching. +When expanding this fragment you expand it without a `$(...)?` repetition block. ```rust macro_rules! visibilities { @@ -289,7 +301,7 @@ macro_rules! visibilities { } visibilities! { - , + , // no vis is fine, as its implicitly pub, pub(crate), pub(in super), From 3469e21185f083267bfb3a7f3e180a1f083e9e4a Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 14 Jul 2021 01:09:18 +0200 Subject: [PATCH 15/92] proc macros are a future thing --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76ceb1b..fb8a414 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ View the [rendered version here](https://veykril.github.io/tlborm/) and the [rep A chinese version of this book can be found [here](https://zjp-cn.github.io/tlborm/). -This book is an attempt to distill the Rust community's collective knowledge of Rust macros, the `Macros by Example` ones as well as procedural macros. +This book is an attempt to distill the Rust community's collective knowledge of Rust macros, the `Macros by Example` ones as well as procedural macros(WIP). As such, both additions (in the form of pull requests) and requests (in the form of issues) are very much welcome. If something's unclear, opens up questions or is not understandable as written down, fear not to make an issue asking for clarification. The goal is for this book to become the best learning resource possible. @@ -22,7 +22,7 @@ In hopes that it will help out all the fresh faces coming to Rust understanding ## Thanks -A big thank you to Daniel Keep for the original work as well as all the contributors that added to the original. +A big thank you to Daniel Keep for the original work as well as all the contributors that added to the original which can be found [here](https://github.com/DanielKeep/tlborm). ## License From 9b3766a3a4389c84c5ce822aed3febfb8a6b4b45 Mon Sep 17 00:00:00 2001 From: Yohannes Kifle Date: Sun, 18 Jul 2021 08:52:15 +0300 Subject: [PATCH 16/92] Fixes possible typo --- src/syntax-extensions/expansion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/expansion.md b/src/syntax-extensions/expansion.md index 135c543..2295814 100644 --- a/src/syntax-extensions/expansion.md +++ b/src/syntax-extensions/expansion.md @@ -20,7 +20,7 @@ In fact, it can turn a syntax extension result into any of the following: In other words, *where* you can invoke a syntax extension determines what its result will be interpreted as. The compiler will take this AST node and completely replace the syntax extension's invocation node with the output node. -*This is a structural operation*, not a textural one! +*This is a structural operation*, not a textual one! For example, consider the following: From a6fdd6f006acb74093a4d4b65d92845bcc77f278 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 21 Jul 2021 16:11:24 +0200 Subject: [PATCH 17/92] Hygiene blocks follow theme highlighting --- res/rust-syntax-bg-highlight.css | 34 -------------------------------- src/macros/minutiae/hygiene.md | 8 ++++---- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/res/rust-syntax-bg-highlight.css b/res/rust-syntax-bg-highlight.css index 67c174b..9c34ae3 100644 --- a/res/rust-syntax-bg-highlight.css +++ b/res/rust-syntax-bg-highlight.css @@ -1,36 +1,2 @@ -/** - * Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT - * file at the top-level directory of this distribution and at - * http://rust-lang.org/COPYRIGHT. - * With elements taken from Bootstrap v3.0.2 (MIT licensed). - * - * Licensed under the Apache License, Version 2.0 or the MIT license - * , at your - * option. This file may not be copied, modified, or distributed - * except according to those terms. - * - * Modified by Daniel Keep. - */ - - -/* Code highlighting */ -pre.rust .kw { color: #f92672; } -pre.rust .kw-2 { color: #f92672; } -pre.rust .prelude-ty { color: #66d9ef; } -pre.rust .number { color: #EFD6AB; } -pre.rust .string { color: #F8E9A5; } -pre.rust .self { color: #f92672; } -pre.rust .boolval { color: #EFD6AB; } -pre.rust .prelude-val { color: #66d9ef; } -pre.rust .attribute { color: #C7AFD1; } -pre.rust .attribute .ident { color: #D4BBDE; } -pre.rust .comment { color: #75715e; } -pre.rust .doccomment { color: #75715e; } -pre.rust .macro { color: #AC7FE7; } -pre.rust .macro-nonterminal { color: #CFC0E2; } -pre.rust .lifetime { color: #B76514; } -pre.rust .op { color: #f92672; } - pre.rust .synctx-0 { background-color: rgba(255, 0, 0, 0.1); } pre.rust .synctx-1 { background-color: rgba(0, 255, 0, 0.1); } diff --git a/src/macros/minutiae/hygiene.md b/src/macros/minutiae/hygiene.md index fb1bd9c..77133db 100644 --- a/src/macros/minutiae/hygiene.md +++ b/src/macros/minutiae/hygiene.md @@ -8,12 +8,12 @@ When two identifiers are compared, *both* the identifiers' textual names *and* s To illustrate this, consider the following code: -
macro_rules! using_a {
    ($e:expr) => {
        {
            let a = 42;
            $e
        }
    }
}

let four = using_a!(a / 10);
+
macro_rules! using_a {
    ($e:expr) => {
        {
            let a = 42;
            $e
        }
    }
}

let four = using_a!(a / 10);
We will use the background colour to denote the syntax context. Now, let's expand the macro invocation: -
let four = {
    let a = 42;
    a / 10
};
+
let four = {
    let a = 42;
    a / 10
};
First, recall that `macro_rules!` invocations effectively *disappear* during expansion. @@ -35,11 +35,11 @@ In other words, a is not the same ide That said, tokens that were substituted *into* the expanded output *retain* their original syntax context (by virtue of having been provided to the macro as opposed to being part of the macro itself). Thus, the solution is to modify the macro as follows: -
macro_rules! using_a {
    ($a:ident, $e:expr) => {
        {
            let $a = 42;
            $e
        }
    }
}

let four = using_a!(a, a / 10);
+
macro_rules! using_a {
    ($a:ident, $e:expr) => {
        {
            let $a = 42;
            $e
        }
    }
}

let four = using_a!(a, a / 10);
Which, upon expansion becomes: -
let four = {
    let a = 42;
    a / 10
};
+
let four = {
    let a = 42;
    a / 10
};
The compiler will accept this code because there is only one `a` being used. From a55add1b99ca0baf7fe48ed2142c20a78589c18d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 22 Jul 2021 01:01:33 +0200 Subject: [PATCH 18/92] Add some redirects so that old links still point to the correct item --- book.toml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/book.toml b/book.toml index 6fa7438..dedf44f 100644 --- a/book.toml +++ b/book.toml @@ -20,3 +20,24 @@ site-url = "/tlborm/" mathjax-support = true git-repository-url = "https://github.com/veykril/tlborm/" additional-css = ["res/rust-syntax-bg-highlight.css"] + +[output.html.redirect] +"/building-blocks/abacus-counting.html" = "../macros/building-blocks/abacus-counting.html" +"/building-blocks/ast-coercion.html" = "../macros/building-blocks/ast-coercion.html" +"/building-blocks/counting.html" = "../macros/building-blocks/counting.html" +"/building-blocks/parsing.html" = "../macros/building-blocks/parsing.html" + +"/macros-practical.html" = "./macros/macros-practical.html" +"/patterns.html" = "./macros/patterns.html" +"/building-blocks.html" = "./macros/building-blocks.html" + +"/patterns/callbacks.html" = "../macros/patterns/callbacks.html" +"/patterns/internal-rules.html" = "../macros/patterns/internal-rules.html" +"/patterns/push-down-acc.html" = "../macros/patterns/push-down-acc.html" +"/patterns/repetition-placement.html" = "../macros/patterns/repetition-placement.html" +"/patterns/tt-bundling.html" = "../macros/patterns/tt-bundling.html" +"/patterns/tt-muncher.html" = "../macros/patterns/tt-muncher.html" + +"/macros/syntax/ast.html" = "../../syntax-extensions/ast.html" +"/macros/syntax/expansion.html" = "../../syntax-extensions/expansion.html" +"/macros/syntax/source-analysis.html" = "../../syntax-extensions/source-analysis.html" From 6ba80d87f5055ac8efe563f873feda596e03671a Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 22 Jul 2021 01:02:04 +0200 Subject: [PATCH 19/92] Add quick link to bit twiddling trick in counting chapter --- src/macros/building-blocks/counting.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/macros/building-blocks/counting.md b/src/macros/building-blocks/counting.md index df0287f..d9cadeb 100644 --- a/src/macros/building-blocks/counting.md +++ b/src/macros/building-blocks/counting.md @@ -1,5 +1,9 @@ # Counting +What follows are several techniques for counting in `macro_rules!` macros: + +> **Note**: If you are just interested in the most efficient way [look here](./counting.md#bit-twiddling) + ## Repetition with replacement Counting things in a macro is a surprisingly tricky task. From c3245a651011348bab1a73a6a2824c65b5de6b26 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 22 Jul 2021 22:29:13 +0200 Subject: [PATCH 20/92] macro_rules macro -> declarative macro --- book.toml | 32 ++++++------ src/SUMMARY.md | 50 +++++++++---------- src/{macros.md => decl-macros.md} | 13 ++--- .../building-blocks.md | 0 .../building-blocks/abacus-counting.md | 0 .../building-blocks/ast-coercion.md | 0 .../building-blocks/counting.md | 0 .../building-blocks/parsing.md | 0 src/{macros => decl-macros}/macro_rules.md | 0 .../macros-methodical.md | 0 .../macros-practical-table.html | 0 .../macros-practical.md | 2 +- src/{macros => decl-macros}/minutiae.md | 0 .../minutiae/debugging.md | 0 .../minutiae/fragment-specifiers.md | 0 .../minutiae/hygiene.md | 0 .../minutiae/identifiers.md | 0 .../minutiae/import-export.md | 0 .../minutiae/metavar-and-expansion.md | 0 .../minutiae/scoping.md | 0 src/{macros => decl-macros}/patterns.md | 0 .../patterns/callbacks.md | 0 .../patterns/internal-rules.md | 0 .../patterns/push-down-acc.md | 0 .../patterns/repetition-replacement.md | 0 .../patterns/tt-bundling.md | 0 .../patterns/tt-muncher.md | 0 src/syntax-extensions/hygiene.md | 4 +- 28 files changed, 51 insertions(+), 50 deletions(-) rename src/{macros.md => decl-macros.md} (80%) rename src/{macros => decl-macros}/building-blocks.md (100%) rename src/{macros => decl-macros}/building-blocks/abacus-counting.md (100%) rename src/{macros => decl-macros}/building-blocks/ast-coercion.md (100%) rename src/{macros => decl-macros}/building-blocks/counting.md (100%) rename src/{macros => decl-macros}/building-blocks/parsing.md (100%) rename src/{macros => decl-macros}/macro_rules.md (100%) rename src/{macros => decl-macros}/macros-methodical.md (100%) rename src/{macros => decl-macros}/macros-practical-table.html (100%) rename src/{macros => decl-macros}/macros-practical.md (99%) rename src/{macros => decl-macros}/minutiae.md (100%) rename src/{macros => decl-macros}/minutiae/debugging.md (100%) rename src/{macros => decl-macros}/minutiae/fragment-specifiers.md (100%) rename src/{macros => decl-macros}/minutiae/hygiene.md (100%) rename src/{macros => decl-macros}/minutiae/identifiers.md (100%) rename src/{macros => decl-macros}/minutiae/import-export.md (100%) rename src/{macros => decl-macros}/minutiae/metavar-and-expansion.md (100%) rename src/{macros => decl-macros}/minutiae/scoping.md (100%) rename src/{macros => decl-macros}/patterns.md (100%) rename src/{macros => decl-macros}/patterns/callbacks.md (100%) rename src/{macros => decl-macros}/patterns/internal-rules.md (100%) rename src/{macros => decl-macros}/patterns/push-down-acc.md (100%) rename src/{macros => decl-macros}/patterns/repetition-replacement.md (100%) rename src/{macros => decl-macros}/patterns/tt-bundling.md (100%) rename src/{macros => decl-macros}/patterns/tt-muncher.md (100%) diff --git a/book.toml b/book.toml index dedf44f..93bdbfa 100644 --- a/book.toml +++ b/book.toml @@ -22,22 +22,22 @@ git-repository-url = "https://github.com/veykril/tlborm/" additional-css = ["res/rust-syntax-bg-highlight.css"] [output.html.redirect] -"/building-blocks/abacus-counting.html" = "../macros/building-blocks/abacus-counting.html" -"/building-blocks/ast-coercion.html" = "../macros/building-blocks/ast-coercion.html" -"/building-blocks/counting.html" = "../macros/building-blocks/counting.html" -"/building-blocks/parsing.html" = "../macros/building-blocks/parsing.html" +"/building-blocks/abacus-counting.html" = "../decl-macros/building-blocks/abacus-counting.html" +"/building-blocks/ast-coercion.html" = "../decl-macros/building-blocks/ast-coercion.html" +"/building-blocks/counting.html" = "../decl-macros/building-blocks/counting.html" +"/building-blocks/parsing.html" = "../decl-macros/building-blocks/parsing.html" -"/macros-practical.html" = "./macros/macros-practical.html" -"/patterns.html" = "./macros/patterns.html" -"/building-blocks.html" = "./macros/building-blocks.html" +"/macros-practical.html" = "./decl-macros/macros-practical.html" +"/patterns.html" = "./decl-macros/patterns.html" +"/building-blocks.html" = "./decl-macros/building-blocks.html" -"/patterns/callbacks.html" = "../macros/patterns/callbacks.html" -"/patterns/internal-rules.html" = "../macros/patterns/internal-rules.html" -"/patterns/push-down-acc.html" = "../macros/patterns/push-down-acc.html" -"/patterns/repetition-placement.html" = "../macros/patterns/repetition-placement.html" -"/patterns/tt-bundling.html" = "../macros/patterns/tt-bundling.html" -"/patterns/tt-muncher.html" = "../macros/patterns/tt-muncher.html" +"/patterns/callbacks.html" = "../decl-macros/patterns/callbacks.html" +"/patterns/internal-rules.html" = "../decl-macros/patterns/internal-rules.html" +"/patterns/push-down-acc.html" = "../decl-macros/patterns/push-down-acc.html" +"/patterns/repetition-placement.html" = "../decl-macros/patterns/repetition-placement.html" +"/patterns/tt-bundling.html" = "../decl-macros/patterns/tt-bundling.html" +"/patterns/tt-muncher.html" = "../decl-macros/patterns/tt-muncher.html" -"/macros/syntax/ast.html" = "../../syntax-extensions/ast.html" -"/macros/syntax/expansion.html" = "../../syntax-extensions/expansion.html" -"/macros/syntax/source-analysis.html" = "../../syntax-extensions/source-analysis.html" +"/decl-macros/syntax/ast.html" = "../../syntax-extensions/ast.html" +"/decl-macros/syntax/expansion.html" = "../../syntax-extensions/expansion.html" +"/decl-macros/syntax/source-analysis.html" = "../../syntax-extensions/source-analysis.html" diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 6ab599d..7d1be37 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -7,30 +7,30 @@ - [Macros in the Ast](./syntax-extensions/ast.md) - [Expansion](./syntax-extensions/expansion.md) - [Hygiene](./syntax-extensions/hygiene.md) -- [Macros-By-Example](./macros.md) - - [A Methodical Introduction](./macros/macros-methodical.md) - - [macro_rules!](./macros/macro_rules.md) - - [Minutiae](./macros/minutiae.md) - - [Fragment Specifiers](./macros/minutiae/fragment-specifiers.md) - - [Metavariables and Expansion Redux](./macros/minutiae/metavar-and-expansion.md) - - [Hygiene](./macros/minutiae/hygiene.md) - - [Non-Identifier Identifiers](./macros/minutiae/identifiers.md) - - [Debugging](./macros/minutiae/debugging.md) - - [Scoping](./macros/minutiae/scoping.md) - - [Import and Export](./macros/minutiae/import-export.md) - - [A Practical Introduction](./macros/macros-practical.md) - - [Patterns](./macros/patterns.md) - - [Callbacks](./macros/patterns/callbacks.md) - - [Incremental TT Munchers](./macros/patterns/tt-muncher.md) - - [Internal Rules](./macros/patterns/internal-rules.md) - - [Push-down Accumulation](./macros/patterns/push-down-acc.md) - - [Repetition Replacement](./macros/patterns/repetition-replacement.md) - - [TT Bundling](./macros/patterns/tt-bundling.md) - - [Building Blocks](./macros/building-blocks.md) - - [AST Coercion](./macros/building-blocks/ast-coercion.md) - - [Counting](./macros/building-blocks/counting.md) - - [Abacus Counting](./macros/building-blocks/abacus-counting.md) - - [Parsing Rust](./macros/building-blocks/parsing.md) - - [Procedural Macros]() +- [Declarative Macros](./decl-macros.md) + - [A Methodical Introduction](./decl-macros/macros-methodical.md) + - [macro_rules!](./decl-macros/macro_rules.md) + - [Minutiae](./decl-macros/minutiae.md) + - [Fragment Specifiers](./decl-macros/minutiae/fragment-specifiers.md) + - [Metavariables and Expansion Redux](./decl-macros/minutiae/metavar-and-expansion.md) + - [Hygiene](./decl-macros/minutiae/hygiene.md) + - [Non-Identifier Identifiers](./decl-macros/minutiae/identifiers.md) + - [Debugging](./decl-macros/minutiae/debugging.md) + - [Scoping](./decl-macros/minutiae/scoping.md) + - [Import and Export](./decl-macros/minutiae/import-export.md) + - [A Practical Introduction](./decl-macros/macros-practical.md) + - [Patterns](./decl-macros/patterns.md) + - [Callbacks](./decl-macros/patterns/callbacks.md) + - [Incremental TT Munchers](./decl-macros/patterns/tt-muncher.md) + - [Internal Rules](./decl-macros/patterns/internal-rules.md) + - [Push-down Accumulation](./decl-macros/patterns/push-down-acc.md) + - [Repetition Replacement](./decl-macros/patterns/repetition-replacement.md) + - [TT Bundling](./decl-macros/patterns/tt-bundling.md) + - [Building Blocks](./decl-macros/building-blocks.md) + - [AST Coercion](./decl-macros/building-blocks/ast-coercion.md) + - [Counting](./decl-macros/building-blocks/counting.md) + - [Abacus Counting](./decl-macros/building-blocks/abacus-counting.md) + - [Parsing Rust](./decl-macros/building-blocks/parsing.md) + - [Procedural Macros]() [Glossary](./glossary.md) diff --git a/src/macros.md b/src/decl-macros.md similarity index 80% rename from src/macros.md rename to src/decl-macros.md index dd86617..34f8148 100644 --- a/src/macros.md +++ b/src/decl-macros.md @@ -1,6 +1,7 @@ -# Macros By Example +# Declarative Macros + +This chapter will introduce Rust's declarative macro system: [`macro_rules!`][mbe]. -This chapter will introduce Rust's [Macro-By-Example][mbe] system: [`macro_rules!`][mbe]. There are two different introductions in this chapter, a [methodical] and a [practical]. The former will attempt to give you a complete and thorough explanation of *how* the system works, while the latter one will cover more practical examples. @@ -14,7 +15,7 @@ Should the information presented here not suffice, then there is also the [Macro [mbe]: https://doc.rust-lang.org/reference/macros-by-example.html [Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html -[practical]: ./macros/macros-practical.md -[methodical]: ./macros/macros-methodical.md -[patterns]: ./macros/patterns.md -[building blocks]: ./macros/building-blocks.md +[practical]: ./decl-macros/macros-practical.md +[methodical]: ./decl-macros/macros-methodical.md +[patterns]: ./decl-macros/patterns.md +[building blocks]: ./decl-macros/building-blocks.md diff --git a/src/macros/building-blocks.md b/src/decl-macros/building-blocks.md similarity index 100% rename from src/macros/building-blocks.md rename to src/decl-macros/building-blocks.md diff --git a/src/macros/building-blocks/abacus-counting.md b/src/decl-macros/building-blocks/abacus-counting.md similarity index 100% rename from src/macros/building-blocks/abacus-counting.md rename to src/decl-macros/building-blocks/abacus-counting.md diff --git a/src/macros/building-blocks/ast-coercion.md b/src/decl-macros/building-blocks/ast-coercion.md similarity index 100% rename from src/macros/building-blocks/ast-coercion.md rename to src/decl-macros/building-blocks/ast-coercion.md diff --git a/src/macros/building-blocks/counting.md b/src/decl-macros/building-blocks/counting.md similarity index 100% rename from src/macros/building-blocks/counting.md rename to src/decl-macros/building-blocks/counting.md diff --git a/src/macros/building-blocks/parsing.md b/src/decl-macros/building-blocks/parsing.md similarity index 100% rename from src/macros/building-blocks/parsing.md rename to src/decl-macros/building-blocks/parsing.md diff --git a/src/macros/macro_rules.md b/src/decl-macros/macro_rules.md similarity index 100% rename from src/macros/macro_rules.md rename to src/decl-macros/macro_rules.md diff --git a/src/macros/macros-methodical.md b/src/decl-macros/macros-methodical.md similarity index 100% rename from src/macros/macros-methodical.md rename to src/decl-macros/macros-methodical.md diff --git a/src/macros/macros-practical-table.html b/src/decl-macros/macros-practical-table.html similarity index 100% rename from src/macros/macros-practical-table.html rename to src/decl-macros/macros-practical-table.html diff --git a/src/macros/macros-practical.md b/src/decl-macros/macros-practical.md similarity index 99% rename from src/macros/macros-practical.md rename to src/decl-macros/macros-practical.md index 4943e90..89f63d4 100644 --- a/src/macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -3,7 +3,7 @@ This chapter will introduce the Rust macro-by-example system using a relatively simple, practical example. It does *not* attempt to explain all of the intricacies of the system; its goal is to get you comfortable with how and why macros are written. -There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-06-macros.html) which is another high-level explanation, and the [methodical introduction](../macros.md) chapter of this book, which explains the macro system in detail. +There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-06-macros.html) which is another high-level explanation, and the [methodical introduction](../decl-macros.md) chapter of this book, which explains the macro system in detail. ## A Little Context diff --git a/src/macros/minutiae.md b/src/decl-macros/minutiae.md similarity index 100% rename from src/macros/minutiae.md rename to src/decl-macros/minutiae.md diff --git a/src/macros/minutiae/debugging.md b/src/decl-macros/minutiae/debugging.md similarity index 100% rename from src/macros/minutiae/debugging.md rename to src/decl-macros/minutiae/debugging.md diff --git a/src/macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md similarity index 100% rename from src/macros/minutiae/fragment-specifiers.md rename to src/decl-macros/minutiae/fragment-specifiers.md diff --git a/src/macros/minutiae/hygiene.md b/src/decl-macros/minutiae/hygiene.md similarity index 100% rename from src/macros/minutiae/hygiene.md rename to src/decl-macros/minutiae/hygiene.md diff --git a/src/macros/minutiae/identifiers.md b/src/decl-macros/minutiae/identifiers.md similarity index 100% rename from src/macros/minutiae/identifiers.md rename to src/decl-macros/minutiae/identifiers.md diff --git a/src/macros/minutiae/import-export.md b/src/decl-macros/minutiae/import-export.md similarity index 100% rename from src/macros/minutiae/import-export.md rename to src/decl-macros/minutiae/import-export.md diff --git a/src/macros/minutiae/metavar-and-expansion.md b/src/decl-macros/minutiae/metavar-and-expansion.md similarity index 100% rename from src/macros/minutiae/metavar-and-expansion.md rename to src/decl-macros/minutiae/metavar-and-expansion.md diff --git a/src/macros/minutiae/scoping.md b/src/decl-macros/minutiae/scoping.md similarity index 100% rename from src/macros/minutiae/scoping.md rename to src/decl-macros/minutiae/scoping.md diff --git a/src/macros/patterns.md b/src/decl-macros/patterns.md similarity index 100% rename from src/macros/patterns.md rename to src/decl-macros/patterns.md diff --git a/src/macros/patterns/callbacks.md b/src/decl-macros/patterns/callbacks.md similarity index 100% rename from src/macros/patterns/callbacks.md rename to src/decl-macros/patterns/callbacks.md diff --git a/src/macros/patterns/internal-rules.md b/src/decl-macros/patterns/internal-rules.md similarity index 100% rename from src/macros/patterns/internal-rules.md rename to src/decl-macros/patterns/internal-rules.md diff --git a/src/macros/patterns/push-down-acc.md b/src/decl-macros/patterns/push-down-acc.md similarity index 100% rename from src/macros/patterns/push-down-acc.md rename to src/decl-macros/patterns/push-down-acc.md diff --git a/src/macros/patterns/repetition-replacement.md b/src/decl-macros/patterns/repetition-replacement.md similarity index 100% rename from src/macros/patterns/repetition-replacement.md rename to src/decl-macros/patterns/repetition-replacement.md diff --git a/src/macros/patterns/tt-bundling.md b/src/decl-macros/patterns/tt-bundling.md similarity index 100% rename from src/macros/patterns/tt-bundling.md rename to src/decl-macros/patterns/tt-bundling.md diff --git a/src/macros/patterns/tt-muncher.md b/src/decl-macros/patterns/tt-muncher.md similarity index 100% rename from src/macros/patterns/tt-muncher.md rename to src/decl-macros/patterns/tt-muncher.md diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index 22440e3..5b85de7 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -1,6 +1,6 @@ # Hygiene -Hygiene is an important concept for macros, it describes the ability for a macro to work in any context, not affecting nor being affected by its surroundings. +Hygiene is an important concept for macros, it describes the ability for a macro to work in its own syntax context, not affecting nor being affected by its surroundings. In other words this means that a syntax extension should be invocable anywhere without interfering with its surrounding context. In a perfect world all syntax extensions in rust would be fully hygienic, unfortunately this isn't the case, so care should be taken to avoid writing syntax extensions that aren't fully hygienic. @@ -32,4 +32,4 @@ In this case for `use_local` to be considered fully hygienic, this snippet again This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and proc-macro `hygiene` chapters, mainly explaining how hygienic these syntax extensions can be, be it fully or only partially. There also exists this [github gist](https://gist.github.com/Kestrer/8c05ebd4e0e9347eb05f265dfb7252e1) that explains how to write hygienic syntax extensions while going into a hygiene a bit overall. -[`macro_rules!` `hygiene`]: ../macros/minutiae/hygiene.md +[`macro_rules!` `hygiene`]: ../decl-macros/minutiae/hygiene.md From 55fb604df90614a3292ee74c15f9bece16371ca3 Mon Sep 17 00:00:00 2001 From: mbknust <48494224+mbknust@users.noreply.github.com> Date: Thu, 29 Jul 2021 10:16:07 +0200 Subject: [PATCH 21/92] Hygenic enum counting We can capture the first identifier with a separate metavariable, and then put it at the end of the enum. --- src/decl-macros/building-blocks/counting.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/decl-macros/building-blocks/counting.md b/src/decl-macros/building-blocks/counting.md index d9cadeb..38c4ea3 100644 --- a/src/decl-macros/building-blocks/counting.md +++ b/src/decl-macros/building-blocks/counting.md @@ -132,11 +132,12 @@ This approach can be used where you need to count a set of mutually distinct ide ```rust macro_rules! count_idents { - ($($idents:ident),* $(,)*) => { + ($(,)*) => {0}; + ($last_ident:ident, $($idents:ident),* $(,)*) => { { #[allow(dead_code, non_camel_case_types)] - enum Idents { $($idents,)* __CountIdentsLast } - const COUNT: u32 = Idents::__CountIdentsLast as u32; + enum Idents { $($idents,)* $last_ident } + const COUNT: u32 = Idents::$last_ident as u32 + 1; COUNT } }; @@ -149,9 +150,7 @@ macro_rules! count_idents { ``` This method does have two drawbacks. -First, as implied above, it can *only* count valid identifiers(which are also not keywords), and it does not allow those identifiers to repeat. - -Secondly, this approach is *not* hygienic, meaning that if whatever identifier you use in place of `__CountIdentsLast` is provided as input, the macro will fail due to the duplicate variants in the `enum`. +As implied above, it can *only* count valid identifiers (which are also not keywords), and it does not allow those identifiers to repeat. ## Bit twiddling From 53d973e4c6b709f1f0c014d1df808ea03b8a32e0 Mon Sep 17 00:00:00 2001 From: mbknust <48494224+mbknust@users.noreply.github.com> Date: Thu, 29 Jul 2021 20:03:48 +0200 Subject: [PATCH 22/92] Apply suggestions from code review Fixed handling of trailing commas in matcher. Co-authored-by: Lukas Wirth --- src/decl-macros/building-blocks/counting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/building-blocks/counting.md b/src/decl-macros/building-blocks/counting.md index 38c4ea3..d1f646e 100644 --- a/src/decl-macros/building-blocks/counting.md +++ b/src/decl-macros/building-blocks/counting.md @@ -132,8 +132,8 @@ This approach can be used where you need to count a set of mutually distinct ide ```rust macro_rules! count_idents { - ($(,)*) => {0}; - ($last_ident:ident, $($idents:ident),* $(,)*) => { + () => {0}; + ($last_ident:ident, $($idents:ident),* $(,)?) => { { #[allow(dead_code, non_camel_case_types)] enum Idents { $($idents,)* $last_ident } From a7bd4025c4b4b49542c94d3b105a48d784c5dbcc Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 3 Aug 2021 16:15:21 +0200 Subject: [PATCH 23/92] Make hyigene syntax context background more opaque on light themes --- res/rust-syntax-bg-highlight.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/res/rust-syntax-bg-highlight.css b/res/rust-syntax-bg-highlight.css index 9c34ae3..1e609bc 100644 --- a/res/rust-syntax-bg-highlight.css +++ b/res/rust-syntax-bg-highlight.css @@ -1,2 +1,6 @@ pre.rust .synctx-0 { background-color: rgba(255, 0, 0, 0.1); } pre.rust .synctx-1 { background-color: rgba(0, 255, 0, 0.1); } +.light pre.rust .synctx-0 { background-color: rgba(255, 0, 0, 0.2); } +.light pre.rust .synctx-1 { background-color: rgba(0, 255, 0, 0.2); } +.rust pre.rust .synctx-0 { background-color: rgba(255, 0, 0, 0.2); } +.rust pre.rust .synctx-1 { background-color: rgba(0, 255, 0, 0.2); } From 47532ab683156988e91072eff18a0958f6c34b26 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 23 Jul 2021 00:10:10 +0200 Subject: [PATCH 24/92] proc-macros first chapter --- src/SUMMARY.md | 6 ++++- src/decl-macros.md | 2 +- src/proc-macros.md | 40 ++++++++++++++++++++++++++++++++ src/proc-macros/attr.md | 1 + src/proc-macros/derive.md | 1 + src/proc-macros/function-like.md | 1 + src/proc-macros/hygiene.md | 1 + 7 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 src/proc-macros.md create mode 100644 src/proc-macros/attr.md create mode 100644 src/proc-macros/derive.md create mode 100644 src/proc-macros/function-like.md create mode 100644 src/proc-macros/hygiene.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 7d1be37..946e20b 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -31,6 +31,10 @@ - [Counting](./decl-macros/building-blocks/counting.md) - [Abacus Counting](./decl-macros/building-blocks/abacus-counting.md) - [Parsing Rust](./decl-macros/building-blocks/parsing.md) - - [Procedural Macros]() + - [Procedural Macros](./proc-macros.md) + - [Hygiene](./proc-macros/hygiene.md) + - [custom!(…)](./proc-macros/function-like.md) + - [#\[Derive(CustomDerive, …)\]](./proc-macros/derive.md) + - [#\[CustomAttr(…)\]](./proc-macros/attr.md) [Glossary](./glossary.md) diff --git a/src/decl-macros.md b/src/decl-macros.md index 34f8148..381b8f8 100644 --- a/src/decl-macros.md +++ b/src/decl-macros.md @@ -9,7 +9,7 @@ As such, the [methodical introduction][methodical] is intended for people who ju Following up the two introductions it offers some generally very useful [patterns] and [building blocks] for creating feature rich macros. -Should the information presented here not suffice, then there is also the [Macros chapter of the Rust Book] which is a more approachable, high-level explanation as well as the reference [chapter](https://doc.rust-lang.org/reference/macros-by-example.html) which goes more into the precise details of things. +Other resources about declarative macros include the [Macros chapter of the Rust Book] which is a more approachable, high-level explanation as well as the reference [chapter](https://doc.rust-lang.org/reference/macros-by-example.html) which goes more into the precise details of things. > **Note**: This book will usually use the term *mbe*(**M**acro-**B**y-**E**xample), *mbe macro* or `macro_rules!` macro when talking about `macro_rules!` macros. diff --git a/src/proc-macros.md b/src/proc-macros.md new file mode 100644 index 0000000..a10acb4 --- /dev/null +++ b/src/proc-macros.md @@ -0,0 +1,40 @@ +# Procedural Macros + +This chapter will introduce Rust's second syntax extension type, *procedural macros*. + +Unlike a [declarative macro](./decl-macros.md), a procedural macro takes the form of a rust function taking in a token stream(or two) and outputting a token stream. +This makes writing procedural macros vastly different from declarative ones and comes with its own share of advantages and disadvantages. + +As noted in the [syntax extension chapter](./syntax-extensions/ast.md) it is possible to implement all three syntaxes of macro invocations with procedural macros, `#[attr]` attributes, `#[derive(…)]` derives and `foo!()` function-like. Each of these have some slight differences which will be explained in their separate chapters. + +All of these still run at the same stage in the compiler expansion-wise as declarative macros. +As mentioned earlier, a procedural macro is just a function that maps a token stream(a stream of token trees) to another token stream. +Thus it is a program the compiler compiles, then executes by feeding the input of the macro invocation, and depending on the type of macro invocation replaces the invocation with the output entirely or appends it. + +So how do we go about creating such a proc-macro? By creating a crate of course! +A proc-macro is at its core just a function exported from a crate with the `proc-macro` [crate type](https://doc.rust-lang.org/reference/linkage.html), so when writing multiple proc macros you can have them all live in one crate. + +> **Note**: When using Cargo, to define a `proc-macro` crate you define and set the `lib.proc-macro` key in the `Cargo.toml` to true. +> ```toml +> [lib] +> proc-macro = true +> ``` + +A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. +This crate also exposes the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html) mentioned earlier, which will be your macro's input and output type. +This type as it's documentation says is just a stream of token trees as we know them from earlier chapters but encoded as rust types. +Another type of interest is the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily used for error reporting and [hygiene](./proc-macros/hygiene.md). Each token has an associated span that may be altered freely. + +With this knowledge we can take a look at the function signatures of procedural macros, which in case of a function-like macro or derive is `fn(TokenStream) -> TokenStream` and in case of an attribute is `fn(TokenStream, TokenStream) -> TokenStream`. +Note how the return type is a `TokenStream` itself for both and not a result or something else that gives the notion of fallible. +This does not mean that proc-macros cannot fail though, in fact they have two ways of reporting errors, the first one being to panic and the second to emit a [`compile_error!`](https://doc.rust-lang.org/std/macro.compile_error.html) invocation. +If a proc-macro panics the compiler will catch it and emit the payload as an error coming from the macro invocation. + +> **Beware**: The compiler will happily hang on endless loops spun up inside proc-macros causing the compilation of crates using the proc-macro to hang as well. + + +With all of this in mind we can now start implementing procedural macros, but as it turns out the `proc_macro` crate does not offer any parsing capabilities, just access to the token stream which means we are required to do the parsing of said tokens ourselves. +This can be quite cumbersome depending on the macro and as such we will make use of the [`syn`](https://docs.rs/syn/1.0.74/syn/) crate to make our lives simpler. +We will quickly go over the aforementioned crate as well as some other helpful ones in the next chapter and then finally begin implementing some macros. + +Other resources about procedural macros include the the reference [chapter](https://doc.rust-lang.org/reference/procedural-macros.html) which this chapter took heavy inspiration from. diff --git a/src/proc-macros/attr.md b/src/proc-macros/attr.md new file mode 100644 index 0000000..ecf69c9 --- /dev/null +++ b/src/proc-macros/attr.md @@ -0,0 +1 @@ +# `#[CustomAttr(…)]` diff --git a/src/proc-macros/derive.md b/src/proc-macros/derive.md new file mode 100644 index 0000000..6c24822 --- /dev/null +++ b/src/proc-macros/derive.md @@ -0,0 +1 @@ +# `#[Derive(CustomDerive, …)]` diff --git a/src/proc-macros/function-like.md b/src/proc-macros/function-like.md new file mode 100644 index 0000000..47d93b9 --- /dev/null +++ b/src/proc-macros/function-like.md @@ -0,0 +1 @@ +# `custom!(…)` diff --git a/src/proc-macros/hygiene.md b/src/proc-macros/hygiene.md new file mode 100644 index 0000000..0e34ab9 --- /dev/null +++ b/src/proc-macros/hygiene.md @@ -0,0 +1 @@ +# Hygiene From 25f26a98d4a39f1065963025fb8d8bef460e0b29 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 23 Jul 2021 01:20:23 +0200 Subject: [PATCH 25/92] Add third party crates section to proc-macros --- src/SUMMARY.md | 9 ++-- src/proc-macros.md | 4 +- src/proc-macros/third-party-crates.md | 67 +++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 src/proc-macros/third-party-crates.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 946e20b..19c5ca6 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -32,9 +32,10 @@ - [Abacus Counting](./decl-macros/building-blocks/abacus-counting.md) - [Parsing Rust](./decl-macros/building-blocks/parsing.md) - [Procedural Macros](./proc-macros.md) - - [Hygiene](./proc-macros/hygiene.md) - - [custom!(…)](./proc-macros/function-like.md) - - [#\[Derive(CustomDerive, …)\]](./proc-macros/derive.md) - - [#\[CustomAttr(…)\]](./proc-macros/attr.md) + - [Third-Party Crates](./proc-macros/third-party-crates.md) + - [Hygiene and Spans]() + - [custom!(…)](./proc-macros/function-like.md) + - [#\[Derive(CustomDerive, …)\]](./proc-macros/derive.md) + - [#\[CustomAttr(…)\]](./proc-macros/attr.md) [Glossary](./glossary.md) diff --git a/src/proc-macros.md b/src/proc-macros.md index a10acb4..f41313b 100644 --- a/src/proc-macros.md +++ b/src/proc-macros.md @@ -23,7 +23,7 @@ A proc-macro is at its core just a function exported from a crate with the `proc A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. This crate also exposes the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html) mentioned earlier, which will be your macro's input and output type. This type as it's documentation says is just a stream of token trees as we know them from earlier chapters but encoded as rust types. -Another type of interest is the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily used for error reporting and [hygiene](./proc-macros/hygiene.md). Each token has an associated span that may be altered freely. +Another type of interest is the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily used for error reporting and hygiene. Each token has an associated span that may be altered freely. With this knowledge we can take a look at the function signatures of procedural macros, which in case of a function-like macro or derive is `fn(TokenStream) -> TokenStream` and in case of an attribute is `fn(TokenStream, TokenStream) -> TokenStream`. Note how the return type is a `TokenStream` itself for both and not a result or something else that gives the notion of fallible. @@ -34,7 +34,7 @@ If a proc-macro panics the compiler will catch it and emit the payload as an err With all of this in mind we can now start implementing procedural macros, but as it turns out the `proc_macro` crate does not offer any parsing capabilities, just access to the token stream which means we are required to do the parsing of said tokens ourselves. -This can be quite cumbersome depending on the macro and as such we will make use of the [`syn`](https://docs.rs/syn/1.0.74/syn/) crate to make our lives simpler. +Parsing tokens to input by hand can be quite cumbersome depending on the macro that is being implemented and as such we will make use of the [`syn`](https://docs.rs/syn/*/syn/) crate to make parsing simpler. We will quickly go over the aforementioned crate as well as some other helpful ones in the next chapter and then finally begin implementing some macros. Other resources about procedural macros include the the reference [chapter](https://doc.rust-lang.org/reference/procedural-macros.html) which this chapter took heavy inspiration from. diff --git a/src/proc-macros/third-party-crates.md b/src/proc-macros/third-party-crates.md new file mode 100644 index 0000000..4b3aed7 --- /dev/null +++ b/src/proc-macros/third-party-crates.md @@ -0,0 +1,67 @@ +# Third-Party Crates + +> **Note**: Crates beyond the automatically linked [`proc_macro`] crate are not required to write procedural macros. +> The crates listed here merely make writing them simpler and possibly more readable, while potentially adding to the compilation time of the procedural macro due to added the dependencies. + +As procedural macros live in a crate they can naturally depend on ([crates.io](https://crates.io/)) crates. +turns out the crate ecosystem has some really helpful crates tailored towards procedural macros that this chapter will quickly go over, most of which will be used in the following chapters to implement the example macros. +As these are merely quick introductions it is advised to look at each crate's documentation for more in-depth information if required. + +## [`proc-macro2`] + +[`proc-macro2`], the successor of the [`proc_macro`] crate! Or so you might think but that is of course not correct, the name might be a bit misleading. +This crate is actually just a wrapper around the [`proc_macro`] crate serving two specific purposes, taken from the documentation: +- Bring proc-macro-like functionality to other contexts like build.rs and main.rs. +- Make procedural macros unit testable. + +As the [`proc_macro`] crate is exclusive to [`proc_macro`] type crates, making them unit testable or accessing them from non-proc macro code is next to impossible. +With that in mind the [`proc-macro2`] crate mimics the original [`proc_macro`] crate's api, acting as a wrapper in proc-macro crates and standing on its own in non-proc-macro crates. +Hence it is advised to build libraries targeting proc-macro code to be built against [`proc-macro2`] instead as that will enable those libraries to be unit testable, which is also the reason why the following listed crates take and emit [`proc-macro2::TokenStream`](https://docs.rs/proc-macro2/1.0.27/proc_macro2/struct.TokenStream.html)s instead. +When a `proc_macro` token stream is required, one can simply `.into()` the `proc-macro2` token stream to get the `proc_macro` version and vice-versa. + +Procedural macros using the `proc-macro2` crate will usually import the `proc-macro2::TokenStream` in an aliased form like `use proc-macro2::TokenStream as TokenStream2`. + +## [`quote`] + +The [`quote`] crate mainly exposes just one macro, the [`quote!`](https://docs.rs/quote/1/quote/macro.quote.html) macro. + +This little macro allows you to easily create token streams by writing the actual source out as syntax while also giving you the power of interpolating tokens right into the written syntax. +[Interpolation](https://docs.rs/quote/1/quote/macro.quote.html#interpolation) can be done by using the `#local` syntax where local refers to a local in the current scope. +Likewise `#( #local )*` can be used to interpolate over an iterator of types that implement [`ToTokens`](https://docs.rs/quote/1/quote/trait.ToTokens.html), this works similar to declarative `macro_rules!` repetitions in that they allow a separator as well as extra tokens inside the repetition. + +```rs +let name = /* some identifier */; +let exprs = /* an iterator over expressions tokenstreams */; +let expanded = quote! { + impl SomeTrait for #name { // #name interpolates the name local from above + fn some_function(&self) -> usize { + #( #exprs )* // #name interpolates exprs by iterating the iterator + } + } +}; +``` + +This a very useful tool when preparing macro output avoiding the need of creating a token stream by inserting tokens one by one. + +> **Note**: As stated earlier, this crate makes use of `proc_macro2` and thus the `quote!` macro returns a `proc-macro2::TokenStream`. + +## [`syn`](https://docs.rs/syn/*/syn/) + +The [`syn`] crate is a parsing library for parsing a stream of Rust tokens into a syntax tree of Rust source code. +It is a very powerful library that makes parsing proc-macro input quite a bit easier, be it Rust syntax or custom syntax. +As the library can be a heavy compilation dependency, it makes heavy use of feature gates to allow users to cut it as small as required. + +So what does it offer? A bunch of things. + +First of all it has definitions and parsing for all standard Rust syntax nodes(when the `full` feature is enabled), as well as a [`DeriveInput`](https://docs.rs/syn/1/syn/struct.DeriveInput.html) type which encapsulates all the information a derive macro gets passed as an input stream as a structured input(requires the `derive` feature, enabled by default). These can be used right out of the box with the [`parse_macro_input!`](https://docs.rs/syn/1/syn/macro.parse_macro_input.html) macro(requires the `parsing` and `proc-macro` features, enabled by default) to parse token streams into these types. + +If rust syntax doesn't cut it, and instead one wishes to parse custom non-rust syntax the crate also offers a generic [parsing API](https://docs.rs/syn/1/syn/parse/index.html), mainly in the form of the [`Parse`](https://docs.rs/syn/1/syn/parse/trait.Parse.html) trait(requires the `parsing` feature, enabled by default). + +Aside from this the types exposed by the library keep location information and spans which allows procedural macros to emit detailed error messages pointing at the macro input at the points of interest. + +As this is again a library for procedural macros, it makes use of the `proc_macro2` token streams and spans and as such, conversions may be required. + +[`proc_macro`]: https://doc.rust-lang.org/proc_macro/ +[`proc-macro2`]: https://docs.rs/proc-macro2/*/proc_macro2/ +[`quote`]: https://docs.rs/quote/*/quote/ +[`syn`]: https://docs.rs/syn/*/syn/ From 983ee40b2802a6aa25cad47505697b2cbdad237c Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 23 Jul 2021 20:11:26 +0200 Subject: [PATCH 26/92] Restructure --- src/SUMMARY.md | 8 ++-- src/proc-macros.md | 62 +++++++++++++++++++-------- src/proc-macros/function-like.md | 22 ++++++++++ src/proc-macros/hygiene.md | 1 + src/proc-macros/third-party-crates.md | 2 +- 5 files changed, 72 insertions(+), 23 deletions(-) diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 19c5ca6..44b195d 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -32,10 +32,12 @@ - [Abacus Counting](./decl-macros/building-blocks/abacus-counting.md) - [Parsing Rust](./decl-macros/building-blocks/parsing.md) - [Procedural Macros](./proc-macros.md) + - [Function-like](./proc-macros/function-like.md) + - [Derive](./proc-macros/derive.md) + - [Attribute](./proc-macros/attr.md) - [Third-Party Crates](./proc-macros/third-party-crates.md) + - [Parsing with Syn]() - [Hygiene and Spans]() - - [custom!(…)](./proc-macros/function-like.md) - - [#\[Derive(CustomDerive, …)\]](./proc-macros/derive.md) - - [#\[CustomAttr(…)\]](./proc-macros/attr.md) + - [Techniques]() [Glossary](./glossary.md) diff --git a/src/proc-macros.md b/src/proc-macros.md index f41313b..de90b52 100644 --- a/src/proc-macros.md +++ b/src/proc-macros.md @@ -1,15 +1,9 @@ # Procedural Macros This chapter will introduce Rust's second syntax extension type, *procedural macros*. +It does so by first going over the common features and requirements shared by the 3 types of proc macros before introducing each type itself. Unlike a [declarative macro](./decl-macros.md), a procedural macro takes the form of a rust function taking in a token stream(or two) and outputting a token stream. -This makes writing procedural macros vastly different from declarative ones and comes with its own share of advantages and disadvantages. - -As noted in the [syntax extension chapter](./syntax-extensions/ast.md) it is possible to implement all three syntaxes of macro invocations with procedural macros, `#[attr]` attributes, `#[derive(…)]` derives and `foo!()` function-like. Each of these have some slight differences which will be explained in their separate chapters. - -All of these still run at the same stage in the compiler expansion-wise as declarative macros. -As mentioned earlier, a procedural macro is just a function that maps a token stream(a stream of token trees) to another token stream. -Thus it is a program the compiler compiles, then executes by feeding the input of the macro invocation, and depending on the type of macro invocation replaces the invocation with the output entirely or appends it. So how do we go about creating such a proc-macro? By creating a crate of course! A proc-macro is at its core just a function exported from a crate with the `proc-macro` [crate type](https://doc.rust-lang.org/reference/linkage.html), so when writing multiple proc macros you can have them all live in one crate. @@ -21,20 +15,50 @@ A proc-macro is at its core just a function exported from a crate with the `proc > ``` A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. -This crate also exposes the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html) mentioned earlier, which will be your macro's input and output type. -This type as it's documentation says is just a stream of token trees as we know them from earlier chapters but encoded as rust types. -Another type of interest is the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily used for error reporting and hygiene. Each token has an associated span that may be altered freely. +The two most important types exposed by the crate are the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html), which are the proc-macro variant of the already familiar token trees as well as the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily for error reporting and hygiene. See the [Hygiene and Spans]() chapter for more information. -With this knowledge we can take a look at the function signatures of procedural macros, which in case of a function-like macro or derive is `fn(TokenStream) -> TokenStream` and in case of an attribute is `fn(TokenStream, TokenStream) -> TokenStream`. -Note how the return type is a `TokenStream` itself for both and not a result or something else that gives the notion of fallible. -This does not mean that proc-macros cannot fail though, in fact they have two ways of reporting errors, the first one being to panic and the second to emit a [`compile_error!`](https://doc.rust-lang.org/std/macro.compile_error.html) invocation. -If a proc-macro panics the compiler will catch it and emit the payload as an error coming from the macro invocation. +As proc-macros therefore are functions living in a crate, they can be addressed as all the other items in a rust project. All thats required to add the crate to the dependency graph of a project and bring the desired item into scope. -> **Beware**: The compiler will happily hang on endless loops spun up inside proc-macros causing the compilation of crates using the proc-macro to hang as well. +> **Note**: Procedural macros invocations still run at the same stage in the compiler expansion-wise as declarative macros, just that they are standalone rust programs that the compiler compiles, runs, and finally either replaces or appends to. + + +## Types of procedural macros +With procedural macros, there are actually 3 different kinds that can be implemented with each having slightly different properties. +- *function-like* proc-macros which are used to implement `$name ! $arg` invocable macros +- *attribute* proc-macros which are used to implement `#[$arg]` and `#![$arg]` attributes +- *derive* proc-macros which are used to implement a derive, a "sub-macro" inside of a `#[derive(…)]` attribute -With all of this in mind we can now start implementing procedural macros, but as it turns out the `proc_macro` crate does not offer any parsing capabilities, just access to the token stream which means we are required to do the parsing of said tokens ourselves. -Parsing tokens to input by hand can be quite cumbersome depending on the macro that is being implemented and as such we will make use of the [`syn`](https://docs.rs/syn/*/syn/) crate to make parsing simpler. -We will quickly go over the aforementioned crate as well as some other helpful ones in the next chapter and then finally begin implementing some macros. +At their core, all 3 work almost the same with a few differences in their inputs and output reflected by their function definition. +As mentioned all a procedural macro really is, is a function that maps a token stream so let's take a quick look at each basic definition and their differences. -Other resources about procedural macros include the the reference [chapter](https://doc.rust-lang.org/reference/procedural-macros.html) which this chapter took heavy inspiration from. +### *function-like* +```rs +#[proc_macro] +pub fn my_proc_macro(input: TokenStream) -> TokenStream { + TokenStream::new() +} +``` + +### *attribute* +```rs +#[proc_macro_attribute] +pub fn my_attribute(input: TokenStream, annotated_item: TokenStream) -> TokenStream { + TokenStream::new() +} +``` + +### *derive* +```rs +#[proc_macro_derive(MyDerive)] +pub fn my_derive(annotated_item: TokenStream) -> TokenStream { + TokenStream::new() +} +``` + +As shown, the basic structure is the same for each, a public function marked with an attribute defining its procedural macro type returning a `TokenStream`. +Note how the return type is a `TokenStream` and not a result or something else that gives the notion of being fallible. +This does not mean that proc-macros cannot fail though, in fact they have two ways of reporting errors, the first one being to panic and the second to emit a [`compile_error!`](https://doc.rust-lang.org/std/macro.compile_error.html) invocation. +If a proc-macro panics the compiler will catch it and emit the payload as an error coming from the macro invocation. + +> **Beware**: The compiler will happily hang on endless loops spun up inside proc-macros causing the compilation of crates using the proc-macro to hang as well. diff --git a/src/proc-macros/function-like.md b/src/proc-macros/function-like.md index 47d93b9..4e4953e 100644 --- a/src/proc-macros/function-like.md +++ b/src/proc-macros/function-like.md @@ -1 +1,23 @@ # `custom!(…)` + +> **Note**: This chapter assumes basic knowledge of the `proc_macro2`, `quote` and `syn` crates introduced in the [`third-party-crates` chapter](./third-party-crates.md). + +Function-like procedural macros are invoked just like declarative macros, the path to to the macro, followed by a bang(!), followed by the input token tree, e.g `makro!(tokentree). +Just from looking at that invocation one cannot differentiate between the two. + +This chapter will introduce this type of procedural macro and implement a basic one to give an example of how to build such a macro. + +Here is a simple skeleton of such a macro: +```rs +use proc_macro::TokenStream; + +#[proc_macro] +pub fn identity(input: TokenStream) -> TokenStream { + input +} +``` + +Note the `pub` visibility and `#[proc_macro]` attribute which are both required, the former as the function has to be exported to be visible to other crates while the latter is required to designate this function as being a function-like procedural macro. +This proc-macro is not that interesting though as all it does is replace its invocation with the input. + +Let's build a small macro that parses some input, does some transformation on the parsed input and then emits the transformed tokens. diff --git a/src/proc-macros/hygiene.md b/src/proc-macros/hygiene.md index 0e34ab9..dd461af 100644 --- a/src/proc-macros/hygiene.md +++ b/src/proc-macros/hygiene.md @@ -1 +1,2 @@ # Hygiene +Each token has an associated span that may be altered freely. diff --git a/src/proc-macros/third-party-crates.md b/src/proc-macros/third-party-crates.md index 4b3aed7..f50fdbf 100644 --- a/src/proc-macros/third-party-crates.md +++ b/src/proc-macros/third-party-crates.md @@ -48,7 +48,7 @@ This a very useful tool when preparing macro output avoiding the need of creatin ## [`syn`](https://docs.rs/syn/*/syn/) The [`syn`] crate is a parsing library for parsing a stream of Rust tokens into a syntax tree of Rust source code. -It is a very powerful library that makes parsing proc-macro input quite a bit easier, be it Rust syntax or custom syntax. +It is a very powerful library that makes parsing proc-macro input quite a bit easier, as the [`proc_macro`] crate itself does not expose any kind of parsing capabilities, merely the tokens. As the library can be a heavy compilation dependency, it makes heavy use of feature gates to allow users to cut it as small as required. So what does it offer? A bunch of things. From cdb8ffe5e07617a2b1a35cd3f62e90a2f3c2a4b8 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 23 Jul 2021 23:56:18 +0200 Subject: [PATCH 27/92] Do a methodical <-> practical split --- src/SUMMARY.md | 8 ++- src/proc-macros.md | 61 +-------------------- src/proc-macros/attr.md | 1 - src/proc-macros/function-like.md | 23 -------- src/proc-macros/methodical.md | 61 +++++++++++++++++++++ src/proc-macros/methodical/attr.md | 1 + src/proc-macros/{ => methodical}/derive.md | 0 src/proc-macros/methodical/function-like.md | 25 +++++++++ 8 files changed, 93 insertions(+), 87 deletions(-) delete mode 100644 src/proc-macros/attr.md delete mode 100644 src/proc-macros/function-like.md create mode 100644 src/proc-macros/methodical.md create mode 100644 src/proc-macros/methodical/attr.md rename src/proc-macros/{ => methodical}/derive.md (100%) create mode 100644 src/proc-macros/methodical/function-like.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 44b195d..3e1c8dc 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -32,12 +32,14 @@ - [Abacus Counting](./decl-macros/building-blocks/abacus-counting.md) - [Parsing Rust](./decl-macros/building-blocks/parsing.md) - [Procedural Macros](./proc-macros.md) - - [Function-like](./proc-macros/function-like.md) - - [Derive](./proc-macros/derive.md) - - [Attribute](./proc-macros/attr.md) + - [A Methodical Introduction](./proc-macros/methodical.md) + - [Function-like](./proc-macros/methodical/function-like.md) + - [Attribute](./proc-macros/methodical/attr.md) + - [Derive](./proc-macros/methodical/derive.md) - [Third-Party Crates](./proc-macros/third-party-crates.md) - [Parsing with Syn]() - [Hygiene and Spans]() - [Techniques]() + - [A Practical Introduction]() [Glossary](./glossary.md) diff --git a/src/proc-macros.md b/src/proc-macros.md index de90b52..60fa00d 100644 --- a/src/proc-macros.md +++ b/src/proc-macros.md @@ -1,64 +1,5 @@ # Procedural Macros This chapter will introduce Rust's second syntax extension type, *procedural macros*. -It does so by first going over the common features and requirements shared by the 3 types of proc macros before introducing each type itself. -Unlike a [declarative macro](./decl-macros.md), a procedural macro takes the form of a rust function taking in a token stream(or two) and outputting a token stream. - -So how do we go about creating such a proc-macro? By creating a crate of course! -A proc-macro is at its core just a function exported from a crate with the `proc-macro` [crate type](https://doc.rust-lang.org/reference/linkage.html), so when writing multiple proc macros you can have them all live in one crate. - -> **Note**: When using Cargo, to define a `proc-macro` crate you define and set the `lib.proc-macro` key in the `Cargo.toml` to true. -> ```toml -> [lib] -> proc-macro = true -> ``` - -A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. -The two most important types exposed by the crate are the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html), which are the proc-macro variant of the already familiar token trees as well as the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily for error reporting and hygiene. See the [Hygiene and Spans]() chapter for more information. - -As proc-macros therefore are functions living in a crate, they can be addressed as all the other items in a rust project. All thats required to add the crate to the dependency graph of a project and bring the desired item into scope. - -> **Note**: Procedural macros invocations still run at the same stage in the compiler expansion-wise as declarative macros, just that they are standalone rust programs that the compiler compiles, runs, and finally either replaces or appends to. - - -## Types of procedural macros - -With procedural macros, there are actually 3 different kinds that can be implemented with each having slightly different properties. -- *function-like* proc-macros which are used to implement `$name ! $arg` invocable macros -- *attribute* proc-macros which are used to implement `#[$arg]` and `#![$arg]` attributes -- *derive* proc-macros which are used to implement a derive, a "sub-macro" inside of a `#[derive(…)]` attribute - -At their core, all 3 work almost the same with a few differences in their inputs and output reflected by their function definition. -As mentioned all a procedural macro really is, is a function that maps a token stream so let's take a quick look at each basic definition and their differences. - -### *function-like* -```rs -#[proc_macro] -pub fn my_proc_macro(input: TokenStream) -> TokenStream { - TokenStream::new() -} -``` - -### *attribute* -```rs -#[proc_macro_attribute] -pub fn my_attribute(input: TokenStream, annotated_item: TokenStream) -> TokenStream { - TokenStream::new() -} -``` - -### *derive* -```rs -#[proc_macro_derive(MyDerive)] -pub fn my_derive(annotated_item: TokenStream) -> TokenStream { - TokenStream::new() -} -``` - -As shown, the basic structure is the same for each, a public function marked with an attribute defining its procedural macro type returning a `TokenStream`. -Note how the return type is a `TokenStream` and not a result or something else that gives the notion of being fallible. -This does not mean that proc-macros cannot fail though, in fact they have two ways of reporting errors, the first one being to panic and the second to emit a [`compile_error!`](https://doc.rust-lang.org/std/macro.compile_error.html) invocation. -If a proc-macro panics the compiler will catch it and emit the payload as an error coming from the macro invocation. - -> **Beware**: The compiler will happily hang on endless loops spun up inside proc-macros causing the compilation of crates using the proc-macro to hang as well. +As with the [declarative macros](./decl-macros.md) chapter, this one is also split into a [methodical](./proc-macros/methodical.md) and a [practical]() subchapter with the former being a more formal introduction and the latter being a more practical oriented one. diff --git a/src/proc-macros/attr.md b/src/proc-macros/attr.md deleted file mode 100644 index ecf69c9..0000000 --- a/src/proc-macros/attr.md +++ /dev/null @@ -1 +0,0 @@ -# `#[CustomAttr(…)]` diff --git a/src/proc-macros/function-like.md b/src/proc-macros/function-like.md deleted file mode 100644 index 4e4953e..0000000 --- a/src/proc-macros/function-like.md +++ /dev/null @@ -1,23 +0,0 @@ -# `custom!(…)` - -> **Note**: This chapter assumes basic knowledge of the `proc_macro2`, `quote` and `syn` crates introduced in the [`third-party-crates` chapter](./third-party-crates.md). - -Function-like procedural macros are invoked just like declarative macros, the path to to the macro, followed by a bang(!), followed by the input token tree, e.g `makro!(tokentree). -Just from looking at that invocation one cannot differentiate between the two. - -This chapter will introduce this type of procedural macro and implement a basic one to give an example of how to build such a macro. - -Here is a simple skeleton of such a macro: -```rs -use proc_macro::TokenStream; - -#[proc_macro] -pub fn identity(input: TokenStream) -> TokenStream { - input -} -``` - -Note the `pub` visibility and `#[proc_macro]` attribute which are both required, the former as the function has to be exported to be visible to other crates while the latter is required to designate this function as being a function-like procedural macro. -This proc-macro is not that interesting though as all it does is replace its invocation with the input. - -Let's build a small macro that parses some input, does some transformation on the parsed input and then emits the transformed tokens. diff --git a/src/proc-macros/methodical.md b/src/proc-macros/methodical.md new file mode 100644 index 0000000..bc7bab6 --- /dev/null +++ b/src/proc-macros/methodical.md @@ -0,0 +1,61 @@ +# A Methodical Introduction + +Unlike a [declarative macro](../decl-macros.md), a procedural macro takes the form of a rust function taking in a token stream(or two) and outputting a token stream. + +A proc-macro is at its core just a function exported from a crate with the `proc-macro` [crate type](https://doc.rust-lang.org/reference/linkage.html), so when writing multiple proc macros you can have them all live in one crate. + +> **Note**: When using Cargo, to define a `proc-macro` crate you define and set the `lib.proc-macro` key in the `Cargo.toml` to true. +> ```toml +> [lib] +> proc-macro = true +> ``` + +A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. +The two most important types exposed by the crate are the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html), which are the proc-macro variant of the already familiar token trees as well as the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily for error reporting and hygiene. See the [Hygiene and Spans]() chapter for more information. + +As proc-macros therefore are functions living in a crate, they can be addressed as all the other items in a rust project. +All thats required to add the crate to the dependency graph of a project and bring the desired item into scope. + +> **Note**: Procedural macros invocations still run at the same stage in the compiler expansion-wise as declarative macros, just that they are standalone rust programs that the compiler compiles, runs, and finally either replaces or appends to. + + +## Types of procedural macros + +With procedural macros, there are actually exist 3 different kinds with each having slightly different properties. +- *function-like* proc-macros which are used to implement `$name ! $arg` invocable macros +- *attribute* proc-macros which are used to implement `#[$arg]` and `#![$arg]` attributes +- *derive* proc-macros which are used to implement a derive, a "sub-macro" inside of a `#[derive(…)]` attribute + +At their core, all 3 work almost the same with a few differences in their inputs and output reflected by their function definition. +As mentioned all a procedural macro really is, is a function that maps a token stream so let's take a quick look at each basic definition and their differences. + +### *function-like* +```rs +#[proc_macro] +pub fn my_proc_macro(input: TokenStream) -> TokenStream { + TokenStream::new() +} +``` + +### *attribute* +```rs +#[proc_macro_attribute] +pub fn my_attribute(input: TokenStream, annotated_item: TokenStream) -> TokenStream { + TokenStream::new() +} +``` + +### *derive* +```rs +#[proc_macro_derive(MyDerive)] +pub fn my_derive(annotated_item: TokenStream) -> TokenStream { + TokenStream::new() +} +``` + +As shown, the basic structure is the same for each, a public function marked with an attribute defining its procedural macro type returning a `TokenStream`. +Note how the return type is a `TokenStream` and not a result or something else that gives the notion of being fallible. +This does not mean that proc-macros cannot fail though, in fact they have two ways of reporting errors, the first one being to panic and the second to emit a [`compile_error!`](https://doc.rust-lang.org/std/macro.compile_error.html) invocation. +If a proc-macro panics the compiler will catch it and emit the payload as an error coming from the macro invocation. + +> **Beware**: The compiler will happily hang on endless loops spun up inside proc-macros causing the compilation of crates using the proc-macro to hang as well. diff --git a/src/proc-macros/methodical/attr.md b/src/proc-macros/methodical/attr.md new file mode 100644 index 0000000..02b61da --- /dev/null +++ b/src/proc-macros/methodical/attr.md @@ -0,0 +1 @@ +# Attribute diff --git a/src/proc-macros/derive.md b/src/proc-macros/methodical/derive.md similarity index 100% rename from src/proc-macros/derive.md rename to src/proc-macros/methodical/derive.md diff --git a/src/proc-macros/methodical/function-like.md b/src/proc-macros/methodical/function-like.md new file mode 100644 index 0000000..3d4a37f --- /dev/null +++ b/src/proc-macros/methodical/function-like.md @@ -0,0 +1,25 @@ +# Function-like + +Function-like procedural macros are invoked just like declarative macros, the path to to the macro, followed by a bang(!), followed by the input token tree, e.g `makro!(tokentree). +This type of macro is the simplest of the three though it is also the only one which you can't differentiate from declarative macros when solely looking at the invocation. + +As was shown in the chapter introduction, a simple skeleton of a function-like procedural macro looks like the following: +```rs +use proc_macro::TokenStream; + +#[proc_macro] +pub fn identity(input: TokenStream) -> TokenStream { + input +} +``` + +As one can see this is in fact just a mapping from one [`TokenStream`] to another where the `input` will be the tokens inside of the invocation delimiters, e.g. for an example invocation `foo!(bar)` the input token stream would consist of the `bar` token. +When invoked the invocation will simply be replaced with the output [`TokenStream`], or error should the expansion fail which may be due to the macro panicking or producing an invalid token stream for the invocation position. + +For this macro type the same placement and expansion rules apply as for declarative macros, that is the macro must output a correct token stream for the invocation location. +Unlike with declarative macros though, function-like procedural macros do not have certain restrictions imposed on their inputs though. +That is the restrictions for what may follow fragment specifiers listed in the [Metavariables and Expansion Redux](../../decl-macros/minutiae/metavar-and-expansion.md) chapter listed is not applicable here, as the procedural macros work on the tokens directly instead of matching them against fragment specifiers or similar. + +With that said it is apparent that the procedural counter part to these macros is more powerful as they can arbitrarily modify their input, and produce any output desired as long as its within the bounds of the language syntax. + +[`TokenStream`]:https://doc.rust-lang.org/proc_macro/struct.TokenStream.html From e19eb0c8d1e5c4addcfaf3b2c26d4319d66b3007 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 31 Jul 2021 22:18:55 +0200 Subject: [PATCH 28/92] Finish methodical introduction for the 3 proc-macro kinds --- src/proc-macros/methodical.md | 4 +- src/proc-macros/methodical/attr.md | 34 +++++++++++ src/proc-macros/methodical/derive.md | 66 ++++++++++++++++++++- src/proc-macros/methodical/function-like.md | 21 +++++-- 4 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src/proc-macros/methodical.md b/src/proc-macros/methodical.md index bc7bab6..82780be 100644 --- a/src/proc-macros/methodical.md +++ b/src/proc-macros/methodical.md @@ -23,8 +23,8 @@ All thats required to add the crate to the dependency graph of a project and bri With procedural macros, there are actually exist 3 different kinds with each having slightly different properties. - *function-like* proc-macros which are used to implement `$name ! $arg` invocable macros -- *attribute* proc-macros which are used to implement `#[$arg]` and `#![$arg]` attributes -- *derive* proc-macros which are used to implement a derive, a "sub-macro" inside of a `#[derive(…)]` attribute +- *attribute* proc-macros which are used to implement `#[$arg]` attributes +- *derive* proc-macros which are used to implement a derive, an *input* to a `#[derive(…)]` attribute At their core, all 3 work almost the same with a few differences in their inputs and output reflected by their function definition. As mentioned all a procedural macro really is, is a function that maps a token stream so let's take a quick look at each basic definition and their differences. diff --git a/src/proc-macros/methodical/attr.md b/src/proc-macros/methodical/attr.md index 02b61da..2c82c22 100644 --- a/src/proc-macros/methodical/attr.md +++ b/src/proc-macros/methodical/attr.md @@ -1 +1,35 @@ # Attribute + +Attribute procedural macros define new *outer* attributes which can be attached to items. +This type can be invoked with the `#[attr]` or `#[attr(…)]` syntax where `…` is an arbitrary token tree. + +A simple skeleton of an attribute procedural macro looks like the following: +```rs +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn tlborm_attribute(input: TokenStream, annotated_item: TokenStream) -> TokenStream { + annotated_item +} +``` + +Of note here is that unlike the other two procedural macro kinds, this one has two input parameters instead of one. +- The first parameter is the delimited token tree following the attribute's, name excluding the delimiters around it. +It is empty if the attribute is written bare, that is just a name without a `(TokenTree)` following it, e.g. `#[attr]`. +- The second token stream is the item the attribute is attached to *without* the attribute this proc macro defines. +As this is an [`active`](https://doc.rust-lang.org/reference/attributes.html#active-and-inert-attributes) attribute, the attribute will be stripped from the item before it is being passed to the proc macro. + +The returned token stream will **replace** the annotated item fully. +Note that the replacement does not have to be a single item, it can be 0 or more. + + +Usage example: +```rs +use tlborm_proc::tlborm_attribute; + +#[tlborm_attribute] +fn foo() {} + +#[tlborm_attribute(attributes are pretty handsome)] +fn bar() {} +``` diff --git a/src/proc-macros/methodical/derive.md b/src/proc-macros/methodical/derive.md index 6c24822..efe8fa0 100644 --- a/src/proc-macros/methodical/derive.md +++ b/src/proc-macros/methodical/derive.md @@ -1 +1,65 @@ -# `#[Derive(CustomDerive, …)]` +# Derive + +Derive procedural macros define new inputs for the `derive` attribute. +This type can be invoked by feeding it to a derive attribute's input, e.g. `#[derive(TlbormDerive)]`. + +A simple skeleton of a derive procedural macro looks like the following: +```rs +use proc_macro::TokenStream; + +#[proc_macro_derive(TlbormDerive)] +pub fn tlborm_derive(item: TokenStream) -> TokenStream { + TokenStream::neW() +} +``` + +The `proc_macro_derive` is a bit more special in that it requires an extra identifier, this identifier will become the actual name of the derive proc macro. +The input token stream is the item the derive attribute is attached to, that is, it will always be an `enum`, `struct` or `union` as these are the only items a derive attribute can annotated. +The returned token stream will be **appended** to the containing block or module of the annotated item with the requirement that the token stream consists of a set of valid items. + +Usage example: +```rs +use tlborm_proc::TlbormDerive; + +#[derive(TlbormDerive)] +struct Foo; +``` + +### Helper Attributes + +Derive proc macros are a bit more special in that they can add additional attributes visible only in the scope of the item definition. +These attributes are called *derive macro helper attributes* and are [inert](https://doc.rust-lang.org/reference/attributes.html#active-and-inert-attributes). +Their purpose is to give derive proc macros additional customizability on a per field or variant basis, that is these attributes can be used to annotate fields or enum variants while having no effect on their own. +As they are `inert` they will not be stripped and are visible to all macros. + +They can be defined by adding an `attributes(helper0, helper1, ..)` argument to the `proc_macro_derive` attribute containing a comma separated list of identifiers which are the names of the helper attributes. + +Thus a simple skeleton of a derive procedural macro with helper attributes looks like the following: +```rs +use proc_macro::TokenStream; + +#[proc_macro_derive(TlbormDerive, attributes(tlborm_helper))] +pub fn tlborm_derive(item: TokenStream) -> TokenStream { + TokenStream::neW() +} +``` + +That is all there is to helper attributes, to consume them in the proc macro the implementation will then have to check the attributes of fields and variants to see whether they are attributed with the corresponding helper. +It is an error to use a helper attribute if none of the used derive macros of the given item declare it as such, as the compiler will then instead try to resolve it as a normal attribute. + +Usage example: +```rs +use tlborm_proc::TlbormDerive; + +#[derive(TlbormDerive)] +struct Foo { + #[tlborm_helper] + field: u32 +} + +#[derive(TlbormDerive)] +enum Bar { + #[tlborm_helper] + Variant { #[tlborm_helper] field: u32 } +} +``` diff --git a/src/proc-macros/methodical/function-like.md b/src/proc-macros/methodical/function-like.md index 3d4a37f..fdf1461 100644 --- a/src/proc-macros/methodical/function-like.md +++ b/src/proc-macros/methodical/function-like.md @@ -1,20 +1,22 @@ # Function-like -Function-like procedural macros are invoked just like declarative macros, the path to to the macro, followed by a bang(!), followed by the input token tree, e.g `makro!(tokentree). -This type of macro is the simplest of the three though it is also the only one which you can't differentiate from declarative macros when solely looking at the invocation. +Function-like procedural macros are invoked like declarative macros that is `makro!(…)`. -As was shown in the chapter introduction, a simple skeleton of a function-like procedural macro looks like the following: +This type of macro is the simplest of the three though. +It is also the only one which you can't differentiate from declarative macros when solely looking at the invocation. + +A simple skeleton of a function-like procedural macro looks like the following: ```rs use proc_macro::TokenStream; #[proc_macro] -pub fn identity(input: TokenStream) -> TokenStream { +pub fn tlborm_fn_macro(input: TokenStream) -> TokenStream { input } ``` As one can see this is in fact just a mapping from one [`TokenStream`] to another where the `input` will be the tokens inside of the invocation delimiters, e.g. for an example invocation `foo!(bar)` the input token stream would consist of the `bar` token. -When invoked the invocation will simply be replaced with the output [`TokenStream`], or error should the expansion fail which may be due to the macro panicking or producing an invalid token stream for the invocation position. +The returned token stream will **replace** the macro invocation. For this macro type the same placement and expansion rules apply as for declarative macros, that is the macro must output a correct token stream for the invocation location. Unlike with declarative macros though, function-like procedural macros do not have certain restrictions imposed on their inputs though. @@ -23,3 +25,12 @@ That is the restrictions for what may follow fragment specifiers listed in the [ With that said it is apparent that the procedural counter part to these macros is more powerful as they can arbitrarily modify their input, and produce any output desired as long as its within the bounds of the language syntax. [`TokenStream`]:https://doc.rust-lang.org/proc_macro/struct.TokenStream.html + +Usage example: +```rs +use tlborm_proc::tlborm_attribute; + +fn foo() { + tlborm_attribute!(be quick; time is mana); +} +``` From 3d3acbd9ea8d50782afaffdcb2dd16c0445739a1 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 31 Jul 2021 23:19:44 +0200 Subject: [PATCH 29/92] proc-macro hygiene --- src/SUMMARY.md | 3 +-- src/decl-macros/minutiae/hygiene.md | 4 ++-- src/proc-macros/hygiene.md | 15 +++++++++++++-- src/proc-macros/methodical/derive.md | 2 +- src/syntax-extensions/hygiene.md | 22 +++++++++++++++------- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 3e1c8dc..2fe2be7 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -37,8 +37,7 @@ - [Attribute](./proc-macros/methodical/attr.md) - [Derive](./proc-macros/methodical/derive.md) - [Third-Party Crates](./proc-macros/third-party-crates.md) - - [Parsing with Syn]() - - [Hygiene and Spans]() + - [Hygiene and Spans](./proc-macros/hygiene.md) - [Techniques]() - [A Practical Introduction]() diff --git a/src/decl-macros/minutiae/hygiene.md b/src/decl-macros/minutiae/hygiene.md index 77133db..cb40491 100644 --- a/src/decl-macros/minutiae/hygiene.md +++ b/src/decl-macros/minutiae/hygiene.md @@ -1,7 +1,7 @@ # Hygiene -`macro_rules!` macros in Rust are *partially* hygienic. -Specifically, they are hygienic when it comes to most identifiers, but *not* when it comes to generic type parameters or lifetimes. +`macro_rules!` macros in Rust are *partially* hygienic, also called mixed hygiene. +Specifically, they are hygienic when it comes to *local variables*, *labels* and `$crate`, but nothing else. Hygiene works by attaching an invisible "syntax context" value to all identifiers. When two identifiers are compared, *both* the identifiers' textual names *and* syntax contexts must be identical for the two to be considered equal. diff --git a/src/proc-macros/hygiene.md b/src/proc-macros/hygiene.md index dd461af..27d2176 100644 --- a/src/proc-macros/hygiene.md +++ b/src/proc-macros/hygiene.md @@ -1,2 +1,13 @@ -# Hygiene -Each token has an associated span that may be altered freely. +# Hygiene and Spans + +This chapter talks about procedural macro [hygiene](../syntax-extensions/hygiene.md) and the type that encodes it, [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html). + +Every token in a [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html) has an associated `Span` holding some additional info. +A span, as its documentation states, is `A region of source code, along with macro expansion information`. +It points into a region of the original source code(important for displaying diagnostics at the correct places) as well as holding the kind of *hygiene* for this location. +The hygiene is relevant mainly for identifiers, as it allows or forbids the identifier from referencing things or being referenced by things defined outside of the invocation. + +There are 3 kinds of hygiene(which can be seen by the constructors of the `Span` type): +- [`definition site`](https://doc.rust-lang.org/proc_macro/struct.Span.html#method.def_site)(***unstable***): A span that resolves at the macro definition site. Identifiers with this span will not be able to reference things defined outside or be referenced by things outside of the invocation. This is what one would call "hygienic". +- [`mixed site`](https://doc.rust-lang.org/proc_macro/struct.Span.html#method.mixed_site): A span that has the same hygiene as `macro_rules` declarative macros, that is it may resolve to definition site or call site depending on the type of identifier. See [here](../decl-macros/minutiae/hygiene.md) for more information. +- [`call site`](https://doc.rust-lang.org/proc_macro/struct.Span.html#method.call_site): A span that resolves to the invocation site. Identifiers in this case will behave as if written directly at the call site, that is they freely resolve to things defined outside of the invocation and can be referenced from the outside as well. This is what one would call "unhygienic". diff --git a/src/proc-macros/methodical/derive.md b/src/proc-macros/methodical/derive.md index efe8fa0..72e5ffd 100644 --- a/src/proc-macros/methodical/derive.md +++ b/src/proc-macros/methodical/derive.md @@ -1,6 +1,6 @@ # Derive -Derive procedural macros define new inputs for the `derive` attribute. +Derive procedural macros define new inputs for the [`derive`](https://doc.rust-lang.org/reference/attributes/derive.html) attribute. This type can be invoked by feeding it to a derive attribute's input, e.g. `#[derive(TlbormDerive)]`. A simple skeleton of a derive procedural macro looks like the following: diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index 5b85de7..f515b86 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -9,27 +9,35 @@ We will go into general hygiene concepts here which will be touched upon in the   Hygiene mainly affects identifiers and paths emitted by syntax extensions. +In short, if an identifier created by a syntax extension cannot be accessed by the environment where the syntax extension has been invoked it is hygienic in regards to that identifier. +Likewise, if an identifier used in a syntax extension cannot reference something defined outside of a syntax extension it is considered hygienic. + +> **Note**: The terms `create` and `use` refer to the position the identifier is in. +> That is the `Foo` in `struct Foo {}` or the `foo` in `let foo = …;` are created in the sense that they introduce something new under the name, +> but the `Foo` in `fn foo(_: Foo) {}` or the `foo` in `foo + 3` are usages in the sense that they are referring to something existing. This is best shown by example: -Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, then given the following snippet: +Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, that is is *creates* the identifier `local`. +Then given the following snippet: ```rust,ignore make_local!(); assert_eq!(local, 0); ``` -For `make_local` to be considered fully hygienic this snippet should not compile. -On the other hand if this example was to compile, the macro couldn't be considered hygienic, as it impacted its surrounding context by introducing a new name into it. +If the `local` in `assert_eq!(local, 0);` resolves to the local defined by the syntax extension, the syntax extension is not hygienic(at least in regards to local names/bindings). -Now let's assume we have some syntax extension `use_local` that expands to `local = 42;`, then given the following snippet: +Now let's assume we have some syntax extension `use_local` that expands to `local = 42;`, that is it makes *use* of the identifier `local`. +Then given the following snippet: ```rust,ignore let mut local = 0; use_local!(); ``` -In this case for `use_local` to be considered fully hygienic, this snippet again should not compile as otherwise it would be affected by its surrounding context and also affect its surrounding context as well. +If the `local` inside of the syntax extension for the given invocation resolves to the local defined before its invocation, the syntax extension is not hygienic either. -This is a rather short introduction to hygiene which will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and proc-macro `hygiene` chapters, mainly explaining how hygienic these syntax extensions can be, be it fully or only partially. -There also exists this [github gist](https://gist.github.com/Kestrer/8c05ebd4e0e9347eb05f265dfb7252e1) that explains how to write hygienic syntax extensions while going into a hygiene a bit overall. +This is a rather short introduction to the general concept of hygiene. +It will be explained in more depth in the corresponding [`macro_rules!` `hygiene`] and [proc-macro `hygiene`] chapters, with their specific peculiarities. [`macro_rules!` `hygiene`]: ../decl-macros/minutiae/hygiene.md +[proc-macro `hygiene`]: ../proc-macros/hygiene.md From f0c10d6535d7979431eca02bbddd0d4f2ee4d65b Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 31 Jul 2021 23:31:17 +0200 Subject: [PATCH 30/92] Split off parts of declarative macro debugging chapter into syntax extensions --- src/SUMMARY.md | 1 + src/decl-macros/minutiae/debugging.md | 60 +-------------------------- src/syntax-extensions/debugging.md | 59 ++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 58 deletions(-) create mode 100644 src/syntax-extensions/debugging.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 2fe2be7..28ff9ee 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -7,6 +7,7 @@ - [Macros in the Ast](./syntax-extensions/ast.md) - [Expansion](./syntax-extensions/expansion.md) - [Hygiene](./syntax-extensions/hygiene.md) + - [Debugging](./syntax-extensions/debugging.md) - [Declarative Macros](./decl-macros.md) - [A Methodical Introduction](./decl-macros/macros-methodical.md) - [macro_rules!](./decl-macros/macro_rules.md) diff --git a/src/decl-macros/minutiae/debugging.md b/src/decl-macros/minutiae/debugging.md index 5f201ea..ab1aa91 100644 --- a/src/decl-macros/minutiae/debugging.md +++ b/src/decl-macros/minutiae/debugging.md @@ -1,6 +1,7 @@ # Debugging -`rustc` provides a number of tools to debug `macro_rules!` macros. +> **Note**: This is a list of debugging tools specifically tailored towards declarative macros, additional means of debugging these can be found in the [debugging chapter](../../syntax-extensions/debugging.md) of syntax extensions. + One of the most useful is [`trace_macros!`], which is a directive to the compiler instructing it to dump every `macro_rules!` macro invocation prior to expansion. For example, given the following: @@ -72,63 +73,6 @@ sing! { This can be used to do slightly more targeted debugging than [`trace_macros!`]. -Sometimes, it is what the macro *expands to* that proves problematic. -For this, the `--pretty` argument to the compiler can be used. -Given the following code: - -```rust,ignore -// Shorthand for initializing a `String`. -macro_rules! S { - ($e:expr) => {String::from($e)}; -} - -fn main() { - let world = S!("World"); - println!("Hello, {}!", world); -} -``` - -compiled with the following command: - -```shell -rustc -Z unstable-options --pretty expanded hello.rs -``` - -produces the following output (modified for formatting): - -```rust,ignore -#![feature(no_std, prelude_import)] -#![no_std] -#[prelude_import] -use std::prelude::v1::*; -#[macro_use] -extern crate std as std; -// Shorthand for initializing a `String`. -fn main() { - let world = String::from("World"); - ::std::io::_print(::std::fmt::Arguments::new_v1( - { - static __STATIC_FMTSTR: &'static [&'static str] - = &["Hello, ", "!\n"]; - __STATIC_FMTSTR - }, - &match (&world,) { - (__arg0,) => [ - ::std::fmt::ArgumentV1::new(__arg0, ::std::fmt::Display::fmt) - ], - } - )); -} -``` - -Other options to `--pretty` can be listed using `rustc -Z unstable-options --help -v`; -a full list is not provided since, as implied by the name, any such list would be subject to change at any time. - -But not just `rustc` exposes means to aid in debugging macros. -For the aforementioned `--pretty=expanded` option, there exists a nice `cargo` addon called [`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) which is basically just a wrapper around it. - -You can also use the [playground](https://play.rust-lang.org/), clicking on its `TOOLS` button in the top right gives you the option to expand macros right there! - Another amazing tool is [`lukaslueg`'s](https://github.com/lukaslueg) [`macro_railroad`](https://github.com/lukaslueg/macro_railroad), a tool that allows you visualize and generate syntax diagrams for Rust's `macro_rules!` macros. It visualizes the accepted macro's grammar as an automata. diff --git a/src/syntax-extensions/debugging.md b/src/syntax-extensions/debugging.md new file mode 100644 index 0000000..13bfdee --- /dev/null +++ b/src/syntax-extensions/debugging.md @@ -0,0 +1,59 @@ +# Debugging + +`rustc` provides a number of tools to debug general syntax extensions, as well as some more specific ones tailored towards declarative and procedural macros respectively. + + +Sometimes, it is what the extension *expands to* that proves problematic as you do not usually see the expanded code. +Fortunately `rustc` offers the ability to look at the expanded code, for this, the `--pretty` argument can be used. +Given the following code: + +```rust,ignore +// Shorthand for initializing a `String`. +macro_rules! S { + ($e:expr) => {String::from($e)}; +} + +fn main() { + let world = S!("World"); + println!("Hello, {}!", world); +} +``` + +compiled with the following command: + +```shell +rustc -Z unstable-options -Zunpretty=expanded hello.rs +``` + +produces the following output (modified for formatting): + +```rust,ignore +#![feature(prelude_import)] +#[prelude_import] +use std::prelude::rust_2018::*; +#[macro_use] +extern crate std; +// Shorthand for initializing a `String`. +macro_rules! S { ($e : expr) => { String :: from($e) } ; } + +fn main() { + let world = String::from("World"); + { + ::std::io::_print( + ::core::fmt::Arguments::new_v1( + &["Hello, ", "!\n"], + &match (&world,) { + (arg0,) => [ + ::core::fmt::ArgumentV1::new(arg0, ::core::fmt::Display::fmt) + ], + } + ) + ); + }; +} +``` + +But not just `rustc` exposes means to aid in debugging syntax extensions. +For the aforementioned `--pretty=expanded` option, there exists a nice `cargo` plugin called [`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) which is basically just a wrapper around it. + +You can also use the [playground](https://play.rust-lang.org/), clicking on its `TOOLS` button in the top right gives you the option to expand syntax extensions as well! From 318898e076c83789ab2e4622fce781458503aa2d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 15 Sep 2021 15:54:57 +0200 Subject: [PATCH 31/92] Minor fixes, basic practical macros structure --- src/SUMMARY.md | 6 +++++- src/decl-macros/macros-methodical.md | 2 +- src/decl-macros/macros-practical.md | 2 +- src/proc-macros.md | 4 +++- src/proc-macros/methodical.md | 2 ++ src/proc-macros/practical.md | 5 +++++ src/proc-macros/practical/attr.md | 1 + src/proc-macros/practical/derive.md | 1 + src/proc-macros/practical/function-like.md | 1 + src/proc-macros/third-party-crates.md | 2 +- 10 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 src/proc-macros/practical.md create mode 100644 src/proc-macros/practical/attr.md create mode 100644 src/proc-macros/practical/derive.md create mode 100644 src/proc-macros/practical/function-like.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 28ff9ee..1d40f2f 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -37,9 +37,13 @@ - [Function-like](./proc-macros/methodical/function-like.md) - [Attribute](./proc-macros/methodical/attr.md) - [Derive](./proc-macros/methodical/derive.md) + - [A Practical Introduction]() + - [Function-like]() + - [Attribute]() + - [Derive]() - [Third-Party Crates](./proc-macros/third-party-crates.md) - [Hygiene and Spans](./proc-macros/hygiene.md) - [Techniques]() - - [A Practical Introduction]() + - [Testing]() [Glossary](./glossary.md) diff --git a/src/decl-macros/macros-methodical.md b/src/decl-macros/macros-methodical.md index 3db1f6d..2801366 100644 --- a/src/decl-macros/macros-methodical.md +++ b/src/decl-macros/macros-methodical.md @@ -1,6 +1,6 @@ # Macros, A Methodical Introduction -This chapter will introduce Rust's [Macro-By-Example][mbe] system by explaining the system as a whole. +This chapter will introduce Rust's declarative [Macro-By-Example][mbe] system by explaining the system as a whole. It will do so by first going into the construct's syntax and its key parts and then following it up with more general information that one should at least be aware of. [mbe]: https://doc.rust-lang.org/reference/macros-by-example.html diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index 89f63d4..eae6cb6 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -1,6 +1,6 @@ # Macros, A Practical Introduction -This chapter will introduce the Rust macro-by-example system using a relatively simple, practical example. +This chapter will introduce Rust's declarative [Macro-By-Example](https://doc.rust-lang.org/reference/macros-by-example.html) system using a relatively simple, practical example. It does *not* attempt to explain all of the intricacies of the system; its goal is to get you comfortable with how and why macros are written. There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-06-macros.html) which is another high-level explanation, and the [methodical introduction](../decl-macros.md) chapter of this book, which explains the macro system in detail. diff --git a/src/proc-macros.md b/src/proc-macros.md index 60fa00d..bd5e727 100644 --- a/src/proc-macros.md +++ b/src/proc-macros.md @@ -2,4 +2,6 @@ This chapter will introduce Rust's second syntax extension type, *procedural macros*. -As with the [declarative macros](./decl-macros.md) chapter, this one is also split into a [methodical](./proc-macros/methodical.md) and a [practical]() subchapter with the former being a more formal introduction and the latter being a more practical oriented one. +As with the [declarative macros](./decl-macros.md) chapter, this one is also split into a [methodical](./proc-macros/methodical.md) and a (WIP) practical subchapter with the former being a more formal introduction and the latter being a more practical oriented one. + +A lot of the basic information covered has been sourced from the [rust reference](https://doc.rust-lang.org/nightly/reference/introduction.html), as most knowledge about procedural macros is currently located there. diff --git a/src/proc-macros/methodical.md b/src/proc-macros/methodical.md index 82780be..f98b763 100644 --- a/src/proc-macros/methodical.md +++ b/src/proc-macros/methodical.md @@ -1,5 +1,7 @@ # A Methodical Introduction +This chapter will introduce Rust's procedural macro system by explaining the system as a whole. + Unlike a [declarative macro](../decl-macros.md), a procedural macro takes the form of a rust function taking in a token stream(or two) and outputting a token stream. A proc-macro is at its core just a function exported from a crate with the `proc-macro` [crate type](https://doc.rust-lang.org/reference/linkage.html), so when writing multiple proc macros you can have them all live in one crate. diff --git a/src/proc-macros/practical.md b/src/proc-macros/practical.md new file mode 100644 index 0000000..7d740b4 --- /dev/null +++ b/src/proc-macros/practical.md @@ -0,0 +1,5 @@ +# A Practical Introduction + +This chapter will introduce Rust's procedural macro system using 3 relatively simple, practical examples, one for each procedural macro type. + +There is also the [proc-macro workshop](https://github.com/dtolnay/proc-macro-workshop) another high-level introduction, and the [methodical introduction](./methodical.md) chapter of this book, which explains the system without practical examples. diff --git a/src/proc-macros/practical/attr.md b/src/proc-macros/practical/attr.md new file mode 100644 index 0000000..02b61da --- /dev/null +++ b/src/proc-macros/practical/attr.md @@ -0,0 +1 @@ +# Attribute diff --git a/src/proc-macros/practical/derive.md b/src/proc-macros/practical/derive.md new file mode 100644 index 0000000..f12a065 --- /dev/null +++ b/src/proc-macros/practical/derive.md @@ -0,0 +1 @@ +# Derive diff --git a/src/proc-macros/practical/function-like.md b/src/proc-macros/practical/function-like.md new file mode 100644 index 0000000..687f0dc --- /dev/null +++ b/src/proc-macros/practical/function-like.md @@ -0,0 +1 @@ +# Function-like diff --git a/src/proc-macros/third-party-crates.md b/src/proc-macros/third-party-crates.md index f50fdbf..8fd2ecc 100644 --- a/src/proc-macros/third-party-crates.md +++ b/src/proc-macros/third-party-crates.md @@ -1,7 +1,7 @@ # Third-Party Crates > **Note**: Crates beyond the automatically linked [`proc_macro`] crate are not required to write procedural macros. -> The crates listed here merely make writing them simpler and possibly more readable, while potentially adding to the compilation time of the procedural macro due to added the dependencies. +> The crates listed here merely make writing them simpler and more concise, while potentially adding to the compilation time of the procedural macro due to added dependencies. As procedural macros live in a crate they can naturally depend on ([crates.io](https://crates.io/)) crates. turns out the crate ecosystem has some really helpful crates tailored towards procedural macros that this chapter will quickly go over, most of which will be used in the following chapters to implement the example macros. From 82338bac58aec0981224d4d42343221ec49f9668 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 15 Sep 2021 16:26:32 +0200 Subject: [PATCH 32/92] Fix hyigene link --- src/proc-macros/methodical.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proc-macros/methodical.md b/src/proc-macros/methodical.md index f98b763..56afcbb 100644 --- a/src/proc-macros/methodical.md +++ b/src/proc-macros/methodical.md @@ -13,7 +13,7 @@ A proc-macro is at its core just a function exported from a crate with the `proc > ``` A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. -The two most important types exposed by the crate are the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html), which are the proc-macro variant of the already familiar token trees as well as the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily for error reporting and hygiene. See the [Hygiene and Spans]() chapter for more information. +The two most important types exposed by the crate are the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html), which are the proc-macro variant of the already familiar token trees as well as the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily for error reporting and hygiene. See the [Hygiene and Spans](./hygiene.md) chapter for more information. As proc-macros therefore are functions living in a crate, they can be addressed as all the other items in a rust project. All thats required to add the crate to the dependency graph of a project and bring the desired item into scope. From e1baa14747c26fc251dc52864dd4211ca79b9c32 Mon Sep 17 00:00:00 2001 From: Goldstein Date: Sun, 26 Sep 2021 17:24:25 +0300 Subject: [PATCH 33/92] Add const generics based counting method It's slower than bit twiddling, but roughly just as fast as slice length method, usable in const contexts, non-recursive and quite simple to read. --- src/decl-macros/building-blocks/counting.md | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/decl-macros/building-blocks/counting.md b/src/decl-macros/building-blocks/counting.md index d1f646e..0ef3a47 100644 --- a/src/decl-macros/building-blocks/counting.md +++ b/src/decl-macros/building-blocks/counting.md @@ -126,6 +126,29 @@ macro_rules! count_tts { This has been tested to work up to 10,000 tokens, and can probably go much higher. +## Array length + +Another modification of the previous approach is to use const generics stabilized in Rust 1.51. +It's only slightly slower than slice length method on 20,000 tokens and works in const contexts. + +```rust +const fn count_helper(_: [(); N]) -> usize { N } + +macro_rules! replace_expr { + ($_t:tt $sub:expr) => { $sub } +} + +macro_rules! count_tts { + ($($smth:tt)*) => { + count_helper([$(replace_expr!($smth ())),*]) + } +} +# +# fn main() { +# assert_eq!(count_tts!(0 1 2), 3); +# } +``` + ## Enum counting This approach can be used where you need to count a set of mutually distinct identifiers. From b1c37fc9c9aaca8c0e93f560983875d96ec83877 Mon Sep 17 00:00:00 2001 From: Onigbinde Oluwamuyiwa Elijah Date: Sat, 18 Sep 2021 00:04:18 -0400 Subject: [PATCH 34/92] Removed unnecessary word. --- src/syntax-extensions/ast.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/ast.md b/src/syntax-extensions/ast.md index 1f8ef95..5e597dd 100644 --- a/src/syntax-extensions/ast.md +++ b/src/syntax-extensions/ast.md @@ -22,7 +22,7 @@ For example, whilst [`format!`] is a `macro_rules!` macro, [`format_args!`] (whi The fourth form is essentially a variation which is *not* available to macros. -In fact, the only case where this form is used *at all* is with the `macro_rules!` construct itself which. +In fact, the only case where this form is used *at all* is with the `macro_rules!` construct itself. So, starting with the third form, how does the Rust parser know what the `$arg` in (`$name ! $arg`) looks like for every possible syntax extension? The answer is that it doesn't *have to*. From 9658cea5ba5c69b1e3372ee556eae740497ae69d Mon Sep 17 00:00:00 2001 From: "Jackson G. Kaindume" Date: Fri, 8 Oct 2021 01:36:43 +0200 Subject: [PATCH 35/92] fix typo --- src/syntax-extensions/hygiene.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index f515b86..3f24340 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -18,7 +18,7 @@ Likewise, if an identifier used in a syntax extension cannot reference something This is best shown by example: -Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, that is is *creates* the identifier `local`. +Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, that is it *creates* the identifier `local`. Then given the following snippet: ```rust,ignore make_local!(); From ebf299ef195fa90969b406d1f500ff8acbc1ed44 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 16 Oct 2021 16:24:53 +0200 Subject: [PATCH 36/92] Fold level 1 and higher in the sidebar --- book.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/book.toml b/book.toml index 93bdbfa..4692c5f 100644 --- a/book.toml +++ b/book.toml @@ -21,6 +21,10 @@ mathjax-support = true git-repository-url = "https://github.com/veykril/tlborm/" additional-css = ["res/rust-syntax-bg-highlight.css"] +[output.html.fold] +enable = true +level = 1 + [output.html.redirect] "/building-blocks/abacus-counting.html" = "../decl-macros/building-blocks/abacus-counting.html" "/building-blocks/ast-coercion.html" = "../decl-macros/building-blocks/ast-coercion.html" From 2d1801fed1c0a6062214a11c159f53be0859ba7d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 27 Oct 2021 14:06:06 +0200 Subject: [PATCH 37/92] Change `paren` to `parenthesis` for clarity --- src/syntax-extensions/source-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/source-analysis.md b/src/syntax-extensions/source-analysis.md index 4454d45..fe2aeb7 100644 --- a/src/syntax-extensions/source-analysis.md +++ b/src/syntax-extensions/source-analysis.md @@ -130,7 +130,7 @@ root node, there are *seven* token trees at the root level. For reference, the A It is important to understand the distinction between the AST and token trees. When writing macros, you have to deal with *both* as distinct things. -One other aspect of this to note: it is *impossible* to have an unpaired paren, bracket or brace; +One other aspect of this to note: it is *impossible* to have an unpaired parenthesis, bracket or brace; nor is it possible to have incorrectly nested groups in a token tree. [tokenization]: https://en.wikipedia.org/wiki/Lexical_analysis#Tokenization From 6d0f8ec064697a4cd5d6445a627abe083fe8bdef Mon Sep 17 00:00:00 2001 From: "D.Schulte" Date: Sat, 13 Nov 2021 02:07:03 +0100 Subject: [PATCH 38/92] Fixed typo --- src/syntax-extensions/ast.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/ast.md b/src/syntax-extensions/ast.md index 5e597dd..44052ae 100644 --- a/src/syntax-extensions/ast.md +++ b/src/syntax-extensions/ast.md @@ -81,7 +81,7 @@ it remembers the tokens it contains, but doesn't try to *understand* them. This means ⬚ can be anything, even invalid Rust! As to why this is a good thing, we will come back to that at a later point. -So, does this also apply to `$arg` in form 1 and 2, and to ther two args in form 4? Kind of. +So, does this also apply to `$arg` in form 1 and 2, and to the two args in form 4? Kind of. The `$arg$` for form 1 and 2 is a bit different in that it is not directly a token tree, but a *simple path* that is either followed by an `=` token and a literal expression, or a token tree. We will explore this more in-depth in the appropriate proc-macro chapter. The important part here is that this form as well, makes use of token trees to describe the input. From 2b3a1285e083e7fdadb65a259ce6090caa5981c6 Mon Sep 17 00:00:00 2001 From: bestgopher <84328409@qq.com> Date: Thu, 25 Nov 2021 19:27:58 +0800 Subject: [PATCH 39/92] Update derive.md --- src/proc-macros/methodical/derive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proc-macros/methodical/derive.md b/src/proc-macros/methodical/derive.md index 72e5ffd..bb2ee9b 100644 --- a/src/proc-macros/methodical/derive.md +++ b/src/proc-macros/methodical/derive.md @@ -40,7 +40,7 @@ use proc_macro::TokenStream; #[proc_macro_derive(TlbormDerive, attributes(tlborm_helper))] pub fn tlborm_derive(item: TokenStream) -> TokenStream { - TokenStream::neW() + TokenStream::new() } ``` From ff4188b438879075311ab5963b72693196fd0a0f Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 2 Jan 2022 17:48:17 +0100 Subject: [PATCH 40/92] Add `pat_param` fragment specifier information --- .../minutiae/fragment-specifiers.md | 25 ++++++++++++++++++- .../minutiae/metavar-and-expansion.md | 3 ++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index 49d657c..ace8c74 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -13,6 +13,7 @@ This section will go a bit more into detail for some of them and shows a few exa * [`literal`](#literal) * [`meta`](#meta) * [`pat`](#pat) +* [`pat_param`](#pat_param) * [`path`](#path) * [`stmt`](#stmt) * [`tt`](#tt) @@ -159,7 +160,7 @@ metas! { ## `pat` -The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html), except or-patterns. +The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html), including or-patterns starting with the 2021 edition. ```rust macro_rules! patterns { @@ -171,6 +172,28 @@ patterns! { _ 0..5 ref mut PatternsAreNice + 0 | 1 | 2 | 3 +} +# fn main() {} +``` + +## `pat_param` + +In the 2021 edition, the behavior for the `pat` fragment type has been changed to allow or-patterns to be parsed. +This changes the follow list of the fragment, preventing such fragment from being followed by a `|` token. +To avoid this problem or to get the old fragment behavior back one can use the `pat_param` fragment which allows `|` to follow it, as it disallows top level or-patterns. + +```rust +macro_rules! patterns { + ($( $( $pat:pat_param )|+ )*) => (); +} + +patterns! { + "literal" + _ + 0..5 + ref mut PatternsAreNice + 0 | 1 | 2 | 3 } # fn main() {} ``` diff --git a/src/decl-macros/minutiae/metavar-and-expansion.md b/src/decl-macros/minutiae/metavar-and-expansion.md index 568e582..0178812 100644 --- a/src/decl-macros/minutiae/metavar-and-expansion.md +++ b/src/decl-macros/minutiae/metavar-and-expansion.md @@ -25,7 +25,8 @@ restricts what can follow various metavariables. The complete list, showing what may follow what fragment specifier, as of Rust 1.46 is as follows: * [`stmt`] and [`expr`]: `=>`, `,`, or `;` -* [`pat`]: `=>`, `,`, `=`, `|`, `if`, `in` +* [`pat`]: `=>`, `,`, `=`, `if`, `in` +* [`pat_param`]: `=>`, `,`, `=`, `|`, `if`, `in` * [`path`] and [`ty`]:`=>`, `,`, `=`, `|`, `;`, `:`, `>`, `>>`, `[`, `{`, `as`, `where`, or a macro variable of the [`block`] fragment specifier. * [`vis`]: `,`, an identifier other than a non-raw `priv`, any token that can begin a type or a From 45ca843ec505f13a3efec37a1d4d7ab08dd018fd Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 19 Jan 2022 19:25:55 +0100 Subject: [PATCH 41/92] Add macros 2.0 chapter with syntax exploration --- src/SUMMARY.md | 1 + src/decl-macros/macros2.md | 45 +++++++++++++++++++ .../minutiae/metavar-and-expansion.md | 1 + 3 files changed, 47 insertions(+) create mode 100644 src/decl-macros/macros2.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 1d40f2f..58ff673 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -32,6 +32,7 @@ - [Counting](./decl-macros/building-blocks/counting.md) - [Abacus Counting](./decl-macros/building-blocks/abacus-counting.md) - [Parsing Rust](./decl-macros/building-blocks/parsing.md) + - [Macros 2.0](./decl-macros/macros2.md) - [Procedural Macros](./proc-macros.md) - [A Methodical Introduction](./proc-macros/methodical.md) - [Function-like](./proc-macros/methodical/function-like.md) diff --git a/src/decl-macros/macros2.md b/src/decl-macros/macros2.md new file mode 100644 index 0000000..070a72e --- /dev/null +++ b/src/decl-macros/macros2.md @@ -0,0 +1,45 @@ +# Macros 2.0 + +> *RFC*: [rfcs#1584](https://github.com/rust-lang/rfcs/blob/master/text/1584-macros.md)\ +> *Tracking Issue*: [rust#39412](https://github.com/rust-lang/rust/issues/39412)\ +> *Feature*: `#![feature(decl_macro)]` + +While not yet stable(or rather far from being finished), there is proposal for a new declarative macro system that is supposed to replace `macro_rules!` dubbed declarative macros 2.0, `macro`, `decl_macro` or confusingly also `macros-by-example. + +This chapter is only meant to quickly glance over the current state, showing how to use this macro system and where it differs. +Nothing described here is final and may be subject to change at any time. + +## Syntax + +We'll do a comparison between the `macro` and `macro_rules` syntax for two macros we have implemented in previous chapters: + +```rust +#![feature(decl_macro)] + +macro_rules! replace_expr_ { + ($_t:tt $sub:expr) => { $sub } +} +macro replace_expr($_t:tt $sub:expr) { + $sub +} + +macro_rules! count_tts_ { + () => { 0 }; + ($odd:tt $($a:tt $b:tt)*) => { (count_tts!($($a)*) << 1) | 1 }; + ($($a:tt $even:tt)*) => { count_tts!($($a)*) << 1 }; +} +macro count_tts { + () => { 0 }, + ($odd:tt $($a:tt $b:tt)*) => { (count_tts!($($a)*) << 1) | 1 }, + ($($a:tt $even:tt)*) => { count_tts!($($a)*) << 1 }, +} +``` + +As can be seen, they look very similar, with just a few differences as well as that `macro`s have two different forms. + +Let's inspect the `count_tts` macro first, as that one looks more like what we are used to. +As can be seen, it practically looks identical to the `macro_rules` version with two exceptions, it uses the `macro` keyword and the rule separator is a `,` instead of a `;`. + +There is a second form to this though, which is a shorthand for macros that only have one rule. +Taking a look at `replace_expr` we can see that in this case we can write the definition in a way that more resembles an ordinary function. +We can write the matcher directly after the name followed by the transcriber, dropping a pair of braces and the `=>` token. diff --git a/src/decl-macros/minutiae/metavar-and-expansion.md b/src/decl-macros/minutiae/metavar-and-expansion.md index 0178812..43b11ed 100644 --- a/src/decl-macros/minutiae/metavar-and-expansion.md +++ b/src/decl-macros/minutiae/metavar-and-expansion.md @@ -135,6 +135,7 @@ Once you capture with anything else, the only thing you can do with the result f [`block`]:./fragment-specifiers.md#block [`stmt`]:./fragment-specifiers.md#stmt [`pat`]:./fragment-specifiers.md#pat +[`pat_param`]:./fragment-specifiers.md#pat_param [`expr`]:./fragment-specifiers.md#expr [`ty`]:./fragment-specifiers.md#ty [`ident`]:./fragment-specifiers.md#ident From 4eaec0a9f762240bb3c71391469ae90eade47da4 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 19 Jan 2022 19:47:49 +0100 Subject: [PATCH 42/92] Add small hygiene section for macros 2.0 --- src/decl-macros/macros2.md | 48 +++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/decl-macros/macros2.md b/src/decl-macros/macros2.md index 070a72e..364d5af 100644 --- a/src/decl-macros/macros2.md +++ b/src/decl-macros/macros2.md @@ -7,7 +7,7 @@ While not yet stable(or rather far from being finished), there is proposal for a new declarative macro system that is supposed to replace `macro_rules!` dubbed declarative macros 2.0, `macro`, `decl_macro` or confusingly also `macros-by-example. This chapter is only meant to quickly glance over the current state, showing how to use this macro system and where it differs. -Nothing described here is final and may be subject to change at any time. +Nothing described here is final or complete, and may be subject to change. ## Syntax @@ -43,3 +43,49 @@ As can be seen, it practically looks identical to the `macro_rules` version with There is a second form to this though, which is a shorthand for macros that only have one rule. Taking a look at `replace_expr` we can see that in this case we can write the definition in a way that more resembles an ordinary function. We can write the matcher directly after the name followed by the transcriber, dropping a pair of braces and the `=>` token. + +Syntax for invoking `macro`s is the same as for `macro_rules` and function-like procedural macros, the name followed by a `!` followed by the macro input token tree. + +## `macro` are proper items + +Unlike with `macro_rules` macros, which are textually scoped and require `#[macro_export]`(and potentially a re-export) to be treated as an item, `macro` macros behave like proper rust items by default. + +As such, you can properly qualify them with visibility specifiers like `pub`, `pub(crate)`, `pub(in path)` and the like. + + +## Hygiene + +Hygiene is by far the biggest difference between the two declarative macro systems. +Unlike `macro_rules` which have [mixed site hygiene], `macro` have definition site hygiene, meaning they do not leak identifiers outside of their invocation. + +As such the following compiles with a `macro_rules` macro, but fails with a `macro` definition: + +```rust +#![feature(decl_macro)] +// try uncommenting the following line, and commenting out the line right after + +macro_rules! foo { +// macro foo { + ($name: ident) => { + pub struct $name; + + impl $name { + pub fn new() -> $name { + $name + } + } + } +} + +foo!(Foo); + +fn main() { + // this fails with a `macro`, but succeeds with a `macro_rules` + let foo = Foo::new(); +} +``` + +There may be plans to allow escaping hygiene for identifiers(hygiene bending) in the future. + + +[mixed site hygiene]: ./minutiae/hygiene.md From de5ca7aa311c0b51e2495f454baac1608e609683 Mon Sep 17 00:00:00 2001 From: Tilman Date: Fri, 18 Feb 2022 10:02:33 +0100 Subject: [PATCH 43/92] Fix typo --- src/decl-macros/macros-practical-table.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/macros-practical-table.html b/src/decl-macros/macros-practical-table.html index e25fd06..1702c74 100644 --- a/src/decl-macros/macros-practical-table.html +++ b/src/decl-macros/macros-practical-table.html @@ -101,7 +101,7 @@ Note: there are two ⌂ here, because the next input token might match either the comma - separator betweenem> elements in the repetition, or the comma after the + separator between elements in the repetition, or the comma after the repetition. The macro system will keep track of both possibilities, until it is able to decide which one to follow. @@ -162,4 +162,4 @@ - \ No newline at end of file + From edf372e4b9a0251718d902120d0f0f320c9bef11 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 10 Mar 2022 16:27:02 +0100 Subject: [PATCH 44/92] Add note regarding push down accumulators being inherently quadratic --- src/decl-macros/patterns/push-down-acc.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/decl-macros/patterns/push-down-acc.md b/src/decl-macros/patterns/push-down-acc.md index 2bab372..ca2102b 100644 --- a/src/decl-macros/patterns/push-down-acc.md +++ b/src/decl-macros/patterns/push-down-acc.md @@ -1,5 +1,7 @@ # Push-down Accumulation +> Note: The pattern described here is inherently quadratic and as such, can worsen compilation times significantly. Therefore using it should be avoided unless necessary. + ```rust macro_rules! init_array { (@accum (0, $_e:expr) -> ($($body:tt)*)) From 227603a159652f6e3aadaa97ab446905f4363dca Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 12 Mar 2022 03:48:43 +0100 Subject: [PATCH 45/92] Add github sponsor button --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..efdee6a --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: veykril From 7bc27786675be2561d74d81b0841931f786d0167 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 12 Mar 2022 03:53:38 +0100 Subject: [PATCH 46/92] Note that the accumulator part of push down accumulators should prefer the end of the rule --- src/decl-macros/patterns/push-down-acc.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/decl-macros/patterns/push-down-acc.md b/src/decl-macros/patterns/push-down-acc.md index ca2102b..1c15536 100644 --- a/src/decl-macros/patterns/push-down-acc.md +++ b/src/decl-macros/patterns/push-down-acc.md @@ -74,5 +74,9 @@ As you can see, each layer adds to the accumulated output until the terminating The only critical part of the above formulation is the use of `$($body:tt)*` to preserve the output without triggering parsing. The use of `($input) -> ($output)` is simply a convention adopted to help clarify the behavior of such macros. +An important fact to note though is that the accumulator should be put at the end of the matcher, not at the start. +The reason for that is simple: Putting it at the start will require the compiler to first match the accumulator before actually matching relevant tokens that will decide whether the rule matches or not. +This means that putting the accumulator part at the start of a rule will drastically increase compilation times and therefor should be avoided. + Push-down accumulation is frequently used as part of [incremental TT munchers](./tt-muncher.md), as it allows arbitrarily complex intermediate results to be constructed. [Internal Rules](./internal-rules.md) were of use here as well, as they simplify creating such macros. From 39530249e1b579997eae652d447d52bc456586dc Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 6 Apr 2022 15:37:10 +1000 Subject: [PATCH 47/92] Fix errors relating to `-Zunpretty=expanded`. - It's called `-Zunpretty`, not `-Zpretty`. - It requires a nightly compiler. - `-Zunstable-options` is not required. --- src/decl-macros/macros-practical.md | 7 +++---- src/syntax-extensions/debugging.md | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index eae6cb6..ddf0b3a 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -972,11 +972,10 @@ error[E0425]: cannot find value `n` in this scope That can't be right... let's check what the macro is expanding to. ```shell -$ rustc -Z unstable-options --pretty expanded recurrence.rs +$ rustc +nightly -Zunpretty=expanded recurrence.rs ``` -The `--pretty expanded` argument tells `rustc` to perform macro expansion, then turn the resulting AST back into source code. -Because this option isn't considered stable yet, we also need `-Z unstable-options`. +The `-Zunpretty=expanded` argument tells `rustc` to perform macro expansion, then turn the resulting AST back into source code. The output (after cleaning up some formatting) is shown below; in particular, note the place in the code where `$recur` was substituted: @@ -1117,7 +1116,7 @@ As you can see, the a that's defined As such, the compiler treats them as completely different identifiers, *even though they have the same lexical appearance*. This is something to be *really* careful of when working on `macro_rules!` macros, syntax extensions in general even: they can produce ASTs which -will not compile, but which *will* compile if written out by hand, or dumped using `--pretty expanded`. +will not compile, but which *will* compile if written out by hand, or dumped using `-Zunpretty=expanded`. The solution to this is to capture the identifier *with the appropriate syntax context*. To do that, we need to again adjust our macro syntax. diff --git a/src/syntax-extensions/debugging.md b/src/syntax-extensions/debugging.md index 13bfdee..c6d2370 100644 --- a/src/syntax-extensions/debugging.md +++ b/src/syntax-extensions/debugging.md @@ -4,7 +4,7 @@ Sometimes, it is what the extension *expands to* that proves problematic as you do not usually see the expanded code. -Fortunately `rustc` offers the ability to look at the expanded code, for this, the `--pretty` argument can be used. +Fortunately `rustc` offers the ability to look at the expanded code via the unstable `-Zunpretty=expanded` argument. Given the following code: ```rust,ignore @@ -22,7 +22,7 @@ fn main() { compiled with the following command: ```shell -rustc -Z unstable-options -Zunpretty=expanded hello.rs +rustc +nightly -Zunpretty=expanded hello.rs ``` produces the following output (modified for formatting): @@ -54,6 +54,6 @@ fn main() { ``` But not just `rustc` exposes means to aid in debugging syntax extensions. -For the aforementioned `--pretty=expanded` option, there exists a nice `cargo` plugin called [`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) which is basically just a wrapper around it. +For the aforementioned `-Zunpretty=expanded` option, there exists a nice `cargo` plugin called [`cargo-expand`](https://github.com/dtolnay/cargo-expand) made by [`dtolnay`](https://github.com/dtolnay) which is basically just a wrapper around it. You can also use the [playground](https://play.rust-lang.org/), clicking on its `TOOLS` button in the top right gives you the option to expand syntax extensions as well! From e0c02facdb836267b19809dbe4150a6428a46082 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 6 Apr 2022 12:53:23 +0200 Subject: [PATCH 48/92] Update github runner OS --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index eaac01f..d2f81ba 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -7,7 +7,7 @@ on: jobs: deploy: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From be78f40f693bb775a94650b3f0151d4c6bbfd389 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 6 Apr 2022 15:17:41 +1000 Subject: [PATCH 49/92] Minor English nitpicks. --- README.md | 4 ++-- src/decl-macros.md | 2 +- src/decl-macros/minutiae/fragment-specifiers.md | 13 +++++-------- src/decl-macros/minutiae/import-export.md | 4 ++-- src/glossary.md | 2 +- src/proc-macros/methodical.md | 6 +++--- src/proc-macros/third-party-crates.md | 2 +- src/syntax-extensions/hygiene.md | 11 ++++++----- 8 files changed, 21 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index fb8a414..9250ee4 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ If something's unclear, opens up questions or is not understandable as written d The goal is for this book to become the best learning resource possible. The [original Little Book of Rust Macros](https://github.com/DanielKeep/tlborm) has helped me immensely with understanding ***Macros by Example*** style macros while I was still learning the language. -Unfortunately, the original book hasn't been updated since april of 2016, while the Rust language as well as it's macro-system keeps evolving. -Which is why I took up the task to update the book and keep it updated as well as I can while also adding new found things to it. +Unfortunately, the original book hasn't been updated since April of 2016, while the Rust language as well as its macro-system keeps evolving. +Which is why I took up the task to update the book and keep it updated as well as I can while also adding newfound things to it. In hopes that it will help out all the fresh faces coming to Rust understanding its macro systems, a part of the language a people tend to have trouble with. > This book expects you to have basic knowledge of Rust, it will not explain language features or constructs that are irrelevant to macros. diff --git a/src/decl-macros.md b/src/decl-macros.md index 381b8f8..e96fa56 100644 --- a/src/decl-macros.md +++ b/src/decl-macros.md @@ -7,7 +7,7 @@ There are two different introductions in this chapter, a [methodical] and a [pra The former will attempt to give you a complete and thorough explanation of *how* the system works, while the latter one will cover more practical examples. As such, the [methodical introduction][methodical] is intended for people who just want the system as a whole explained, while the [practical introduction][practical] guides one through the implementation of a single macro. -Following up the two introductions it offers some generally very useful [patterns] and [building blocks] for creating feature rich macros. +Following up the two introductions it offers some generally very useful [patterns] and [building blocks] for creating feature-rich macros. Other resources about declarative macros include the [Macros chapter of the Rust Book] which is a more approachable, high-level explanation as well as the reference [chapter](https://doc.rust-lang.org/reference/macros-by-example.html) which goes more into the precise details of things. diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index ace8c74..430e3b8 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -41,7 +41,7 @@ blocks! { ## `expr` -The `expr` fragment matches any kind of [expression](https://doc.rust-lang.org/reference/expressions.html)(Rust has a lot of them, given it *is* an expression orientated language). +The `expr` fragment matches any kind of [expression](https://doc.rust-lang.org/reference/expressions.html) (Rust has a lot of them, given it *is* an expression orientated language). ```rust macro_rules! expressions { @@ -137,7 +137,7 @@ literals! { ## `meta` -The `meta` fragment matches an [attribute](https://doc.rust-lang.org/reference/attributes.html), to be more precise, the contents of an attribute. +The `meta` fragment matches the contents of an [attribute](https://doc.rust-lang.org/reference/attributes.html). That is, it will match a simple path, one without generic arguments followed by a delimited token tree or an `=` followed by a literal expression. > **Note**: You will usually see this fragment being used in a matcher like `#[$meta:meta]` or `#![$meta:meta]` to actually capture an attribute. @@ -219,10 +219,7 @@ paths! { ## `stmt` -The `statement` fragment solely matches a [statement](https://doc.rust-lang.org/reference/statements.html) without its trailing semicolon, unless its an item statement that requires one. - -What would be an item statement that requires one? -A Unit-Struct would be a simple one, as defining one requires a trailing semicolon. +The `statement` fragment solely matches a [statement](https://doc.rust-lang.org/reference/statements.html) without its trailing semicolon, unless it is an item statement that requires one (such as a Unit-Struct). Let's use a simple example to show exactly what is meant with this. We use a macro that merely emits what it captures: @@ -266,7 +263,7 @@ fn main() { } ``` -From this we can tell a few things: +From this we can tell a few things. The first you should be able to see immediately is that while the `stmt` fragment doesn't capture trailing semicolons, it still emits them when required, even if the statement is already followed by one. The simple reason for that is that semicolons on their own are already valid statements which the fragment captures eagerly. @@ -324,7 +321,7 @@ macro_rules! visibilities { } visibilities! { - , // no vis is fine, as its implicitly + , // no vis is fine, due to the implicit `?` pub, pub(crate), pub(in super), diff --git a/src/decl-macros/minutiae/import-export.md b/src/decl-macros/minutiae/import-export.md index 014f605..fc931e0 100644 --- a/src/decl-macros/minutiae/import-export.md +++ b/src/decl-macros/minutiae/import-export.md @@ -68,9 +68,9 @@ fn main() {} When exporting `macro_rules!` macros, it is often useful to refer to non-macro symbols in the defining crate. Because crates can be renamed, there is a special substitution variable available: [`$crate`]. -This will *always* expand to an absolute path prefix to the containing crate(*e.g.* `:: macs`). +This will *always* expand to an absolute path prefix to the containing crate (*e.g.* `:: macs`). -Note that unless your compiler version is >= 1.30, this does *not* work for `macro_rules!` macros, since `macro_rules!` macros do not interact with regular name resolution in any way. +Note that unless your compiler version is >= 1.30, this does *not* work for `macro_rules!` macros, because `macro_rules!` macros do not interact with regular name resolution in any way. Otherwise, you cannot use something like `$crate::Y!` to refer to a particular macro within your crate. The implication, combined with selective imports via `#[macro_use]` is that there is currently *no way* to guarantee any given macro will be available when imported by another crate. diff --git a/src/glossary.md b/src/glossary.md index 9cc9ebe..0be96df 100644 --- a/src/glossary.md +++ b/src/glossary.md @@ -8,4 +8,4 @@ A function like macro describes a syntax extension that can be invoked via the f It is called this way due to its resemblance of a function call. ## Syntax Extension -The mechanism rust's `macro_rules!` and procedural macros are built on. +The mechanism Rust's `macro_rules!` and procedural macros are built on. diff --git a/src/proc-macros/methodical.md b/src/proc-macros/methodical.md index 56afcbb..12442fc 100644 --- a/src/proc-macros/methodical.md +++ b/src/proc-macros/methodical.md @@ -2,7 +2,7 @@ This chapter will introduce Rust's procedural macro system by explaining the system as a whole. -Unlike a [declarative macro](../decl-macros.md), a procedural macro takes the form of a rust function taking in a token stream(or two) and outputting a token stream. +Unlike a [declarative macro](../decl-macros.md), a procedural macro takes the form of a Rust function taking in a token stream(or two) and outputting a token stream. A proc-macro is at its core just a function exported from a crate with the `proc-macro` [crate type](https://doc.rust-lang.org/reference/linkage.html), so when writing multiple proc macros you can have them all live in one crate. @@ -15,10 +15,10 @@ A proc-macro is at its core just a function exported from a crate with the `proc A `proc-macro` type crate implicitly links to the compiler-provided [proc_macro](https://doc.rust-lang.org/proc_macro/index.html) crate, which contains all the things you need to get going with developing procedural macros. The two most important types exposed by the crate are the [`TokenStream`](https://doc.rust-lang.org/proc_macro/struct.TokenStream.html), which are the proc-macro variant of the already familiar token trees as well as the [`Span`](https://doc.rust-lang.org/proc_macro/struct.Span.html), which describes a part of source code used primarily for error reporting and hygiene. See the [Hygiene and Spans](./hygiene.md) chapter for more information. -As proc-macros therefore are functions living in a crate, they can be addressed as all the other items in a rust project. +As proc-macros therefore are functions living in a crate, they can be addressed as all the other items in a Rust project. All thats required to add the crate to the dependency graph of a project and bring the desired item into scope. -> **Note**: Procedural macros invocations still run at the same stage in the compiler expansion-wise as declarative macros, just that they are standalone rust programs that the compiler compiles, runs, and finally either replaces or appends to. +> **Note**: Procedural macros invocations still run at the same stage in the compiler expansion-wise as declarative macros, just that they are standalone Rust programs that the compiler compiles, runs, and finally either replaces or appends to. ## Types of procedural macros diff --git a/src/proc-macros/third-party-crates.md b/src/proc-macros/third-party-crates.md index 8fd2ecc..76f71d8 100644 --- a/src/proc-macros/third-party-crates.md +++ b/src/proc-macros/third-party-crates.md @@ -55,7 +55,7 @@ So what does it offer? A bunch of things. First of all it has definitions and parsing for all standard Rust syntax nodes(when the `full` feature is enabled), as well as a [`DeriveInput`](https://docs.rs/syn/1/syn/struct.DeriveInput.html) type which encapsulates all the information a derive macro gets passed as an input stream as a structured input(requires the `derive` feature, enabled by default). These can be used right out of the box with the [`parse_macro_input!`](https://docs.rs/syn/1/syn/macro.parse_macro_input.html) macro(requires the `parsing` and `proc-macro` features, enabled by default) to parse token streams into these types. -If rust syntax doesn't cut it, and instead one wishes to parse custom non-rust syntax the crate also offers a generic [parsing API](https://docs.rs/syn/1/syn/parse/index.html), mainly in the form of the [`Parse`](https://docs.rs/syn/1/syn/parse/trait.Parse.html) trait(requires the `parsing` feature, enabled by default). +If Rust syntax doesn't cut it, and instead one wishes to parse custom non-Rust syntax the crate also offers a generic [parsing API](https://docs.rs/syn/1/syn/parse/index.html), mainly in the form of the [`Parse`](https://docs.rs/syn/1/syn/parse/trait.Parse.html) trait(requires the `parsing` feature, enabled by default). Aside from this the types exposed by the library keep location information and spans which allows procedural macros to emit detailed error messages pointing at the macro input at the points of interest. diff --git a/src/syntax-extensions/hygiene.md b/src/syntax-extensions/hygiene.md index 3f24340..610bef1 100644 --- a/src/syntax-extensions/hygiene.md +++ b/src/syntax-extensions/hygiene.md @@ -1,10 +1,11 @@ # Hygiene -Hygiene is an important concept for macros, it describes the ability for a macro to work in its own syntax context, not affecting nor being affected by its surroundings. +Hygiene is an important concept for macros. +It describes the ability for a macro to work in its own syntax context, not affecting nor being affected by its surroundings. In other words this means that a syntax extension should be invocable anywhere without interfering with its surrounding context. -In a perfect world all syntax extensions in rust would be fully hygienic, unfortunately this isn't the case, so care should be taken to avoid writing syntax extensions that aren't fully hygienic. -We will go into general hygiene concepts here which will be touched upon in the corresponding hygiene chapters for the different syntax extensions rust has to offer. +In a perfect world all syntax extensions in Rust would be fully hygienic, unfortunately this isn't the case, so care should be taken to avoid writing syntax extensions that aren't fully hygienic. +We will go into general hygiene concepts here which will be touched upon in the corresponding hygiene chapters for the different syntax extensions Rust has to offer.   @@ -16,7 +17,7 @@ Likewise, if an identifier used in a syntax extension cannot reference something > That is the `Foo` in `struct Foo {}` or the `foo` in `let foo = …;` are created in the sense that they introduce something new under the name, > but the `Foo` in `fn foo(_: Foo) {}` or the `foo` in `foo + 3` are usages in the sense that they are referring to something existing. -This is best shown by example: +This is best shown by example. Let's assume we have some syntax extension `make_local` that expands to `let local = 0;`, that is it *creates* the identifier `local`. Then given the following snippet: @@ -25,7 +26,7 @@ make_local!(); assert_eq!(local, 0); ``` -If the `local` in `assert_eq!(local, 0);` resolves to the local defined by the syntax extension, the syntax extension is not hygienic(at least in regards to local names/bindings). +If the `local` in `assert_eq!(local, 0);` resolves to the local defined by the syntax extension, the syntax extension is not hygienic (at least in regards to local names/bindings). Now let's assume we have some syntax extension `use_local` that expands to `local = 42;`, that is it makes *use* of the identifier `local`. Then given the following snippet: From 42076c31980b4b61e19f3a3002bf6a64860c7609 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 7 Apr 2022 10:08:41 +1000 Subject: [PATCH 50/92] Remove incorrect sentence. It contradicts the previous sentence. --- src/decl-macros/macros-practical.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index ddf0b3a..abf9673 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -69,7 +69,6 @@ Finally, the rule says that *if* the input matches this rule, then the invocatio It's worth noting that `inits`, as implied by the name, actually contains *all* the expressions that match in this position, not just the first or last. What's more, it captures them *as a sequence* as opposed to, say, irreversibly pasting them all together. Also note that you can do "zero or more" with a repetition by using `*` instead of `+` and even optional, "zero or one" with `?`. -There is no support for "zero or one" or more specific numbers of repetitions. As an exercise, let's take the proposed input and feed it through the rule, to see how it is processed. The "Position" column will show which part of the syntax pattern needs to be matched against next, denoted by a "⌂". From 8d4fe82eb2a86cdfd083b926b55955baac4cce2e Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 7 Apr 2022 19:16:27 +0200 Subject: [PATCH 51/92] Document oddity of `vis` fragment in combination with `tt` fragment --- .../minutiae/fragment-specifiers.md | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index 430e3b8..ca83db3 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -309,10 +309,10 @@ types! { ## `vis` -The `vis` fragment matches a *possibly empty* [Visibility qualifier](https://doc.rust-lang.org/reference/visibility-and-privacy.html). -Emphasis lies on the *possibly empty* part. -You can kind of think of this fragment as having an implicit `?` repetition to it, meaning you don't, and in fact cannot, wrap it in a direct repetition, though this only applies for matching. -When expanding this fragment you expand it without a `$(...)?` repetition block. +The `vis` fragment matches a *possibly empty* [Visibility qualifier]. + +This fragment specifier acts a bit differently than the other ones in that is is allowed to match an empty sequence of tokens on its own, as long as it is not the last part of a matcher. +Note that this ability has the quirk that when the fragment captures an empty sequence, it will still turn it into an opaque capture, meaning it will be possible to match it with a `tt` fragment even though it is empty! ```rust macro_rules! visibilities { @@ -327,7 +327,24 @@ visibilities! { pub(in super), pub(in some_path), } + +macro_rules! non_optional_vis { + ($vis:vis) => (); +} +// vvvvvvvvvvvvvvvvvvvv~~ this is a compile error as the vis fragment is trying to capture at the end +// and therefor loses its empty sequence matching ability +// non_optional_vis!(); +non_optional_vis!(pub); + +macro_rules! it_is_opaque { + (()) => {}; + ($vis:vis ,) => { is_is_opaque!( ($vis) ) } +} +is_is_opaque!(pub); +is_is_opaque!(); // this works, even though it is empty, as the capture becomes opaque to the following expansions + # fn main() {} ``` [`macro_rules`]: ../macro_rules.md +[Visibility qualifier]: https://doc.rust-lang.org/reference/visibility-and-privacy.html From d112f17ebe98d6958ceec0798a530c84660260eb Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 8 Apr 2022 07:26:32 +1000 Subject: [PATCH 52/92] Add some performance advice. About TT munchers, internal rules, and push-down accumulators. --- src/decl-macros/patterns/internal-rules.md | 12 +++++ src/decl-macros/patterns/push-down-acc.md | 26 +++++++--- src/decl-macros/patterns/tt-muncher.md | 56 +++++++++++++++++++++- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/src/decl-macros/patterns/internal-rules.md b/src/decl-macros/patterns/internal-rules.md index 7d1b3ee..06b145d 100644 --- a/src/decl-macros/patterns/internal-rules.md +++ b/src/decl-macros/patterns/internal-rules.md @@ -81,4 +81,16 @@ There other symbols or unique prefixes may be used as desired, but use of `@` ha Additionally, internal rules will often come *before* any "bare" rules, to avoid issues with `macro_rules!` incorrectly attempting to parse an internal invocation as something it cannot possibly be, such as an expression. +## Performance + +One downside of internal rules is that they can hurt compile times. +Only one macro rule can match any (valid) macro invocation, but the compiler must try to match all rules in order. +If a macro has many rules, there can be many such failures, and the use of internal rules will increase the number of such failures. + +Also, the `@as_expr`-style identifier makes rules longer, slightly increasing +the amount of work the compiler must do when matching. + +Therefore, for best performance, avoiding internal rules is best. +Avoiding them often makes complex macros easier to read, too. + [TT Munchers]:./tt-muncher.html diff --git a/src/decl-macros/patterns/push-down-acc.md b/src/decl-macros/patterns/push-down-acc.md index 1c15536..5065f7a 100644 --- a/src/decl-macros/patterns/push-down-acc.md +++ b/src/decl-macros/patterns/push-down-acc.md @@ -1,7 +1,5 @@ # Push-down Accumulation -> Note: The pattern described here is inherently quadratic and as such, can worsen compilation times significantly. Therefore using it should be avoided unless necessary. - ```rust macro_rules! init_array { (@accum (0, $_e:expr) -> ($($body:tt)*)) @@ -74,9 +72,25 @@ As you can see, each layer adds to the accumulated output until the terminating The only critical part of the above formulation is the use of `$($body:tt)*` to preserve the output without triggering parsing. The use of `($input) -> ($output)` is simply a convention adopted to help clarify the behavior of such macros. -An important fact to note though is that the accumulator should be put at the end of the matcher, not at the start. -The reason for that is simple: Putting it at the start will require the compiler to first match the accumulator before actually matching relevant tokens that will decide whether the rule matches or not. -This means that putting the accumulator part at the start of a rule will drastically increase compilation times and therefor should be avoided. - Push-down accumulation is frequently used as part of [incremental TT munchers](./tt-muncher.md), as it allows arbitrarily complex intermediate results to be constructed. [Internal Rules](./internal-rules.md) were of use here as well, as they simplify creating such macros. + +## Performance + +Push-down accumulation is inherently quadratic. +Consider a push-down accumulation rule that builds up an accumulator of 100 token trees, one token tree per invocation. +- The initial invocation will match against the empty accumulator. +- The first recursive invocation will match against the accumulator of 1 token tree. +- The next recursive invocation will match against the accumulator of 2 token trees. + +And so on, up to 100. +This is a classic quadratic pattern, and long inputs can cause macro expansion to blow out compile times. +Furthermore, TT munchers are also inherently quadratic over their input, so a macro that uses both TT munching *and* push-down accumulation will be doubly quadratic! + +All the [performance advice](./tt-muncher.md#performance) about TT munchers holds for push-down accumulation. +In general, avoid using them too much, and keep them as simple as possible. + +Finally, make sure you put the accumulator at the *end* of rules, rather than the beginning. +That way, if a rule fails, the compiler won't have had to match the (potentially long) accumulator before hitting the part of the rule that fails to match. +This can make a large difference to compile times. + diff --git a/src/decl-macros/patterns/tt-muncher.md b/src/decl-macros/patterns/tt-muncher.md index 4198439..fb89faf 100644 --- a/src/decl-macros/patterns/tt-muncher.md +++ b/src/decl-macros/patterns/tt-muncher.md @@ -33,8 +33,10 @@ macro_rules! mixed_rules { ``` This pattern is perhaps the *most powerful* macro parsing technique available, allowing one to parse grammars of significant complexity. +However, it can increase compile times if used excessively, so should be used +with care. -A `TT muncher` is a recursive `macro_rules!` macro that works by incrementally processing its input one step at a time. +A *TT muncher* is a recursive `macro_rules!` macro that works by incrementally processing its input one step at a time. At each step, it matches and removes (munches) some sequence of tokens from the start of its input, generates some intermediate output, then recurses on the input tail. The reason for "TT" in the name specifically is that the unprocessed part of the input is *always* captured as `$($tail:tt)*`. @@ -51,3 +53,55 @@ It is recommended that, when writing a TT muncher, you make reasonable efforts t This can be done by adding additional rules to account for variation in the input (as opposed to recursion into an intermediate layer), or by making compromises on the input syntax to make using standard repetitions more tractable. [`tt`]: ../minutiae/fragment-specifiers.html#tt + +## Performance + +TT munchers are inherently quadratic. +Consider a TT muncher rule that consumes one token tree and then recursively calls itself on the remaining input. +If it is passed 100 token trees: +- The initial invocation will match against all 100 token trees. +- The first recursive invocation will match against 99 token trees. +- The next recursive invocation will match against 98 token trees. + +And so on, down to 1. +This is a classic quadratic pattern, and long inputs can cause macro expansion to blow out compile times. + +Try to avoid using TT munchers too much, especially with long inputs. +The default value of the `recursion_limit` attribute is a good sanity check; if you have to exceed it, you might be heading for trouble. + +If you have the choice between writing a TT muncher that can be called once to handle multiple things, or a simpler macro that can be called multiple times to handle a single thing, prefer the latter. +For example, you could change a macro that is called like this: +```rust +# macro_rules! f { ($($tt:tt)*) => {} } +f! { + fn f_u8(x: u32) -> u8; + fn f_u16(x: u32) -> u16; + fn f_u32(x: u32) -> u32; + fn f_u64(x: u64) -> u64; + fn f_u128(x: u128) -> u128; +} +``` +To one that is called like this: +```rust +# macro_rules! f { ($($tt:tt)*) => {} } +f! { fn f_u8(x: u32) -> u8; } +f! { fn f_u16(x: u32) -> u16; } +f! { fn f_u32(x: u32) -> u32; } +f! { fn f_u64(x: u64) -> u64; } +f! { fn f_u128(x: u128) -> u128; } +``` +The longer the input, the more likely this will improve compile times. + +Also, if a TT muncher macro has many rules, put the most frequently matched +rules as early as possible. +This avoids unnecessary matching failures. +(In fact, this is good advice for any kind of declarative macro, not just TT munchers.) + +Finally, if you can write a macro using normal repetition via `*` or `+`, that should be preferred to a TT muncher. +This is most likely if each invocation of the TT muncher would only process one token at a time. +In more complicated cases, there is an advanced technique used within the `quote` crate that can avoid the quadratic behaviour, at the cost of some conceptual complexity. +See [this comment] for details. + +[this comment]: https://github.com/dtolnay/quote/blob/31c3be473d0457e29c4f47ab9cff73498ac804a7/src/lib.rs#L664-L746 + + From 2f13d95ca60da3e6aadfa9ffaf9966737f65c442 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 8 Apr 2022 11:19:26 +1000 Subject: [PATCH 53/92] Clarify the push-down accumulator example. - Use brackets instead of parens in the push-down accumulator. - Don't use internal rules, especially given that it's now recommended against doing so for performance reasons. - Remove the `@as_expr` rule, it's not necessary. The final output can be produced directly from the `0` accumulator rule. - Reorder the rules so they are in the easiest-to-follow order. - Generally reformat to improve readability. I found this example hard to read when I first encountered it. I think these changes make it much easier to understand. --- src/decl-macros/patterns/push-down-acc.md | 58 +++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/decl-macros/patterns/push-down-acc.md b/src/decl-macros/patterns/push-down-acc.md index 5065f7a..8febe52 100644 --- a/src/decl-macros/patterns/push-down-acc.md +++ b/src/decl-macros/patterns/push-down-acc.md @@ -1,23 +1,22 @@ # Push-down Accumulation +The following macro uses *push-down accumulation*. + ```rust macro_rules! init_array { - (@accum (0, $_e:expr) -> ($($body:tt)*)) - => {init_array!(@as_expr [$($body)*])}; - (@accum (1, $e:expr) -> ($($body:tt)*)) - => {init_array!(@accum (0, $e) -> ($($body)* $e,))}; - (@accum (2, $e:expr) -> ($($body:tt)*)) - => {init_array!(@accum (1, $e) -> ($($body)* $e,))}; - (@accum (3, $e:expr) -> ($($body:tt)*)) - => {init_array!(@accum (2, $e) -> ($($body)* $e,))}; - (@as_expr $e:expr) => {$e}; - [$e:expr; $n:tt] => { - { - let e = $e; - init_array!(@accum ($n, e.clone()) -> ()) + [$e:expr; $n:tt] => { + { + let e = $e; + accum!([$n, e.clone()] -> []) } }; } +macro_rules! accum { + ([3, $e:expr] -> [$($body:tt)*]) => { accum!([2, $e] -> [$($body)* $e,]) }; + ([2, $e:expr] -> [$($body:tt)*]) => { accum!([1, $e] -> [$($body)* $e,]) }; + ([1, $e:expr] -> [$($body:tt)*]) => { accum!([0, $e] -> [$($body)* $e,]) }; + ([0, $_:expr] -> [$($body:tt)*]) => { [$($body)*] }; +} let strings: [String; 3] = init_array![String::from("hi!"); 3]; # assert_eq!(format!("{:?}", strings), "[\"hi!\", \"hi!\", \"hi!\"]"); @@ -28,28 +27,29 @@ This means that it is impossible to have a syntax extension expand to a partial One might hope that the above example could be more directly expressed like so: -```ignore +```rust,ignore macro_rules! init_array { - (@accum 0, $_e:expr) => {/* empty */}; - (@accum 1, $e:expr) => {$e}; - (@accum 2, $e:expr) => {$e, init_array!(@accum 1, $e)}; - (@accum 3, $e:expr) => {$e, init_array!(@accum 2, $e)}; [$e:expr; $n:tt] => { { let e = $e; - [init_array!(@accum $n, e)] + [accum!($n, e.clone())] } }; } +macro_rules! accum { + (3, $e:expr) => { $e, accum!(2, $e) }; + (2, $e:expr) => { $e, accum!(1, $e) }; + (1, $e:expr) => { $e }; +} ``` The expectation is that the expansion of the array literal would proceed as follows: ```rust,ignore - [init_array!(@accum 3, e)] - [e, init_array!(@accum 2, e)] - [e, e, init_array!(@accum 1, e)] - [e, e, e] + [accum!(3, e.clone())] + [e.clone(), accum!(2, e.clone())] + [e.clone(), e.clone(), accum!(1, e.clone())] + [e.clone(), e.clone(), e.clone()] ``` However, this would require each intermediate step to expand to an incomplete expression. @@ -59,12 +59,12 @@ Push-down, however, allows us to incrementally build up a sequence of tokens wit In the example given at the top, the sequence of invocations proceeds as follows: ```rust,ignore -init_array! { String:: from ( "hi!" ) ; 3 } -init_array! { @ accum ( 3 , e . clone ( ) ) -> ( ) } -init_array! { @ accum ( 2 , e.clone() ) -> ( e.clone() , ) } -init_array! { @ accum ( 1 , e.clone() ) -> ( e.clone() , e.clone() , ) } -init_array! { @ accum ( 0 , e.clone() ) -> ( e.clone() , e.clone() , e.clone() , ) } -init_array! { @ as_expr [ e.clone() , e.clone() , e.clone() , ] } +init_array!(String::from("hi!"); 3) +accum!([3, e.clone()] -> []) +accum!([2, e.clone()] -> [e.clone(),]) +accum!([1, e.clone()] -> [e.clone(), e.clone(),]) +accum!([0, e.clone()] -> [e.clone(), e.clone(), e.clone(),]) +[e.clone(), e.clone(), e.clone(),] ``` As you can see, each layer adds to the accumulated output until the terminating rule finally emits it as a complete construct. From 96514b83e5413388cf802f3c1c313beed6df8875 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 8 Apr 2022 14:12:02 +0200 Subject: [PATCH 54/92] Expand the vis fragment section --- .../minutiae/fragment-specifiers.md | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index ca83db3..f80579e 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -311,9 +311,6 @@ types! { The `vis` fragment matches a *possibly empty* [Visibility qualifier]. -This fragment specifier acts a bit differently than the other ones in that is is allowed to match an empty sequence of tokens on its own, as long as it is not the last part of a matcher. -Note that this ability has the quirk that when the fragment captures an empty sequence, it will still turn it into an opaque capture, meaning it will be possible to match it with a `tt` fragment even though it is empty! - ```rust macro_rules! visibilities { // ∨~~Note this comma, since we cannot repeat a `vis` fragment on its own @@ -327,23 +324,49 @@ visibilities! { pub(in super), pub(in some_path), } +``` + +While able to match empty sequences of tokens, the fragment specifier still acts quite different from [optional repetitions](../macro_rules.md#repetitions) which is described in the following: +If it is being matched against no left over tokens the entire macro matching fails. +```rust macro_rules! non_optional_vis { ($vis:vis) => (); } -// vvvvvvvvvvvvvvvvvvvv~~ this is a compile error as the vis fragment is trying to capture at the end -// and therefor loses its empty sequence matching ability -// non_optional_vis!(); -non_optional_vis!(pub); +non_optional_vis!(); +// ^^^^^^^^^^^^^^^^ error: missing tokens in macro arguments +``` -macro_rules! it_is_opaque { - (()) => {}; - ($vis:vis ,) => { is_is_opaque!( ($vis) ) } +`$vis:vis $ident:ident` matches fine, unlike `$(pub)? $ident:ident` which is ambiguous, as `pub` denotes a valid identifier. +```rust +macro_rules! vis_ident { + ($vis:vis $ident:ident) => (); } -is_is_opaque!(pub); -is_is_opaque!(); // this works, even though it is empty, as the capture becomes opaque to the following expansions +vis_ident!(pub foo); // this works fine -# fn main() {} +macro_rules! pub_ident { + ($(pub)? $ident:ident) => (); +} +pub_ident!(pub foo); + // ^^^ error: local ambiguity when calling macro `pub_ident`: multiple parsing options: built-in NTs ident ('ident') or 1 other option. +``` + +Being a fragment that matches the empty token sequence also gives it a very interesting quirk in combination with `tt` fragments and recursive expansions. + +When matching the empty token sequence, the metavariable will still count as a capture and since it is not a `tt`, `ident` or `lifetime` fragment it will become opaque to further expansions. +This means if this capture is passed onto another macro invocation that captures it as a `tt` you effectively end up with token tree that contains nothing! + +```rust +macro_rules! it_is_opaque { + (()) => { "()" }; + (($tt:tt)) => { concat!("$tt is ", stringify!($tt)) }; + ($vis:vis ,) => { it_is_opaque!( ($vis) ); } +} +fn main() { + // this prints "$tt is ", as the recursive calls hits the second branch with an empty + // tt, opposed to matching with the first branch! + println!("{}", it_is_opaque!(,)); +} ``` [`macro_rules`]: ../macro_rules.md From ce81d52e8f42d9e938ef2910ca55f89f0e6f2cec Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 10 Apr 2022 00:25:55 +0200 Subject: [PATCH 55/92] Link fragment specifiers from macro-rules chapter to minutiae listing --- src/decl-macros/macro_rules.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/decl-macros/macro_rules.md b/src/decl-macros/macro_rules.md index 69ac1d4..243b103 100644 --- a/src/decl-macros/macro_rules.md +++ b/src/decl-macros/macro_rules.md @@ -74,19 +74,19 @@ These allow input to be matched based on some general grammar category, with the Captures are written as a dollar (`$`) followed by an identifier, a colon (`:`), and finally the kind of capture which is also called the fragment-specifier, which must be one of the following: -* `block`: a block (i.e. a block of statements and/or an expression, surrounded by braces) -* `expr`: an expression -* `ident`: an identifier (this includes keywords) -* `item`: an item, like a function, struct, module, impl, etc. -* `lifetime`: a lifetime (e.g. `'foo`, `'static`, ...) -* `literal`: a literal (e.g. `"Hello World!"`, `3.14`, `'🦀'`, ...) -* `meta`: a meta item; the things that go inside the `#[...]` and `#![...]` attributes -* `pat`: a pattern -* `path`: a path (e.g. `foo`, `::std::mem::replace`, `transmute::<_, int>`, …) -* `stmt`: a statement -* `tt`: a single token tree -* `ty`: a type -* `vis`: a possible empty visibility qualifier (e.g. `pub`, `pub(in crate)`, ...) +* [`block`](./minutiae/fragment-specifiers.md#block): a block (i.e. a block of statements and/or an expression, surrounded by braces) +* [`expr`](./minutiae/fragment-specifiers.md#expr): an expression +* [`ident`](./minutiae/fragment-specifiers.md#ident): an identifier (this includes keywords) +* [`item`](./minutiae/fragment-specifiers.md#item): an item, like a function, struct, module, impl, etc. +* [`lifetime`](./minutiae/fragment-specifiers.md#lifetime): a lifetime (e.g. `'foo`, `'static`, ...) +* [`literal`](./minutiae/fragment-specifiers.md#literal): a literal (e.g. `"Hello World!"`, `3.14`, `'🦀'`, ...) +* [`meta`](./minutiae/fragment-specifiers.md#meta): a meta item; the things that go inside the `#[...]` and `#![...]` attributes +* [`pat`](./minutiae/fragment-specifiers.md#pat): a pattern +* [`path`](./minutiae/fragment-specifiers.md#path): a path (e.g. `foo`, `::std::mem::replace`, `transmute::<_, int>`, …) +* [`stmt`](./minutiae/fragment-specifiers.md#stmt): a statement +* [`tt`](./minutiae/fragment-specifiers.md#tt): a single token tree +* [`ty`](./minutiae/fragment-specifiers.md#ty): a type +* [`vis`](./minutiae/fragment-specifiers.md#vis): a possible empty visibility qualifier (e.g. `pub`, `pub(in crate)`, ...) For more in-depth description of the fragment specifiers, check out the [Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. From 0f94b037d5a437e80abc9dcf8bf1c1f6973d71ee Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 10 Apr 2022 02:44:06 +0200 Subject: [PATCH 56/92] Add `Metavariable Expressions` to the macro-rules chapter --- src/decl-macros/macro_rules.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/decl-macros/macro_rules.md b/src/decl-macros/macro_rules.md index 243b103..99ef1c1 100644 --- a/src/decl-macros/macro_rules.md +++ b/src/decl-macros/macro_rules.md @@ -232,6 +232,28 @@ error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``` +## Metavariable Expressions + +> *RFC*: [rfcs#1584](https://github.com/rust-lang/rfcs/blob/master/text/3086-macro-metavar-expr.md)\ +> *Tracking Issue*: [rust#83527](https://github.com/rust-lang/rust/issues/83527)\ +> *Feature*: `#![feature(macro_metavar_expr)]` + +Matchers can contain what is called metavariable expressions. +Metavariable expressions provide matchers with information about metavariables that are otherwise not easily obtainable. +With the exception of the `$$` expression, these have the general form `$ { op(...) }`. +Currently all metavariable expressions but `$$` deal with repetitions. + +The following expressions are available with `ident` being the name of a bound metavariable and `depth` being an integer literal: + +- `${count(ident)}`: The number of times `$ident` repeats in the inner-most repetition in total. This is equivalent to `${count(ident, 0)}`. +- `${count(ident, depth)}`: The number of times `$ident` repeats in the repetition at `depth`, counting outwards. +- `${index()}`: The current repetition index of the inner-most repetition. This is equivalent to `${index(0)}`. +- `${index(depth)}`: The current index of the repetition at `depth`, counting outwards. +- `${length()}`: The number of times the inner-most repetition will repeat for. This is equivalent to `${length(0)}`. +- `${length(depth)}`: The number of times the repetition at `depth` will repeat for, counting outwards. +- `${ignore(ident)}`: Binds `$ident` for repetition, while expanding to nothing. +- `$$`: Expands to a single `$`, effectively escaping the `$` token so it won't be transcribed. +   For the complete grammar definition you may want to consult the [Macros By Example](https://doc.rust-lang.org/reference/macros-by-example.html#macros-by-example) chapter of the Rust reference. From 00fe9fce808f806859c38c5fffbe2a4ccd73d6ba Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 10 Apr 2022 15:12:21 +0200 Subject: [PATCH 57/92] Add metavariable expressions chapter --- .cargo/config.toml | 3 + src/SUMMARY.md | 1 + src/decl-macros/macro_rules.md | 6 +- .../minutiae/fragment-specifiers.md | 6 +- src/decl-macros/minutiae/metavar-expr.md | 193 ++++++++++++++++++ 5 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 src/decl-macros/minutiae/metavar-expr.md diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..7b34c00 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[alias] +# xtask = "run --package xtask --bin xtask --" +md-test = "run --package xtask --bin xtask" diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 58ff673..57d3be8 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -14,6 +14,7 @@ - [Minutiae](./decl-macros/minutiae.md) - [Fragment Specifiers](./decl-macros/minutiae/fragment-specifiers.md) - [Metavariables and Expansion Redux](./decl-macros/minutiae/metavar-and-expansion.md) + - [Metavariable Expressions](./decl-macros/minutiae/metavar-expr.md) - [Hygiene](./decl-macros/minutiae/hygiene.md) - [Non-Identifier Identifiers](./decl-macros/minutiae/identifiers.md) - [Debugging](./decl-macros/minutiae/debugging.md) diff --git a/src/decl-macros/macro_rules.md b/src/decl-macros/macro_rules.md index 99ef1c1..9e8e9c3 100644 --- a/src/decl-macros/macro_rules.md +++ b/src/decl-macros/macro_rules.md @@ -238,15 +238,15 @@ error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times > *Tracking Issue*: [rust#83527](https://github.com/rust-lang/rust/issues/83527)\ > *Feature*: `#![feature(macro_metavar_expr)]` -Matchers can contain what is called metavariable expressions. -Metavariable expressions provide matchers with information about metavariables that are otherwise not easily obtainable. +Transcriber can contain what is called metavariable expressions. +Metavariable expressions provide transcribers with information about metavariables that are otherwise not easily obtainable. With the exception of the `$$` expression, these have the general form `$ { op(...) }`. Currently all metavariable expressions but `$$` deal with repetitions. The following expressions are available with `ident` being the name of a bound metavariable and `depth` being an integer literal: - `${count(ident)}`: The number of times `$ident` repeats in the inner-most repetition in total. This is equivalent to `${count(ident, 0)}`. -- `${count(ident, depth)}`: The number of times `$ident` repeats in the repetition at `depth`, counting outwards. +- `${count(ident, depth)}`: The number of times `$ident` repeats in the repetition at `depth`. - `${index()}`: The current repetition index of the inner-most repetition. This is equivalent to `${index(0)}`. - `${index(depth)}`: The current index of the repetition at `depth`, counting outwards. - `${length()}`: The number of times the inner-most repetition will repeat for. This is equivalent to `${length(0)}`. diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index f80579e..949f928 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -1,6 +1,6 @@ # Fragment Specifiers -As shown in the [`macro_rules`] chapter, Rust, as of 1.53, has 13 fragment specifiers. +As shown in the [`macro_rules`] chapter, Rust, as of 1.60, has 14 fragment specifiers. This section will go a bit more into detail for some of them and shows a few example inputs of what each matcher matches. > **Note**: Capturing with anything but the `ident`, `lifetime` and `tt` fragments will render the captured AST opaque, making it impossible to further match it with other fragment specifiers in future macro invocations. @@ -363,8 +363,8 @@ macro_rules! it_is_opaque { ($vis:vis ,) => { it_is_opaque!( ($vis) ); } } fn main() { - // this prints "$tt is ", as the recursive calls hits the second branch with an empty - // tt, opposed to matching with the first branch! + // this prints "$tt is ", as the recursive calls hits the second branch with + // an empty tt, opposed to matching with the first branch! println!("{}", it_is_opaque!(,)); } ``` diff --git a/src/decl-macros/minutiae/metavar-expr.md b/src/decl-macros/minutiae/metavar-expr.md new file mode 100644 index 0000000..beed3ac --- /dev/null +++ b/src/decl-macros/minutiae/metavar-expr.md @@ -0,0 +1,193 @@ +# Metavariable Expressions + +> *RFC*: [rfcs#1584](https://github.com/rust-lang/rfcs/blob/master/text/3086-macro-metavar-expr.md)\ +> *Tracking Issue*: [rust#83527](https://github.com/rust-lang/rust/issues/83527)\ +> *Feature*: `#![feature(macro_metavar_expr)]` + +> Note: The example code snippets are very bare bones, trying to show off how they work. If you think you got small snippets with proper isolated usage of these expression please submit them! + + +As shown in the [`macro_rules`](../macro_rules.md) chapter, Rust has special expressions that can be used by macro transcribers to obtain information about metavariables that are otherwise difficult or even impossible to get. +This chapter will introduce them more in-depth together with usage examples. + +- [`$$`](#dollar-dollar-) +- [`${count(ident, depth)}`](#countident-depth) +- [`${index(depth)}`](#index-depth) +- [`${length(depth)}`](#length-depth) +- [`${ignore(ident)}`](#ignore-ident) + +## Dollar Dollar (`$$`) + +The `$$` expression expands to a single `$`, making it effectively an escaped `$`. +This enables the ability in writing macros emitting new macros as the former macro won't transcribe metavariables, repetitions and metavariable expressions that have an escaped `$`. + +We can see the problem without using `$$` in the following snippet: +```rust +macro_rules! foo { + () => { + macro_rules! bar { + ( $( $any:tt )* ) => { $( $any )* }; + // ^^^^^^^^^^^ error: attempted to repeat an expression containing no syntax variables matched as repeating at this depth + } + }; +} + +foo!(); +# fn main() {} +``` + +The problem is obvious, the transcriber of foo sees a repetition and tries to repeat it when transcribing, but there is no `$any` metavariable in its scope causing it to fail. +With `$$` we can get around this as the transcriber of `foo` will no longer try to do the repetition. + +```rust +#![feature(macro_metavar_expr)] + +macro_rules! foo { + () => { + macro_rules! bar { + ( $$( $$any:tt )* ) => { $$( $$any )* }; + } + }; +} + +foo!(); +bar!(); +# fn main() {} +``` + +## `count(ident, depth)` + +The `count` metavariable expression expands to the repetition count of the metavariable `$ident` at the given repetition depth. + +- The `ident` argument must be a declared metavariable in the scope of the rule. +- The `depth` argument must be an integer literal of value less or equal to the maximum repetition depth that the `$ident` metavariable appears in. +- The expression expands to an unsuffixed integer literal token. + +The `count(ident)` expression defaults `depth` to `0`, making it a shorthand for `count(ident, 0)`. + +```rust +#![feature(macro_metavar_expr)] + +macro_rules! foo { + ( $( $outer:ident ( $( $inner:ident ),* ) ; )* ) => { + println!("count(outer, 0): $outer repeats {} times", ${count(outer)}); + println!("count(inner, 0): The $inner repetition repeats {} times in the outer repetition", ${count(inner, 0)}); + println!("count(inner, 1): $inner repeats {} times in the inner repetitions", ${count(inner, 1)}); + }; +} + +fn main() { + foo! { + outer () ; + outer ( inner , inner ) ; + outer () ; + outer ( inner ) ; + }; +} +``` + +## `index(depth)` + +The `index(depth)` metavariable expression expands to the current iteration index of the repetition at the given depth. + +- The `depth` argument targets the repetition at `depth` counting outwards from the inner-most repetition where the expression is invoked. +- The expression expands to an unsuffixed integer literal token. + +The `index()` expression defaults `depth` to `0`, making it a shorthand for `index(0)`. + +```rust +#![feature(macro_metavar_expr)] + +macro_rules! attach_iteration_counts { + ( $( ( $( $inner:ident ),* ) ; )* ) => { + ( $( + $(( + stringify!($inner), + ${index(1)}, // this targets the outer repetition + ${index()} // and this, being an alias for `index(0)` targets the inner repetition + ),)* + )* ) + }; +} + +fn main() { + let v = attach_iteration_counts! { + ( hello ) ; + ( indices , of ) ; + () ; + ( these, repetitions ) ; + }; + println!("{v:?}"); +} +``` + + +## `length(depth)` + +The `length(depth)` metavariable expression expands to the iteration count of the repetition at the given depth. + +- The `depth` argument targets the repetition at `depth` counting outwards from the inner-most repetition where the expression is invoked. +- The expression expands to an unsuffixed integer literal token. + +The `length()` expression defaults `depth` to `0`, making it a shorthand for `length(0)`. + + +```rust +#![feature(macro_metavar_expr)] + +macro_rules! lets_count { + ( $( $outer:ident ( $( $inner:ident ),* ) ; )* ) => { + $( + $( + println!( + "'{}' in inner iteration {}/{} with '{}' in outer iteration {}/{} ", + stringify!($inner), ${index()}, ${length()}, + stringify!($outer), ${index(1)}, ${length(1)}, + ); + )* + )* + }; +} + +fn main() { + lets_count!( + many (small , things) ; + none () ; + exactly ( one ) ; + ); +} +``` + +## `ignore(ident)` + +The `ignore(ident)` metavariable expression expands to nothing, making it possible to expand something as often as a metavariable repeats without expanding the metavariable. + +- The `ident` argument must be a declared metavariable in the scope of the rule. + +```rust +#![feature(macro_metavar_expr)] + +macro_rules! repetition_tuples { + ( $( ( $( $inner:ident ),* ) ; )* ) => { + ($( + $( + ( + ${index()}, + ${index(1)} + ${ignore(inner)} // without this metavariable expression, compilation would fail + ), + )* + )*) + }; +} + +fn main() { + let tuple = repetition_tuples!( + ( one, two ) ; + () ; + ( one ) ; + ( one, two, three ) ; + ); + println!("{tuple:?}"); +} +``` From 8ff8752dab95fd2f06d1971e4cddc9181fdb6ee6 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 10 Apr 2022 15:26:24 +0200 Subject: [PATCH 58/92] Move minutiae chapters out of the methodical introduction --- src/SUMMARY.md | 19 +- src/decl-macros/macro_rules.md | 259 ----------------- src/decl-macros/macros-methodical.md | 260 ++++++++++++++++++ src/decl-macros/macros-practical.md | 8 +- .../minutiae/fragment-specifiers.md | 4 +- src/decl-macros/minutiae/metavar-expr.md | 3 +- 6 files changed, 276 insertions(+), 277 deletions(-) delete mode 100644 src/decl-macros/macro_rules.md diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 57d3be8..282283d 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -10,17 +10,16 @@ - [Debugging](./syntax-extensions/debugging.md) - [Declarative Macros](./decl-macros.md) - [A Methodical Introduction](./decl-macros/macros-methodical.md) - - [macro_rules!](./decl-macros/macro_rules.md) - - [Minutiae](./decl-macros/minutiae.md) - - [Fragment Specifiers](./decl-macros/minutiae/fragment-specifiers.md) - - [Metavariables and Expansion Redux](./decl-macros/minutiae/metavar-and-expansion.md) - - [Metavariable Expressions](./decl-macros/minutiae/metavar-expr.md) - - [Hygiene](./decl-macros/minutiae/hygiene.md) - - [Non-Identifier Identifiers](./decl-macros/minutiae/identifiers.md) - - [Debugging](./decl-macros/minutiae/debugging.md) - - [Scoping](./decl-macros/minutiae/scoping.md) - - [Import and Export](./decl-macros/minutiae/import-export.md) - [A Practical Introduction](./decl-macros/macros-practical.md) + - [Minutiae](./decl-macros/minutiae.md) + - [Fragment Specifiers](./decl-macros/minutiae/fragment-specifiers.md) + - [Metavariables and Expansion Redux](./decl-macros/minutiae/metavar-and-expansion.md) + - [Metavariable Expressions](./decl-macros/minutiae/metavar-expr.md) + - [Hygiene](./decl-macros/minutiae/hygiene.md) + - [Non-Identifier Identifiers](./decl-macros/minutiae/identifiers.md) + - [Debugging](./decl-macros/minutiae/debugging.md) + - [Scoping](./decl-macros/minutiae/scoping.md) + - [Import and Export](./decl-macros/minutiae/import-export.md) - [Patterns](./decl-macros/patterns.md) - [Callbacks](./decl-macros/patterns/callbacks.md) - [Incremental TT Munchers](./decl-macros/patterns/tt-muncher.md) diff --git a/src/decl-macros/macro_rules.md b/src/decl-macros/macro_rules.md deleted file mode 100644 index 9e8e9c3..0000000 --- a/src/decl-macros/macro_rules.md +++ /dev/null @@ -1,259 +0,0 @@ -# `macro_rules!` - -With all that in mind, we can introduce `macro_rules!` itself. -As noted previously, `macro_rules!` is *itself* a syntax extension, meaning it is *technically* not part of the Rust syntax. -It uses the following forms: - -```rust,ignore -macro_rules! $name { - $rule0 ; - $rule1 ; - // … - $ruleN ; -} -``` - -There must be *at least* one rule, and you can omit the semicolon after the last rule. -You can use brackets(`[]`), parentheses(`()`) or braces(`{}`). - -Each *"rule"* looks like the following: - -```ignore - ($matcher) => {$expansion} -``` - -Like before, the types of parentheses used can be any kind, but parentheses around the matcher and braces around the expansion are somewhat conventional. -The expansion part of a rule is also called its *transcriber*. - -Note that the choice of the parentheses does not matter in regards to how the mbe macro may be invoked. -In fact, function-like macros can be invoked with any kind of parentheses as well, but invocations with `{ .. }` and `( ... );`, notice the trailing semicolon, are special in that their expansion will *always* be parsed as an *item*. - -If you are wondering, the `macro_rules!` invocation expands to... *nothing*. -At least, nothing that appears in the AST; rather, it manipulates compiler-internal structures to register the mbe macro. -As such, you can *technically* use `macro_rules!` in any position where an empty expansion is valid. - -## Matching - -When a `macro_rules!` macro is invoked, the `macro_rules!` interpreter goes through the rules one by one, in declaration order. -For each rule, it tries to match the contents of the input token tree against that rule's `matcher`. -A matcher must match the *entirety* of the input to be considered a match. - -If the input matches the matcher, the invocation is replaced by the `expansion`; otherwise, the next rule is tried. -If all rules fail to match, the expansion fails with an error. - -The simplest example is of an empty matcher: - -```rust,ignore -macro_rules! four { - () => { 1 + 3 }; -} -``` - -This matches if and only if the input is also empty (*i.e.* `four!()`, `four![]` or `four!{}`). - -Note that the specific grouping tokens you use when you invoke the function-like macro *are not* matched, they are in fact not passed to the invocation at all. -That is, you can invoke the above macro as `four![]` and it will still match. -Only the *contents* of the input token tree are considered. - -Matchers can also contain literal token trees, which must be matched exactly. -This is done by simply writing the token trees normally. -For example, to match the sequence `4 fn ['spang "whammo"] @_@`, you would write: - -```rust,ignore -macro_rules! gibberish { - (4 fn ['spang "whammo"] @_@) => {...}; -} -``` - -You can use any token tree that you can write. - -## Metavariables - -Matchers can also contain captures. -These allow input to be matched based on some general grammar category, with the result captured to a metavariable which can then be substituted into the output. - -Captures are written as a dollar (`$`) followed by an identifier, a colon (`:`), and finally the kind of capture which is also called the fragment-specifier, which must be one of the following: - -* [`block`](./minutiae/fragment-specifiers.md#block): a block (i.e. a block of statements and/or an expression, surrounded by braces) -* [`expr`](./minutiae/fragment-specifiers.md#expr): an expression -* [`ident`](./minutiae/fragment-specifiers.md#ident): an identifier (this includes keywords) -* [`item`](./minutiae/fragment-specifiers.md#item): an item, like a function, struct, module, impl, etc. -* [`lifetime`](./minutiae/fragment-specifiers.md#lifetime): a lifetime (e.g. `'foo`, `'static`, ...) -* [`literal`](./minutiae/fragment-specifiers.md#literal): a literal (e.g. `"Hello World!"`, `3.14`, `'🦀'`, ...) -* [`meta`](./minutiae/fragment-specifiers.md#meta): a meta item; the things that go inside the `#[...]` and `#![...]` attributes -* [`pat`](./minutiae/fragment-specifiers.md#pat): a pattern -* [`path`](./minutiae/fragment-specifiers.md#path): a path (e.g. `foo`, `::std::mem::replace`, `transmute::<_, int>`, …) -* [`stmt`](./minutiae/fragment-specifiers.md#stmt): a statement -* [`tt`](./minutiae/fragment-specifiers.md#tt): a single token tree -* [`ty`](./minutiae/fragment-specifiers.md#ty): a type -* [`vis`](./minutiae/fragment-specifiers.md#vis): a possible empty visibility qualifier (e.g. `pub`, `pub(in crate)`, ...) - -For more in-depth description of the fragment specifiers, check out the [Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. - -For example, here is a `macro_rules!` macro which captures its input as an expression under the metavariable `$e`: - -```rust,ignore -macro_rules! one_expression { - ($e:expr) => {...}; -} -``` - -These metavariables leverage the Rust compiler's parser, ensuring that they are always "correct". -An `expr` metavariables will *always* capture a complete, valid expression for the version of Rust being compiled. - -You can mix literal token trees and metavariables, within limits (explained in [Metavariables and Expansion Redux]). - -To refer to a metavariable you simply write `$name`, as the type of the variable is already specified in the matcher. For example: - -```rust,ignore -macro_rules! times_five { - ($e:expr) => { 5 * $e }; -} -``` - -Much like macro expansion, metavariables are substituted as complete AST nodes. -This means that no matter what sequence of tokens is captured by `$e`, it will be interpreted as a single, complete expression. - -You can also have multiple metavariables in a single matcher: - -```rust,ignore -macro_rules! multiply_add { - ($a:expr, $b:expr, $c:expr) => { $a * ($b + $c) }; -} -``` - -And use them as often as you like in the expansion: - -```rust,ignore -macro_rules! discard { - ($e:expr) => {}; -} -macro_rules! repeat { - ($e:expr) => { $e; $e; $e; }; -} -``` - -There is also a special metavariable called [`$crate`] which can be used to refer to the current crate. - -[Metavariables and Expansion Redux]: ./minutiae/metavar-and-expansion.md -[`$crate`]: ./minutiae/hygiene.md#crate - -## Repetitions - -Matchers can contain repetitions. These allow a sequence of tokens to be matched. -These have the general form `$ ( ... ) sep rep`. - -* `$` is a literal dollar token. -* `( ... )` is the paren-grouped matcher being repeated. -* `sep` is an *optional* separator token. It may not be a delimiter or one - of the repetition operators. Common examples are `,` and `;`. -* `rep` is the *required* repeat operator. Currently, this can be: - * `?`: indicating at most one repetition - * `*`: indicating zero or more repetitions - * `+`: indicating one or more repetitions - - Since `?` represents at most one occurrence, it cannot be used with a separator. - -Repetitions can contain any other valid matcher, including literal token trees, metavariables, and other repetitions allowing arbitrary nesting. - -Repetitions use the same syntax in the expansion and repeated metavariables can only be accessed inside of repetitions in the expansion. - -For example, below is a mbe macro which formats each element as a string. -It matches zero or more comma-separated expressions and expands to an expression that constructs a vector. - -```rust -macro_rules! vec_strs { - ( - // Start a repetition: - $( - // Each repeat must contain an expression... - $element:expr - ) - // ...separated by commas... - , - // ...zero or more times. - * - ) => { - // Enclose the expansion in a block so that we can use - // multiple statements. - { - let mut v = Vec::new(); - - // Start a repetition: - $( - // Each repeat will contain the following statement, with - // $element replaced with the corresponding expression. - v.push(format!("{}", $element)); - )* - - v - } - }; -} - -fn main() { - let s = vec_strs![1, "a", true, 3.14159f32]; - assert_eq!(s, &["1", "a", "true", "3.14159"]); -} -``` - -You can repeat multiple metavariables in a single repetition as long as all metavariables repeat equally often. -So this invocation of the following macro works: - -```rust -macro_rules! repeat_two { - ($($i:ident)*, $($i2:ident)*) => { - $( let $i: (); let $i2: (); )* - } -} - -repeat_two!( a b c d e f, u v w x y z ); -``` - -But this does not: - -```rust -# macro_rules! repeat_two { -# ($($i:ident)*, $($i2:ident)*) => { -# $( let $i: (); let $i2: (); )* -# } -# } - -repeat_two!( a b c d e f, x y z ); -``` - -failing with the following error - -``` -error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times - --> src/main.rs:6:10 - | -6 | $( let $i: (); let $i2: (); )* - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``` - -## Metavariable Expressions - -> *RFC*: [rfcs#1584](https://github.com/rust-lang/rfcs/blob/master/text/3086-macro-metavar-expr.md)\ -> *Tracking Issue*: [rust#83527](https://github.com/rust-lang/rust/issues/83527)\ -> *Feature*: `#![feature(macro_metavar_expr)]` - -Transcriber can contain what is called metavariable expressions. -Metavariable expressions provide transcribers with information about metavariables that are otherwise not easily obtainable. -With the exception of the `$$` expression, these have the general form `$ { op(...) }`. -Currently all metavariable expressions but `$$` deal with repetitions. - -The following expressions are available with `ident` being the name of a bound metavariable and `depth` being an integer literal: - -- `${count(ident)}`: The number of times `$ident` repeats in the inner-most repetition in total. This is equivalent to `${count(ident, 0)}`. -- `${count(ident, depth)}`: The number of times `$ident` repeats in the repetition at `depth`. -- `${index()}`: The current repetition index of the inner-most repetition. This is equivalent to `${index(0)}`. -- `${index(depth)}`: The current index of the repetition at `depth`, counting outwards. -- `${length()}`: The number of times the inner-most repetition will repeat for. This is equivalent to `${length(0)}`. -- `${length(depth)}`: The number of times the repetition at `depth` will repeat for, counting outwards. -- `${ignore(ident)}`: Binds `$ident` for repetition, while expanding to nothing. -- `$$`: Expands to a single `$`, effectively escaping the `$` token so it won't be transcribed. - -  - -For the complete grammar definition you may want to consult the [Macros By Example](https://doc.rust-lang.org/reference/macros-by-example.html#macros-by-example) chapter of the Rust reference. diff --git a/src/decl-macros/macros-methodical.md b/src/decl-macros/macros-methodical.md index 2801366..8937425 100644 --- a/src/decl-macros/macros-methodical.md +++ b/src/decl-macros/macros-methodical.md @@ -5,3 +5,263 @@ It will do so by first going into the construct's syntax and its key parts and t [mbe]: https://doc.rust-lang.org/reference/macros-by-example.html [Macros chapter of the Rust Book]: https://doc.rust-lang.org/book/ch19-06-macros.html + +# `macro_rules!` + +With all that in mind, we can introduce `macro_rules!` itself. +As noted previously, `macro_rules!` is *itself* a syntax extension, meaning it is *technically* not part of the Rust syntax. +It uses the following forms: + +```rust,ignore +macro_rules! $name { + $rule0 ; + $rule1 ; + // … + $ruleN ; +} +``` + +There must be *at least* one rule, and you can omit the semicolon after the last rule. +You can use brackets(`[]`), parentheses(`()`) or braces(`{}`). + +Each *"rule"* looks like the following: + +```ignore + ($matcher) => {$expansion} +``` + +Like before, the types of parentheses used can be any kind, but parentheses around the matcher and braces around the expansion are somewhat conventional. +The expansion part of a rule is also called its *transcriber*. + +Note that the choice of the parentheses does not matter in regards to how the mbe macro may be invoked. +In fact, function-like macros can be invoked with any kind of parentheses as well, but invocations with `{ .. }` and `( ... );`, notice the trailing semicolon, are special in that their expansion will *always* be parsed as an *item*. + +If you are wondering, the `macro_rules!` invocation expands to... *nothing*. +At least, nothing that appears in the AST; rather, it manipulates compiler-internal structures to register the mbe macro. +As such, you can *technically* use `macro_rules!` in any position where an empty expansion is valid. + +## Matching + +When a `macro_rules!` macro is invoked, the `macro_rules!` interpreter goes through the rules one by one, in declaration order. +For each rule, it tries to match the contents of the input token tree against that rule's `matcher`. +A matcher must match the *entirety* of the input to be considered a match. + +If the input matches the matcher, the invocation is replaced by the `expansion`; otherwise, the next rule is tried. +If all rules fail to match, the expansion fails with an error. + +The simplest example is of an empty matcher: + +```rust,ignore +macro_rules! four { + () => { 1 + 3 }; +} +``` + +This matches if and only if the input is also empty (*i.e.* `four!()`, `four![]` or `four!{}`). + +Note that the specific grouping tokens you use when you invoke the function-like macro *are not* matched, they are in fact not passed to the invocation at all. +That is, you can invoke the above macro as `four![]` and it will still match. +Only the *contents* of the input token tree are considered. + +Matchers can also contain literal token trees, which must be matched exactly. +This is done by simply writing the token trees normally. +For example, to match the sequence `4 fn ['spang "whammo"] @_@`, you would write: + +```rust,ignore +macro_rules! gibberish { + (4 fn ['spang "whammo"] @_@) => {...}; +} +``` + +You can use any token tree that you can write. + +## Metavariables + +Matchers can also contain captures. +These allow input to be matched based on some general grammar category, with the result captured to a metavariable which can then be substituted into the output. + +Captures are written as a dollar (`$`) followed by an identifier, a colon (`:`), and finally the kind of capture which is also called the fragment-specifier, which must be one of the following: + +* [`block`](./minutiae/fragment-specifiers.md#block): a block (i.e. a block of statements and/or an expression, surrounded by braces) +* [`expr`](./minutiae/fragment-specifiers.md#expr): an expression +* [`ident`](./minutiae/fragment-specifiers.md#ident): an identifier (this includes keywords) +* [`item`](./minutiae/fragment-specifiers.md#item): an item, like a function, struct, module, impl, etc. +* [`lifetime`](./minutiae/fragment-specifiers.md#lifetime): a lifetime (e.g. `'foo`, `'static`, ...) +* [`literal`](./minutiae/fragment-specifiers.md#literal): a literal (e.g. `"Hello World!"`, `3.14`, `'🦀'`, ...) +* [`meta`](./minutiae/fragment-specifiers.md#meta): a meta item; the things that go inside the `#[...]` and `#![...]` attributes +* [`pat`](./minutiae/fragment-specifiers.md#pat): a pattern +* [`path`](./minutiae/fragment-specifiers.md#path): a path (e.g. `foo`, `::std::mem::replace`, `transmute::<_, int>`, …) +* [`stmt`](./minutiae/fragment-specifiers.md#stmt): a statement +* [`tt`](./minutiae/fragment-specifiers.md#tt): a single token tree +* [`ty`](./minutiae/fragment-specifiers.md#ty): a type +* [`vis`](./minutiae/fragment-specifiers.md#vis): a possible empty visibility qualifier (e.g. `pub`, `pub(in crate)`, ...) + +For more in-depth description of the fragment specifiers, check out the [Fragment Specifiers](./minutiae/fragment-specifiers.md) chapter. + +For example, here is a `macro_rules!` macro which captures its input as an expression under the metavariable `$e`: + +```rust,ignore +macro_rules! one_expression { + ($e:expr) => {...}; +} +``` + +These metavariables leverage the Rust compiler's parser, ensuring that they are always "correct". +An `expr` metavariables will *always* capture a complete, valid expression for the version of Rust being compiled. + +You can mix literal token trees and metavariables, within limits (explained in [Metavariables and Expansion Redux]). + +To refer to a metavariable you simply write `$name`, as the type of the variable is already specified in the matcher. For example: + +```rust,ignore +macro_rules! times_five { + ($e:expr) => { 5 * $e }; +} +``` + +Much like macro expansion, metavariables are substituted as complete AST nodes. +This means that no matter what sequence of tokens is captured by `$e`, it will be interpreted as a single, complete expression. + +You can also have multiple metavariables in a single matcher: + +```rust,ignore +macro_rules! multiply_add { + ($a:expr, $b:expr, $c:expr) => { $a * ($b + $c) }; +} +``` + +And use them as often as you like in the expansion: + +```rust,ignore +macro_rules! discard { + ($e:expr) => {}; +} +macro_rules! repeat { + ($e:expr) => { $e; $e; $e; }; +} +``` + +There is also a special metavariable called [`$crate`] which can be used to refer to the current crate. + +[Metavariables and Expansion Redux]: ./minutiae/metavar-and-expansion.md +[`$crate`]: ./minutiae/hygiene.md#crate + +## Repetitions + +Matchers can contain repetitions. These allow a sequence of tokens to be matched. +These have the general form `$ ( ... ) sep rep`. + +* `$` is a literal dollar token. +* `( ... )` is the paren-grouped matcher being repeated. +* `sep` is an *optional* separator token. It may not be a delimiter or one + of the repetition operators. Common examples are `,` and `;`. +* `rep` is the *required* repeat operator. Currently, this can be: + * `?`: indicating at most one repetition + * `*`: indicating zero or more repetitions + * `+`: indicating one or more repetitions + + Since `?` represents at most one occurrence, it cannot be used with a separator. + +Repetitions can contain any other valid matcher, including literal token trees, metavariables, and other repetitions allowing arbitrary nesting. + +Repetitions use the same syntax in the expansion and repeated metavariables can only be accessed inside of repetitions in the expansion. + +For example, below is a mbe macro which formats each element as a string. +It matches zero or more comma-separated expressions and expands to an expression that constructs a vector. + +```rust +macro_rules! vec_strs { + ( + // Start a repetition: + $( + // Each repeat must contain an expression... + $element:expr + ) + // ...separated by commas... + , + // ...zero or more times. + * + ) => { + // Enclose the expansion in a block so that we can use + // multiple statements. + { + let mut v = Vec::new(); + + // Start a repetition: + $( + // Each repeat will contain the following statement, with + // $element replaced with the corresponding expression. + v.push(format!("{}", $element)); + )* + + v + } + }; +} + +fn main() { + let s = vec_strs![1, "a", true, 3.14159f32]; + assert_eq!(s, &["1", "a", "true", "3.14159"]); +} +``` + +You can repeat multiple metavariables in a single repetition as long as all metavariables repeat equally often. +So this invocation of the following macro works: + +```rust +macro_rules! repeat_two { + ($($i:ident)*, $($i2:ident)*) => { + $( let $i: (); let $i2: (); )* + } +} + +repeat_two!( a b c d e f, u v w x y z ); +``` + +But this does not: + +```rust +# macro_rules! repeat_two { +# ($($i:ident)*, $($i2:ident)*) => { +# $( let $i: (); let $i2: (); )* +# } +# } + +repeat_two!( a b c d e f, x y z ); +``` + +failing with the following error + +``` +error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times + --> src/main.rs:6:10 + | +6 | $( let $i: (); let $i2: (); )* + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``` + +## Metavariable Expressions + +> *RFC*: [rfcs#1584](https://github.com/rust-lang/rfcs/blob/master/text/3086-macro-metavar-expr.md)\ +> *Tracking Issue*: [rust#83527](https://github.com/rust-lang/rust/issues/83527)\ +> *Feature*: `#![feature(macro_metavar_expr)]` + +Transcriber can contain what is called metavariable expressions. +Metavariable expressions provide transcribers with information about metavariables that are otherwise not easily obtainable. +With the exception of the `$$` expression, these have the general form `$ { op(...) }`. +Currently all metavariable expressions but `$$` deal with repetitions. + +The following expressions are available with `ident` being the name of a bound metavariable and `depth` being an integer literal: + +- `${count(ident)}`: The number of times `$ident` repeats in the inner-most repetition in total. This is equivalent to `${count(ident, 0)}`. +- `${count(ident, depth)}`: The number of times `$ident` repeats in the repetition at `depth`. +- `${index()}`: The current repetition index of the inner-most repetition. This is equivalent to `${index(0)}`. +- `${index(depth)}`: The current index of the repetition at `depth`, counting outwards. +- `${length()}`: The number of times the inner-most repetition will repeat for. This is equivalent to `${length(0)}`. +- `${length(depth)}`: The number of times the repetition at `depth` will repeat for, counting outwards. +- `${ignore(ident)}`: Binds `$ident` for repetition, while expanding to nothing. +- `$$`: Expands to a single `$`, effectively escaping the `$` token so it won't be transcribed. + +  + +For the complete grammar definition you may want to consult the [Macros By Example](https://doc.rust-lang.org/reference/macros-by-example.html#macros-by-example) chapter of the Rust reference. diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index abf9673..ca1b0ab 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -61,8 +61,8 @@ That rule says the input to the invocation must match: - the literal token sequence `,` `...` `,`, - a valid *expression* captured into the [metavariable] `recur` (`$recur:expr`). -[repeating]: ./macro_rules.md#repetitions -[metavariable]: ./macro_rules.md#metavariables +[repeating]: ./macros-methodical.md#repetitions +[metavariable]: ./macros-methodical.md#metavariables Finally, the rule says that *if* the input matches this rule, then the invocation should be replaced by the token sequence `/* ... */`. @@ -200,7 +200,7 @@ Here, I've added a new metavariable: `sty` which should be a type. > **Aside**: if you're wondering, the bit after the colon in a metavariable can be one of several kinds of syntax matchers. > The most common ones are `item`, `expr`, and `ty`. -> A complete explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macro_rules.md#Metavariables). +> A complete explanation can be found in [Macros, A Methodical Introduction; `macro_rules!` (Matchers)](./macros-methodical.md#metavariables). > > There's one other thing to be aware of: in the interests of future-proofing the language, the compiler restricts what tokens you're allowed to put *after* a matcher, depending on what kind it is. > Typically, this comes up when trying to match expressions or statements; @@ -1320,4 +1320,4 @@ Which gives us: Success! -[`macro_rules!`]: ./macro_rules.md +[`macro_rules!`]: ./macros-methodical.md diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index 949f928..a01428d 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -1,6 +1,6 @@ # Fragment Specifiers -As shown in the [`macro_rules`] chapter, Rust, as of 1.60, has 14 fragment specifiers. +As mentioned in the [`methodical introduction`](../macros-methodical.md) chapter, Rust, as of 1.60, has 14 fragment specifiers. This section will go a bit more into detail for some of them and shows a few example inputs of what each matcher matches. > **Note**: Capturing with anything but the `ident`, `lifetime` and `tt` fragments will render the captured AST opaque, making it impossible to further match it with other fragment specifiers in future macro invocations. @@ -326,7 +326,7 @@ visibilities! { } ``` -While able to match empty sequences of tokens, the fragment specifier still acts quite different from [optional repetitions](../macro_rules.md#repetitions) which is described in the following: +While able to match empty sequences of tokens, the fragment specifier still acts quite different from [optional repetitions](../macros-methodical.md#repetitions) which is described in the following: If it is being matched against no left over tokens the entire macro matching fails. ```rust diff --git a/src/decl-macros/minutiae/metavar-expr.md b/src/decl-macros/minutiae/metavar-expr.md index beed3ac..8ea419c 100644 --- a/src/decl-macros/minutiae/metavar-expr.md +++ b/src/decl-macros/minutiae/metavar-expr.md @@ -6,8 +6,7 @@ > Note: The example code snippets are very bare bones, trying to show off how they work. If you think you got small snippets with proper isolated usage of these expression please submit them! - -As shown in the [`macro_rules`](../macro_rules.md) chapter, Rust has special expressions that can be used by macro transcribers to obtain information about metavariables that are otherwise difficult or even impossible to get. +As mentioned in the [`methodical introduction`](../macros-methodical.md), Rust has special expressions that can be used by macro transcribers to obtain information about metavariables that are otherwise difficult or even impossible to get. This chapter will introduce them more in-depth together with usage examples. - [`$$`](#dollar-dollar-) From 7f27999721726fd4d1eab855ea6200eb00a128f2 Mon Sep 17 00:00:00 2001 From: zjp Date: Sun, 17 Apr 2022 22:21:07 +0800 Subject: [PATCH 59/92] Change recursion limit number to 128 close https://github.com/Veykril/tlborm/issues/63 --- src/syntax-extensions/expansion.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/syntax-extensions/expansion.md b/src/syntax-extensions/expansion.md index 2295814..bb84787 100644 --- a/src/syntax-extensions/expansion.md +++ b/src/syntax-extensions/expansion.md @@ -106,8 +106,8 @@ as many as is needed to completely expand all invocations. Well, not *quite*. In fact, the compiler imposes an upper limit on the number of such recursive passes it is willing to run before giving up. -This is known as the syntax extension recursion limit and defaults to 32. -If the 32nd expansion contains a syntax extension invocation, the compiler will abort with an error indicating that the recursion limit was exceeded. +This is known as the syntax extension recursion limit and defaults to 128. +If the 128nd expansion contains a syntax extension invocation, the compiler will abort with an error indicating that the recursion limit was exceeded. This limit can be raised using the `#![recursion_limit="…"]` [attribute][recursion_limit], though it *must* be done crate-wide. Generally, it is recommended to try and keep syntax extension below this limit wherever possible as it may impact compilation times. From f7ef4fad36d8a81926d6bff590f842b7c8a818fb Mon Sep 17 00:00:00 2001 From: zjp Date: Sun, 17 Apr 2022 22:25:21 +0800 Subject: [PATCH 60/92] Change the captured number of `stmt` example in fragment-specifiers.md --- src/decl-macros/minutiae/fragment-specifiers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index a01428d..4695b20 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -267,7 +267,7 @@ From this we can tell a few things. The first you should be able to see immediately is that while the `stmt` fragment doesn't capture trailing semicolons, it still emits them when required, even if the statement is already followed by one. The simple reason for that is that semicolons on their own are already valid statements which the fragment captures eagerly. -So our macro isn't capturing 8 times, but 11! +So our macro isn't capturing 8 times, but 10! This can be important when doing multiples repetitions and expanding these in one repetition expansion, as the repetition numbers have to match in those cases. Another thing you should be able to notice here is that the trailing semicolon of the `struct Foo;` item statement is being matched, otherwise we would've seen an extra one like in the other cases. From 4ee483e55f6653e0d1f8bf4335e202f39661b8fe Mon Sep 17 00:00:00 2001 From: zjp Date: Sun, 17 Apr 2022 22:29:14 +0800 Subject: [PATCH 61/92] Update the default edition for code blocks to 2021 This is important because the example on https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#pat is failed due to or-patterns only allowed since 2021 edition. --- book.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/book.toml b/book.toml index 4692c5f..3c7d79c 100644 --- a/book.toml +++ b/book.toml @@ -45,3 +45,7 @@ level = 1 "/decl-macros/syntax/ast.html" = "../../syntax-extensions/ast.html" "/decl-macros/syntax/expansion.html" = "../../syntax-extensions/expansion.html" "/decl-macros/syntax/source-analysis.html" = "../../syntax-extensions/source-analysis.html" + +[rust] +# see: https://rust-lang.github.io/mdBook/format/configuration/general.html#rust-options +edition = "2021" From 7d7f7274f8df59f76a8b70bdc0e5863d1f0ea09f Mon Sep 17 00:00:00 2001 From: ZJPzjp Date: Mon, 18 Apr 2022 00:39:33 +0800 Subject: [PATCH 62/92] Remove the needless whitespace Co-authored-by: Lukas Wirth --- src/syntax-extensions/expansion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/expansion.md b/src/syntax-extensions/expansion.md index bb84787..a19ba13 100644 --- a/src/syntax-extensions/expansion.md +++ b/src/syntax-extensions/expansion.md @@ -106,7 +106,7 @@ as many as is needed to completely expand all invocations. Well, not *quite*. In fact, the compiler imposes an upper limit on the number of such recursive passes it is willing to run before giving up. -This is known as the syntax extension recursion limit and defaults to 128. +This is known as the syntax extension recursion limit and defaults to 128. If the 128nd expansion contains a syntax extension invocation, the compiler will abort with an error indicating that the recursion limit was exceeded. This limit can be raised using the `#![recursion_limit="…"]` [attribute][recursion_limit], though it *must* be done crate-wide. From 5c5332d7a7ed2ba2eb826b0dc5b60d8240cbaf2a Mon Sep 17 00:00:00 2001 From: ZJPzjp Date: Mon, 18 Apr 2022 00:40:11 +0800 Subject: [PATCH 63/92] Fix a typo Co-authored-by: Lukas Wirth --- src/syntax-extensions/expansion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/expansion.md b/src/syntax-extensions/expansion.md index a19ba13..ed22854 100644 --- a/src/syntax-extensions/expansion.md +++ b/src/syntax-extensions/expansion.md @@ -107,7 +107,7 @@ as many as is needed to completely expand all invocations. Well, not *quite*. In fact, the compiler imposes an upper limit on the number of such recursive passes it is willing to run before giving up. This is known as the syntax extension recursion limit and defaults to 128. -If the 128nd expansion contains a syntax extension invocation, the compiler will abort with an error indicating that the recursion limit was exceeded. +If the 128th expansion contains a syntax extension invocation, the compiler will abort with an error indicating that the recursion limit was exceeded. This limit can be raised using the `#![recursion_limit="…"]` [attribute][recursion_limit], though it *must* be done crate-wide. Generally, it is recommended to try and keep syntax extension below this limit wherever possible as it may impact compilation times. From 46648036bf68970541832372a0681130fdcb9318 Mon Sep 17 00:00:00 2001 From: Nihaal Sangha Date: Thu, 21 Apr 2022 01:16:45 +0100 Subject: [PATCH 64/92] Add missing backtick --- src/decl-macros/macros2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/macros2.md b/src/decl-macros/macros2.md index 364d5af..e3883c0 100644 --- a/src/decl-macros/macros2.md +++ b/src/decl-macros/macros2.md @@ -4,7 +4,7 @@ > *Tracking Issue*: [rust#39412](https://github.com/rust-lang/rust/issues/39412)\ > *Feature*: `#![feature(decl_macro)]` -While not yet stable(or rather far from being finished), there is proposal for a new declarative macro system that is supposed to replace `macro_rules!` dubbed declarative macros 2.0, `macro`, `decl_macro` or confusingly also `macros-by-example. +While not yet stable(or rather far from being finished), there is proposal for a new declarative macro system that is supposed to replace `macro_rules!` dubbed declarative macros 2.0, `macro`, `decl_macro` or confusingly also `macros-by-example`. This chapter is only meant to quickly glance over the current state, showing how to use this macro system and where it differs. Nothing described here is final or complete, and may be subject to change. From 244cd8e948a57888ab8c17eaf89891d1b9b364dd Mon Sep 17 00:00:00 2001 From: zjp Date: Tue, 19 Apr 2022 00:38:25 +0800 Subject: [PATCH 65/92] Add Edition Differences statement for `pat` src: https://doc.rust-lang.org/reference/macros-by-example.html#follow-set-ambiguity-restrictions --- src/decl-macros/minutiae/metavar-and-expansion.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/decl-macros/minutiae/metavar-and-expansion.md b/src/decl-macros/minutiae/metavar-and-expansion.md index 43b11ed..c22b274 100644 --- a/src/decl-macros/minutiae/metavar-and-expansion.md +++ b/src/decl-macros/minutiae/metavar-and-expansion.md @@ -25,7 +25,7 @@ restricts what can follow various metavariables. The complete list, showing what may follow what fragment specifier, as of Rust 1.46 is as follows: * [`stmt`] and [`expr`]: `=>`, `,`, or `;` -* [`pat`]: `=>`, `,`, `=`, `if`, `in` +* [`pat`]: `=>`, `,`, `=`, `if`, `in`[^pat-edition] * [`pat_param`]: `=>`, `,`, `=`, `|`, `if`, `in` * [`path`] and [`ty`]:`=>`, `,`, `=`, `|`, `;`, `:`, `>`, `>>`, `[`, `{`, `as`, `where`, or a macro variable of the [`block`] fragment specifier. @@ -33,6 +33,8 @@ The complete list, showing what may follow what fragment specifier, as of Rust 1 metavariable with an [`ident`], [`ty`], or [`path`] fragment specifier. * All other fragment specifiers have no restrictions. +[^pat-edition]: **Edition Differences**: Before the 2021 edition, `pat` may also be followed by `|`. + Repetitions also adhere to these restrictions, meaning if a repetition can repeat multiple times(`*` or `+`), then the contents must be able to follow themselves. If a repetition can repeat zero times (`?` or `*`) then what comes after the repetition must be able to follow what comes before. From b077c4ab62a15968981e78e7d878dc757cbf4fd1 Mon Sep 17 00:00:00 2001 From: zjp Date: Tue, 19 Apr 2022 00:40:26 +0800 Subject: [PATCH 66/92] Fix broken links and Add a hack example before `$$` occurs The hack example comes from https://github.com/rust-lang/rust/pull/95860. --- src/decl-macros/minutiae/metavar-expr.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/decl-macros/minutiae/metavar-expr.md b/src/decl-macros/minutiae/metavar-expr.md index 8ea419c..6338096 100644 --- a/src/decl-macros/minutiae/metavar-expr.md +++ b/src/decl-macros/minutiae/metavar-expr.md @@ -11,9 +11,9 @@ This chapter will introduce them more in-depth together with usage examples. - [`$$`](#dollar-dollar-) - [`${count(ident, depth)}`](#countident-depth) -- [`${index(depth)}`](#index-depth) -- [`${length(depth)}`](#length-depth) -- [`${ignore(ident)}`](#ignore-ident) +- [`${index(depth)}`](#indexdepth) +- [`${length(depth)}`](#lengthdepth) +- [`${ignore(ident)}`](#ignoreident) ## Dollar Dollar (`$$`) @@ -36,7 +36,7 @@ foo!(); ``` The problem is obvious, the transcriber of foo sees a repetition and tries to repeat it when transcribing, but there is no `$any` metavariable in its scope causing it to fail. -With `$$` we can get around this as the transcriber of `foo` will no longer try to do the repetition. +With `$$` we can get around this as the transcriber of `foo` will no longer try to do the repetition.[^tt-$] ```rust #![feature(macro_metavar_expr)] @@ -54,6 +54,9 @@ bar!(); # fn main() {} ``` +[^tt-$]: Before `$$` occurs, users must resort to a tricky and not so well-known hack to declare nested macros with repetitions + [via using `$tt` like this](https://play.rust-lang.org/?version=nightly&mode=debug&edition=2021&gist=9ce18fc79ce17c77d20e74f3c46ee13c). + ## `count(ident, depth)` The `count` metavariable expression expands to the repetition count of the metavariable `$ident` at the given repetition depth. From 7b7b1dbc7596b5e8d545c93766c85a003ed3d3b3 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 23 Apr 2022 01:55:12 +0200 Subject: [PATCH 67/92] Widen the table in the practical decl macro introduction --- src/decl-macros/macros-practical-table.html | 52 ++++++++++++++------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/decl-macros/macros-practical-table.html b/src/decl-macros/macros-practical-table.html index 1702c74..c436bac 100644 --- a/src/decl-macros/macros-practical-table.html +++ b/src/decl-macros/macros-practical-table.html @@ -5,6 +5,12 @@ font-size: 70%; } + .parse-table { + --table-width: 900px; + width: var(--table-width); + margin-left: calc((var(--content-max-width) - var(--table-width)) / 2); + } + table.parse-table code { white-space: pre-wrap; background-color: transparent; @@ -29,9 +35,9 @@ - + - + @@ -44,56 +50,64 @@ + ⌂ + + ⌂ + + ⌂ + + ⌂ + + ⌂ + + ⌂ + + ⌂ + + ⌂ ⌂ + @@ -108,14 +122,16 @@ + ⌂ ⌂ + + ⌂ ⌂ ⌂ + @@ -128,28 +144,32 @@ + ⌂ ⌂ + + ⌂ ⌂ + + ⌂ + + ⌂ + From d102afbfea0274f79fc49626ba668d48d0865fd1 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 11 May 2022 16:06:41 +0200 Subject: [PATCH 68/92] fix: Fix incorrect description of count metavar expression --- src/decl-macros/minutiae/metavar-expr.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/minutiae/metavar-expr.md b/src/decl-macros/minutiae/metavar-expr.md index 6338096..f93c30c 100644 --- a/src/decl-macros/minutiae/metavar-expr.md +++ b/src/decl-macros/minutiae/metavar-expr.md @@ -59,13 +59,13 @@ bar!(); ## `count(ident, depth)` -The `count` metavariable expression expands to the repetition count of the metavariable `$ident` at the given repetition depth. +The `count` metavariable expression expands to the repetition count of the metavariable `$ident` up to the given repetition depth. - The `ident` argument must be a declared metavariable in the scope of the rule. - The `depth` argument must be an integer literal of value less or equal to the maximum repetition depth that the `$ident` metavariable appears in. - The expression expands to an unsuffixed integer literal token. -The `count(ident)` expression defaults `depth` to `0`, making it a shorthand for `count(ident, 0)`. +The `count(ident)` expression defaults `depth` to the maximum valid depth, making it count the total repetitions for the given metavariable. ```rust #![feature(macro_metavar_expr)] From bfa419d1632e749329a0c6a61018a3f1994a237b Mon Sep 17 00:00:00 2001 From: Marijn Schouten Date: Tue, 21 Jun 2022 19:01:05 +0200 Subject: [PATCH 69/92] fix grammar around 'consist' --- src/decl-macros/patterns/internal-rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/patterns/internal-rules.md b/src/decl-macros/patterns/internal-rules.md index 06b145d..7b4edcf 100644 --- a/src/decl-macros/patterns/internal-rules.md +++ b/src/decl-macros/patterns/internal-rules.md @@ -45,7 +45,7 @@ macro_rules! foo { This is definitely not the nicest solution we could have for this macro, as it pollutes the global macro namespace as mentioned earlier. In this specific case `as_expr` is also a very simple macro that we only used once, so let's "embed" this macro in our `foo` macro with internal rules! -To do so we simply prepend a new matcher for our macro consists of the matcher used in the `as_expr` macro, but with a small addition. +To do so, we simply prepend a new matcher for our macro, which consists of the matcher used in the `as_expr` macro, but with a small addition. We prepend a tokentree that makes it match only when specifically asked to. In this case we can for example use `@as_expr`, so our matcher becomes `(@as_expr $e:expr) => {$e};`. With this we get the macro that was defined at the very top of this page: From 0897de12772c1acdc373d4067f2b29d8bfe473a7 Mon Sep 17 00:00:00 2001 From: Marijn Schouten Date: Tue, 21 Jun 2022 18:57:25 +0200 Subject: [PATCH 70/92] expand 'mbe' --- src/decl-macros/patterns/internal-rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/patterns/internal-rules.md b/src/decl-macros/patterns/internal-rules.md index 7b4edcf..dca37b8 100644 --- a/src/decl-macros/patterns/internal-rules.md +++ b/src/decl-macros/patterns/internal-rules.md @@ -17,7 +17,7 @@ macro_rules! foo { Internal rules can be used to unify multiple `macro_rules!` macros into one, or to make it easier to read and write [TT Munchers] by explicitly naming what rule you wish to call in a macro. -So why is it useful to unify multiple mbe macros into one? +So why is it useful to unify multiple macros-by-example into one? The main reasoning for this is how they are handled in the 2015 Edition of Rust due to `macro_rules!` macros not being namespaced in said edition. This gives one the troubles of having to re-export all the internal `macro_rules!` macros as well polluting the global macro namespace or even worse, macro name collisions with other crates. In short, it's quite a hassle. From b2aa1dbf4d14f2ecd70efba772503d6283f5a6f9 Mon Sep 17 00:00:00 2001 From: Marijn Schouten Date: Tue, 21 Jun 2022 18:53:09 +0200 Subject: [PATCH 71/92] add missing 'as' --- src/decl-macros/patterns/internal-rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/patterns/internal-rules.md b/src/decl-macros/patterns/internal-rules.md index dca37b8..1389183 100644 --- a/src/decl-macros/patterns/internal-rules.md +++ b/src/decl-macros/patterns/internal-rules.md @@ -19,7 +19,7 @@ Internal rules can be used to unify multiple `macro_rules!` macros into one, or So why is it useful to unify multiple macros-by-example into one? The main reasoning for this is how they are handled in the 2015 Edition of Rust due to `macro_rules!` macros not being namespaced in said edition. -This gives one the troubles of having to re-export all the internal `macro_rules!` macros as well polluting the global macro namespace or even worse, macro name collisions with other crates. +This gives one the troubles of having to re-export all the internal `macro_rules!` macros as well as polluting the global macro namespace or even worse, macro name collisions with other crates. In short, it's quite a hassle. This fortunately isn't really a problem anymore nowadays with a rustc version >= 1.30, for more information consult the [Import and Export chapter](../minutiae/import-export.html). From 5d9ab7aef2aac940993ec8a6f3661a08a86ab721 Mon Sep 17 00:00:00 2001 From: Heinenen Date: Sun, 26 Jun 2022 14:57:08 +0200 Subject: [PATCH 72/92] fix typos --- src/decl-macros/building-blocks/parsing.md | 2 +- src/decl-macros/minutiae/scoping.md | 2 +- src/decl-macros/patterns/internal-rules.md | 2 +- src/proc-macros/methodical/attr.md | 2 +- src/proc-macros/methodical/derive.md | 2 +- src/syntax-extensions/ast.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/decl-macros/building-blocks/parsing.md b/src/decl-macros/building-blocks/parsing.md index 8d200f6..ad8eda2 100644 --- a/src/decl-macros/building-blocks/parsing.md +++ b/src/decl-macros/building-blocks/parsing.md @@ -40,7 +40,7 @@ macro_rules! function_item_matcher { # } ``` -A simple function matcher that ignores qualifiers like `unsafe`, `async`, ... as well a generics and where clauses. +A simple function matcher that ignores qualifiers like `unsafe`, `async`, ... as well as generics and where clauses. If parsing those is required it is likely that you are better off using a proc-macro instead. This lets you for example, inspect the function signature, generate some extra things from it and then re-emit the entire function again. diff --git a/src/decl-macros/minutiae/scoping.md b/src/decl-macros/minutiae/scoping.md index e886206..190e9ea 100644 --- a/src/decl-macros/minutiae/scoping.md +++ b/src/decl-macros/minutiae/scoping.md @@ -132,7 +132,7 @@ mod c { # fn main() {} ``` -Another complication is that `#[macro_use]` applied to an `extern crate` *does not* behave this way: such declarations are effectively *hoisted* to the top of the module. Thus, assuming `X!` is defined in an external crate called `mac`, the following holds: +Another complication is that `#[macro_use]` applied to an `extern crate` *does not* behave this way: such declarations are effectively *hoisted* to the top of the module. Thus, assuming `X!` is defined in an external crate called `macs`, the following holds: ```rust,ignore mod a { diff --git a/src/decl-macros/patterns/internal-rules.md b/src/decl-macros/patterns/internal-rules.md index 1389183..811d253 100644 --- a/src/decl-macros/patterns/internal-rules.md +++ b/src/decl-macros/patterns/internal-rules.md @@ -71,7 +71,7 @@ If you look closely you might even see that this pattern can be combined quite n The reason for using `@` was that, as of Rust 1.2, the `@` token is *not* used in prefix position; as such, it cannot conflict with anything. This reasoning became obsolete later on when in Rust 1.7 macro matchers got future proofed by emitting a warning to prevent certain tokens from being allowed to follow certain fragments[^ambiguity-restrictions], which in Rust 1.12 became a hard-error. -There other symbols or unique prefixes may be used as desired, but use of `@` has started to become widespread, so using it may aid readers in understanding your macro. +Other symbols or unique prefixes may be used as desired, but use of `@` has started to become widespread, so using it may aid readers in understanding your macro. [^ambiguity-restrictions]:[ambiguity-restrictions](../minutiae/metavar-and-expansion.html) diff --git a/src/proc-macros/methodical/attr.md b/src/proc-macros/methodical/attr.md index 2c82c22..5a0e6c3 100644 --- a/src/proc-macros/methodical/attr.md +++ b/src/proc-macros/methodical/attr.md @@ -14,7 +14,7 @@ pub fn tlborm_attribute(input: TokenStream, annotated_item: TokenStream) -> Toke ``` Of note here is that unlike the other two procedural macro kinds, this one has two input parameters instead of one. -- The first parameter is the delimited token tree following the attribute's, name excluding the delimiters around it. +- The first parameter is the delimited token tree following the attribute's name, excluding the delimiters around it. It is empty if the attribute is written bare, that is just a name without a `(TokenTree)` following it, e.g. `#[attr]`. - The second token stream is the item the attribute is attached to *without* the attribute this proc macro defines. As this is an [`active`](https://doc.rust-lang.org/reference/attributes.html#active-and-inert-attributes) attribute, the attribute will be stripped from the item before it is being passed to the proc macro. diff --git a/src/proc-macros/methodical/derive.md b/src/proc-macros/methodical/derive.md index bb2ee9b..82708f4 100644 --- a/src/proc-macros/methodical/derive.md +++ b/src/proc-macros/methodical/derive.md @@ -14,7 +14,7 @@ pub fn tlborm_derive(item: TokenStream) -> TokenStream { ``` The `proc_macro_derive` is a bit more special in that it requires an extra identifier, this identifier will become the actual name of the derive proc macro. -The input token stream is the item the derive attribute is attached to, that is, it will always be an `enum`, `struct` or `union` as these are the only items a derive attribute can annotated. +The input token stream is the item the derive attribute is attached to, that is, it will always be an `enum`, `struct` or `union` as these are the only items a derive attribute can annotate. The returned token stream will be **appended** to the containing block or module of the annotated item with the requirement that the token stream consists of a set of valid items. Usage example: diff --git a/src/syntax-extensions/ast.md b/src/syntax-extensions/ast.md index 44052ae..6acec8d 100644 --- a/src/syntax-extensions/ast.md +++ b/src/syntax-extensions/ast.md @@ -82,7 +82,7 @@ This means ⬚ can be anything, even invalid Rust! As to why this is a good thing, we will come back to that at a later point. So, does this also apply to `$arg` in form 1 and 2, and to the two args in form 4? Kind of. -The `$arg$` for form 1 and 2 is a bit different in that it is not directly a token tree, but a *simple path* that is either followed by an `=` token and a literal expression, or a token tree. +The `$arg` for form 1 and 2 is a bit different in that it is not directly a token tree, but a *simple path* that is either followed by an `=` token and a literal expression, or a token tree. We will explore this more in-depth in the appropriate proc-macro chapter. The important part here is that this form as well, makes use of token trees to describe the input. The 4th form in general is more special and accepts a very specific grammar that also makes use of token trees though. From 8557f4d04308afef8c8bafd99038af6c3c804f95 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 14 Sep 2022 18:26:02 +0200 Subject: [PATCH 73/92] Add note about state of the proc-macro chapter --- src/proc-macros.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/proc-macros.md b/src/proc-macros.md index bd5e727..9e6a580 100644 --- a/src/proc-macros.md +++ b/src/proc-macros.md @@ -1,5 +1,7 @@ # Procedural Macros +> Note: This section is still very incomplete! + This chapter will introduce Rust's second syntax extension type, *procedural macros*. As with the [declarative macros](./decl-macros.md) chapter, this one is also split into a [methodical](./proc-macros/methodical.md) and a (WIP) practical subchapter with the former being a more formal introduction and the latter being a more practical oriented one. From dc2ab49f7970498744ded1bed1835899cef312b3 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Tue, 11 Oct 2022 10:17:37 -0700 Subject: [PATCH 74/92] Change `..., n-1, n-2` to `..., n-2, n-1` Better reflects natural ordering of numbers --- src/decl-macros/macros-practical.md | 44 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index ca1b0ab..2d5d414 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -13,7 +13,7 @@ There is also the [Macros chapter of the Rust Book](https://doc.rust-lang.org/bo If you aren't familiar, a recurrence relation is a sequence where each value is defined in terms of one or more *previous* values, with one or more initial values to get the whole thing started. For example, the [Fibonacci sequence](https://en.wikipedia.org/wiki/Fibonacci_number) can be defined by the relation: -\\[F_{n} = 0, 1, ..., F_{n-1} + F_{n-2}\\] +\\[F_{n} = 0, 1, ..., F_{n-2} + F_{n-1}\\] Thus, the first two numbers in the sequence are 0 and 1, with the third being \\( F_{0} + F_{1} = 0 + 1 = 1\\), the fourth \\( F_{1} + F_{2} = 1 + 1 = 2\\), and so on forever. @@ -35,7 +35,7 @@ Usually, when working on a new `macro_rules!` macro, the first thing I do is dec In this specific case, my first attempt looked like this: ```rust,ignore -let fib = recurrence![a[n] = 0, 1, ..., a[n-1] + a[n-2]]; +let fib = recurrence![a[n] = 0, 1, ..., a[n-2] + a[n-1]]; for e in fib.take(10) { println!("{}", e) } ``` @@ -116,7 +116,7 @@ We need a branch to yield the initial values of the sequence; nothing tricky. } else { let a = /* something */; let n = self.pos; - let next_val = a[n-1] + a[n-2]; + let next_val = a[n-2] + a[n-1]; self.mem.TODO_shuffle_down_and_append(next_val); @@ -160,7 +160,7 @@ let fib = { } else { let a = /* something */; let n = self.pos; - let next_val = (a[n-1] + a[n-2]); + let next_val = (a[n-2] + a[n-1]); self.mem.TODO_shuffle_down_and_append(next_val.clone()); @@ -189,7 +189,7 @@ macro_rules! recurrence { } /* -let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-1] + a[n-2]]; +let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-2] + a[n-1]]; for e in fib.take(10) { println!("{}", e) } */ @@ -278,7 +278,7 @@ macro_rules! recurrence { fn main() { /* - let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-1] + a[n-2]]; + let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-2] + a[n-1]]; for e in fib.take(10) { println!("{}", e) } */ @@ -324,7 +324,7 @@ fn main() { let next_val = { let n = self.pos; let a = IndexOffset { slice: &self.mem, offset: n }; - (a[n-1] + a[n-2]) + (a[n-2] + a[n-1]) }; { @@ -423,7 +423,7 @@ macro_rules! recurrence { let next_val = { let n = self.pos; let a = IndexOffset { slice: &self.mem, offset: n }; - (a[n-1] + a[n-2]) + (a[n-2] + a[n-1]) }; { @@ -447,7 +447,7 @@ macro_rules! recurrence { } fn main() { - let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-1] + a[n-2]]; + let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-2] + a[n-1]]; for e in fib.take(10) { println!("{}", e) } } @@ -460,7 +460,7 @@ However, if we try to compile this, `rustc` aborts, telling us: error: local ambiguity: multiple parsing options: built-in NTs expr ('inits') or 1 other option. --> src/main.rs:75:45 | -75 | let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-1] + a[n-2]]; +75 | let fib = recurrence![a[n]: u64 = 0, 1, ..., a[n-2] + a[n-1]]; | ``` @@ -500,7 +500,7 @@ macro_rules! recurrence { } fn main() { - let fib = recurrence![a[n]: u64 = 0, 1 ... a[n-1] + a[n-2]]; + let fib = recurrence![a[n]: u64 = 0, 1 ... a[n-2] + a[n-1]]; // ^~~ changed for e in fib.take(10) { println!("{}", e) } @@ -523,7 +523,7 @@ macro_rules! recurrence { } fn main() { - let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-1] + a[n-2]]; + let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-2] + a[n-1]]; // ^~~~~^ changed for e in fib.take(10) { println!("{}", e) } @@ -590,7 +590,7 @@ macro_rules! recurrence { # let next_val = { # let n = self.pos; # let a = IndexOffset { slice: &self.mem, offset: n }; -# (a[n-1] + a[n-2]) +# (a[n-2] + a[n-1]) # }; # # { @@ -614,7 +614,7 @@ macro_rules! recurrence { } fn main() { - let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-1] + a[n-2]]; + let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-2] + a[n-1]]; for e in fib.take(10) { println!("{}", e) } } @@ -832,7 +832,7 @@ macro_rules! recurrence { let next_val = { let n = self.pos; let a = IndexOffset { slice: &self.mem, offset: n }; - (a[n-1] + a[n-2]) + (a[n-2] + a[n-1]) }; { @@ -858,7 +858,7 @@ macro_rules! recurrence { /* ... */ # # fn main() { -# let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-1] + a[n-2]]; +# let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-2] + a[n-1]]; # # for e in fib.take(10) { println!("{}", e) } # } @@ -934,7 +934,7 @@ With that done, we can now substitute the last thing: the `recur` expression. # }; # } # fn main() { -# let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-1] + a[n-2]]; +# let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-2] + a[n-1]]; # for e in fib.take(10) { println!("{}", e) } # } ``` @@ -945,25 +945,25 @@ And, when we compile our finished `macro_rules!` macro... error[E0425]: cannot find value `a` in this scope --> src/main.rs:68:50 | -68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-1] + a[n-2]]; +68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-2] + a[n-1]]; | ^ not found in this scope error[E0425]: cannot find value `n` in this scope --> src/main.rs:68:52 | -68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-1] + a[n-2]]; +68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-2] + a[n-1]]; | ^ not found in this scope error[E0425]: cannot find value `a` in this scope --> src/main.rs:68:59 | -68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-1] + a[n-2]]; +68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-2] + a[n-1]]; | ^ not found in this scope error[E0425]: cannot find value `n` in this scope --> src/main.rs:68:61 | -68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-1] + a[n-2]]; +68 | let fib = recurrence![a[n]: u64 = 1, 1; ...; a[n-2] + a[n-1]]; | ^ not found in this scope ``` @@ -1211,7 +1211,7 @@ macro_rules! recurrence { } fn main() { - let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-1] + a[n-2]]; + let fib = recurrence![a[n]: u64 = 0, 1; ...; a[n-2] + a[n-1]]; for e in fib.take(10) { println!("{}", e) } } From 0c25dffbe0f637aa7ab9fc1d39eb7e40a8570a97 Mon Sep 17 00:00:00 2001 From: amab8901 <83634595+amab8901@users.noreply.github.com> Date: Wed, 12 Oct 2022 05:14:36 +0200 Subject: [PATCH 75/92] n-2 comes before n-1 --- src/decl-macros/macros-practical-table.html | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/decl-macros/macros-practical-table.html b/src/decl-macros/macros-practical-table.html index c436bac..63c1ad0 100644 --- a/src/decl-macros/macros-practical-table.html +++ b/src/decl-macros/macros-practical-table.html @@ -52,7 +52,7 @@ - + @@ -60,7 +60,7 @@ - + @@ -68,7 +68,7 @@ - + @@ -76,7 +76,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -92,7 +92,7 @@ - + @@ -100,7 +100,7 @@ - + @@ -108,7 +108,7 @@ - + @@ -124,7 +124,7 @@ - + @@ -132,7 +132,7 @@ - + @@ -146,7 +146,7 @@ - + @@ -154,7 +154,7 @@ - + @@ -162,7 +162,7 @@ - + @@ -172,7 +172,7 @@ - +
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ a[n] = 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ [n] = 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ n] = 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ ] = 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ = 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ 0, 1, ..., a[n-1] + a[n-2]
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ ⌂ , 1, ..., a[n-1] + a[n-2] 0
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ ⌂ 1, ..., a[n-1] + a[n-2] 0
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ ⌂ ⌂ , ..., a[n-1] + a[n-2] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ ⌂ ..., a[n-1] + a[n-2] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ ⌂ , a[n-1] + a[n-2] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ a[n-1] + a[n-2] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr - ⌂ 0, 1 a[n-1] + a[n-2]a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ a[n] = 0, 1, ..., a[n-1] + a[n-2]a[n] = 0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ [n] = 0, 1, ..., a[n-1] + a[n-2][n] = 0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ n] = 0, 1, ..., a[n-1] + a[n-2]n] = 0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ ] = 0, 1, ..., a[n-1] + a[n-2]] = 0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ = 0, 1, ..., a[n-1] + a[n-2]= 0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ 0, 1, ..., a[n-1] + a[n-2]0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ 0, 1, ..., a[n-1] + a[n-2]0, 1, ..., a[n-2] + a[n-1]
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ ⌂ , 1, ..., a[n-1] + a[n-2], 1, ..., a[n-2] + a[n-1] 0
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ ⌂ 1, ..., a[n-1] + a[n-2]1, ..., a[n-2] + a[n-1] 0
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ ⌂ ⌂ , ..., a[n-1] + a[n-2], ..., a[n-2] + a[n-1] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ ⌂ ..., a[n-1] + a[n-2]..., a[n-2] + a[n-1] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ ⌂ , a[n-1] + a[n-2], a[n-2] + a[n-1] 0, 1
a[n] = $($inits:expr),+ , ... , $recur:expr ⌂ a[n-1] + a[n-2]a[n-2] + a[n-1] 0, 1
0, 1a[n-1] + a[n-2]a[n-2] + a[n-1]
From ae96cb66fff3d07e8778b8080eb44df85a3bde26 Mon Sep 17 00:00:00 2001 From: amab8901 <83634595+amab8901@users.noreply.github.com> Date: Wed, 12 Oct 2022 09:20:22 +0200 Subject: [PATCH 76/92] simplify the code --- src/decl-macros/macros-practical.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index 2d5d414..f75bedb 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -324,14 +324,14 @@ fn main() { let next_val = { let n = self.pos; let a = IndexOffset { slice: &self.mem, offset: n }; - (a[n-2] + a[n-1]) + a[n-2] + a[n-1] }; { use std::mem::swap; let mut swap_tmp = next_val; - for i in (0..2).rev() { + for i in [1,0] { swap(&mut swap_tmp, &mut self.mem[i]); } } From ff7d20f7a8fe4b6e9e40dedc3f1464c9ec423955 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 16 Oct 2022 13:54:48 +0200 Subject: [PATCH 77/92] Fix tt-bundling example breaking due to 2021 edition changes --- src/decl-macros/patterns/tt-bundling.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/decl-macros/patterns/tt-bundling.md b/src/decl-macros/patterns/tt-bundling.md index 631bfb5..06fbeed 100644 --- a/src/decl-macros/patterns/tt-bundling.md +++ b/src/decl-macros/patterns/tt-bundling.md @@ -29,7 +29,7 @@ fn main() { call_a_or_b_on_tail!( (a: compute_len, b: show_tail), the recursive part that skips over all these - tokens doesn't much care whether we will call a + tokens does not much care whether we will call a or call b: only the terminal rules care. ), None @@ -39,10 +39,10 @@ fn main() { (a: compute_len, b: show_tail), and now, to justify the existence of two paths we will also call a: its input should somehow - be self-referential, so let's make it return - some eighty-six! + be self-referential, so let us make it return + some ninety-one! ), - Some(92) + Some(91) ); } ``` From 2b247ecec8cdbecf8cee6727cbe2ff60f8b171f2 Mon Sep 17 00:00:00 2001 From: OccupyMars2025 <31559413+OccupyMars2025@users.noreply.github.com> Date: Mon, 8 May 2023 21:59:00 +0800 Subject: [PATCH 78/92] fix a typo --- src/decl-macros/macros-practical.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index f75bedb..36a79b3 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -1089,7 +1089,7 @@ To show the difference, let's take a simpler example. macro_rules! using_a { ($e:expr) => { { - let a = 42i; + let a = 42; $e } } From 0bb9ed08ddd709ec619213e9b80ba83f12e471d5 Mon Sep 17 00:00:00 2001 From: Joshua Price Date: Sun, 14 May 2023 00:42:38 -0400 Subject: [PATCH 79/92] fix typo --- src/proc-macros/methodical/derive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proc-macros/methodical/derive.md b/src/proc-macros/methodical/derive.md index 82708f4..6f74e2e 100644 --- a/src/proc-macros/methodical/derive.md +++ b/src/proc-macros/methodical/derive.md @@ -9,7 +9,7 @@ use proc_macro::TokenStream; #[proc_macro_derive(TlbormDerive)] pub fn tlborm_derive(item: TokenStream) -> TokenStream { - TokenStream::neW() + TokenStream::new() } ``` From 4323e6304caf13fa486bfebf707c8d0893ec1dff Mon Sep 17 00:00:00 2001 From: Eduardo Menges Mattje <50274155+EduMenges@users.noreply.github.com> Date: Fri, 7 Apr 2023 19:37:51 -0300 Subject: [PATCH 80/92] Fix typos --- src/proc-macros/methodical.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proc-macros/methodical.md b/src/proc-macros/methodical.md index 12442fc..fb87709 100644 --- a/src/proc-macros/methodical.md +++ b/src/proc-macros/methodical.md @@ -23,7 +23,7 @@ All thats required to add the crate to the dependency graph of a project and bri ## Types of procedural macros -With procedural macros, there are actually exist 3 different kinds with each having slightly different properties. +With procedural macros, there actually exists 3 different kinds with each having slightly different properties. - *function-like* proc-macros which are used to implement `$name ! $arg` invocable macros - *attribute* proc-macros which are used to implement `#[$arg]` attributes - *derive* proc-macros which are used to implement a derive, an *input* to a `#[derive(…)]` attribute From b31bb992cebaa8522c3ba41455929e9bfc4a603c Mon Sep 17 00:00:00 2001 From: smheidrich Date: Sat, 13 May 2023 13:40:54 +0200 Subject: [PATCH 81/92] Separate naive and working callback snippets --- src/decl-macros/patterns/callbacks.md | 39 ++++++++++++++------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/decl-macros/patterns/callbacks.md b/src/decl-macros/patterns/callbacks.md index 2ee59fd..d63fa41 100644 --- a/src/decl-macros/patterns/callbacks.md +++ b/src/decl-macros/patterns/callbacks.md @@ -1,14 +1,8 @@ # Callbacks -```rust -macro_rules! call_with_larch { - ($callback:ident) => { $callback!(larch) }; -} - -macro_rules! expand_to_larch { - () => { larch }; -} +Due to the order that macros are expanded in, it is (as of Rust 1.2) impossible to pass information to a macro from the expansion of *another* macro: +```rust macro_rules! recognize_tree { (larch) => { println!("#1, the Larch.") }; (redwood) => { println!("#2, the Mighty Redwood.") }; @@ -18,27 +12,34 @@ macro_rules! recognize_tree { ($($other:tt)*) => { println!("I don't know; some kind of birch maybe?") }; } +macro_rules! expand_to_larch { + () => { larch }; +} + fn main() { recognize_tree!(expand_to_larch!()); - call_with_larch!(recognize_tree); + // first expands to: recognize_tree! { expand_to_larch ! ( ) } + // and then: println! { "I don't know; some kind of birch maybe?" } } ``` -Due to the order that macros are expanded in, it is (as of Rust 1.2) impossible to pass information to a macro from the expansion of *another* macro. This can make modularizing macros very difficult. -An alternative is to use recursion and pass a callback. -Here is a trace of the above example to demonstrate how this takes place: +An alternative is to use recursion and pass a callback: -```rust,ignore -recognize_tree! { expand_to_larch ! ( ) } -println! { "I don't know; some kind of birch maybe?" } +```rust // ... -call_with_larch! { recognize_tree } -recognize_tree! { larch } -println! { "#1, the Larch." } -// ... +macro_rules! call_with_larch { + ($callback:ident) => { $callback!(larch) }; +} + +fn main() { + call_with_larch!(recognize_tree); + // first expands to: call_with_larch! { recognize_tree } + // then: recognize_tree! { larch } + // and finally: println! { "#1, the Larch." } +} ``` Using a `tt` repetition, one can also forward arbitrary arguments to a callback. From a96b9c65efdf320d0be3ee37da7b95cc8588987b Mon Sep 17 00:00:00 2001 From: Menachem Adin Date: Thu, 15 Jun 2023 23:45:57 +0300 Subject: [PATCH 82/92] Fix C snippet --- src/syntax-extensions/source-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syntax-extensions/source-analysis.md b/src/syntax-extensions/source-analysis.md index fe2aeb7..fa5b508 100644 --- a/src/syntax-extensions/source-analysis.md +++ b/src/syntax-extensions/source-analysis.md @@ -34,7 +34,7 @@ Rust does *not*. For example, C/C++ macros are *effectively* processed at this p [^cpp-it-seemed-like-a-good-idea-at-the-time] ```c -#define SUB void +#define SUB int #define BEGIN { #define END } From 7716ac04845b57ed231db925285ff912f4885a23 Mon Sep 17 00:00:00 2001 From: "Jesus Guzman, Jr" Date: Thu, 3 Aug 2023 01:33:13 -0700 Subject: [PATCH 83/92] Fix typo with inner line doc comment. --- src/decl-macros/minutiae/fragment-specifiers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index 4695b20..b1f3b27 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -156,7 +156,7 @@ metas! { # fn main() {} ``` -> **Doc-Comment Fact**: Doc-Comments like `/// ...` and `!// ...` are actually syntax sugar for attributes! They desugar to `#[doc="..."]` and `#![doc="..."]` respectively, meaning you can match on them like with attributes! +> **Doc-Comment Fact**: Doc-Comments like `/// ...` and `//! ...` are actually syntax sugar for attributes! They desugar to `#[doc="..."]` and `#![doc="..."]` respectively, meaning you can match on them like with attributes! ## `pat` From 99a3c3b3350016d3b82e8bb28cb461806d44899a Mon Sep 17 00:00:00 2001 From: Guilliam Xavier Date: Wed, 6 Sep 2023 14:10:31 +0200 Subject: [PATCH 84/92] Fix AST w.r.t. associativity in Source Analysis --- src/syntax-extensions/source-analysis.md | 36 +++++++++++------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/syntax-extensions/source-analysis.md b/src/syntax-extensions/source-analysis.md index fa5b508..3176078 100644 --- a/src/syntax-extensions/source-analysis.md +++ b/src/syntax-extensions/source-analysis.md @@ -102,29 +102,27 @@ Note that this has *no relationship* to the AST the expression would produce; in root node, there are *seven* token trees at the root level. For reference, the AST would be: ```text - ┌─────────┐ - │ BinOp │ - │ op: Add │ - ┌╴│ lhs: ◌ │ -┌─────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ -│ Var │╶┘ └─────────┘ └╴│ BinOp │ -│ name: a │ │ op: Add │ -└─────────┘ ┌╴│ lhs: ◌ │ - ┌─────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ - │ Var │╶┘ └─────────┘ └╴│ BinOp │ - │ name: b │ │ op: Add │ - └─────────┘ ┌╴│ lhs: ◌ │ + ┌─────────┐ + │ BinOp │ + │ op: Add │ + ┌╴│ lhs: ◌ │ ┌─────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ │ BinOp │╶┘ └─────────┘ └╴│ Var │ │ op: Add │ │ name: e │ ┌╴│ lhs: ◌ │ └─────────┘ - ┌─────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ - │ Var │╶┘ └─────────┘ └╴│ Index │ - │ name: c │ ┌╴│ arr: ◌ │ - └─────────┘ ┌─────────┐ │ │ ind: ◌ │╶┐ ┌─────────┐ - │ Var │╶┘ └─────────┘ └╴│ LitInt │ - │ name: d │ │ val: 0 │ - └─────────┘ └─────────┘ +┌─────────┐ ┌─────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ +│ Var │╶┐ │ BinOp │╶┘ └─────────┘ └╴│ BinOp │ +│ name: a │ │ │ op: Add │ │ op: Add │ +└─────────┘ └╴│ lhs: ◌ │ ┌╴│ lhs: ◌ │ +┌─────────┐ ┌╴│ rhs: ◌ │ ┌─────────┐ │ │ rhs: ◌ │╶┐ ┌─────────┐ +│ Var │╶┘ └─────────┘ │ Var │╶┘ └─────────┘ └╴│ Index │ +│ name: b │ │ name: c │ ┌╴│ arr: ◌ │ +└─────────┘ └─────────┘ ┌─────────┐ │ │ ind: ◌ │╶┐ + │ Var │╶┘ └─────────┘ │ + │ name: d │ ┌─────────┐ │ + └─────────┘ │ LitInt │╶┘ + │ val: 0 │ + └─────────┘ ``` It is important to understand the distinction between the AST and token trees. When writing macros, From acd3623e637637d19a320f2bd23bd7dd18ebdc34 Mon Sep 17 00:00:00 2001 From: Guilliam Xavier Date: Wed, 6 Sep 2023 17:16:00 +0200 Subject: [PATCH 85/92] Fix macro expansion in Counting # Bit twiddling --- src/decl-macros/building-blocks/counting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/building-blocks/counting.md b/src/decl-macros/building-blocks/counting.md index 0ef3a47..cff6b46 100644 --- a/src/decl-macros/building-blocks/counting.md +++ b/src/decl-macros/building-blocks/counting.md @@ -213,7 +213,7 @@ count_tts!(0 0 0 0 0) << 1; ``` This invocation will now match the second rule as its input is an uneven amount of token trees. In this case the first token tree is discarded to make the input even again, then we also do the halving step in this invocation again since we know the input would be even now anyways. -Therefor we can count 1 for the uneven discard and multiply by 2 again since we also halved. +Therefore we can count 1 for the uneven discard and multiply by 2 again since we also halved. ```rust,ignore ((count_tts!(0 0) << 1) | 1) << 1; ``` @@ -221,7 +221,7 @@ Therefor we can count 1 for the uneven discard and multiply by 2 again since we ((count_tts!(0) << 1 << 1) | 1) << 1; ``` ```rust,ignore -(((count_tts!() | 1) << 1 << 1) | 1) << 1; +((((count_tts!() << 1) | 1) << 1 << 1) | 1) << 1; ``` ```rust,ignore ((((0 << 1) | 1) << 1 << 1) | 1) << 1; From 6ef830c4c9b7405db7f0822b5d320526a7c7a8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Aguil=C3=B3=20Dom=C3=ADnguez?= Date: Tue, 9 Jan 2024 11:56:49 +0100 Subject: [PATCH 86/92] Added extra 'changed' annotation in macros-practical since when reading it I missed this change due to the annotation not being there --- src/decl-macros/macros-practical.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index 36a79b3..14bc654 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -1186,7 +1186,7 @@ macro_rules! recurrence { let $ind = self.pos; // ^~~~ changed let $seq = IndexOffset { slice: &self.mem, offset: $ind }; -// ^~~~ changed +// ^~~~ changed ^~~~ changed $recur }; From 9d6fa654f11047c7f26eca2210e1591172ea17a2 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 8 Feb 2024 08:07:54 +1100 Subject: [PATCH 87/92] Fix tests in `parsing.md`. For code hidden with `#` there must be a space after the `#`. --- src/decl-macros/building-blocks/parsing.md | 94 +++++++++++----------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/src/decl-macros/building-blocks/parsing.md b/src/decl-macros/building-blocks/parsing.md index ad8eda2..5312133 100644 --- a/src/decl-macros/building-blocks/parsing.md +++ b/src/decl-macros/building-blocks/parsing.md @@ -27,13 +27,13 @@ macro_rules! function_item_matcher { } } -#function_item_matcher!( -# #[inline] -# #[cold] -# pub fn foo(bar: i32, baz: i32, ) -> String { -# format!("{} {}", bar, baz) -# } -#); +# function_item_matcher!( +# #[inline] +# #[cold] +# pub fn foo(bar: i32, baz: i32, ) -> String { +# format!("{} {}", bar, baz) +# } +# ); # # fn main() { # assert_eq!(foo(13, 37), "13 37"); @@ -114,27 +114,27 @@ macro_rules! struct_item_matcher { } } -#struct_item_matcher!( -# #[derive(Copy, Clone)] -# pub(crate) struct Foo { -# pub bar: i32, -# baz: &'static str, -# qux: f32 -# } -#); -#struct_item_matcher!( -# #[derive(Copy, Clone)] -# pub(crate) struct Bar; -#); -#struct_item_matcher!( -# #[derive(Clone)] -# pub(crate) struct Baz (i32, pub f32, String); -#); -#fn main() { -# let _: Foo = Foo { bar: 42, baz: "macros can be nice", qux: 3.14, }; -# let _: Bar = Bar; -# let _: Baz = Baz(2, 0.1234, String::new()); -#} +# struct_item_matcher!( +# #[derive(Copy, Clone)] +# pub(crate) struct Foo { +# pub bar: i32, +# baz: &'static str, +# qux: f32 +# } +# ); +# struct_item_matcher!( +# #[derive(Copy, Clone)] +# pub(crate) struct Bar; +# ); +# struct_item_matcher!( +# #[derive(Clone)] +# pub(crate) struct Baz (i32, pub f32, String); +# ); +# fn main() { +# let _: Foo = Foo { bar: 42, baz: "macros can be nice", qux: 3.14, }; +# let _: Bar = Bar; +# let _: Baz = Baz(2, 0.1234, String::new()); +# } ``` # Enum @@ -191,25 +191,25 @@ macro_rules! enum_item_matcher { }; } -#enum_item_matcher!( -# #[derive(Copy, Clone)] -# pub(crate) enum Foo { -# Bar, -# Baz, -# } -#); -#enum_item_matcher!( -# #[derive(Copy, Clone)] -# pub(crate) enum Bar { -# Foo(i32, f32), -# Bar, -# Baz(), -# } -#); -#enum_item_matcher!( -# #[derive(Clone)] -# pub(crate) enum Baz {} -#); +# enum_item_matcher!( +# #[derive(Copy, Clone)] +# pub(crate) enum Foo { +# Bar, +# Baz, +# } +# ); +# enum_item_matcher!( +# #[derive(Copy, Clone)] +# pub(crate) enum Bar { +# Foo(i32, f32), +# Bar, +# Baz(), +# } +# ); +# enum_item_matcher!( +# #[derive(Clone)] +# pub(crate) enum Baz {} +# ); ``` [patterns]: ../patterns.md From 9abe6ddbe154616be7bf6bd89b8e0280c9f19e4b Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 7 Feb 2024 16:58:51 +1100 Subject: [PATCH 88/92] Explain how to parse methods. I had to do this myself recently and when I looked in this chapter it was "WIP" so I had to work it out myself. Might as well record it for the benefit of others. --- src/decl-macros/building-blocks/parsing.md | 83 +++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/src/decl-macros/building-blocks/parsing.md b/src/decl-macros/building-blocks/parsing.md index 5312133..e7e026f 100644 --- a/src/decl-macros/building-blocks/parsing.md +++ b/src/decl-macros/building-blocks/parsing.md @@ -16,7 +16,7 @@ macro_rules! function_item_matcher { $( #[$meta:meta] )* // ^~~~attributes~~~~^ $vis:vis fn $name:ident ( $( $arg_name:ident : $arg_ty:ty ),* $(,)? ) - // ^~~~~~~~~~~~~~~~argument list!~~~~~~~~~~~~~~^ + // ^~~~~~~~~~~~~~~~argument list~~~~~~~~~~~~~~~^ $( -> $ret_ty:ty )? // ^~~~return type~~~^ { $($tt:tt)* } @@ -53,7 +53,86 @@ Kind of like a `Derive` proc-macro but weaker and for functions. The macro for parsing basic functions is nice and all, but sometimes we would like to also parse methods, functions that refer to their object via some form of `self` usage. This makes things a bit trickier: -> WIP +```rust +macro_rules! method_item_matcher { + // self + ( + $( #[$meta:meta] )* + // ^~~~attributes~~~~^ + $vis:vis fn $name:ident ( $self:ident $(, $arg_name:ident : $arg_ty:ty )* $(,)? ) + // ^~~~~~~~~~~~~~~~~~~~~argument list~~~~~~~~~~~~~~~~~~~~~~^ + $( -> $ret_ty:ty )? + // ^~~~return type~~~^ + { $($tt:tt)* } + // ^~~~~body~~~~^ + ) => { + $( #[$meta] )* + $vis fn $name ( $self $(, $arg_name : $arg_ty )* ) $( -> $ret_ty )? { $($tt)* } + }; + + // mut self + ( + $( #[$meta:meta] )* + $vis:vis fn $name:ident ( mut $self:ident $(, $arg_name:ident : $arg_ty:ty )* $(,)? ) + $( -> $ret_ty:ty )? + { $($tt:tt)* } + ) => { + $( #[$meta] )* + $vis fn $name ( mut $self $(, $arg_name : $arg_ty )* ) $( -> $ret_ty )? { $($tt)* } + }; + + // &self + ( + $( #[$meta:meta] )* + $vis:vis fn $name:ident ( & $self:ident $(, $arg_name:ident : $arg_ty:ty )* $(,)? ) + $( -> $ret_ty:ty )? + { $($tt:tt)* } + ) => { + $( #[$meta] )* + $vis fn $name ( & $self $(, $arg_name : $arg_ty )* ) $( -> $ret_ty )? { $($tt)* } + }; + + // &mut self + ( + $( #[$meta:meta] )* + $vis:vis fn $name:ident ( &mut $self:ident $(, $arg_name:ident : $arg_ty:ty )* $(,)? ) + $( -> $ret_ty:ty )? + { $($tt:tt)* } + ) => { + $( #[$meta] )* + $vis fn $name ( &mut $self $(, $arg_name : $arg_ty )* ) $( -> $ret_ty )? { $($tt)* } + } +} + +# struct T(i32); +# impl T { +# method_item_matcher!( +# #[inline] +# pub fn s(self, x: i32) -> String { format!("{}", x) } +# ); +# method_item_matcher!( +# pub fn ms(mut self, x: i32,) -> String { format!("{}", x) } +# ); +# method_item_matcher!( +# pub fn rs(&self, x: i32, y: i32) -> String { format!("{}", self.0 + x + y) } +# ); +# method_item_matcher!( +# pub fn rms(&mut self) -> String { self.0.to_string() } +# ); +# } +# +# fn main() { +# assert_eq!({ let t = T(11); t.s(11) }, "11"); +# assert_eq!({ let t = T(22); t.ms(22) }, "22"); +# assert_eq!({ let t = T(30); t.rs(1, 2) }, "33"); +# assert_eq!({ let mut t = T(44); t.rms() }, "44"); +# } +``` +The four rules are identical except for the `self` receiver on both sides of the rule, which is `self`, `mut self`, `&self`, and `&mut self`. +You might not need all four rules. + +`$self:ident` must be used in the matcher instead of a bare `self`. +Without that, uses of `self` in the body will cause compile errors, because a macro invocation can only access identifiers it receives from parameters. ## Struct From 764adb3bcab5f917402b5a14be105357bf08ce01 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 7 Feb 2024 17:38:30 +1100 Subject: [PATCH 89/92] Fix test failures. `mdbook test` reports lots of errors. This commit fixes them, in various ways. - Add space after `#` on hidden lines. - Add `compile_fail` to all examples that are intended to not compile. - Add `text` to non-code blocks. - Add `ignore` to examples using `#![feature(...)]`, which requires nightly. Unfortunate, but I can't see a better way to handle these. - Add `edition2021` where necessary. - Add necessary hidden code. --- src/decl-macros/macros-methodical.md | 4 ++-- src/decl-macros/macros-practical.md | 6 +++--- src/decl-macros/macros2.md | 6 ++++-- src/decl-macros/minutiae/debugging.md | 4 ++-- src/decl-macros/minutiae/fragment-specifiers.md | 9 ++++++--- src/decl-macros/minutiae/identifiers.md | 8 ++++---- .../minutiae/metavar-and-expansion.md | 2 +- src/decl-macros/minutiae/metavar-expr.md | 17 +++++++++++------ src/decl-macros/patterns/callbacks.md | 8 ++++++++ 9 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/decl-macros/macros-methodical.md b/src/decl-macros/macros-methodical.md index 8937425..931d7e1 100644 --- a/src/decl-macros/macros-methodical.md +++ b/src/decl-macros/macros-methodical.md @@ -220,7 +220,7 @@ repeat_two!( a b c d e f, u v w x y z ); But this does not: -```rust +```rust,compile_fail # macro_rules! repeat_two { # ($($i:ident)*, $($i2:ident)*) => { # $( let $i: (); let $i2: (); )* @@ -232,7 +232,7 @@ repeat_two!( a b c d e f, x y z ); failing with the following error -``` +```text error: meta-variable `i` repeats 6 times, but `i2` repeats 3 times --> src/main.rs:6:10 | diff --git a/src/decl-macros/macros-practical.md b/src/decl-macros/macros-practical.md index 14bc654..917a689 100644 --- a/src/decl-macros/macros-practical.md +++ b/src/decl-macros/macros-practical.md @@ -374,7 +374,7 @@ Success! Now, let's copy & paste this into the macro expansion, and replace the expanded code with an invocation. This gives us: -```rust +```rust,compile_fail macro_rules! recurrence { ( a[n]: $sty:ty = $($inits:expr),+ , ... , $recur:expr ) => { { @@ -489,7 +489,7 @@ Theoretically, this *should* work as desired, but currently doesn't. Thankfully, the fix is relatively simple: we remove the comma from the syntax. To keep things balanced, we'll remove *both* commas around `...`: -```rust +```rust,compile_fail macro_rules! recurrence { ( a[n]: $sty:ty = $($inits:expr),+ ... $recur:expr ) => { // ^~~ changed @@ -866,7 +866,7 @@ macro_rules! recurrence { With that done, we can now substitute the last thing: the `recur` expression. -```rust +```rust,compile_fail # macro_rules! count_exprs { # () => (0); # ($head:expr $(, $tail:expr)*) => (1 + count_exprs!($($tail),*)); diff --git a/src/decl-macros/macros2.md b/src/decl-macros/macros2.md index e3883c0..0e8723e 100644 --- a/src/decl-macros/macros2.md +++ b/src/decl-macros/macros2.md @@ -13,7 +13,8 @@ Nothing described here is final or complete, and may be subject to change. We'll do a comparison between the `macro` and `macro_rules` syntax for two macros we have implemented in previous chapters: -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(decl_macro)] macro_rules! replace_expr_ { @@ -60,7 +61,8 @@ Unlike `macro_rules` which have [mixed site hygiene], `macro` have definition si As such the following compiles with a `macro_rules` macro, but fails with a `macro` definition: -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(decl_macro)] // try uncommenting the following line, and commenting out the line right after diff --git a/src/decl-macros/minutiae/debugging.md b/src/decl-macros/minutiae/debugging.md index ab1aa91..643fbe3 100644 --- a/src/decl-macros/minutiae/debugging.md +++ b/src/decl-macros/minutiae/debugging.md @@ -50,8 +50,8 @@ You can also enable this from the command-line by adding `-Z trace-macros` to th Secondly, there is [`log_syntax!`] which causes the compiler to output all tokens passed to it. For example, this makes the compiler sing a song: -```rust -# // Note: make sure to use a nightly channel compiler. +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(log_syntax)] macro_rules! sing { diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index b1f3b27..1fa12b2 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -162,7 +162,7 @@ metas! { The `pat` fragment matches any kind of [pattern](https://doc.rust-lang.org/reference/patterns.html), including or-patterns starting with the 2021 edition. -```rust +```rust,edition2021 macro_rules! patterns { ($($pat:pat)*) => (); } @@ -329,7 +329,7 @@ visibilities! { While able to match empty sequences of tokens, the fragment specifier still acts quite different from [optional repetitions](../macros-methodical.md#repetitions) which is described in the following: If it is being matched against no left over tokens the entire macro matching fails. -```rust +```rust,compile_fail macro_rules! non_optional_vis { ($vis:vis) => (); } @@ -337,13 +337,16 @@ non_optional_vis!(); // ^^^^^^^^^^^^^^^^ error: missing tokens in macro arguments ``` -`$vis:vis $ident:ident` matches fine, unlike `$(pub)? $ident:ident` which is ambiguous, as `pub` denotes a valid identifier. +`$vis:vis $ident:ident` matches fine. ```rust macro_rules! vis_ident { ($vis:vis $ident:ident) => (); } vis_ident!(pub foo); // this works fine +``` +In contrast, `$(pub)? $ident:ident` is ambiguous, as `pub` denotes a valid identifier. +```rust,compile_fail macro_rules! pub_ident { ($(pub)? $ident:ident) => (); } diff --git a/src/decl-macros/minutiae/identifiers.md b/src/decl-macros/minutiae/identifiers.md index f05b536..633fc3c 100644 --- a/src/decl-macros/minutiae/identifiers.md +++ b/src/decl-macros/minutiae/identifiers.md @@ -36,7 +36,7 @@ So `self` is both a keyword and not a keyword at the same time. You might wonder how this is in any way important. Take this example: -```rust +```rust,compile_fail macro_rules! make_mutable { ($i:ident) => {let mut $i = $i;}; } @@ -74,7 +74,7 @@ error: `mut` must be followed by a named binding So the macro will happily match `self` as an identifier, allowing you to use it in cases where you can't actually use it. But, fine; it somehow remembers that `self` is a keyword even when it's an identifier, so you *should* be able to do this, right? -```rust +```rust,compile_fail macro_rules! make_self_mutable { ($i:ident) => {let mut $i = self;}; } @@ -117,7 +117,7 @@ Now the compiler thinks we refer to our module with `self`, but that doesn't mak We already have a `self` right there, in the function signature which is definitely not a module. It's almost like it's complaining that the `self` it's trying to use isn't the *same* `self`... as though the `self` keyword has hygiene, like an... identifier. -```rust +```rust,compile_fail macro_rules! double_method { ($body:expr) => { fn double(mut self) -> Dummy { @@ -169,7 +169,7 @@ At last, *this works*. So `self` is both a keyword *and* an identifier when it feels like it. Surely this works for other, similar constructs, right? -```rust +```rust,compile_fail macro_rules! double_method { ($self_:ident, $body:expr) => { fn double($self_) -> Dummy { diff --git a/src/decl-macros/minutiae/metavar-and-expansion.md b/src/decl-macros/minutiae/metavar-and-expansion.md index c22b274..f1b821d 100644 --- a/src/decl-macros/minutiae/metavar-and-expansion.md +++ b/src/decl-macros/minutiae/metavar-and-expansion.md @@ -42,7 +42,7 @@ The parser also does not perform any kind of lookahead. That means if the compiler cannot unambiguously determine how to parse the macro invocation one token at a time, it will abort with an ambiguity error. A simple example that triggers this: -```rust +```rust,compile_fail macro_rules! ambiguity { ($($i:ident)* $i2:ident) => { }; } diff --git a/src/decl-macros/minutiae/metavar-expr.md b/src/decl-macros/minutiae/metavar-expr.md index f93c30c..6eab9b1 100644 --- a/src/decl-macros/minutiae/metavar-expr.md +++ b/src/decl-macros/minutiae/metavar-expr.md @@ -21,7 +21,7 @@ The `$$` expression expands to a single `$`, making it effectively an escaped `$ This enables the ability in writing macros emitting new macros as the former macro won't transcribe metavariables, repetitions and metavariable expressions that have an escaped `$`. We can see the problem without using `$$` in the following snippet: -```rust +```rust,compile_fail macro_rules! foo { () => { macro_rules! bar { @@ -38,7 +38,8 @@ foo!(); The problem is obvious, the transcriber of foo sees a repetition and tries to repeat it when transcribing, but there is no `$any` metavariable in its scope causing it to fail. With `$$` we can get around this as the transcriber of `foo` will no longer try to do the repetition.[^tt-$] -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(macro_metavar_expr)] macro_rules! foo { @@ -67,7 +68,8 @@ The `count` metavariable expression expands to the repetition count of the metav The `count(ident)` expression defaults `depth` to the maximum valid depth, making it count the total repetitions for the given metavariable. -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(macro_metavar_expr)] macro_rules! foo { @@ -97,7 +99,8 @@ The `index(depth)` metavariable expression expands to the current iteration inde The `index()` expression defaults `depth` to `0`, making it a shorthand for `index(0)`. -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(macro_metavar_expr)] macro_rules! attach_iteration_counts { @@ -134,7 +137,8 @@ The `length(depth)` metavariable expression expands to the iteration count of th The `length()` expression defaults `depth` to `0`, making it a shorthand for `length(0)`. -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(macro_metavar_expr)] macro_rules! lets_count { @@ -166,7 +170,8 @@ The `ignore(ident)` metavariable expression expands to nothing, making it possib - The `ident` argument must be a declared metavariable in the scope of the rule. -```rust +```rust,ignore +# // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. #![feature(macro_metavar_expr)] macro_rules! repetition_tuples { diff --git a/src/decl-macros/patterns/callbacks.md b/src/decl-macros/patterns/callbacks.md index d63fa41..4eb4608 100644 --- a/src/decl-macros/patterns/callbacks.md +++ b/src/decl-macros/patterns/callbacks.md @@ -29,6 +29,14 @@ An alternative is to use recursion and pass a callback: ```rust // ... +# macro_rules! recognize_tree { +# (larch) => { println!("#1, the Larch.") }; +# (redwood) => { println!("#2, the Mighty Redwood.") }; +# (fir) => { println!("#3, the Fir.") }; +# (chestnut) => { println!("#4, the Horse Chestnut.") }; +# (pine) => { println!("#5, the Scots Pine.") }; +# ($($other:tt)*) => { println!("I don't know; some kind of birch maybe?") }; +# } macro_rules! call_with_larch { ($callback:ident) => { $callback!(larch) }; From 14eb2d7151bd48cf5b60e0063407c2f334575a03 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 8 Feb 2024 14:36:13 +1100 Subject: [PATCH 90/92] Improve GitHub actions. `mdbook test` isn't run on CI! It should be. Plus some other CI improvements, based on the equivalent file at `github.com/nnethercote/perf-book/`: - Run actions on pull requests, to allow tests to run before merging. - Add some missing names for steps. - Update `actions/checkout` from v2 to v3. --- .github/workflows/gh-pages.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index d2f81ba..ea7f184 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,6 +1,7 @@ name: github pages on: + pull_request: push: branches: - master @@ -9,14 +10,19 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Clone repository + uses: actions/checkout@v3 - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 with: mdbook-version: 'latest' - - run: mdbook build + - name: Build + run: mdbook build + + - name: Test + run: mdbook test - name: Deploy uses: peaceiris/actions-gh-pages@v3 @@ -26,3 +32,5 @@ jobs: force_orphan: true user_name: 'github-actions[bot]' user_email: 'github-actions[bot]@users.noreply.github.com' + # Only deploy on a push to master, not on a pull request. + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'Veykril/tlborm' From 768bafb7245c05249ef867c76c4d537220441e38 Mon Sep 17 00:00:00 2001 From: Emanuel Czirai Date: Mon, 27 May 2024 11:24:55 +0200 Subject: [PATCH 91/92] typo: expression orientated -> expression-oriented --- src/decl-macros/minutiae/fragment-specifiers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decl-macros/minutiae/fragment-specifiers.md b/src/decl-macros/minutiae/fragment-specifiers.md index 1fa12b2..61e9222 100644 --- a/src/decl-macros/minutiae/fragment-specifiers.md +++ b/src/decl-macros/minutiae/fragment-specifiers.md @@ -41,7 +41,7 @@ blocks! { ## `expr` -The `expr` fragment matches any kind of [expression](https://doc.rust-lang.org/reference/expressions.html) (Rust has a lot of them, given it *is* an expression orientated language). +The `expr` fragment matches any kind of [expression](https://doc.rust-lang.org/reference/expressions.html) (Rust has a lot of them, given it *is* an expression-oriented language). ```rust macro_rules! expressions { From a70defd2ec7685d332de64cc5645e28f5ec7a973 Mon Sep 17 00:00:00 2001 From: Emanuel Czirai Date: Mon, 27 May 2024 21:01:29 +0200 Subject: [PATCH 92/92] `ignore` and `count` require `$ident` instead of just `ident` this is also seen in the recent stabilization PR attempt: https://github.com/rust-lang/rust/pull/122808 and rust playground confirms it. --- src/decl-macros/minutiae/metavar-expr.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/decl-macros/minutiae/metavar-expr.md b/src/decl-macros/minutiae/metavar-expr.md index 6eab9b1..d1d03ed 100644 --- a/src/decl-macros/minutiae/metavar-expr.md +++ b/src/decl-macros/minutiae/metavar-expr.md @@ -10,10 +10,10 @@ As mentioned in the [`methodical introduction`](../macros-methodical.md), Rust h This chapter will introduce them more in-depth together with usage examples. - [`$$`](#dollar-dollar-) -- [`${count(ident, depth)}`](#countident-depth) +- [`${count($ident, depth)}`](#countident-depth) - [`${index(depth)}`](#indexdepth) - [`${length(depth)}`](#lengthdepth) -- [`${ignore(ident)}`](#ignoreident) +- [`${ignore($ident)}`](#ignoreident) ## Dollar Dollar (`$$`) @@ -58,15 +58,15 @@ bar!(); [^tt-$]: Before `$$` occurs, users must resort to a tricky and not so well-known hack to declare nested macros with repetitions [via using `$tt` like this](https://play.rust-lang.org/?version=nightly&mode=debug&edition=2021&gist=9ce18fc79ce17c77d20e74f3c46ee13c). -## `count(ident, depth)` +## `count($ident, depth)` The `count` metavariable expression expands to the repetition count of the metavariable `$ident` up to the given repetition depth. -- The `ident` argument must be a declared metavariable in the scope of the rule. +- The `$ident` argument must be a declared metavariable in the scope of the rule. - The `depth` argument must be an integer literal of value less or equal to the maximum repetition depth that the `$ident` metavariable appears in. - The expression expands to an unsuffixed integer literal token. -The `count(ident)` expression defaults `depth` to the maximum valid depth, making it count the total repetitions for the given metavariable. +The `count($ident)` expression defaults `depth` to the maximum valid depth, making it count the total repetitions for the given metavariable. ```rust,ignore # // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. @@ -74,9 +74,9 @@ The `count(ident)` expression defaults `depth` to the maximum valid depth, makin macro_rules! foo { ( $( $outer:ident ( $( $inner:ident ),* ) ; )* ) => { - println!("count(outer, 0): $outer repeats {} times", ${count(outer)}); - println!("count(inner, 0): The $inner repetition repeats {} times in the outer repetition", ${count(inner, 0)}); - println!("count(inner, 1): $inner repeats {} times in the inner repetitions", ${count(inner, 1)}); + println!("count(outer, 0): $outer repeats {} times", ${count($outer)}); + println!("count(inner, 0): The $inner repetition repeats {} times in the outer repetition", ${count($inner, 0)}); + println!("count(inner, 1): $inner repeats {} times in the inner repetitions", ${count($inner, 1)}); }; } @@ -164,11 +164,11 @@ fn main() { } ``` -## `ignore(ident)` +## `ignore($ident)` -The `ignore(ident)` metavariable expression expands to nothing, making it possible to expand something as often as a metavariable repeats without expanding the metavariable. +The `ignore($ident)` metavariable expression expands to nothing, making it possible to expand something as often as a metavariable repeats without expanding the metavariable. -- The `ident` argument must be a declared metavariable in the scope of the rule. +- The `$ident` argument must be a declared metavariable in the scope of the rule. ```rust,ignore # // This code block marked `ignore` because mdbook can't handle `#![feature(...)]`. @@ -181,7 +181,7 @@ macro_rules! repetition_tuples { ( ${index()}, ${index(1)} - ${ignore(inner)} // without this metavariable expression, compilation would fail + ${ignore($inner)} // without this metavariable expression, compilation would fail ), )* )*)