From 963f84a8e94eed6f3c9e6820911215d03ef52965 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 31 Dec 2024 02:54:34 +0000 Subject: [PATCH 001/126] Add baseline for expressions --- .github/workflows/ci.yml | 11 + .github/workflows/publish_book.yml | 44 ++ Cargo.toml | 3 +- KNOWN_ISSUES.md | 6 + README.md | 133 ++-- book/.gitignore | 2 + book/book.toml | 6 + book/build.sh | 10 + book/download-mdbook.sh | 33 + book/src/SUMMARY.md | 16 + book/src/command-reference.md | 3 + book/src/command-reference/control-flow.md | 1 + book/src/command-reference/expressions.md | 1 + book/src/command-reference/strings.md | 1 + book/src/concepts.md | 3 + book/src/concepts/commands.md | 1 + book/src/concepts/expressions.md | 1 + book/src/concepts/parsing.md | 1 + book/src/concepts/variables.md | 1 + book/src/introduction.md | 3 + book/src/introduction/examples.md | 1 + book/src/introduction/motivation.md | 1 + book/src/introduction/quick-reference.md | 1 + book/test.sh | 14 + book/watch.sh | 10 + src/command.rs | 42 +- src/commands/control_flow_commands.rs | 61 ++ src/commands/core_commands.rs | 12 +- src/commands/expression_commands.rs | 44 ++ src/commands/mod.rs | 11 + src/expressions/boolean.rs | 105 ++++ src/expressions/evaluation_tree.rs | 456 ++++++++++++++ src/expressions/float.rs | 344 +++++++++++ src/expressions/integer.rs | 680 +++++++++++++++++++++ src/expressions/mod.rs | 46 ++ src/expressions/operations.rs | 362 +++++++++++ src/expressions/value.rs | 114 ++++ src/internal_prelude.rs | 195 +++++- src/interpreter.rs | 232 ++++++- src/lib.rs | 2 +- src/parsing.rs | 101 --- tests/control_flow.rs | 20 + tests/evaluate.rs | 57 ++ 43 files changed, 2997 insertions(+), 194 deletions(-) create mode 100644 .github/workflows/publish_book.yml create mode 100644 KNOWN_ISSUES.md create mode 100644 book/.gitignore create mode 100644 book/book.toml create mode 100755 book/build.sh create mode 100755 book/download-mdbook.sh create mode 100644 book/src/SUMMARY.md create mode 100644 book/src/command-reference.md create mode 100644 book/src/command-reference/control-flow.md create mode 100644 book/src/command-reference/expressions.md create mode 100644 book/src/command-reference/strings.md create mode 100644 book/src/concepts.md create mode 100644 book/src/concepts/commands.md create mode 100644 book/src/concepts/expressions.md create mode 100644 book/src/concepts/parsing.md create mode 100644 book/src/concepts/variables.md create mode 100644 book/src/introduction.md create mode 100644 book/src/introduction/examples.md create mode 100644 book/src/introduction/motivation.md create mode 100644 book/src/introduction/quick-reference.md create mode 100755 book/test.sh create mode 100755 book/watch.sh create mode 100644 src/commands/control_flow_commands.rs create mode 100644 src/commands/expression_commands.rs create mode 100644 src/expressions/boolean.rs create mode 100644 src/expressions/evaluation_tree.rs create mode 100644 src/expressions/float.rs create mode 100644 src/expressions/integer.rs create mode 100644 src/expressions/mod.rs create mode 100644 src/expressions/operations.rs create mode 100644 src/expressions/value.rs delete mode 100644 src/parsing.rs create mode 100644 tests/control_flow.rs create mode 100644 tests/evaluate.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dffae887..2bf38cc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,17 @@ jobs: toolchain: stable - run: ./style-check.sh + book: + name: Rust Book Test + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - run: ./book/test.sh + miri: name: Miri runs-on: ubuntu-latest diff --git a/.github/workflows/publish_book.yml b/.github/workflows/publish_book.yml new file mode 100644 index 00000000..a8416d28 --- /dev/null +++ b/.github/workflows/publish_book.yml @@ -0,0 +1,44 @@ +name: Publish book to Github Pages + +on: + push: + branches: + - main + +permissions: + contents: read # to read the docs and build job + pages: write # to deploy to pages + id-token: write # to verify the deployment originates from an appropriate source + +jobs: + # Build job - https://github.com/actions/upload-pages-artifact + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + # Sets up the repository configuration for github pages to be deployed from an action + - name: Setup Pages + uses: actions/configure-pages@v5 + # Builds the book and outputs to book/book + - name: Build book + run: ./book/build.sh + # Creates a correctly compressed archive from the build HTML file, and outputs it as + # a `github-pages` artifact + - name: Upload static files as artifact + uses: actions/upload-pages-artifact@v3 + with: + path: book/book + + # Deployment job - https://github.com/actions/deploy-pages + deploy: + runs-on: ubuntu-latest + needs: build + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + # Deploys the artifact `github-pages` created by the `upload-pages-artifact` job + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 25735ffe..314a15ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,4 +20,5 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0" } -syn = { version = "2.0", default-features = false, features = ["parsing"] } +syn = { version = "2.0", default-features = false, features = ["full", "parsing", "derive", "printing"] } +quote = { version = "1.0", default-features = false } diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md new file mode 100644 index 00000000..3c4e546d --- /dev/null +++ b/KNOWN_ISSUES.md @@ -0,0 +1,6 @@ +# Since Major Version 0.3 + +* `[MAX LITERAL]` - In an expression, the maximum negative integer literal is not valid, for example `-128i8`. + * By comparison, in `rustc`, there is special handling for such literals, so that e.g. `-128i8` and `--(-128i8)` are accepted. + * In rust analyzer, before the full pass, it seems to store literals as u128 and perform wrapping arithmetic on them. On a save / full pass, it does appear to reject literals which are out of bounds (e.g. 150i8). + * We could fix this by special casing maximal signed integer literals (e.g. 128 for an i8) which have yet to be involved in a non-unary expression. \ No newline at end of file diff --git a/README.md b/README.md index d68d9a39..3b438fd8 100644 --- a/README.md +++ b/README.md @@ -359,7 +359,7 @@ The idea is that we create two new tools: In more detail: -* `[!parse! (DESTRUCTURING) = (INPUT)]` is a more general `[!set!]` which acts like a `let = else { panic!() }`. It takes a `()`-wrapped parse destructuring on the left and a token stream as input on the right. Any `#x` in the parse definition acts as a binding rather than as a substitution. Parsing will handled commas intelligently, and accept intelligent parse operations to do heavy-lifting for the user. Parse operations look like `[!OPERATION! DESTRUCTURING]` with the operation name in `UPPER_SNAKE_CASE`. Some examples might be: +* `[!let! (DESTRUCTURING) = (INPUT)]` (or maybe `!let!` or `!set!`) is a more general `[!set!]` which acts like a `let = else { panic!() }`. It takes a `()`-wrapped parse destructuring on the left and a `()`-wrapped token stream as input on the right. Any `#x` in the parse definition acts as a binding rather than as a substitution. Parsing will handled commas intelligently, and accept intelligent parse operations to do heavy-lifting for the user. Parse operations look like `[!OPERATION! DESTRUCTURING]` with the operation name in `UPPER_SNAKE_CASE`. Some examples might be: * `[!FIELDS! { hello: #a, world?: #b }]` - which can be parsed in any order, cope with trailing commas, and forbid fields in the source stream which aren't in the destructuring. * `[!SUBFIELDS! { hello: #a, world?: #b }]` - which can parse fields in any order, cope with trailing commas, and allow fields in the source stream which aren't in the destructuring. * `[!ITEM! { #ident, #impl_generics, ... }]` - which calls syn's parse item on the token @@ -367,9 +367,11 @@ In more detail: * More tailored examples, such as `[!GENERICS! { impl: #x, type: #y, where: #z }]` which uses syn to parse the generics, and then uses subfields on the result. * Possibly `[!GROUPED! #x]` to parse a group with no brackets, to avoid parser ambiguity in some cases * `[!OPTIONAL! ...]` might be supported, but other complex logic (loops, matching) is delayed lazily until interpretation time - which feels more intuitive. -* `[!for! (DESTRUCTURING) in (INPUT) { ... }]` which operates like the rust `for` loop, and uses a parse destructuring on the left, and has support for optional commas between values -* `[!match! (INPUT) => { (DESTRUCTURING_1) => { ... }, (DESTRUCTURING_2) => { ... }, (#fallback) => { ... } }]` which operates like a rust `match` expression, and can replace the function of the branches of declarative macro inputs. -* `[!macro_rules! name!(DESTRUCTURING) = { ... }]` which can define a declarative macro, but just parses its inputs as a token stream, and uses preinterpret for its heavy lifting. +* `[!for! (DESTRUCTURING) in [!split_at! INPUT ,] { ... }]` which operates like the rust `for` loop over token trees, allowing them to be parsed. Instead of the destructuring, you can also just use a variable e.g. `#x` Operates well with the `[!split_at! ..]` command. +* `[!while! [!parse_start_from! #variable DESTRUCTURING] { ... }]` can be used to consume a destructuring from the start of #variable. `[!parse_start_from! ..]` returns true if the parse succeeded, false if the variable is an empty token stream, and errors otherwise. +* `[!parse_loop! #variable as (DESTRUCTURING), { ... }]` where the `,` is a separator and cleverly handles trailing `,` or `;` - similar to `syn::Punctuated` +* `[!match! (INPUT) => { (DESTRUCTURING_1) => { ... }, (DESTRUCTURING_2) => { ... }, #fallback => { ... } }]` which operates like a rust `match` expression, and can replace the function of the branches of declarative macro inputs. +* `[!macro_rules! name!(DESTRUCTURING) = { ... }]` which can define a declarative macro, but just parses its inputs as a token stream, and uses preinterpret for its heavy lifting. This could alternatively exist as `preinterpret::define_macro!{ my_macro!(DESTRUCTURING) = { ... }}`. And then we can end up with syntax like the following: @@ -381,7 +383,7 @@ And then we can end up with syntax like the following: // A simple macro can just take a token stream as input preinterpret::preinterpret! { [!macro_rules! my_macro!(#input) { - [!for! (#trait for #type) in (#input) { + [!parse_loop! #input as (#trait for #type), { impl #trait for #type }] }] @@ -403,9 +405,9 @@ preinterpret::preinterpret! { punctuation?: #punct = ("!") // Default }] ) = { - [!for! ( + [!parse_loop! #type_list as ( #type [!GENERICS! { impl: #impl_generics, type: #type_generics }] - ) in (#type_list) { + ) { impl<#impl_generics> SuperDuper for #type #type_generics { const Hello: &'static str = [!string! #hello " " #world #punct]; } @@ -414,26 +416,66 @@ preinterpret::preinterpret! { } ``` -### Possible extension: Integer commands +### Possible extension: Numeric commands -Each of these commands functions in three steps: -* Apply the interpreter to the token stream, which recursively executes preinterpret commands. -* Iterate over each token (recursing into groups), expecting each to be an integer literal. -* Apply some command-specific mapping to this stream of integer literals, and output a single integer literal without its type suffix. The suffix can be added back manually if required with a wrapper such as `[!literal! [!add! 1 2] u64]`. + +These might be introduced behind a default-enabled feature flag. -* `[!add! 5u64 9 32]` outputs `46`. It takes any number of integers and outputs their sum. The calculation operates in `u128` space. -* `[!sub! 64u32 1u32]` outputs `63`. It takes two integers and outputs their difference. The calculation operates in `i128` space. -* `[!mod! $length 2]` outputs `0` if `$length` is even, else `1`. It takes two integers `a` and `b`, and outputs `a mod b`. +Each numeric command would have an implicit configuration: +* A calculation space (e.g. as `i32` or `f64`) +* An output literal suffix (e.g. output no suffix or add `i32`) -We also support the following assignment commands: +We could support: +* Inferred configurations (like rustc, by looking at the suffices of literals or inferring them) + * `calc` (inferred space, explicit suffix or no suffix) +* Non-specific configurations, such as: + * `int` (`i128` space, no suffix) + * `float` (`f64` space, no suffix) +* Specific configurations, such as: + * `i32` (`i32` space, `i32` suffix) + * `usize` (`u64` space, `usize` suffix) + +#### Mathematical interpretation + +This would support calculator style expressions: +* `[!calc! (5 + 10) / 2]` outputs `7` as an integer space is inferred +* `[!int! (5 + 10) / 2]` outputs `7` +* `[!f64! (5 + 10) / 2]` outputs `7.5f64` + +These commands would execute in these steps: +* Apply the interpreter to the token stream, which recursively executes preinterpret commands, wrapping their outputs in transparent groups. +* Iterate over each token (or groups with `(..)` or transparent delimeters), expecting numeric literals, or operators `+` / `-` / `*` / `/`. +* Evaluate the mathematical expression in the given calculation space. Any literals are cast to the given calculation space. +* Output a single literal, with its output literal suffix. + +Extra details: +* Overflows would default to outputting a compile error, with a possible option to reconfigure the interpreter to use different overflow behaviour. +* A suffix could be stripped with e.g. `[!strip_suffix! 7.5f32]` giving `7.5`. +* A sum over some unknown number of items could be achieved with e.g. `[!int! 0 $(+ $x)*]` -* `[!increment! #i]` is shorthand for `[!set! #i = [!add! #i 1]]` and outputs no tokens. +#### Numeric functions -Even better - we could even support calculator-style expression interpretation: +A functions with N parameters works in three steps: +* Apply the interpreter to the token stream, which recursively executes preinterpret commands, wrapping their outputs in transparent groups. +* Expect N remaining token trees as the arguments. Each should be evaluated as a mathematical expression, using an inferred calculation space +* Output a single literal, with its calculation space suffix. -* `[!usize! (5 + 10) / mod(4, 2)]` outputs `7usize` +Example commands could be: + +* `[!mod! $length 2]` outputs `0` if `$length` is even, else `1`. It takes two integers `a` and `b`, and outputs `a mod b`. The calculation operates in the space of `$length` if it has a suffix, else in `int` space. +* `[!sum! 5u64 9 32]` outputs `46u64`. It takes any number of integers and outputs their sum. The calculation operates in `u64` space. + +We also support the following assignment commands: + +* `[!increment! #i]` is shorthand for `[!set! #i = [!calc! #i + 1]]` and outputs no tokens. ### Possible extension: User-defined commands @@ -441,41 +483,56 @@ Even better - we could even support calculator-style expression interpretation: ### Possible extension: Boolean commands +Similar to numeric commands, these could be an interpretation mode with e.g. +* `[!bool! (true || false) && !true || #x <= 3]` + Each of these commands functions in three steps: -* Apply the interpreter to the token stream, which recursively executes preinterpret commands. -* Expects to read exactly two token trees (unless otherwise specified) +* Apply the interpreter to the token stream, which recursively executes preinterpret commands, wrapping their outputs in transparent groups. +* Iterate over each token (or groups with `(..)` or transparent delimeters), expecting boolean literals, boolean operators, or comparison statements. * Apply some command-specific comparison, and outputs the boolean literal `true` or `false`. -Comparison commands under consideration are: -* `[!eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example: - * `[!eq! (3 4) (3 4)]` outputs `true` because the token stream ignores spacing. - * `[!eq! 1u64 1]` outputs `false` because these are different literals. -* `[!lt! #foo #bar]` outputs `true` if `#foo` is an integer literal and less than `#bar` -* `[!gt! #foo #bar]` outputs `true` if `#foo` is an integer literal and greater than `#bar` -* `[!lte! #foo #bar]` outputs `true` if `#foo` is an integer literal and less than or equal to `#bar` -* `[!gte! #foo #bar]` outputs `true` if `#foo` is an integer literal and greater than or equal to `#bar` -* `[!not! #foo]` expects a single boolean literal, and outputs the negation of `#foo` +Comparison statements could look like the following and operate on literals. For each, a comparison space (e.g. `string` or `u32`) needs to be inferrable from the literals. A compile error is thrown if a comparison space cannot be inferred: +* `#foo == #bar` outputs `true` if `#foo` and `#bar` are exactly the same literal. +* `#foo <= #bar` outputs `true` if `#foo` is less than or equal to `#bar` +* `#foo >= #bar` outputs `true` if `#foo` is greater than or equal to `#bar` +* `#foo > #bar` outputs `true` if `#foo` is greater than `#bar` +* `#foo < #bar` outputs `true` if `#foo` is less than `#bar` + +Other boolean commands could be possible, similar to numeric commands: +* `[!tokens_eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example: + * `[!tokens_eq! (3 4) (3 4)]` outputs `true` because the token stream ignores spacing. + * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. ### Possible extension: Token stream commands -* `[!skip! 4 from [#stream]]` reads and drops the first 4 token trees from the stream, and outputs the rest -* `[!ungroup! (#stream)]` outputs `#stream`. It expects to receive a single group (i.e. wrapped in brackets), and unwraps it. +* `[!split_at! #stream ,]` expects a token tree, and then some punct. Returns a stream split into transparent groups by the given token. Ignores a trailing empty stream. +* `[!skip! #stream 4]` expects to receive a (possibly transparent) group, and reads and drops the first 4 token trees from the group's stream, and outputs the rest +* `[!ungroup! [(#stream)]]` outputs `#stream` without any groups. It expects to receive a single group, and if the resulting token stream is a single group, it unwraps again +* `[!flatten! #stream]` removes all groups from `#stream`, leaving a token stream of idents, literals and punctuation +* `[!at! #stream 2]` takes the 2nd token tree from the stream +* `[!zip! #a #b #c]` returns a transparent group tuple of the nth token in each of `#a`, `#b` and `#c`. Errors if they are of different lengths. + +We could support a piped calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream as #x |> [!skip! #x 4] |> [!ungroup! #x]]` ### Possible extension: Control flow commands #### If statement -`[!if! #cond then { #a } else { #b }]` outputs `#a` if `#cond` is `true`, else `#b` if `#cond` is false. +`[!if! (XXX) { #a } else { #b }]` outputs `#a` if `XXX` is a boolean expression evaluating to `true`, else outputs `#b`. The `if` command works as follows: -* It starts by only interpreting its first token tree, and expects to see a single `true` or `false` literal. -* It then expects to reads an unintepreted `then` ident, following by a single `{ .. }` group, whose contents get interpreted and output only if the condition was `true`. +* It starts by only interpreting its first token tree, and expects to see a `(..)` group which can be evaluated as a boolean expression. +* It then expects a single `{ .. }` group, whose contents get interpreted and output only if the condition was `true`. * It optionally also reads an `else` ident and a by a single `{ .. }` group, whose contents get interpreted and output only if the condition was `false`. +#### While loop + +Is discussed in macro 2.0 above. + #### For loop -* `[!for! #token_tree in [#stream] { ... }]` +Is discussed in macro 2.0 above. #### Goto and label @@ -489,7 +546,7 @@ preinterpret::preinterpret!{ [!label! loop] const [!ident! AB #i]: u8 = 0; [!increment! #i] - [!if! [!lte! #i 100] then { [!goto! loop] }] + [!if! (#i <= 100) { [!goto! loop] }] } ``` diff --git a/book/.gitignore b/book/.gitignore new file mode 100644 index 00000000..80dde542 --- /dev/null +++ b/book/.gitignore @@ -0,0 +1,2 @@ +book +bin \ No newline at end of file diff --git a/book/book.toml b/book/book.toml new file mode 100644 index 00000000..b93b3826 --- /dev/null +++ b/book/book.toml @@ -0,0 +1,6 @@ +[book] +authors = ["David Edey"] +language = "en" +multilingual = false +src = "src" +title = "The Preinterpret Guide" diff --git a/book/build.sh b/book/build.sh new file mode 100755 index 00000000..efc16516 --- /dev/null +++ b/book/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e pipefail + +# Ensure we're in the book directory +cd "$(dirname "$0")" + +# Ensure mdbook has been downloaded +/usr/bin/env bash ./download-mdbook.sh + +./bin/mdbook build diff --git a/book/download-mdbook.sh b/book/download-mdbook.sh new file mode 100755 index 00000000..0c81772a --- /dev/null +++ b/book/download-mdbook.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -e pipefail + +# Ensure we're in the book directory +cd "$(dirname "$0")" + +MDBOOK_VER="v0.4.31" + +## Check if the right version of mdbook is already downloaded +if [ -f ./bin/mdbook ]; then + if ./bin/mdbook --version | grep $MDBOOK_VER > /dev/null; then + echo "bin/mdbook @ $MDBOOK_VER is already downloaded" + exit 0 + fi + echo "bin/mdbook is already downloaded but the wrong version, so downloading the correct version" +fi + +# Download and extract the mdbook binary into the bin folder +mkdir -p bin && cd bin + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + TAR_FILE="mdbook-${MDBOOK_VER}-x86_64-unknown-linux-musl.tar.gz" +else + TAR_FILE="mdbook-${MDBOOK_VER}-x86_64-apple-darwin.tar.gz" +fi + +echo "Downloading $TAR_FILE..." +curl -OL "https://github.com/rust-lang/mdBook/releases/download/${MDBOOK_VER}/${TAR_FILE}" +tar -xf "${TAR_FILE}" +rm "${TAR_FILE}" + +echo "Extracted to bin/mdbook" +cd .. \ No newline at end of file diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md new file mode 100644 index 00000000..5d8bc280 --- /dev/null +++ b/book/src/SUMMARY.md @@ -0,0 +1,16 @@ +# Summary + +- [Introduction](./introduction.md) + - [Quick Reference](./introduction/quick-reference.md) + - [Motivation](./introduction/motivation.md) + - [Examples](./introduction/examples.md) +- [Concepts](./concepts.md) + - [Commands](./concepts/commands.md) + - [Variables](./concepts/variables.md) + - [Expressions](./concepts/expressions.md) + - [Parsing](./concepts/parsing.md) +- [Command Reference](./command-reference.md) + - [Strings and Idents](./command-reference/strings.md) + - [Expressions](./command-reference/expressions.md) + - [Control Flow](./command-reference/control-flow.md) + diff --git a/book/src/command-reference.md b/book/src/command-reference.md new file mode 100644 index 00000000..f91c8879 --- /dev/null +++ b/book/src/command-reference.md @@ -0,0 +1,3 @@ +# Command Reference + +Coming soon... \ No newline at end of file diff --git a/book/src/command-reference/control-flow.md b/book/src/command-reference/control-flow.md new file mode 100644 index 00000000..0ca4f252 --- /dev/null +++ b/book/src/command-reference/control-flow.md @@ -0,0 +1 @@ +# Control Flow diff --git a/book/src/command-reference/expressions.md b/book/src/command-reference/expressions.md new file mode 100644 index 00000000..d52b5748 --- /dev/null +++ b/book/src/command-reference/expressions.md @@ -0,0 +1 @@ +# Expressions diff --git a/book/src/command-reference/strings.md b/book/src/command-reference/strings.md new file mode 100644 index 00000000..69131c15 --- /dev/null +++ b/book/src/command-reference/strings.md @@ -0,0 +1 @@ +# Strings and Idents diff --git a/book/src/concepts.md b/book/src/concepts.md new file mode 100644 index 00000000..839917fc --- /dev/null +++ b/book/src/concepts.md @@ -0,0 +1,3 @@ +# Concepts + +Coming soon... \ No newline at end of file diff --git a/book/src/concepts/commands.md b/book/src/concepts/commands.md new file mode 100644 index 00000000..61c515e7 --- /dev/null +++ b/book/src/concepts/commands.md @@ -0,0 +1 @@ +# Commands diff --git a/book/src/concepts/expressions.md b/book/src/concepts/expressions.md new file mode 100644 index 00000000..d52b5748 --- /dev/null +++ b/book/src/concepts/expressions.md @@ -0,0 +1 @@ +# Expressions diff --git a/book/src/concepts/parsing.md b/book/src/concepts/parsing.md new file mode 100644 index 00000000..dc018419 --- /dev/null +++ b/book/src/concepts/parsing.md @@ -0,0 +1 @@ +# Parsing diff --git a/book/src/concepts/variables.md b/book/src/concepts/variables.md new file mode 100644 index 00000000..ee1fba42 --- /dev/null +++ b/book/src/concepts/variables.md @@ -0,0 +1 @@ +# Variables diff --git a/book/src/introduction.md b/book/src/introduction.md new file mode 100644 index 00000000..5eca8c1d --- /dev/null +++ b/book/src/introduction.md @@ -0,0 +1,3 @@ +# Introduction + +Book coming soon... \ No newline at end of file diff --git a/book/src/introduction/examples.md b/book/src/introduction/examples.md new file mode 100644 index 00000000..df635b4e --- /dev/null +++ b/book/src/introduction/examples.md @@ -0,0 +1 @@ +# Examples diff --git a/book/src/introduction/motivation.md b/book/src/introduction/motivation.md new file mode 100644 index 00000000..6d769339 --- /dev/null +++ b/book/src/introduction/motivation.md @@ -0,0 +1 @@ +# Motivation diff --git a/book/src/introduction/quick-reference.md b/book/src/introduction/quick-reference.md new file mode 100644 index 00000000..fa95ef17 --- /dev/null +++ b/book/src/introduction/quick-reference.md @@ -0,0 +1 @@ +# Quick Reference diff --git a/book/test.sh b/book/test.sh new file mode 100755 index 00000000..c8d2f8cf --- /dev/null +++ b/book/test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e pipefail + +# Ensure we're in the book directory +cd "$(dirname "$0")" + +# Ensure mdbook has been downloaded +/usr/bin/env bash ./download-mdbook.sh + +# TODO +# Replace with mdbook test once it actually works with external libraries +# See issue: https://github.com/rust-lang/mdBook/issues/394#issuecomment-2234216353 +# I'm hopeful this will work once https://github.com/rust-lang/mdBook/pull/2503 is merged +bin/mdbook build diff --git a/book/watch.sh b/book/watch.sh new file mode 100755 index 00000000..2a5561fe --- /dev/null +++ b/book/watch.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e pipefail + +# Ensure we're in the book directory +cd "$(dirname "$0")" + +# Ensure mdbook has been downloaded +/usr/bin/env bash ./download-mdbook.sh + +./bin/mdbook watch --open diff --git a/src/command.rs b/src/command.rs index 68ec39d5..06cc4993 100644 --- a/src/command.rs +++ b/src/command.rs @@ -75,12 +75,18 @@ impl CommandInvocation { } } -pub(crate) struct VariableSubstitution { +impl HasSpanRange for CommandInvocation { + fn span_range(&self) -> SpanRange { + self.command_span.span_range() + } +} + +pub(crate) struct Variable { marker: Punct, // # variable_name: Ident, } -impl VariableSubstitution { +impl Variable { pub(crate) fn new(marker: Punct, variable_name: Ident) -> Self { Self { marker, @@ -88,8 +94,15 @@ impl VariableSubstitution { } } - pub(crate) fn execute(self, interpreter: &mut Interpreter) -> Result { - let VariableSubstitution { + pub(crate) fn variable_name(&self) -> &Ident { + &self.variable_name + } + + pub(crate) fn execute_substitution( + &self, + interpreter: &mut Interpreter, + ) -> Result { + let Variable { marker, variable_name, } = self; @@ -99,8 +112,7 @@ impl VariableSubstitution { let marker = marker.as_char(); let name_str = variable_name.to_string(); let name_str = &name_str; - Err(Error::new( - variable_name.span(), + variable_name.span().err( format!( "The variable {}{} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}{}]", marker, @@ -108,12 +120,24 @@ impl VariableSubstitution { marker, name_str, ), - )) + ) } } } } +impl core::fmt::Display for Variable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}{}", self.marker.as_char(), self.variable_name) + } +} + +impl HasSpanRange for Variable { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.marker.span(), self.variable_name.span()) + } +} + pub(crate) struct CommandArgumentStream { tokens: Tokens, } @@ -123,6 +147,10 @@ impl CommandArgumentStream { Self { tokens } } + pub(crate) fn interpret(self, interpreter: &mut Interpreter) -> Result { + interpreter.interpret_tokens(self.tokens) + } + pub(crate) fn interpret_and_concat_to_string( self, interpreter: &mut Interpreter, diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs new file mode 100644 index 00000000..577e7987 --- /dev/null +++ b/src/commands/control_flow_commands.rs @@ -0,0 +1,61 @@ +use crate::internal_prelude::*; + +pub(crate) struct IfCommand; + +impl CommandDefinition for IfCommand { + const COMMAND_NAME: &'static str = "if"; + + fn execute( + interpreter: &mut Interpreter, + argument: CommandArgumentStream, + command_span: Span, + ) -> Result { + let Some(parsed) = parse_if_statement(&mut argument.tokens()) else { + return command_span.span_range().err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); + }; + + let interpreted_condition = interpreter.interpret_item(parsed.condition)?; + let evaluated_condition = evaluate_expression( + interpreted_condition, + ExpressionParsingMode::BeforeCurlyBraces, + )? + .expect_bool("An if condition must evaluate to a boolean")? + .value(); + + if evaluated_condition { + interpreter.interpret_token_stream(parsed.true_code) + } else if let Some(false_code) = parsed.false_code { + interpreter.interpret_token_stream(false_code) + } else { + Ok(TokenStream::new()) + } + } +} + +struct IfStatement { + condition: NextItem, + true_code: TokenStream, + false_code: Option, +} + +fn parse_if_statement(tokens: &mut Tokens) -> Option { + let condition = tokens.next_item().ok()??; + let true_code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); + let false_code = if tokens.peek().is_some() { + tokens.next_as_punct_matching('!')?; + let else_word = tokens.next_as_ident()?; + tokens.next_as_punct_matching('!')?; + if else_word != "else" { + return None; + } + Some(tokens.next_as_kinded_group(Delimiter::Brace)?.stream()) + } else { + None + }; + tokens.check_end()?; + Some(IfStatement { + condition, + true_code, + false_code, + }) +} diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 1ab44080..14bc2523 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -12,10 +12,10 @@ impl CommandDefinition for SetCommand { ) -> Result { let mut argument_tokens = argument.tokens(); let variable_name = match parse_variable_set(&mut argument_tokens) { - Some(ident) => ident.to_string(), + Some(variable) => variable.variable_name().to_string(), None => { - return Err(command_span - .error("A set call is expected to start with `#variable_name = ..`")); + return command_span + .err("A set call is expected to start with `#variable_name = ..`"); } }; @@ -26,6 +26,12 @@ impl CommandDefinition for SetCommand { } } +pub(crate) fn parse_variable_set(tokens: &mut Tokens) -> Option { + let variable = tokens.next_item_as_variable("").ok()?; + tokens.next_as_punct_matching('=')?; + Some(variable) +} + pub(crate) struct RawCommand; impl CommandDefinition for RawCommand { diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs new file mode 100644 index 00000000..b59b9dc9 --- /dev/null +++ b/src/commands/expression_commands.rs @@ -0,0 +1,44 @@ +use crate::internal_prelude::*; + +pub(crate) struct EvaluateCommand; + +impl CommandDefinition for EvaluateCommand { + const COMMAND_NAME: &'static str = "evaluate"; + + fn execute( + interpreter: &mut Interpreter, + argument: CommandArgumentStream, + _command_span: Span, + ) -> Result { + let token_stream = argument.interpret(interpreter)?; + Ok(evaluate_expression(token_stream, ExpressionParsingMode::Standard)?.into_token_stream()) + } +} + +pub(crate) struct IncrementCommand; + +impl CommandDefinition for IncrementCommand { + const COMMAND_NAME: &'static str = "increment"; + + fn execute( + interpreter: &mut Interpreter, + argument: CommandArgumentStream, + command_span: Span, + ) -> Result { + let error_message = "Expected [!increment! #variable]"; + let mut tokens = argument.tokens(); + let variable = tokens.next_item_as_variable(error_message)?; + tokens.assert_end(error_message)?; + let variable_contents = variable.execute_substitution(interpreter)?; + let evaluated_integer = + evaluate_expression(variable_contents, ExpressionParsingMode::Standard)? + .expect_integer(&format!("Expected {variable} to evaluate to an integer"))?; + interpreter.set_variable( + variable.variable_name().to_string(), + evaluated_integer + .increment(command_span.span_range())? + .to_token_stream(), + ); + Ok(TokenStream::new()) + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index b1971577..fd72142e 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,9 +1,13 @@ mod concat_commands; +mod control_flow_commands; mod core_commands; +mod expression_commands; use crate::internal_prelude::*; use concat_commands::*; +use control_flow_commands::*; use core_commands::*; +use expression_commands::*; define_commands! { pub(crate) enum CommandKind { @@ -34,5 +38,12 @@ define_commands! { DecapitalizeCommand, TitleCommand, InsertSpacesCommand, + + // Expression Commands + EvaluateCommand, + IncrementCommand, + + // Control flow commands + IfCommand, } } diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs new file mode 100644 index 00000000..08846ddb --- /dev/null +++ b/src/expressions/boolean.rs @@ -0,0 +1,105 @@ +use super::*; + +pub(crate) struct EvaluationBoolean { + pub(super) source_span: SpanRange, + pub(super) value: bool, +} + +impl EvaluationBoolean { + pub(super) fn new(value: bool, source_span: SpanRange) -> Self { + Self { value, source_span } + } + + pub(crate) fn value(&self) -> bool { + self.value + } + + pub(super) fn for_litbool(lit: &syn::LitBool) -> Self { + Self { + source_span: lit.span().span_range(), + value: lit.value, + } + } + + pub(super) fn handle_unary_operation( + self, + operation: UnaryOperation, + ) -> Result { + let input = self.value; + match operation.operator { + UnaryOperator::Neg => operation.unsupported_for_value_type_err("boolean"), + UnaryOperator::Not => operation.output(!input), + UnaryOperator::NoOp => operation.output(input), + UnaryOperator::Cast(target) => match target { + ValueKind::Integer(IntegerKind::Untyped) => { + operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) + } + ValueKind::Integer(IntegerKind::I8) => operation.output(input as i8), + ValueKind::Integer(IntegerKind::I16) => operation.output(input as i16), + ValueKind::Integer(IntegerKind::I32) => operation.output(input as i32), + ValueKind::Integer(IntegerKind::I64) => operation.output(input as i64), + ValueKind::Integer(IntegerKind::I128) => operation.output(input as i128), + ValueKind::Integer(IntegerKind::Isize) => operation.output(input as isize), + ValueKind::Integer(IntegerKind::U8) => operation.output(input as u8), + ValueKind::Integer(IntegerKind::U16) => operation.output(input as u16), + ValueKind::Integer(IntegerKind::U32) => operation.output(input as u32), + ValueKind::Integer(IntegerKind::U64) => operation.output(input as u64), + ValueKind::Integer(IntegerKind::U128) => operation.output(input as u128), + ValueKind::Integer(IntegerKind::Usize) => operation.output(input as usize), + ValueKind::Float(_) => operation.err("This cast is not supported"), + ValueKind::Boolean => operation.output(self.value), + }, + } + } + + pub(super) fn handle_integer_binary_operation( + self, + _right: EvaluationInteger, + operation: BinaryOperation, + ) -> Result { + match operation.integer_operator() { + IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + operation.unsupported_for_value_type_err("boolean") + } + } + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: &BinaryOperation, + ) -> Result { + let lhs = self.value; + let rhs = rhs.value; + match operation.paired_operator() { + PairedBinaryOperator::Addition + | PairedBinaryOperator::Subtraction + | PairedBinaryOperator::Multiplication + | PairedBinaryOperator::Division => operation.unsupported_for_value_type_err("boolean"), + PairedBinaryOperator::LogicalAnd => operation.output(lhs && rhs), + PairedBinaryOperator::LogicalOr => operation.output(lhs || rhs), + PairedBinaryOperator::Remainder => operation.unsupported_for_value_type_err("boolean"), + PairedBinaryOperator::BitXor => operation.output(lhs ^ rhs), + PairedBinaryOperator::BitAnd => operation.output(lhs & rhs), + PairedBinaryOperator::BitOr => operation.output(lhs | rhs), + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(!lhs & rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs & !rhs), + } + } +} + +impl ToEvaluationOutput for bool { + fn to_output(self, span: SpanRange) -> EvaluationOutput { + EvaluationValue::Boolean(EvaluationBoolean::new(self, span)).into() + } +} + +impl quote::ToTokens for EvaluationBoolean { + fn to_tokens(&self, tokens: &mut TokenStream) { + LitBool::new(self.value, self.source_span.span()).to_tokens(tokens) + } +} diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs new file mode 100644 index 00000000..4602cb2e --- /dev/null +++ b/src/expressions/evaluation_tree.rs @@ -0,0 +1,456 @@ +use super::*; + +pub(super) struct EvaluationTree { + /// We store the tree as a normalized stack of nodes to make it easier to evaluate + /// without risking hitting stack overflow issues for deeply unbalanced trees + evaluation_stack: Vec, +} + +impl EvaluationTree { + pub(super) fn build_from(expression: &Expr) -> Result { + EvaluationTreeBuilder::new(expression).build() + } + + pub(super) fn evaluate(mut self) -> Result { + loop { + let EvaluationNode { result_placement, content } = self.evaluation_stack.pop() + .expect("The builder should ensure that the stack is non-empty and has a final element of a RootResult which results in a return below."); + let result = content.evaluate()?; + match result_placement { + ResultPlacement::RootResult => return Ok(result), + ResultPlacement::UnaryOperationInput { + parent_node_stack_index, + } => { + self.evaluation_stack[parent_node_stack_index] + .content + .set_unary_input(result); + } + ResultPlacement::BinaryOperationLeftChild { + parent_node_stack_index, + } => { + self.evaluation_stack[parent_node_stack_index] + .content + .set_binary_left_input(result); + } + ResultPlacement::BinaryOperationRightChild { + parent_node_stack_index, + } => { + self.evaluation_stack[parent_node_stack_index] + .content + .set_binary_right_input(result); + } + } + } + } +} + +struct EvaluationTreeBuilder<'a> { + work_stack: Vec<(&'a Expr, ResultPlacement)>, + evaluation_stack: Vec, +} + +impl<'a> EvaluationTreeBuilder<'a> { + fn new(expression: &'a Expr) -> Self { + Self { + work_stack: vec![(expression, ResultPlacement::RootResult)], + evaluation_stack: Vec::new(), + } + } + + /// Attempts to construct a preinterpret expression tree from a syn [Expr]. + /// It tries to align with the [rustc expression] building approach. + /// [rustc expression]: https://doc.rust-lang.org/reference/expressions.html + fn build(mut self) -> Result { + while let Some((expression, placement)) = self.work_stack.pop() { + match expression { + Expr::Binary(expr) => { + self.add_binary_operation( + placement, + BinaryOperation::for_binary_expression(expr)?, + &expr.left, + &expr.right, + ); + } + Expr::Cast(expr) => { + self.add_unary_operation( + placement, + UnaryOperation::for_cast_expression(expr)?, + &expr.expr, + ); + } + Expr::Group(expr) => { + self.add_unary_operation( + placement, + UnaryOperation::for_group_expression(expr)?, + &expr.expr, + ); + } + Expr::Lit(expr) => { + self.add_literal(placement, EvaluationValue::for_literal_expression(expr)?); + } + Expr::Paren(expr) => { + self.add_unary_operation( + placement, + UnaryOperation::for_paren_expression(expr)?, + &expr.expr, + ); + } + Expr::Unary(expr) => { + self.add_unary_operation( + placement, + UnaryOperation::for_unary_expression(expr)?, + &expr.expr, + ); + } + other_expression => { + return other_expression + .span_range() + .err("This expression is not supported in preinterpret expressions"); + } + } + } + Ok(EvaluationTree { + evaluation_stack: self.evaluation_stack, + }) + } + + fn add_binary_operation( + &mut self, + placement: ResultPlacement, + operation: BinaryOperation, + lhs: &'a Expr, + rhs: &'a Expr, + ) { + let parent_node_stack_index = self.evaluation_stack.len(); + self.evaluation_stack.push(EvaluationNode { + result_placement: placement, + content: EvaluationNodeContent::Operator(EvaluationOperator::Binary { + operation, + left_input: None, + right_input: None, + }), + }); + // Note - we put the lhs towards the end of the stack so it's evaluated first + self.work_stack.push(( + rhs, + ResultPlacement::BinaryOperationRightChild { + parent_node_stack_index, + }, + )); + self.work_stack.push(( + lhs, + ResultPlacement::BinaryOperationLeftChild { + parent_node_stack_index, + }, + )); + } + + fn add_unary_operation( + &mut self, + placement: ResultPlacement, + operation: UnaryOperation, + input: &'a Expr, + ) { + let parent_node_stack_index = self.evaluation_stack.len(); + self.evaluation_stack.push(EvaluationNode { + result_placement: placement, + content: EvaluationNodeContent::Operator(EvaluationOperator::Unary { + operation, + input: None, + }), + }); + self.work_stack.push(( + input, + ResultPlacement::UnaryOperationInput { + parent_node_stack_index, + }, + )); + } + + fn add_literal(&mut self, placement: ResultPlacement, literal: EvaluationValue) { + self.evaluation_stack.push(EvaluationNode { + result_placement: placement, + content: EvaluationNodeContent::Literal(literal), + }); + } +} + +struct EvaluationNode { + result_placement: ResultPlacement, + content: EvaluationNodeContent, +} + +enum EvaluationNodeContent { + Literal(EvaluationValue), + Operator(EvaluationOperator), +} + +impl EvaluationNodeContent { + fn evaluate(self) -> Result { + match self { + Self::Literal(literal) => Ok(EvaluationOutput::Value(literal)), + Self::Operator(operator) => operator.evaluate(), + } + } + + fn set_unary_input(&mut self, input: EvaluationOutput) { + match self { + Self::Operator(EvaluationOperator::Unary { + input: existing_input, + .. + }) => *existing_input = Some(input), + _ => panic!("Attempted to set unary input on a non-unary operator"), + } + } + + fn set_binary_left_input(&mut self, input: EvaluationOutput) { + match self { + Self::Operator(EvaluationOperator::Binary { + left_input: existing_input, + .. + }) => *existing_input = Some(input), + _ => panic!("Attempted to set binary left input on a non-binary operator"), + } + } + + fn set_binary_right_input(&mut self, input: EvaluationOutput) { + match self { + Self::Operator(EvaluationOperator::Binary { + right_input: existing_input, + .. + }) => *existing_input = Some(input), + _ => panic!("Attempted to set binary right input on a non-binary operator"), + } + } +} + +pub(crate) enum EvaluationOutput { + Value(EvaluationValue), +} + +pub(super) trait ToEvaluationOutput: Sized { + fn to_output(self, span: SpanRange) -> EvaluationOutput; +} + +impl EvaluationOutput { + pub(super) fn expect_value_pair( + self, + operator: PairedBinaryOperator, + right: EvaluationOutput, + operator_span: SpanRange, + ) -> Result { + let left_lit = self.into_value(); + let right_lit = right.into_value(); + Ok(match (left_lit, right_lit) { + (EvaluationValue::Integer(left), EvaluationValue::Integer(right)) => { + let integer_pair = match (left.value, right.value) { + (EvaluationIntegerValue::Untyped(untyped_lhs), rhs) => match rhs { + EvaluationIntegerValue::Untyped(untyped_rhs) => { + EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationIntegerValue::U8(rhs) => { + EvaluationIntegerValuePair::U8(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U16(rhs) => { + EvaluationIntegerValuePair::U16(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U32(rhs) => { + EvaluationIntegerValuePair::U32(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U64(rhs) => { + EvaluationIntegerValuePair::U64(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U128(rhs) => { + EvaluationIntegerValuePair::U128(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::Usize(rhs) => { + EvaluationIntegerValuePair::Usize(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I8(rhs) => { + EvaluationIntegerValuePair::I8(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I16(rhs) => { + EvaluationIntegerValuePair::I16(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I32(rhs) => { + EvaluationIntegerValuePair::I32(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I64(rhs) => { + EvaluationIntegerValuePair::I64(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I128(rhs) => { + EvaluationIntegerValuePair::I128(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::Isize(rhs) => { + EvaluationIntegerValuePair::Isize(untyped_lhs.parse_as()?, rhs) + } + }, + (lhs, EvaluationIntegerValue::Untyped(untyped_rhs)) => match lhs { + EvaluationIntegerValue::Untyped(untyped_lhs) => { + EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationIntegerValue::U8(lhs) => { + EvaluationIntegerValuePair::U8(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U16(lhs) => { + EvaluationIntegerValuePair::U16(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U32(lhs) => { + EvaluationIntegerValuePair::U32(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U64(lhs) => { + EvaluationIntegerValuePair::U64(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U128(lhs) => { + EvaluationIntegerValuePair::U128(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::Usize(lhs) => { + EvaluationIntegerValuePair::Usize(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I8(lhs) => { + EvaluationIntegerValuePair::I8(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I16(lhs) => { + EvaluationIntegerValuePair::I16(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I32(lhs) => { + EvaluationIntegerValuePair::I32(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I64(lhs) => { + EvaluationIntegerValuePair::I64(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I128(lhs) => { + EvaluationIntegerValuePair::I128(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::Isize(lhs) => { + EvaluationIntegerValuePair::Isize(lhs, untyped_rhs.parse_as()?) + } + }, + (EvaluationIntegerValue::U8(lhs), EvaluationIntegerValue::U8(rhs)) => { + EvaluationIntegerValuePair::U8(lhs, rhs) + } + (EvaluationIntegerValue::U16(lhs), EvaluationIntegerValue::U16(rhs)) => { + EvaluationIntegerValuePair::U16(lhs, rhs) + } + (EvaluationIntegerValue::U32(lhs), EvaluationIntegerValue::U32(rhs)) => { + EvaluationIntegerValuePair::U32(lhs, rhs) + } + (EvaluationIntegerValue::U64(lhs), EvaluationIntegerValue::U64(rhs)) => { + EvaluationIntegerValuePair::U64(lhs, rhs) + } + (EvaluationIntegerValue::U128(lhs), EvaluationIntegerValue::U128(rhs)) => { + EvaluationIntegerValuePair::U128(lhs, rhs) + } + (EvaluationIntegerValue::Usize(lhs), EvaluationIntegerValue::Usize(rhs)) => { + EvaluationIntegerValuePair::Usize(lhs, rhs) + } + (EvaluationIntegerValue::I8(lhs), EvaluationIntegerValue::I8(rhs)) => { + EvaluationIntegerValuePair::I8(lhs, rhs) + } + (EvaluationIntegerValue::I16(lhs), EvaluationIntegerValue::I16(rhs)) => { + EvaluationIntegerValuePair::I16(lhs, rhs) + } + (EvaluationIntegerValue::I32(lhs), EvaluationIntegerValue::I32(rhs)) => { + EvaluationIntegerValuePair::I32(lhs, rhs) + } + (EvaluationIntegerValue::I64(lhs), EvaluationIntegerValue::I64(rhs)) => { + EvaluationIntegerValuePair::I64(lhs, rhs) + } + (EvaluationIntegerValue::I128(lhs), EvaluationIntegerValue::I128(rhs)) => { + EvaluationIntegerValuePair::I128(lhs, rhs) + } + (EvaluationIntegerValue::Isize(lhs), EvaluationIntegerValue::Isize(rhs)) => { + EvaluationIntegerValuePair::Isize(lhs, rhs) + } + (left_value, right_value) => { + return operator_span.err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + } + }; + EvaluationLiteralPair::Integer(integer_pair) + } + (EvaluationValue::Boolean(left), EvaluationValue::Boolean(right)) => { + EvaluationLiteralPair::BooleanPair(left, right) + } + (EvaluationValue::Float(left), EvaluationValue::Float(right)) => { + let float_pair = match (left.value, right.value) { + (EvaluationFloatValue::Untyped(untyped_lhs), rhs) => match rhs { + EvaluationFloatValue::Untyped(untyped_rhs) => { + EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationFloatValue::F32(rhs) => { + EvaluationFloatValuePair::F32(untyped_lhs.parse_as()?, rhs) + } + EvaluationFloatValue::F64(rhs) => { + EvaluationFloatValuePair::F64(untyped_lhs.parse_as()?, rhs) + } + }, + (lhs, EvaluationFloatValue::Untyped(untyped_rhs)) => match lhs { + EvaluationFloatValue::Untyped(untyped_lhs) => { + EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationFloatValue::F32(lhs) => { + EvaluationFloatValuePair::F32(lhs, untyped_rhs.parse_as()?) + } + EvaluationFloatValue::F64(lhs) => { + EvaluationFloatValuePair::F64(lhs, untyped_rhs.parse_as()?) + } + }, + (EvaluationFloatValue::F32(lhs), EvaluationFloatValue::F32(rhs)) => { + EvaluationFloatValuePair::F32(lhs, rhs) + } + (EvaluationFloatValue::F64(lhs), EvaluationFloatValue::F64(rhs)) => { + EvaluationFloatValuePair::F64(lhs, rhs) + } + (left_value, right_value) => { + return operator_span.err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + } + }; + EvaluationLiteralPair::Float(float_pair) + } + (left, right) => { + return operator_span.err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); + } + }) + } + + pub(crate) fn expect_integer(self, error_message: &str) -> Result { + match self.into_value() { + EvaluationValue::Integer(value) => Ok(value), + other => other.source_span().err(error_message), + } + } + + pub(crate) fn expect_bool(self, error_message: &str) -> Result { + match self.into_value() { + EvaluationValue::Boolean(value) => Ok(value), + other => other.source_span().err(error_message), + } + } + + pub(super) fn into_value(self) -> EvaluationValue { + match self { + Self::Value(literal) => literal, + } + } +} + +impl From for EvaluationOutput { + fn from(literal: EvaluationValue) -> Self { + Self::Value(literal) + } +} + +impl quote::ToTokens for EvaluationOutput { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + EvaluationOutput::Value(evaluation_literal) => evaluation_literal.to_tokens(tokens), + } + } +} + +enum ResultPlacement { + RootResult, + UnaryOperationInput { parent_node_stack_index: usize }, + BinaryOperationLeftChild { parent_node_stack_index: usize }, + BinaryOperationRightChild { parent_node_stack_index: usize }, +} diff --git a/src/expressions/float.rs b/src/expressions/float.rs new file mode 100644 index 00000000..925fcbcd --- /dev/null +++ b/src/expressions/float.rs @@ -0,0 +1,344 @@ +use super::*; +use crate::internal_prelude::*; + +pub(crate) struct EvaluationFloat { + pub(super) source_span: SpanRange, + pub(super) value: EvaluationFloatValue, +} + +impl EvaluationFloat { + pub(super) fn new(value: EvaluationFloatValue, source_span: SpanRange) -> Self { + Self { value, source_span } + } + + pub(super) fn for_litfloat(lit: &syn::LitFloat) -> Result { + Ok(Self { + source_span: lit.span().span_range(), + value: EvaluationFloatValue::for_litfloat(lit)?, + }) + } + + pub(super) fn handle_unary_operation( + self, + operation: UnaryOperation, + ) -> Result { + match self.value { + EvaluationFloatValue::Untyped(input) => input.handle_unary_operation(&operation), + EvaluationFloatValue::F32(input) => input.handle_unary_operation(&operation), + EvaluationFloatValue::F64(input) => input.handle_unary_operation(&operation), + } + } + + pub(super) fn handle_integer_binary_operation( + self, + right: EvaluationInteger, + operation: BinaryOperation, + ) -> Result { + match self.value { + EvaluationFloatValue::Untyped(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationFloatValue::F32(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationFloatValue::F64(input) => { + input.handle_integer_binary_operation(right, &operation) + } + } + } +} + +impl quote::ToTokens for EvaluationFloat { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.value + .to_unspanned_literal() + .with_span(self.source_span.start()) + .to_tokens(tokens) + } +} + +pub(super) enum EvaluationFloatValuePair { + Untyped(UntypedFloat, UntypedFloat), + F32(f32, f32), + F64(f64, f64), +} + +impl EvaluationFloatValuePair { + pub(super) fn handle_paired_binary_operation( + self, + operation: &BinaryOperation, + ) -> Result { + match self { + Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::F32(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::F64(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + } + } +} + +pub(super) enum EvaluationFloatValue { + Untyped(UntypedFloat), + F32(f32), + F64(f64), +} + +impl EvaluationFloatValue { + pub(super) fn for_litfloat(lit: &syn::LitFloat) -> Result { + Ok(match lit.suffix() { + "" => Self::Untyped(UntypedFloat::new_from_lit_float(lit)), + "f32" => Self::F32(lit.base10_parse()?), + "f64" => Self::F64(lit.base10_parse()?), + suffix => { + return lit.span().err(format!( + "The literal suffix {suffix} is not supported in preinterpret expressions" + )); + } + }) + } + + pub(super) fn describe_type(&self) -> &'static str { + match self { + EvaluationFloatValue::Untyped(_) => "untyped float", + EvaluationFloatValue::F32(_) => "f32", + EvaluationFloatValue::F64(_) => "f64", + } + } + + fn to_unspanned_literal(&self) -> Literal { + match self { + EvaluationFloatValue::Untyped(float) => float.to_unspanned_literal(), + EvaluationFloatValue::F32(float) => Literal::f32_suffixed(*float), + EvaluationFloatValue::F64(float) => Literal::f64_suffixed(*float), + } + } +} + +#[derive(Copy, Clone)] +pub(super) enum FloatKind { + Untyped, + F32, + F64, +} + +pub(super) struct UntypedFloat( + /// The span of the literal is ignored, and will be set when converted to an output. + LitFloat, +); +pub(super) type FallbackFloat = f64; + +impl UntypedFloat { + pub(super) fn new_from_lit_float(lit_float: &LitFloat) -> Self { + // LitFloat doesn't support Clone, so we have to do this + Self::new_from_literal(lit_float.token()) + } + + pub(super) fn new_from_literal(literal: Literal) -> Self { + Self(syn::LitFloat::from(literal)) + } + + pub(super) fn handle_unary_operation( + self, + operation: &UnaryOperation, + ) -> Result { + let input = self.parse_fallback()?; + match operation.operator { + UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), + UnaryOperator::Not => operation.unsupported_for_value_type_err("untyped float"), + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Cast(target) => match target { + ValueKind::Integer(IntegerKind::Untyped) => { + operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) + } + ValueKind::Integer(IntegerKind::I8) => operation.output(input as i8), + ValueKind::Integer(IntegerKind::I16) => operation.output(input as i16), + ValueKind::Integer(IntegerKind::I32) => operation.output(input as i32), + ValueKind::Integer(IntegerKind::I64) => operation.output(input as i64), + ValueKind::Integer(IntegerKind::I128) => operation.output(input as i128), + ValueKind::Integer(IntegerKind::Isize) => operation.output(input as isize), + ValueKind::Integer(IntegerKind::U8) => operation.output(input as u8), + ValueKind::Integer(IntegerKind::U16) => operation.output(input as u16), + ValueKind::Integer(IntegerKind::U32) => operation.output(input as u32), + ValueKind::Integer(IntegerKind::U64) => operation.output(input as u64), + ValueKind::Integer(IntegerKind::U128) => operation.output(input as u128), + ValueKind::Integer(IntegerKind::Usize) => operation.output(input as usize), + ValueKind::Float(FloatKind::Untyped) => { + operation.output(UntypedFloat::from_fallback(input as FallbackFloat)) + } + ValueKind::Float(FloatKind::F32) => operation.output(input as f32), + ValueKind::Float(FloatKind::F64) => operation.output(input), + ValueKind::Boolean => operation.err("This cast is not supported"), + }, + } + } + + pub(super) fn handle_integer_binary_operation( + self, + _rhs: EvaluationInteger, + operation: &BinaryOperation, + ) -> Result { + match operation.integer_operator() { + IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + operation.unsupported_for_value_type_err("untyped float") + } + } + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: &BinaryOperation, + ) -> Result { + let lhs = self.parse_fallback()?; + let rhs = rhs.parse_fallback()?; + match operation.paired_operator() { + PairedBinaryOperator::Addition => operation.output(Self::from_fallback(lhs + rhs)), + PairedBinaryOperator::Subtraction => operation.output(Self::from_fallback(lhs - rhs)), + PairedBinaryOperator::Multiplication => { + operation.output(Self::from_fallback(lhs * rhs)) + } + PairedBinaryOperator::Division => operation.output(Self::from_fallback(lhs / rhs)), + PairedBinaryOperator::LogicalAnd | PairedBinaryOperator::LogicalOr => { + operation.unsupported_for_value_type_err(stringify!($float_type)) + } + PairedBinaryOperator::Remainder => operation.output(Self::from_fallback(lhs % rhs)), + PairedBinaryOperator::BitXor + | PairedBinaryOperator::BitAnd + | PairedBinaryOperator::BitOr => { + operation.unsupported_for_value_type_err("untyped float") + } + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + pub(super) fn from_fallback(value: FallbackFloat) -> Self { + Self::new_from_literal(Literal::f64_unsuffixed(value)) + } + + fn parse_fallback(&self) -> Result { + self.0.base10_digits().parse().map_err(|err| { + self.0.span().error(format!( + "Could not parse as the default inferred type {}: {}", + core::any::type_name::(), + err + )) + }) + } + + pub(super) fn parse_as(&self) -> Result + where + N: FromStr, + N::Err: core::fmt::Display, + { + self.0.base10_digits().parse().map_err(|err| { + self.0.span().error(format!( + "Could not parse as {}: {}", + core::any::type_name::(), + err + )) + }) + } + + fn to_unspanned_literal(&self) -> Literal { + self.0.token() + } +} + +impl ToEvaluationOutput for UntypedFloat { + fn to_output(self, span_range: SpanRange) -> EvaluationOutput { + EvaluationValue::Float(EvaluationFloat::new( + EvaluationFloatValue::Untyped(self), + span_range, + )) + .into() + } +} + +macro_rules! impl_float_operations { + ( + $($float_enum_variant:ident($float_type:ident)),* $(,)? + ) => {$( + impl ToEvaluationOutput for $float_type { + fn to_output(self, span_range: SpanRange) -> EvaluationOutput { + EvaluationValue::Float(EvaluationFloat::new(EvaluationFloatValue::$float_enum_variant(self), span_range)).into() + } + } + + impl HandleUnaryOperation for $float_type { + fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + match operation.operator { + UnaryOperator::Neg => operation.output(-self), + UnaryOperator::Not => operation.unsupported_for_value_type_err(stringify!($float_type)), + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Cast(target) => match target { + ValueKind::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), + ValueKind::Integer(IntegerKind::I8) => operation.output(self as i8), + ValueKind::Integer(IntegerKind::I16) => operation.output(self as i16), + ValueKind::Integer(IntegerKind::I32) => operation.output(self as i32), + ValueKind::Integer(IntegerKind::I64) => operation.output(self as i64), + ValueKind::Integer(IntegerKind::I128) => operation.output(self as i128), + ValueKind::Integer(IntegerKind::Isize) => operation.output(self as isize), + ValueKind::Integer(IntegerKind::U8) => operation.output(self as u8), + ValueKind::Integer(IntegerKind::U16) => operation.output(self as u16), + ValueKind::Integer(IntegerKind::U32) => operation.output(self as u32), + ValueKind::Integer(IntegerKind::U64) => operation.output(self as u64), + ValueKind::Integer(IntegerKind::U128) => operation.output(self as u128), + ValueKind::Integer(IntegerKind::Usize) => operation.output(self as usize), + ValueKind::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), + ValueKind::Float(FloatKind::F32) => operation.output(self as f32), + ValueKind::Float(FloatKind::F64) => operation.output(self as f64), + ValueKind::Boolean => operation.err("This cast is not supported"), + } + } + } + } + + impl HandleBinaryOperation for $float_type { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { + // Unlike integer arithmetic, float arithmetic does not overflow + // and instead falls back to NaN or infinity. In future we could + // allow trapping on these codes, but for now this is good enough + let lhs = self; + match operation.paired_operator() { + PairedBinaryOperator::Addition => operation.output(lhs + rhs), + PairedBinaryOperator::Subtraction => operation.output(lhs - rhs), + PairedBinaryOperator::Multiplication => operation.output(lhs * rhs), + PairedBinaryOperator::Division => operation.output(lhs / rhs), + PairedBinaryOperator::LogicalAnd + | PairedBinaryOperator::LogicalOr => { + operation.unsupported_for_value_type_err(stringify!($float_type)) + } + PairedBinaryOperator::Remainder => operation.output(lhs % rhs), + PairedBinaryOperator::BitXor + | PairedBinaryOperator::BitAnd + | PairedBinaryOperator::BitOr => { + operation.unsupported_for_value_type_err(stringify!($float_type)) + } + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + fn handle_integer_binary_operation( + self, + _rhs: EvaluationInteger, + operation: &BinaryOperation, + ) -> Result { + match operation.integer_operator() { + IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + operation.unsupported_for_value_type_err(stringify!($float_type)) + }, + } + } + } + )*}; +} +impl_float_operations!(F32(f32), F64(f64)); diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs new file mode 100644 index 00000000..101a9016 --- /dev/null +++ b/src/expressions/integer.rs @@ -0,0 +1,680 @@ +use super::*; + +pub(crate) struct EvaluationInteger { + pub(super) source_span: SpanRange, + pub(super) value: EvaluationIntegerValue, +} + +impl EvaluationInteger { + pub(super) fn new(value: EvaluationIntegerValue, source_span: SpanRange) -> Self { + Self { value, source_span } + } + + pub(super) fn for_litint(lit: &syn::LitInt) -> Result { + Ok(Self { + source_span: lit.span().span_range(), + value: EvaluationIntegerValue::for_litint(lit)?, + }) + } + + pub(super) fn handle_unary_operation( + self, + operation: UnaryOperation, + ) -> Result { + match self.value { + EvaluationIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::U8(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::U16(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::U32(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::U64(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::U128(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::Usize(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::I8(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::I16(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::I32(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::I64(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::I128(input) => input.handle_unary_operation(&operation), + EvaluationIntegerValue::Isize(input) => input.handle_unary_operation(&operation), + } + } + + pub(super) fn handle_integer_binary_operation( + self, + right: EvaluationInteger, + operation: BinaryOperation, + ) -> Result { + match self.value { + EvaluationIntegerValue::Untyped(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::U8(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::U16(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::U32(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::U64(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::U128(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::Usize(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::I8(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::I16(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::I32(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::I64(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::I128(input) => { + input.handle_integer_binary_operation(right, &operation) + } + EvaluationIntegerValue::Isize(input) => { + input.handle_integer_binary_operation(right, &operation) + } + } + } + + pub(crate) fn increment(self, operator_span: SpanRange) -> Result { + let span_for_output = self.source_span; + EvaluationOutput::Value(EvaluationValue::Integer(self)) + .expect_value_pair( + PairedBinaryOperator::Addition, + EvaluationOutput::Value(EvaluationValue::Integer(EvaluationInteger::new( + EvaluationIntegerValue::Untyped(UntypedInteger::from_fallback(1)), + operator_span, + ))), + operator_span, + )? + .handle_paired_binary_operation(BinaryOperation { + span_for_output, + operator_span, + operator: BinaryOperator::Paired(PairedBinaryOperator::Addition), + })? + .expect_integer("Integer should be created by summing too integers") + } +} + +impl quote::ToTokens for EvaluationInteger { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.value + .to_unspanned_literal() + .with_span(self.source_span.start()) + .to_tokens(tokens) + } +} + +pub(super) enum EvaluationIntegerValuePair { + Untyped(UntypedInteger, UntypedInteger), + U8(u8, u8), + U16(u16, u16), + U32(u32, u32), + U64(u64, u64), + U128(u128, u128), + Usize(usize, usize), + I8(i8, i8), + I16(i16, i16), + I32(i32, i32), + I64(i64, i64), + I128(i128, i128), + Isize(isize, isize), +} + +impl EvaluationIntegerValuePair { + pub(super) fn handle_paired_binary_operation( + self, + operation: &BinaryOperation, + ) -> Result { + match self { + Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::U8(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::U16(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::U32(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::U64(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::U128(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::Usize(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::I8(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::I16(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::I32(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::I64(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::I128(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::Isize(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + } + } +} + +#[derive(Copy, Clone)] +pub(super) enum IntegerKind { + Untyped, + I8, + I16, + I32, + I64, + I128, + Isize, + U8, + U16, + U32, + U64, + U128, + Usize, +} + +pub(super) enum EvaluationIntegerValue { + Untyped(UntypedInteger), + U8(u8), + U16(u16), + U32(u32), + U64(u64), + U128(u128), + Usize(usize), + I8(i8), + I16(i16), + I32(i32), + I64(i64), + I128(i128), + Isize(isize), +} + +impl EvaluationIntegerValue { + pub(super) fn for_litint(lit: &syn::LitInt) -> Result { + Ok(match lit.suffix() { + "" => Self::Untyped(UntypedInteger::new_from_lit_int(lit)), + "u8" => Self::U8(lit.base10_parse()?), + "u16" => Self::U16(lit.base10_parse()?), + "u32" => Self::U32(lit.base10_parse()?), + "u64" => Self::U64(lit.base10_parse()?), + "u128" => Self::U128(lit.base10_parse()?), + "usize" => Self::Usize(lit.base10_parse()?), + "i8" => Self::I8(lit.base10_parse()?), + "i16" => Self::I16(lit.base10_parse()?), + "i32" => Self::I32(lit.base10_parse()?), + "i64" => Self::I64(lit.base10_parse()?), + "i128" => Self::I128(lit.base10_parse()?), + "isize" => Self::Isize(lit.base10_parse()?), + suffix => { + return lit.span().err(format!( + "The literal suffix {suffix} is not supported in preinterpret expressions" + )); + } + }) + } + + pub(super) fn describe_type(&self) -> &'static str { + match self { + EvaluationIntegerValue::Untyped(_) => "untyped integer", + EvaluationIntegerValue::U8(_) => "u8", + EvaluationIntegerValue::U16(_) => "u16", + EvaluationIntegerValue::U32(_) => "u32", + EvaluationIntegerValue::U64(_) => "u64", + EvaluationIntegerValue::U128(_) => "u128", + EvaluationIntegerValue::Usize(_) => "usize", + EvaluationIntegerValue::I8(_) => "i8", + EvaluationIntegerValue::I16(_) => "i16", + EvaluationIntegerValue::I32(_) => "i32", + EvaluationIntegerValue::I64(_) => "i64", + EvaluationIntegerValue::I128(_) => "i128", + EvaluationIntegerValue::Isize(_) => "isize", + } + } + + fn to_unspanned_literal(&self) -> Literal { + match self { + EvaluationIntegerValue::Untyped(int) => int.to_unspanned_literal(), + EvaluationIntegerValue::U8(int) => Literal::u8_suffixed(*int), + EvaluationIntegerValue::U16(int) => Literal::u16_suffixed(*int), + EvaluationIntegerValue::U32(int) => Literal::u32_suffixed(*int), + EvaluationIntegerValue::U64(int) => Literal::u64_suffixed(*int), + EvaluationIntegerValue::U128(int) => Literal::u128_suffixed(*int), + EvaluationIntegerValue::Usize(int) => Literal::usize_suffixed(*int), + EvaluationIntegerValue::I8(int) => Literal::i8_suffixed(*int), + EvaluationIntegerValue::I16(int) => Literal::i16_suffixed(*int), + EvaluationIntegerValue::I32(int) => Literal::i32_suffixed(*int), + EvaluationIntegerValue::I64(int) => Literal::i64_suffixed(*int), + EvaluationIntegerValue::I128(int) => Literal::i128_suffixed(*int), + EvaluationIntegerValue::Isize(int) => Literal::isize_suffixed(*int), + } + } +} + +pub(super) struct UntypedInteger( + /// The span of the literal is ignored, and will be set when converted to an output. + syn::LitInt, +); +pub(super) type FallbackInteger = i128; + +impl UntypedInteger { + pub(super) fn new_from_lit_int(lit_int: &LitInt) -> Self { + // LitInt doesn't support Clone, so we have to do this + Self::new_from_literal(lit_int.token()) + } + + pub(super) fn new_from_literal(literal: Literal) -> Self { + Self(syn::LitInt::from(literal)) + } + + pub(super) fn handle_unary_operation( + self, + operation: &UnaryOperation, + ) -> Result { + let input = self.parse_fallback()?; + match operation.operator { + UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), + UnaryOperator::Not => operation.unsupported_for_value_type_err("untyped integer"), + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Cast(target) => match target { + ValueKind::Integer(IntegerKind::Untyped) => { + operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) + } + ValueKind::Integer(IntegerKind::I8) => operation.output(input as i8), + ValueKind::Integer(IntegerKind::I16) => operation.output(input as i16), + ValueKind::Integer(IntegerKind::I32) => operation.output(input as i32), + ValueKind::Integer(IntegerKind::I64) => operation.output(input as i64), + ValueKind::Integer(IntegerKind::I128) => operation.output(input), + ValueKind::Integer(IntegerKind::Isize) => operation.output(input as isize), + ValueKind::Integer(IntegerKind::U8) => operation.output(input as u8), + ValueKind::Integer(IntegerKind::U16) => operation.output(input as u16), + ValueKind::Integer(IntegerKind::U32) => operation.output(input as u32), + ValueKind::Integer(IntegerKind::U64) => operation.output(input as u64), + ValueKind::Integer(IntegerKind::U128) => operation.output(input as u128), + ValueKind::Integer(IntegerKind::Usize) => operation.output(input as usize), + ValueKind::Float(FloatKind::Untyped) => { + operation.output(UntypedFloat::from_fallback(input as FallbackFloat)) + } + ValueKind::Float(FloatKind::F32) => operation.output(input as f32), + ValueKind::Float(FloatKind::F64) => operation.output(input as f64), + ValueKind::Boolean => operation.err("This cast is not supported"), + }, + } + } + + pub(super) fn handle_integer_binary_operation( + self, + rhs: EvaluationInteger, + operation: &BinaryOperation, + ) -> Result { + let lhs = self.parse_fallback()?; + match operation.integer_operator() { + IntegerBinaryOperator::ShiftLeft => match rhs.value { + EvaluationIntegerValue::Untyped(rhs) => { + operation.output(lhs << rhs.parse_fallback()?) + } + EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U16(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U32(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U64(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U128(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::Usize(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I8(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I16(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I32(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I64(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), + }, + IntegerBinaryOperator::ShiftRight => match rhs.value { + EvaluationIntegerValue::Untyped(rhs) => { + operation.output(lhs >> rhs.parse_fallback()?) + } + EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U16(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U32(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U64(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U128(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I8(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I16(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I32(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I64(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I128(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), + }, + } + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: &BinaryOperation, + ) -> Result { + let lhs = self.parse_fallback()?; + let rhs = rhs.parse_fallback()?; + let overflow_error = || { + format!( + "The untyped integer operation {:?} {} {:?} overflowed in i128 space", + lhs, + operation.operator.symbol(), + rhs + ) + }; + match operation.paired_operator() { + PairedBinaryOperator::Addition => operation.output_if_some( + lhs.checked_add(rhs).map(Self::from_fallback), + overflow_error, + ), + PairedBinaryOperator::Subtraction => operation.output_if_some( + lhs.checked_sub(rhs).map(Self::from_fallback), + overflow_error, + ), + PairedBinaryOperator::Multiplication => operation.output_if_some( + lhs.checked_mul(rhs).map(Self::from_fallback), + overflow_error, + ), + PairedBinaryOperator::Division => operation.output_if_some( + lhs.checked_div(rhs).map(Self::from_fallback), + overflow_error, + ), + PairedBinaryOperator::LogicalAnd | PairedBinaryOperator::LogicalOr => { + operation.unsupported_for_value_type_err("untyped integer") + } + PairedBinaryOperator::Remainder => operation.output_if_some( + lhs.checked_rem(rhs).map(Self::from_fallback), + overflow_error, + ), + PairedBinaryOperator::BitXor => operation.output(Self::from_fallback(lhs ^ rhs)), + PairedBinaryOperator::BitAnd => operation.output(Self::from_fallback(lhs & rhs)), + PairedBinaryOperator::BitOr => operation.output(Self::from_fallback(lhs | rhs)), + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + pub(super) fn from_fallback(value: FallbackInteger) -> Self { + Self::new_from_literal(Literal::i128_unsuffixed(value)) + } + + pub(super) fn parse_fallback(&self) -> Result { + self.0.base10_digits().parse().map_err(|err| { + self.0.span().error(format!( + "Could not parse as the default inferred type {}: {}", + core::any::type_name::(), + err + )) + }) + } + + pub(super) fn parse_as(&self) -> Result + where + N: FromStr, + N::Err: core::fmt::Display, + { + self.0.base10_digits().parse().map_err(|err| { + self.0.span().error(format!( + "Could not parse as {}: {}", + core::any::type_name::(), + err + )) + }) + } + + pub(super) fn to_unspanned_literal(&self) -> Literal { + self.0.token() + } +} + +impl ToEvaluationOutput for UntypedInteger { + fn to_output(self, span_range: SpanRange) -> EvaluationOutput { + EvaluationValue::Integer(EvaluationInteger::new( + EvaluationIntegerValue::Untyped(self), + span_range, + )) + .into() + } +} + +// We have to use a macro because we don't have checked xx traits :( +macro_rules! impl_signed_int_operations { + ( + $($integer_enum_variant:ident($integer_type:ident)),* $(,)? + ) => {$( + impl ToEvaluationOutput for $integer_type { + fn to_output(self, span_range: SpanRange) -> EvaluationOutput { + EvaluationValue::Integer(EvaluationInteger::new(EvaluationIntegerValue::$integer_enum_variant(self), span_range)).into() + } + } + + impl HandleUnaryOperation for $integer_type { + fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + match operation.operator { + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Neg => operation.output(-self), + UnaryOperator::Not => { + operation.unsupported_for_value_type_err(stringify!($integer_type)) + }, + UnaryOperator::Cast(target) => match target { + ValueKind::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), + ValueKind::Integer(IntegerKind::I8) => operation.output(self as i8), + ValueKind::Integer(IntegerKind::I16) => operation.output(self as i16), + ValueKind::Integer(IntegerKind::I32) => operation.output(self as i32), + ValueKind::Integer(IntegerKind::I64) => operation.output(self as i64), + ValueKind::Integer(IntegerKind::I128) => operation.output(self as i128), + ValueKind::Integer(IntegerKind::Isize) => operation.output(self as isize), + ValueKind::Integer(IntegerKind::U8) => operation.output(self as u8), + ValueKind::Integer(IntegerKind::U16) => operation.output(self as u16), + ValueKind::Integer(IntegerKind::U32) => operation.output(self as u32), + ValueKind::Integer(IntegerKind::U64) => operation.output(self as u64), + ValueKind::Integer(IntegerKind::U128) => operation.output(self as u128), + ValueKind::Integer(IntegerKind::Usize) => operation.output(self as usize), + ValueKind::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), + ValueKind::Float(FloatKind::F32) => operation.output(self as f32), + ValueKind::Float(FloatKind::F64) => operation.output(self as f64), + ValueKind::Boolean => operation.err("This cast is not supported"), + } + } + } + } + + impl HandleBinaryOperation for $integer_type { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { + let lhs = self; + let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.operator.symbol(), rhs); + match operation.paired_operator() { + PairedBinaryOperator::Addition => operation.output_if_some(lhs.checked_add(rhs), overflow_error), + PairedBinaryOperator::Subtraction => operation.output_if_some(lhs.checked_sub(rhs), overflow_error), + PairedBinaryOperator::Multiplication => operation.output_if_some(lhs.checked_mul(rhs), overflow_error), + PairedBinaryOperator::Division => operation.output_if_some(lhs.checked_div(rhs), overflow_error), + PairedBinaryOperator::LogicalAnd + | PairedBinaryOperator::LogicalOr => operation.unsupported_for_value_type_err(stringify!($integer_type)), + PairedBinaryOperator::Remainder => operation.output_if_some(lhs.checked_rem(rhs), overflow_error), + PairedBinaryOperator::BitXor => operation.output(lhs ^ rhs), + PairedBinaryOperator::BitAnd => operation.output(lhs & rhs), + PairedBinaryOperator::BitOr => operation.output(lhs | rhs), + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + fn handle_integer_binary_operation( + self, + rhs: EvaluationInteger, + operation: &BinaryOperation, + ) -> Result { + let lhs = self; + match operation.integer_operator() { + IntegerBinaryOperator::ShiftLeft => { + match rhs.value { + EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), + EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U16(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U32(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U64(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U128(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::Usize(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I8(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I16(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I32(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I64(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), + } + }, + IntegerBinaryOperator::ShiftRight => { + match rhs.value { + EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs >> rhs.parse_fallback()?), + EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U16(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U32(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U64(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U128(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I8(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I16(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I32(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I64(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I128(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), + } + }, + } + } + } + )*}; +} + +macro_rules! impl_unsigned_int_operations { + ( + $($integer_enum_variant:ident($integer_type:ident)),* $(,)? + ) => {$( + impl ToEvaluationOutput for $integer_type { + fn to_output(self, span_range: SpanRange) -> EvaluationOutput { + EvaluationValue::Integer(EvaluationInteger::new(EvaluationIntegerValue::$integer_enum_variant(self), span_range)).into() + } + } + + impl HandleUnaryOperation for $integer_type { + fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + match operation.operator { + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Neg + | UnaryOperator::Not => { + operation.unsupported_for_value_type_err(stringify!($integer_type)) + }, + UnaryOperator::Cast(target) => match target { + ValueKind::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), + ValueKind::Integer(IntegerKind::I8) => operation.output(self as i8), + ValueKind::Integer(IntegerKind::I16) => operation.output(self as i16), + ValueKind::Integer(IntegerKind::I32) => operation.output(self as i32), + ValueKind::Integer(IntegerKind::I64) => operation.output(self as i64), + ValueKind::Integer(IntegerKind::I128) => operation.output(self as i128), + ValueKind::Integer(IntegerKind::Isize) => operation.output(self as isize), + ValueKind::Integer(IntegerKind::U8) => operation.output(self as u8), + ValueKind::Integer(IntegerKind::U16) => operation.output(self as u16), + ValueKind::Integer(IntegerKind::U32) => operation.output(self as u32), + ValueKind::Integer(IntegerKind::U64) => operation.output(self as u64), + ValueKind::Integer(IntegerKind::U128) => operation.output(self as u128), + ValueKind::Integer(IntegerKind::Usize) => operation.output(self as usize), + ValueKind::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), + ValueKind::Float(FloatKind::F32) => operation.output(self as f32), + ValueKind::Float(FloatKind::F64) => operation.output(self as f64), + ValueKind::Boolean => operation.err("This cast is not supported"), + } + } + } + } + + impl HandleBinaryOperation for $integer_type { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { + let lhs = self; + let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.operator.symbol(), rhs); + match operation.paired_operator() { + PairedBinaryOperator::Addition => operation.output_if_some(lhs.checked_add(rhs), overflow_error), + PairedBinaryOperator::Subtraction => operation.output_if_some(lhs.checked_sub(rhs), overflow_error), + PairedBinaryOperator::Multiplication => operation.output_if_some(lhs.checked_mul(rhs), overflow_error), + PairedBinaryOperator::Division => operation.output_if_some(lhs.checked_div(rhs), overflow_error), + PairedBinaryOperator::LogicalAnd + | PairedBinaryOperator::LogicalOr => operation.unsupported_for_value_type_err(stringify!($integer_type)), + PairedBinaryOperator::Remainder => operation.output_if_some(lhs.checked_rem(rhs), overflow_error), + PairedBinaryOperator::BitXor => operation.output(lhs ^ rhs), + PairedBinaryOperator::BitAnd => operation.output(lhs & rhs), + PairedBinaryOperator::BitOr => operation.output(lhs | rhs), + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + fn handle_integer_binary_operation( + self, + rhs: EvaluationInteger, + operation: &BinaryOperation, + ) -> Result { + let lhs = self; + match operation.integer_operator() { + IntegerBinaryOperator::ShiftLeft => { + match rhs.value { + EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), + EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U16(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U32(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U64(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::U128(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::Usize(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I8(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I16(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I32(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I64(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), + EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), + } + }, + IntegerBinaryOperator::ShiftRight => { + match rhs.value { + EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs >> rhs.parse_fallback()?), + EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U16(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U32(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U64(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::U128(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I8(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I16(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I32(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I64(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::I128(rhs) => operation.output(lhs >> rhs), + EvaluationIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), + } + }, + } + } + } + )*}; +} + +impl_signed_int_operations!( + I8(i8), + I16(i16), + I32(i32), + I64(i64), + I128(i128), + Isize(isize) +); +impl_unsigned_int_operations!( + U8(u8), + U16(u16), + U32(u32), + U64(u64), + U128(u128), + Usize(usize) +); diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs new file mode 100644 index 00000000..7f8b5da8 --- /dev/null +++ b/src/expressions/mod.rs @@ -0,0 +1,46 @@ +mod boolean; +mod evaluation_tree; +mod float; +mod integer; +mod operations; +mod value; + +use crate::internal_prelude::*; +use boolean::*; +use evaluation_tree::*; +use float::*; +use integer::*; +use operations::*; +use value::*; + +#[allow(unused)] +pub(crate) enum ExpressionParsingMode { + Standard, + BeforeCurlyBraces, + StartOfStatement, +} + +pub(crate) fn evaluate_expression( + token_stream: TokenStream, + mode: ExpressionParsingMode, +) -> Result { + use syn::parse::{Parse, Parser}; + + // Parsing into a rust expression is overkill here. + // In future we could choose to implement a subset of the grammar which we actually can use/need. + // + // That said, it's useful for now for two reasons: + // * Aligning with rust syntax + // * Saving implementation work, particularly given that syn is likely pulled into most code bases anyway + let expression = match mode { + ExpressionParsingMode::Standard => Expr::parse.parse2(token_stream)?, + ExpressionParsingMode::BeforeCurlyBraces => { + (Expr::parse_without_eager_brace).parse2(token_stream)? + } + ExpressionParsingMode::StartOfStatement => { + (Expr::parse_with_earlier_boundary_rule).parse2(token_stream)? + } + }; + + EvaluationTree::build_from(&expression)?.evaluate() +} diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs new file mode 100644 index 00000000..355d7815 --- /dev/null +++ b/src/expressions/operations.rs @@ -0,0 +1,362 @@ +use super::*; + +pub(super) enum EvaluationOperator { + Unary { + operation: UnaryOperation, + input: Option, + }, + Binary { + operation: BinaryOperation, + left_input: Option, + right_input: Option, + }, +} + +impl EvaluationOperator { + pub(super) fn evaluate(self) -> Result { + const OPERATOR_INPUT_EXPECT_STR: &str = "Handling children on the stack ordering should ensure the parent input is always set when the parent is evaluated"; + + match self { + Self::Unary { + operation: operator, + input, + } => operator.evaluate(input.expect(OPERATOR_INPUT_EXPECT_STR)), + Self::Binary { + operation: operator, + left_input, + right_input, + } => operator.evaluate( + left_input.expect(OPERATOR_INPUT_EXPECT_STR), + right_input.expect(OPERATOR_INPUT_EXPECT_STR), + ), + } + } +} + +pub(super) struct UnaryOperation { + pub(super) span_for_output: SpanRange, + pub(super) operator_span: SpanRange, + pub(super) operator: UnaryOperator, +} + +impl UnaryOperation { + fn error(&self, error_message: &str) -> syn::Error { + self.operator_span.error(error_message) + } + + pub(super) fn unsupported_for_value_type_err( + &self, + value_type: &'static str, + ) -> Result { + Err(self.error(&format!( + "The {} operator is not supported for {} values", + self.operator.symbol(), + value_type, + ))) + } + + pub(super) fn err(&self, error_message: &'static str) -> Result { + Err(self.error(error_message)) + } + + pub(super) fn output(&self, output_value: impl ToEvaluationOutput) -> Result { + Ok(output_value.to_output(self.span_for_output)) + } + + pub(super) fn for_cast_expression(expr: &syn::ExprCast) -> Result { + fn extract_type(ty: &syn::Type) -> Result { + match ty { + syn::Type::Group(group) => extract_type(&group.elem), + syn::Type::Path(type_path) + if type_path.qself.is_none() + && type_path.path.leading_colon.is_none() + && type_path.path.segments.len() == 1 => + { + let Some(ident) = type_path.path.get_ident() else { + return type_path + .span_range() + .err("This type is not supported in preinterpret cast expressions"); + }; + match ident.to_string().as_str() { + "int" | "integer" => Ok(ValueKind::Integer(IntegerKind::Untyped)), + "u8" => Ok(ValueKind::Integer(IntegerKind::U8)), + "u16" => Ok(ValueKind::Integer(IntegerKind::U16)), + "u32" => Ok(ValueKind::Integer(IntegerKind::U32)), + "u64" => Ok(ValueKind::Integer(IntegerKind::U64)), + "u128" => Ok(ValueKind::Integer(IntegerKind::U128)), + "usize" => Ok(ValueKind::Integer(IntegerKind::Usize)), + "i8" => Ok(ValueKind::Integer(IntegerKind::I8)), + "i16" => Ok(ValueKind::Integer(IntegerKind::I16)), + "i32" => Ok(ValueKind::Integer(IntegerKind::I32)), + "i64" => Ok(ValueKind::Integer(IntegerKind::I64)), + "i128" => Ok(ValueKind::Integer(IntegerKind::I128)), + "isize" => Ok(ValueKind::Integer(IntegerKind::Isize)), + "float" => Ok(ValueKind::Float(FloatKind::Untyped)), + "f32" => Ok(ValueKind::Float(FloatKind::F32)), + "f64" => Ok(ValueKind::Float(FloatKind::F64)), + "bool" => Ok(ValueKind::Boolean), + _ => ident + .span() + .span_range() + .err("This type is not supported in preinterpret cast expressions"), + } + } + other => other + .span_range() + .err("This type is not supported in preinterpret cast expressions"), + } + } + + Ok(Self { + span_for_output: expr.expr.span_range(), + operator_span: expr.as_token.span.span_range(), + operator: UnaryOperator::Cast(extract_type(&expr.ty)?), + }) + } + + pub(super) fn for_group_expression(expr: &syn::ExprGroup) -> Result { + Ok(Self { + span_for_output: expr.group_token.span.span_range(), + operator_span: expr.group_token.span.span_range(), + operator: UnaryOperator::NoOp, + }) + } + + pub(super) fn for_paren_expression(expr: &syn::ExprParen) -> Result { + Ok(Self { + span_for_output: expr.paren_token.span.span_range(), + operator_span: expr.paren_token.span.span_range(), + operator: UnaryOperator::NoOp, + }) + } + + pub(super) fn for_unary_expression(expr: &syn::ExprUnary) -> Result { + let operator = match &expr.op { + UnOp::Neg(_) => UnaryOperator::Neg, + UnOp::Not(_) => UnaryOperator::Not, + other_unary_op => { + return other_unary_op + .span_range() + .err("This unary operator is not supported in preinterpret expressions"); + } + }; + Ok(Self { + span_for_output: expr.span_range(), + operator_span: expr.op.span_range(), + operator, + }) + } + + pub(super) fn evaluate(self, input: EvaluationOutput) -> Result { + input.into_value().handle_unary_operation(self) + } +} + +#[derive(Copy, Clone)] +pub(super) enum UnaryOperator { + Neg, + Not, + NoOp, + Cast(ValueKind), +} + +impl UnaryOperator { + pub(crate) fn symbol(&self) -> &'static str { + match self { + UnaryOperator::Neg => "-", + UnaryOperator::Not => "!", + UnaryOperator::NoOp => "", + UnaryOperator::Cast(_) => "as", + } + } +} + +pub(super) trait HandleUnaryOperation: Sized { + fn handle_unary_operation(self, operation: &UnaryOperation) -> Result; +} + +pub(super) struct BinaryOperation { + pub(super) span_for_output: SpanRange, + pub(super) operator_span: SpanRange, + pub(super) operator: BinaryOperator, +} + +impl BinaryOperation { + fn error(&self, error_message: &str) -> syn::Error { + self.operator_span.error(error_message) + } + + pub(super) fn unsupported_for_value_type_err( + &self, + value_type: &'static str, + ) -> Result { + Err(self.error(&format!( + "The {} operator is not supported for {} values", + self.operator.symbol(), + value_type, + ))) + } + + pub(super) fn output(&self, output_value: impl ToEvaluationOutput) -> Result { + Ok(output_value.to_output(self.span_for_output)) + } + + pub(super) fn output_if_some( + &self, + output_value: Option, + error_message: impl FnOnce() -> String, + ) -> Result { + match output_value { + Some(output_value) => self.output(output_value), + None => Err(self.operator_span.error(error_message())), + } + } + + pub(super) fn for_binary_expression(expr: &syn::ExprBinary) -> Result { + let operator = match &expr.op { + syn::BinOp::Add(_) => BinaryOperator::Paired(PairedBinaryOperator::Addition), + syn::BinOp::Sub(_) => BinaryOperator::Paired(PairedBinaryOperator::Subtraction), + syn::BinOp::Mul(_) => BinaryOperator::Paired(PairedBinaryOperator::Multiplication), + syn::BinOp::Div(_) => BinaryOperator::Paired(PairedBinaryOperator::Division), + syn::BinOp::Rem(_) => BinaryOperator::Paired(PairedBinaryOperator::Remainder), + syn::BinOp::And(_) => BinaryOperator::Paired(PairedBinaryOperator::LogicalAnd), + syn::BinOp::Or(_) => BinaryOperator::Paired(PairedBinaryOperator::LogicalOr), + syn::BinOp::BitXor(_) => BinaryOperator::Paired(PairedBinaryOperator::BitXor), + syn::BinOp::BitAnd(_) => BinaryOperator::Paired(PairedBinaryOperator::BitAnd), + syn::BinOp::BitOr(_) => BinaryOperator::Paired(PairedBinaryOperator::BitOr), + syn::BinOp::Shl(_) => BinaryOperator::Integer(IntegerBinaryOperator::ShiftLeft), + syn::BinOp::Shr(_) => BinaryOperator::Integer(IntegerBinaryOperator::ShiftRight), + syn::BinOp::Eq(_) => BinaryOperator::Paired(PairedBinaryOperator::Equal), + syn::BinOp::Lt(_) => BinaryOperator::Paired(PairedBinaryOperator::LessThan), + syn::BinOp::Le(_) => BinaryOperator::Paired(PairedBinaryOperator::LessThanOrEqual), + syn::BinOp::Ne(_) => BinaryOperator::Paired(PairedBinaryOperator::NotEqual), + syn::BinOp::Ge(_) => BinaryOperator::Paired(PairedBinaryOperator::GreaterThanOrEqual), + syn::BinOp::Gt(_) => BinaryOperator::Paired(PairedBinaryOperator::GreaterThan), + other_binary_operation => { + return other_binary_operation + .span_range() + .err("This operation is not supported in preinterpret expressions") + } + }; + Ok(Self { + span_for_output: expr.span_range(), + operator_span: expr.op.span_range(), + operator, + }) + } + + fn evaluate(self, left: EvaluationOutput, right: EvaluationOutput) -> Result { + match self.operator { + BinaryOperator::Paired(operator) => { + let value_pair = left.expect_value_pair(operator, right, self.operator_span)?; + value_pair.handle_paired_binary_operation(self) + } + BinaryOperator::Integer(_) => { + let right = right.expect_integer("The shift amount must be an integer")?; + left.into_value() + .handle_integer_binary_operation(right, self) + } + } + } + + pub(super) fn paired_operator(&self) -> PairedBinaryOperator { + match self.operator { + BinaryOperator::Paired(operator) => operator, + BinaryOperator::Integer(_) => panic!("Expected a paired operator"), + } + } + + pub(super) fn integer_operator(&self) -> IntegerBinaryOperator { + match self.operator { + BinaryOperator::Paired(_) => panic!("Expected an integer operator"), + BinaryOperator::Integer(operator) => operator, + } + } +} + +#[derive(Copy, Clone)] +pub(super) enum BinaryOperator { + Paired(PairedBinaryOperator), + Integer(IntegerBinaryOperator), +} + +impl BinaryOperator { + pub(super) fn symbol(&self) -> &'static str { + match self { + BinaryOperator::Paired(paired) => paired.symbol(), + BinaryOperator::Integer(integer) => integer.symbol(), + } + } +} + +#[derive(Copy, Clone)] +pub(super) enum PairedBinaryOperator { + Addition, + Subtraction, + Multiplication, + Division, + Remainder, + LogicalAnd, + LogicalOr, + BitXor, + BitAnd, + BitOr, + Equal, + LessThan, + LessThanOrEqual, + NotEqual, + GreaterThanOrEqual, + GreaterThan, +} + +impl PairedBinaryOperator { + pub(super) fn symbol(&self) -> &'static str { + match self { + PairedBinaryOperator::Addition => "+", + PairedBinaryOperator::Subtraction => "-", + PairedBinaryOperator::Multiplication => "*", + PairedBinaryOperator::Division => "/", + PairedBinaryOperator::Remainder => "%", + PairedBinaryOperator::LogicalAnd => "&&", + PairedBinaryOperator::LogicalOr => "||", + PairedBinaryOperator::BitXor => "^", + PairedBinaryOperator::BitAnd => "&", + PairedBinaryOperator::BitOr => "|", + PairedBinaryOperator::Equal => "==", + PairedBinaryOperator::LessThan => "<", + PairedBinaryOperator::LessThanOrEqual => "<=", + PairedBinaryOperator::NotEqual => "!=", + PairedBinaryOperator::GreaterThanOrEqual => ">=", + PairedBinaryOperator::GreaterThan => ">", + } + } +} + +#[derive(Copy, Clone)] +pub(super) enum IntegerBinaryOperator { + ShiftLeft, + ShiftRight, +} + +impl IntegerBinaryOperator { + pub(super) fn symbol(&self) -> &'static str { + match self { + IntegerBinaryOperator::ShiftLeft => "<<", + IntegerBinaryOperator::ShiftRight => ">>", + } + } +} + +pub(super) trait HandleBinaryOperation: Sized { + fn handle_paired_binary_operation( + self, + rhs: Self, + operation: &BinaryOperation, + ) -> Result; + + fn handle_integer_binary_operation( + self, + rhs: EvaluationInteger, + operation: &BinaryOperation, + ) -> Result; +} diff --git a/src/expressions/value.rs b/src/expressions/value.rs new file mode 100644 index 00000000..114277a7 --- /dev/null +++ b/src/expressions/value.rs @@ -0,0 +1,114 @@ +use super::*; + +pub(crate) enum EvaluationValue { + Integer(EvaluationInteger), + Float(EvaluationFloat), + Boolean(EvaluationBoolean), +} + +impl EvaluationValue { + pub(super) fn for_literal_expression(expr: &ExprLit) -> Result { + // https://docs.rs/syn/latest/syn/enum.Lit.html + Ok(match &expr.lit { + Lit::Int(lit) => Self::Integer(EvaluationInteger::for_litint(lit)?), + Lit::Float(lit) => Self::Float(EvaluationFloat::for_litfloat(lit)?), + Lit::Bool(lit) => Self::Boolean(EvaluationBoolean::for_litbool(lit)), + other_literal => { + return other_literal + .span() + .err("This literal is not supported in preinterpret expressions"); + } + }) + } + + pub(super) fn describe_type(&self) -> &'static str { + match self { + Self::Integer(int) => int.value.describe_type(), + Self::Float(float) => float.value.describe_type(), + Self::Boolean(_) => "bool", + } + } + + pub(super) fn handle_unary_operation( + self, + operation: UnaryOperation, + ) -> Result { + match self { + EvaluationValue::Integer(value) => value.handle_unary_operation(operation), + EvaluationValue::Float(value) => value.handle_unary_operation(operation), + EvaluationValue::Boolean(value) => value.handle_unary_operation(operation), + } + } + + pub(super) fn handle_integer_binary_operation( + self, + right: EvaluationInteger, + operation: BinaryOperation, + ) -> Result { + match self { + EvaluationValue::Integer(value) => { + value.handle_integer_binary_operation(right, operation) + } + EvaluationValue::Float(value) => { + value.handle_integer_binary_operation(right, operation) + } + EvaluationValue::Boolean(value) => { + value.handle_integer_binary_operation(right, operation) + } + } + } + + pub(super) fn source_span(&self) -> SpanRange { + match self { + EvaluationValue::Integer(integer) => integer.source_span, + EvaluationValue::Float(float) => float.source_span, + EvaluationValue::Boolean(boolean) => boolean.source_span, + } + } +} + +impl HasSpanRange for EvaluationValue { + fn span_range(&self) -> SpanRange { + match self { + Self::Integer(int) => int.source_span, + Self::Float(float) => float.source_span, + Self::Boolean(bool) => bool.source_span, + } + } +} + +impl quote::ToTokens for EvaluationValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Integer(int) => int.to_tokens(tokens), + Self::Float(float) => float.to_tokens(tokens), + Self::Boolean(bool) => bool.to_tokens(tokens), + } + } +} + +#[derive(Copy, Clone)] +pub(super) enum ValueKind { + Integer(IntegerKind), + Float(FloatKind), + Boolean, +} + +pub(super) enum EvaluationLiteralPair { + Integer(EvaluationIntegerValuePair), + Float(EvaluationFloatValuePair), + BooleanPair(EvaluationBoolean, EvaluationBoolean), +} + +impl EvaluationLiteralPair { + pub(super) fn handle_paired_binary_operation( + self, + operation: BinaryOperation, + ) -> Result { + match self { + Self::Integer(pair) => pair.handle_paired_binary_operation(&operation), + Self::Float(pair) => pair.handle_paired_binary_operation(&operation), + Self::BooleanPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), + } + } +} diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index b91ed286..102b619c 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -1,54 +1,195 @@ pub(crate) use core::iter; +use extra::DelimSpan; pub(crate) use proc_macro2::*; +pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; -pub(crate) use syn::{parse_str, Error, Lit, Result}; +pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, UnOp}; pub(crate) use crate::command::*; pub(crate) use crate::commands::*; +pub(crate) use crate::expressions::*; pub(crate) use crate::interpreter::*; -pub(crate) use crate::parsing::*; pub(crate) use crate::string_conversion::*; -pub(crate) struct Tokens(iter::Peekable<::IntoIter>); +pub(crate) trait TokenStreamExt: Sized { + fn push_token_tree(&mut self, token_tree: TokenTree); + fn push_new_group( + &mut self, + span_range: SpanRange, + delimiter: Delimiter, + inner_tokens: TokenStream, + ); +} -impl Tokens { - pub(crate) fn new(tokens: TokenStream) -> Self { - Self(tokens.into_iter().peekable()) +impl TokenStreamExt for TokenStream { + fn push_token_tree(&mut self, token_tree: TokenTree) { + self.extend(iter::once(token_tree)); } - pub(crate) fn peek(&mut self) -> Option<&TokenTree> { - self.0.peek() + fn push_new_group( + &mut self, + span_range: SpanRange, + delimiter: Delimiter, + inner_tokens: TokenStream, + ) { + let group = Group::new(delimiter, inner_tokens).with_span(span_range.span()); + self.push_token_tree(TokenTree::Group(group)); } +} - pub(crate) fn next(&mut self) -> Option { - self.0.next() +pub(crate) trait SpanErrorExt: Sized { + fn err(self, message: impl std::fmt::Display) -> syn::Result { + Err(self.error(message)) } - pub(crate) fn next_as_ident(&mut self) -> Option { - match self.next() { - Some(TokenTree::Ident(ident)) => Some(ident), - _ => None, - } + fn error(self, message: impl std::fmt::Display) -> syn::Error; +} + +impl SpanErrorExt for Span { + fn error(self, message: impl std::fmt::Display) -> syn::Error { + syn::Error::new(self, message) } +} - pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { - match self.next() { - Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), - _ => None, - } +impl SpanErrorExt for SpanRange { + fn error(self, message: impl std::fmt::Display) -> syn::Error { + syn::Error::new_spanned(self, message) } +} - pub(crate) fn into_token_stream(self) -> TokenStream { - self.0.collect() +pub(crate) trait WithSpanExt { + fn with_span(self, span: Span) -> Self; +} + +impl WithSpanExt for Literal { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self } } -pub(crate) trait SpanErrorExt { - fn error(self, message: impl std::fmt::Display) -> Error; +impl WithSpanExt for Punct { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } } -impl SpanErrorExt for Span { - fn error(self, message: impl std::fmt::Display) -> Error { - Error::new(self, message) +impl WithSpanExt for Group { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +pub(crate) trait HasSpanRange { + fn span_range(&self) -> SpanRange; +} + +/// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] +/// and falls back to the span of the first token when not available. +/// +/// Instead, [`syn::Error`] uses a trick involving a span range. This effectively +/// allows capturing this trick when we're not immediately creating an error. +/// +/// When [`proc_macro::Span::join`] is stabilised and [`syn::spanned`] works, +/// we can swap [`SpanRange`] contents for [`Span`] (or even remove it and [`HasSpanRange`]). +#[derive(Copy, Clone)] +pub(crate) struct SpanRange { + start: Span, + end: Span, +} + +#[allow(unused)] +impl SpanRange { + pub(crate) fn new_between(start: Span, end: Span) -> Self { + Self { start, end } + } + + /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) + /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) + pub(crate) fn span(&self) -> Span { + ::span(self) + } + + pub(crate) fn start(&self) -> Span { + self.start + } + + pub(crate) fn end(&self) -> Span { + self.end + } + + pub(crate) fn replace_start(self, start: Span) -> Self { + Self { start, ..self } } + + pub(crate) fn replace_end(self, end: Span) -> Self { + Self { end, ..self } + } +} + +// This is implemented so we can create an error from it using `Error::new_spanned(..)` +impl ToTokens for SpanRange { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend([ + TokenTree::Punct(Punct::new('<', Spacing::Alone).with_span(self.start)), + TokenTree::Punct(Punct::new('>', Spacing::Alone).with_span(self.end)), + ]); + } +} + +impl HasSpanRange for Span { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(*self, *self) + } +} + +impl HasSpanRange for TokenTree { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl HasSpanRange for Group { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl HasSpanRange for DelimSpan { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.open(), self.close()) + } +} + +impl HasSpanRange for T { + fn span_range(&self) -> SpanRange { + let mut iter = self.into_token_stream().into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + SpanRange { start, end } + } +} + +/// This should only be used for syn built-ins or when there isn't a better +/// span range available +trait AutoSpanRange {} + +macro_rules! impl_auto_span_range { + ($($ty:ty),* $(,)?) => { + $( + impl AutoSpanRange for $ty {} + )* + }; +} + +impl_auto_span_range! { + syn::Expr, + syn::ExprBinary, + syn::ExprUnary, + syn::BinOp, + syn::UnOp, + syn::Type, + syn::TypePath, } diff --git a/src/interpreter.rs b/src/interpreter.rs index c120b82f..93674ad4 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -23,28 +23,228 @@ impl Interpreter { self.variables.get(name) } + pub(crate) fn interpret_token_stream( + &mut self, + token_stream: TokenStream, + ) -> Result { + self.interpret_tokens(Tokens::new(token_stream)) + } + + pub(crate) fn interpret_item(&mut self, item: NextItem) -> Result { + let mut expanded = TokenStream::new(); + self.interpret_next_item(item, &mut expanded)?; + Ok(expanded) + } + pub(crate) fn interpret_tokens(&mut self, mut source_tokens: Tokens) -> Result { let mut expanded = TokenStream::new(); loop { - match parse_next_item(&mut source_tokens)? { - NextItem::Leaf(token_tree) => { - expanded.extend(iter::once(token_tree)); - } - NextItem::Group(group) => { - expanded.extend(iter::once(TokenTree::Group(Group::new( - group.delimiter(), - // If it's a group, run interpret on its contents recursively. - self.interpret_tokens(Tokens::new(group.stream()))?, - )))); - } - NextItem::VariableSubstitution(variable_substitution) => { - expanded.extend(variable_substitution.execute(self)?); + match source_tokens.next_item()? { + Some(next_item) => self.interpret_next_item(next_item, &mut expanded)?, + None => return Ok(expanded), + } + } + } + + fn interpret_next_item(&mut self, next_item: NextItem, output: &mut TokenStream) -> Result<()> { + // We wrap command/variable substitutions in a transparent group so that they + // can be treated as a single item in other commands. + // e.g. if #x = 1 + 1, then [!math! #x * #x] should be 4. + // Note that such groups are ignored in the macro output, due to this + // issue in rustc: https://github.com/rust-lang/rust/issues/67062 + match next_item { + NextItem::Leaf(token_tree) => { + output.push_token_tree(token_tree); + } + NextItem::Group(group) => { + // If it's a group, run interpret on its contents recursively. + output.push_new_group( + group.span_range(), + group.delimiter(), + self.interpret_tokens(Tokens::new(group.stream()))?, + ); + } + NextItem::Variable(variable_substitution) => { + // We wrap substituted variables in a transparent group so that + // they can be used collectively in future expressions, so that + // e.g. if #x = 1 + 1 then #x * #x = 4 rather than 3 + output.push_new_group( + variable_substitution.span_range(), + Delimiter::None, + variable_substitution.execute_substitution(self)?, + ); + } + NextItem::CommandInvocation(command_invocation) => { + output.push_new_group( + command_invocation.span_range(), + Delimiter::None, + command_invocation.execute(self)?, + ); + } + } + Ok(()) + } +} + +pub(crate) struct Tokens(iter::Peekable<::IntoIter>); + +impl Tokens { + pub(crate) fn new(tokens: TokenStream) -> Self { + Self(tokens.into_iter().peekable()) + } + + pub(crate) fn peek(&mut self) -> Option<&TokenTree> { + self.0.peek() + } + + pub(crate) fn next(&mut self) -> Option { + self.0.next() + } + + pub(crate) fn next_as_ident(&mut self) -> Option { + match self.next() { + Some(TokenTree::Ident(ident)) => Some(ident), + _ => None, + } + } + + pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { + match self.next() { + Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), + _ => None, + } + } + + pub(crate) fn next_as_kinded_group(&mut self, delimiter: Delimiter) -> Option { + match self.next() { + Some(TokenTree::Group(group)) if group.delimiter() == delimiter => Some(group), + _ => None, + } + } + + pub(crate) fn check_end(&mut self) -> Option<()> { + if self.peek().is_none() { + Some(()) + } else { + None + } + } + + pub(crate) fn assert_end(&mut self, error_message: &'static str) -> Result<()> { + match self.next() { + Some(token) => token.span_range().err(error_message), + None => Ok(()), + } + } + + pub(crate) fn next_item(&mut self) -> Result> { + let Some(next) = self.next() else { + return Ok(None); + }; + Ok(Some(match next { + TokenTree::Group(group) => { + if let Some(command_invocation) = parse_command_invocation(&group)? { + NextItem::CommandInvocation(command_invocation) + } else { + NextItem::Group(group) } - NextItem::CommandInvocation(command_invocation) => { - expanded.extend(command_invocation.execute(self)?); + } + TokenTree::Punct(punct) => { + if let Some(variable_substitution) = + parse_only_if_variable_substitution(&punct, self) + { + NextItem::Variable(variable_substitution) + } else { + NextItem::Leaf(TokenTree::Punct(punct)) } - NextItem::EndOfStream => return Ok(expanded), } + leaf => NextItem::Leaf(leaf), + })) + } + + pub(crate) fn next_item_as_variable( + &mut self, + error_message: &'static str, + ) -> Result { + match self.next_item()? { + Some(NextItem::Variable(variable_substitution)) => Ok(variable_substitution), + Some(item) => item.span_range().err(error_message), + None => Span::call_site().span_range().err(error_message), } } + + pub(crate) fn into_token_stream(self) -> TokenStream { + self.0.collect() + } +} + +pub(crate) enum NextItem { + CommandInvocation(CommandInvocation), + Variable(Variable), + Group(Group), + Leaf(TokenTree), +} + +impl HasSpanRange for NextItem { + fn span_range(&self) -> SpanRange { + match self { + NextItem::CommandInvocation(command_invocation) => command_invocation.span_range(), + NextItem::Variable(variable_substitution) => variable_substitution.span_range(), + NextItem::Group(group) => group.span_range(), + NextItem::Leaf(token_tree) => token_tree.span_range(), + } + } +} + +fn parse_command_invocation(group: &Group) -> Result> { + fn consume_command_start(group: &Group) -> Option<(Ident, Tokens)> { + if group.delimiter() != Delimiter::Bracket { + return None; + } + let mut tokens = Tokens::new(group.stream()); + tokens.next_as_punct_matching('!')?; + let ident = tokens.next_as_ident()?; + Some((ident, tokens)) + } + + fn consume_command_end(command_ident: &Ident, tokens: &mut Tokens) -> Option { + let command_kind = CommandKind::attempt_parse(command_ident)?; + tokens.next_as_punct_matching('!')?; + Some(command_kind) + } + + // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, + // so return `Ok(None)` + let (command_ident, mut remaining_tokens) = match consume_command_start(group) { + Some(command_start) => command_start, + None => return Ok(None), + }; + + // We have now checked enough that we're confident the user is pretty intentionally using + // the call convention. Any issues we hit from this point will be a helpful compiler error. + match consume_command_end(&command_ident, &mut remaining_tokens) { + Some(command_kind) => Ok(Some(CommandInvocation::new(command_kind, group, remaining_tokens))), + None => Err(command_ident.span().error( + format!( + "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", + CommandKind::list_all(), + command_ident, + ), + )), + } +} + +// We ensure we don't consume any tokens unless we have a variable substitution +fn parse_only_if_variable_substitution(punct: &Punct, tokens: &mut Tokens) -> Option { + if punct.as_char() != '#' { + return None; + } + match tokens.peek() { + Some(TokenTree::Ident(_)) => {} + _ => return None, + } + match tokens.next() { + Some(TokenTree::Ident(variable_name)) => Some(Variable::new(punct.clone(), variable_name)), + _ => unreachable!("We just peeked a token of this type"), + } } diff --git a/src/lib.rs b/src/lib.rs index 7f89bfd6..96dce47c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -512,9 +512,9 @@ //! mod command; mod commands; +mod expressions; mod internal_prelude; mod interpreter; -mod parsing; mod string_conversion; use internal_prelude::*; diff --git a/src/parsing.rs b/src/parsing.rs deleted file mode 100644 index 445e4e26..00000000 --- a/src/parsing.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) enum NextItem { - CommandInvocation(CommandInvocation), - VariableSubstitution(VariableSubstitution), - Group(Group), - Leaf(TokenTree), - EndOfStream, -} - -pub(crate) fn parse_next_item(tokens: &mut Tokens) -> Result { - Ok(match tokens.next() { - Some(TokenTree::Group(group)) => { - if let Some(command_invocation) = parse_command_invocation(&group)? { - NextItem::CommandInvocation(command_invocation) - } else { - NextItem::Group(group) - } - } - Some(TokenTree::Punct(punct)) => { - if let Some(variable_substitution) = parse_only_if_variable_substitution(&punct, tokens) - { - NextItem::VariableSubstitution(variable_substitution) - } else { - NextItem::Leaf(TokenTree::Punct(punct)) - } - } - Some(leaf) => NextItem::Leaf(leaf), - None => NextItem::EndOfStream, - }) -} - -pub(crate) fn parse_variable_set(tokens: &mut Tokens) -> Option { - let variable_name = parse_variable(tokens)?; - tokens.next_as_punct_matching('=')?; - Some(variable_name) -} - -pub(crate) fn parse_variable(tokens: &mut Tokens) -> Option { - tokens.next_as_punct_matching('#')?; - tokens.next_as_ident() -} - -fn parse_command_invocation(group: &Group) -> Result> { - fn consume_command_start(group: &Group) -> Option<(Ident, Tokens)> { - if group.delimiter() != Delimiter::Bracket { - return None; - } - let mut tokens = Tokens::new(group.stream()); - tokens.next_as_punct_matching('!')?; - let ident = tokens.next_as_ident()?; - Some((ident, tokens)) - } - - fn consume_command_end(command_ident: &Ident, tokens: &mut Tokens) -> Option { - let command_kind = CommandKind::attempt_parse(command_ident)?; - tokens.next_as_punct_matching('!')?; - Some(command_kind) - } - - // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, - // so return `Ok(None)` - let (command_ident, mut remaining_tokens) = match consume_command_start(group) { - Some(command_start) => command_start, - None => return Ok(None), - }; - - // We have now checked enough that we're confident the user is pretty intentionally using - // the call convention. Any issues we hit from this point will be a helpful compiler error. - match consume_command_end(&command_ident, &mut remaining_tokens) { - Some(command_kind) => Ok(Some(CommandInvocation::new(command_kind, group, remaining_tokens))), - None => Err(Error::new( - command_ident.span(), - format!( - "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", - CommandKind::list_all(), - command_ident, - ), - )), - } -} - -// We ensure we don't consume any tokens unless we have a variable substitution -fn parse_only_if_variable_substitution( - punct: &Punct, - tokens: &mut Tokens, -) -> Option { - if punct.as_char() != '#' { - return None; - } - match tokens.peek() { - Some(TokenTree::Ident(_)) => {} - _ => return None, - } - match tokens.next() { - Some(TokenTree::Ident(variable_name)) => { - Some(VariableSubstitution::new(punct.clone(), variable_name)) - } - _ => unreachable!("We just peeked a token of this type"), - } -} diff --git a/tests/control_flow.rs b/tests/control_flow.rs new file mode 100644 index 00000000..cb7250c4 --- /dev/null +++ b/tests/control_flow.rs @@ -0,0 +1,20 @@ +use preinterpret::preinterpret; + +macro_rules! assert_preinterpret_eq { + ($input:tt, $($output:tt)*) => { + assert_eq!(preinterpret!($input), $($output)*); + }; +} + +#[test] +fn test_basic_evaluate_works() { + assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); + assert_preinterpret_eq!({ + [!set! #x = 1 == 2] + [!if! (#x) { "YES" } !else! { "NO" }] + }, "NO"); + assert_preinterpret_eq!({ + 0 + [!if! false { + 1 }] + }, 0); +} diff --git a/tests/evaluate.rs b/tests/evaluate.rs new file mode 100644 index 00000000..4ecc32e2 --- /dev/null +++ b/tests/evaluate.rs @@ -0,0 +1,57 @@ +use preinterpret::preinterpret; + +macro_rules! assert_preinterpret_eq { + ($input:tt, $($output:tt)*) => { + assert_eq!(preinterpret!($input), $($output)*); + }; +} + +#[test] +fn test_basic_evaluate_works() { + assert_preinterpret_eq!([!evaluate! !!(!!(true))], true); + assert_preinterpret_eq!([!evaluate! 1 + 5], 6u8); + assert_preinterpret_eq!([!evaluate! 1 + 5], 6i128); + assert_preinterpret_eq!([!evaluate! 1 + 5u16], 6u16); + assert_preinterpret_eq!([!evaluate! 127i8 + (-127i8) + (-127i8)], -127i8); + assert_preinterpret_eq!([!evaluate! 3.0 + 3.2], 6.2); + assert_preinterpret_eq!([!evaluate! 3.6 + 3999999999999999992.0], 3.6 + 3999999999999999992.0); + assert_preinterpret_eq!([!evaluate! -3.2], -3.2); + assert_preinterpret_eq!([!evaluate! true && true || false], true); + assert_preinterpret_eq!([!evaluate! true as u32 + 2], 3); + assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u32); + assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u64); + assert_preinterpret_eq!([!evaluate! 0b1000 & 0b1101], 0b1000); + assert_preinterpret_eq!([!evaluate! 0b1000 | 0b1101], 0b1101); + assert_preinterpret_eq!([!evaluate! 0b1000 ^ 0b1101], 0b101); + assert_preinterpret_eq!([!evaluate! 5 << 2], 20); + assert_preinterpret_eq!([!evaluate! 5 >> 1], 2); + assert_preinterpret_eq!([!evaluate! 123 == 456], false); + assert_preinterpret_eq!([!evaluate! 123 < 456], true); + assert_preinterpret_eq!([!evaluate! 123 <= 456], true); + assert_preinterpret_eq!([!evaluate! 123 != 456], true); + assert_preinterpret_eq!([!evaluate! 123 >= 456], false); + assert_preinterpret_eq!([!evaluate! 123 > 456], false); + assert_preinterpret_eq!( + { + [!set! #six_as_sum = 3 + 3] // The token stream '3 + 3'. They're not evaluated to 6 (yet). + [!evaluate! #six_as_sum * #six_as_sum] + }, + 36 + ); +} + +#[test] +fn increment_works() { + assert_preinterpret_eq!( + { + [!set! #x = 2 + 2] + [!increment! #x] + [!increment! #x] + #x + }, + 6 + ); +} + +// TODO - Add failing tests for these: +// assert_preinterpret_eq!([!evaluate! !!(!!({true}))], true); From c734c83f8185c63056f8bb974efd2c3c089650fc Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 31 Dec 2024 13:40:33 +0000 Subject: [PATCH 002/126] fix: Fix MSRV --- src/commands/control_flow_commands.rs | 7 +++++-- src/expressions/operations.rs | 11 +++++++---- src/interpreter.rs | 5 +++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 577e7987..c0be06df 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -10,8 +10,11 @@ impl CommandDefinition for IfCommand { argument: CommandArgumentStream, command_span: Span, ) -> Result { - let Some(parsed) = parse_if_statement(&mut argument.tokens()) else { - return command_span.span_range().err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); + let parsed = match parse_if_statement(&mut argument.tokens()) { + Some(parsed) => parsed, + None => { + return command_span.span_range().err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); + } }; let interpreted_condition = interpreter.interpret_item(parsed.condition)?; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 355d7815..082caaf6 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -72,10 +72,13 @@ impl UnaryOperation { && type_path.path.leading_colon.is_none() && type_path.path.segments.len() == 1 => { - let Some(ident) = type_path.path.get_ident() else { - return type_path - .span_range() - .err("This type is not supported in preinterpret cast expressions"); + let ident = match type_path.path.get_ident() { + Some(ident) => ident, + None => { + return type_path + .span_range() + .err("This type is not supported in preinterpret cast expressions") + } }; match ident.to_string().as_str() { "int" | "integer" => Ok(ValueKind::Integer(IntegerKind::Untyped)), diff --git a/src/interpreter.rs b/src/interpreter.rs index 93674ad4..b8183470 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -138,8 +138,9 @@ impl Tokens { } pub(crate) fn next_item(&mut self) -> Result> { - let Some(next) = self.next() else { - return Ok(None); + let next = match self.next() { + Some(next) => next, + None => return Ok(None), }; Ok(Some(match next { TokenTree::Group(group) => { From 8fc41219bbdaf6634597cc03cc75734adcadb1ff Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 31 Dec 2024 14:07:30 +0000 Subject: [PATCH 003/126] tweak: Doc tweaks --- CHANGELOG.md | 26 +++++++++++ KNOWN_ISSUES.md | 6 ++- README.md | 122 +++++++----------------------------------------- 3 files changed, 47 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5a420e2..be17dbec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Major Version 0.2 +## 0.2.1 + +### New Commands + +* `[!evaluate! ...]` +* `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` +* `[!increment! ...]` + +### To come + +* `[!while! cond {}]` +* `[!for! #x in [#y] {}]`... and make it so that whether commands or variable substitutions get replaced by groups depends on the interpreter context (likely only expressions should use groups) +* `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. +* Remove `[!increment! ...]` and replace with `[!set! #x += 1]` +* Support `!else if!` in `!if!` +* Token stream manipulation... and make it performant + * Extend token stream + * Consume from start of token stream +* Support string & char literals (for comparisons & casts) in expressions +* Add more tests + * e.g. for various expressions + * e.g. for long sums + * Add compile failure tests +* Work on book + * Including documenting expressions + ## 0.2.0 * Rename the string case conversion commands to be less noisy by getting rid of the case suffix diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 3c4e546d..1191c900 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -1,6 +1,8 @@ -# Since Major Version 0.3 +# Since Major Version 0.2.1 * `[MAX LITERAL]` - In an expression, the maximum negative integer literal is not valid, for example `-128i8`. * By comparison, in `rustc`, there is special handling for such literals, so that e.g. `-128i8` and `--(-128i8)` are accepted. * In rust analyzer, before the full pass, it seems to store literals as u128 and perform wrapping arithmetic on them. On a save / full pass, it does appear to reject literals which are out of bounds (e.g. 150i8). - * We could fix this by special casing maximal signed integer literals (e.g. 128 for an i8) which have yet to be involved in a non-unary expression. \ No newline at end of file + * We could fix this by special casing maximal signed integer literals (e.g. 128 for an i8) which have yet to be involved in a non-unary expression. +* `&&` and `||` are not lazy in expressions + * This needs a custom expression parser, rather than just leveraging syn diff --git a/README.md b/README.md index 3b438fd8..a5940293 100644 --- a/README.md +++ b/README.md @@ -334,10 +334,6 @@ macro_rules! impl_new_type { ## Future Extension Possibilities -### Add github docs page / rust book - -Add a github docs page / rust book at this repository, to allow us to build out a suite of examples, like `serde` or the little book of macros. - ### Destructuring / Parsing Syntax, and Declarative Macros 2.0 I have a vision for having preinterpret effectively replace the use of declarative macros in the Rust ecosystem, by: @@ -416,87 +412,31 @@ preinterpret::preinterpret! { } ``` -### Possible extension: Numeric commands - - -These might be introduced behind a default-enabled feature flag. - -Each numeric command would have an implicit configuration: -* A calculation space (e.g. as `i32` or `f64`) -* An output literal suffix (e.g. output no suffix or add `i32`) - -We could support: -* Inferred configurations (like rustc, by looking at the suffices of literals or inferring them) - * `calc` (inferred space, explicit suffix or no suffix) -* Non-specific configurations, such as: - * `int` (`i128` space, no suffix) - * `float` (`f64` space, no suffix) -* Specific configurations, such as: - * `i32` (`i32` space, `i32` suffix) - * `usize` (`u64` space, `usize` suffix) - -#### Mathematical interpretation - -This would support calculator style expressions: -* `[!calc! (5 + 10) / 2]` outputs `7` as an integer space is inferred -* `[!int! (5 + 10) / 2]` outputs `7` -* `[!f64! (5 + 10) / 2]` outputs `7.5f64` - -These commands would execute in these steps: -* Apply the interpreter to the token stream, which recursively executes preinterpret commands, wrapping their outputs in transparent groups. -* Iterate over each token (or groups with `(..)` or transparent delimeters), expecting numeric literals, or operators `+` / `-` / `*` / `/`. -* Evaluate the mathematical expression in the given calculation space. Any literals are cast to the given calculation space. -* Output a single literal, with its output literal suffix. - -Extra details: -* Overflows would default to outputting a compile error, with a possible option to reconfigure the interpreter to use different overflow behaviour. -* A suffix could be stripped with e.g. `[!strip_suffix! 7.5f32]` giving `7.5`. -* A sum over some unknown number of items could be achieved with e.g. `[!int! 0 $(+ $x)*]` - -#### Numeric functions +### Possible extension: Token stream commands -A functions with N parameters works in three steps: -* Apply the interpreter to the token stream, which recursively executes preinterpret commands, wrapping their outputs in transparent groups. -* Expect N remaining token trees as the arguments. Each should be evaluated as a mathematical expression, using an inferred calculation space -* Output a single literal, with its calculation space suffix. +* `[!split! #stream ,]` expects a token tree, and then some punct. Returns a stream split into transparent groups by the given token. Ignores a trailing empty stream. +* `[!skip! #stream 4]` expects to receive a (possibly transparent) group, and reads and drops the first 4 token trees from the group's stream, and outputs the rest +* `[!ungroup! #stream]` expects `#stream` to be a single group and unwraps it once +* `[!flatten! #stream]` removes all singleton groups from `#stream`, leaving a token stream of idents, literals and punctuation +* `[!at! #stream 2]` takes the 2nd token tree from the stream +* `[!zip! #a #b #c]` returns a transparent group tuple of the nth token in each of `#a`, `#b` and `#c`. Errors if they are of different lengths. -Example commands could be: +We could support a piped calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream as #x |> [!skip! #x 4] |> [!ungroup! #x]]` -* `[!mod! $length 2]` outputs `0` if `$length` is even, else `1`. It takes two integers `a` and `b`, and outputs `a mod b`. The calculation operates in the space of `$length` if it has a suffix, else in `int` space. -* `[!sum! 5u64 9 32]` outputs `46u64`. It takes any number of integers and outputs their sum. The calculation operates in `u64` space. +#### Possible extension: Better performance -We also support the following assignment commands: +Do some testing and ensure there are tools to avoid `N^2` performance when doing token manipulation, e.g. with: -* `[!increment! #i]` is shorthand for `[!set! #i = [!calc! #i + 1]]` and outputs no tokens. +* `[!push! #stream new tokens...]` +* `[!take! #stream #x]` where `#x` is read as the first token tree from `#stream` +* `[!take! #stream ()]` where the parser is read greadily from `#stream` +* `[!is_empty! #stream]` ### Possible extension: User-defined commands * `[!define! [!my_command! ] { }]` -### Possible extension: Boolean commands - -Similar to numeric commands, these could be an interpretation mode with e.g. -* `[!bool! (true || false) && !true || #x <= 3]` - -Each of these commands functions in three steps: -* Apply the interpreter to the token stream, which recursively executes preinterpret commands, wrapping their outputs in transparent groups. -* Iterate over each token (or groups with `(..)` or transparent delimeters), expecting boolean literals, boolean operators, or comparison statements. -* Apply some command-specific comparison, and outputs the boolean literal `true` or `false`. - -Comparison statements could look like the following and operate on literals. For each, a comparison space (e.g. `string` or `u32`) needs to be inferrable from the literals. A compile error is thrown if a comparison space cannot be inferred: -* `#foo == #bar` outputs `true` if `#foo` and `#bar` are exactly the same literal. -* `#foo <= #bar` outputs `true` if `#foo` is less than or equal to `#bar` -* `#foo >= #bar` outputs `true` if `#foo` is greater than or equal to `#bar` -* `#foo > #bar` outputs `true` if `#foo` is greater than `#bar` -* `#foo < #bar` outputs `true` if `#foo` is less than `#bar` +### Possible extension: Further utility commands Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example: @@ -504,37 +444,9 @@ Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. -### Possible extension: Token stream commands - -* `[!split_at! #stream ,]` expects a token tree, and then some punct. Returns a stream split into transparent groups by the given token. Ignores a trailing empty stream. -* `[!skip! #stream 4]` expects to receive a (possibly transparent) group, and reads and drops the first 4 token trees from the group's stream, and outputs the rest -* `[!ungroup! [(#stream)]]` outputs `#stream` without any groups. It expects to receive a single group, and if the resulting token stream is a single group, it unwraps again -* `[!flatten! #stream]` removes all groups from `#stream`, leaving a token stream of idents, literals and punctuation -* `[!at! #stream 2]` takes the 2nd token tree from the stream -* `[!zip! #a #b #c]` returns a transparent group tuple of the nth token in each of `#a`, `#b` and `#c`. Errors if they are of different lengths. - -We could support a piped calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream as #x |> [!skip! #x 4] |> [!ungroup! #x]]` - -### Possible extension: Control flow commands - -#### If statement - -`[!if! (XXX) { #a } else { #b }]` outputs `#a` if `XXX` is a boolean expression evaluating to `true`, else outputs `#b`. - -The `if` command works as follows: -* It starts by only interpreting its first token tree, and expects to see a `(..)` group which can be evaluated as a boolean expression. -* It then expects a single `{ .. }` group, whose contents get interpreted and output only if the condition was `true`. -* It optionally also reads an `else` ident and a by a single `{ .. }` group, whose contents get interpreted and output only if the condition was `false`. - -#### While loop - -Is discussed in macro 2.0 above. - -#### For loop - -Is discussed in macro 2.0 above. +### Possible extension: Goto -#### Goto and label +_This probably isn't needed_. * `[!label! loop_start]` - defines a label which can be returned to. Effectively, it takes a clones of the remaining token stream after the label in the interpreter. * `[!goto! loop_start]` - jumps to the last execution of `[!label! loop_start]`. It unrolls the preinterpret stack (dropping all unwritten token streams) until it finds a stackframe in which the interpreter has the defined label, and continues the token stream from there. From 2127299c319ef295598b1b30594bb4c5d2d43663 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 31 Dec 2024 14:33:39 +0000 Subject: [PATCH 004/126] docs: Further docs and update planning --- CHANGELOG.md | 13 ++++++++++--- KNOWN_ISSUES.md | 2 +- README.md | 13 +++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be17dbec..ccc5015c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ -# Major Version 0.2 +# Major Version 0.3 -## 0.2.1 +## 0.3.0 ### New Commands @@ -10,10 +10,15 @@ ### To come +* Use `[!let! #x = 12]` instead of `[!set! ..]`. +* Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). * `[!while! cond {}]` * `[!for! #x in [#y] {}]`... and make it so that whether commands or variable substitutions get replaced by groups depends on the interpreter context (likely only expressions should use groups) * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. -* Remove `[!increment! ...]` and replace with `[!set! #x += 1]` +* `[!empty!]` +* `[!is_empty! #stream]` +* `[!range! 0..5]` +* Remove `[!increment! ...]` and replace with `[!assign! #x += 1]` * Support `!else if!` in `!if!` * Token stream manipulation... and make it performant * Extend token stream @@ -26,6 +31,8 @@ * Work on book * Including documenting expressions +# Major Version 0.2 + ## 0.2.0 * Rename the string case conversion commands to be less noisy by getting rid of the case suffix diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 1191c900..16151473 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -1,4 +1,4 @@ -# Since Major Version 0.2.1 +# Since Major Version 0.3.0 * `[MAX LITERAL]` - In an expression, the maximum negative integer literal is not valid, for example `-128i8`. * By comparison, in `rustc`, there is special handling for such literals, so that e.g. `-128i8` and `--(-128i8)` are accepted. diff --git a/README.md b/README.md index a5940293..6c44bfd2 100644 --- a/README.md +++ b/README.md @@ -418,8 +418,8 @@ preinterpret::preinterpret! { * `[!skip! #stream 4]` expects to receive a (possibly transparent) group, and reads and drops the first 4 token trees from the group's stream, and outputs the rest * `[!ungroup! #stream]` expects `#stream` to be a single group and unwraps it once * `[!flatten! #stream]` removes all singleton groups from `#stream`, leaving a token stream of idents, literals and punctuation -* `[!at! #stream 2]` takes the 2nd token tree from the stream -* `[!zip! #a #b #c]` returns a transparent group tuple of the nth token in each of `#a`, `#b` and `#c`. Errors if they are of different lengths. +* `[!index! #stream 0]` takes the 0th token tree from the stream +* `[!zip! #a #b #c]` returns a transparent group tuple of the nth token in each of `#a`, `#b` and `#c`. Errors if they are of different lengths We could support a piped calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream as #x |> [!skip! #x 4] |> [!ungroup! #x]]` @@ -427,10 +427,11 @@ We could support a piped calling convention, such as the `[!pipe! ...]` special Do some testing and ensure there are tools to avoid `N^2` performance when doing token manipulation, e.g. with: -* `[!push! #stream new tokens...]` -* `[!take! #stream #x]` where `#x` is read as the first token tree from `#stream` -* `[!take! #stream ()]` where the parser is read greadily from `#stream` -* `[!is_empty! #stream]` +* `[!append! #stream += new tokens...]` +* `[!consume_from! #stream #x]` where `#x` is read as the first token tree from `#stream` +* `[!consume_from! #stream ()]` where the parser is read greedily from `#stream` + +We could consider tweaking some commands to execute lazily, i.e. change the infrastructure to operate over a `TokenIterable` which is either a `TokenStream` or `LazyTokenStream`. Things like `[!zip! ...]` or `[!range! ...]` could then output a `LazyTokenStream`. ### Possible extension: User-defined commands From 8eb0f471b75fb580434e8c2bf420d5fd3fa8f9cc Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 1 Jan 2025 20:26:01 +0000 Subject: [PATCH 005/126] WIP: More commands --- CHANGELOG.md | 17 ++- src/command.rs | 145 ++++++++---------- src/commands/concat_commands.rs | 206 +++++++++++++++----------- src/commands/control_flow_commands.rs | 13 +- src/commands/core_commands.rs | 18 +-- src/commands/expression_commands.rs | 15 +- src/commands/mod.rs | 8 + src/commands/token_commands.rs | 63 ++++++++ src/internal_prelude.rs | 10 ++ src/interpreter.rs | 90 ++++++++--- tests/control_flow.rs | 13 +- tests/tokens.rs | 47 ++++++ 12 files changed, 419 insertions(+), 226 deletions(-) create mode 100644 src/commands/token_commands.rs create mode 100644 tests/tokens.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc5015c..fa5c02f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,16 @@ ### New Commands -* `[!evaluate! ...]` -* `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` -* `[!increment! ...]` +* Expression commands: + * `[!evaluate! ...]` + * `[!increment! ...]` +* Control flow commands: + * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` +* Token-stream utility commands: + * `[!empty!]` + * `[!is_empty! #stream]` + * `[!length! #stream]` which gives the number of token trees in the token stream. + * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. ### To come @@ -14,10 +21,8 @@ * Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). * `[!while! cond {}]` * `[!for! #x in [#y] {}]`... and make it so that whether commands or variable substitutions get replaced by groups depends on the interpreter context (likely only expressions should use groups) -* `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. -* `[!empty!]` -* `[!is_empty! #stream]` * `[!range! 0..5]` +* `[!error! "message" token stream for span]` * Remove `[!increment! ...]` and replace with `[!assign! #x += 1]` * Support `!else if!` in `!if!` * Token stream manipulation... and make it performant diff --git a/src/command.rs b/src/command.rs index 06cc4993..dcb467ff 100644 --- a/src/command.rs +++ b/src/command.rs @@ -5,8 +5,7 @@ pub(crate) trait CommandDefinition { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + argument: Command, ) -> Result; } @@ -26,10 +25,10 @@ macro_rules! define_commands { } impl $enum_name { - pub(crate) fn execute(self, interpreter: &mut Interpreter, argument_stream: CommandArgumentStream, command_span: Span) -> Result { + pub(crate) fn execute(self, interpreter: &mut Interpreter, command: Command) -> Result { match self { $( - Self::$command => $command::execute(interpreter, argument_stream, command_span), + Self::$command => $command::execute(interpreter, command), )* } } @@ -56,28 +55,30 @@ pub(crate) use define_commands; pub(crate) struct CommandInvocation { command_kind: CommandKind, - argument_stream: CommandArgumentStream, - command_span: Span, + command: Command, } impl CommandInvocation { - pub(crate) fn new(command_kind: CommandKind, group: &Group, argument_tokens: Tokens) -> Self { + pub(crate) fn new(command_ident: Ident, command_kind: CommandKind, group: &Group, argument_tokens: Tokens) -> Self { Self { command_kind, - argument_stream: CommandArgumentStream::new(argument_tokens), - command_span: group.span(), + command: Command::new( + command_ident, + group.span(), + argument_tokens, + ), } } pub(crate) fn execute(self, interpreter: &mut Interpreter) -> Result { self.command_kind - .execute(interpreter, self.argument_stream, self.command_span) + .execute(interpreter, self.command) } } impl HasSpanRange for CommandInvocation { fn span_range(&self) -> SpanRange { - self.command_span.span_range() + self.command.span_range() } } @@ -103,22 +104,17 @@ impl Variable { interpreter: &mut Interpreter, ) -> Result { let Variable { - marker, variable_name, + .. } = self; match interpreter.get_variable(&variable_name.to_string()) { Some(variable_value) => Ok(variable_value.clone()), None => { - let marker = marker.as_char(); - let name_str = variable_name.to_string(); - let name_str = &name_str; - variable_name.span().err( + self.span_range().err( format!( - "The variable {}{} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}{}]", - marker, - name_str, - marker, - name_str, + "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", + self, + self, ), ) } @@ -126,89 +122,66 @@ impl Variable { } } +impl HasSpanRange for Variable { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.marker.span(), self.variable_name.span()) + } +} + impl core::fmt::Display for Variable { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{}{}", self.marker.as_char(), self.variable_name) } } -impl HasSpanRange for Variable { - fn span_range(&self) -> SpanRange { - SpanRange::new_between(self.marker.span(), self.variable_name.span()) - } +pub(crate) struct Command { + command_ident: Ident, + command_span: Span, + argument_tokens: Tokens, } -pub(crate) struct CommandArgumentStream { - tokens: Tokens, -} +impl Command { + fn new(command_ident: Ident, command_span: Span, argument_tokens: Tokens) -> Self { + Self { command_ident, command_span, argument_tokens } + } -impl CommandArgumentStream { - fn new(tokens: Tokens) -> Self { - Self { tokens } + #[allow(unused)] // Likely useful in future + pub(crate) fn ident_span(&self) -> Span { + self.command_ident.span() } - pub(crate) fn interpret(self, interpreter: &mut Interpreter) -> Result { - interpreter.interpret_tokens(self.tokens) + pub(crate) fn span(&self) -> Span { + self.command_span } - pub(crate) fn interpret_and_concat_to_string( - self, - interpreter: &mut Interpreter, - ) -> Result { - let interpreted = interpreter.interpret_tokens(self.tokens)?; - Ok(concat_recursive(interpreted)) + pub(crate) fn error(&self, message: impl core::fmt::Display) -> syn::Error { + self.command_span.error(message) } - pub(crate) fn tokens(self) -> Tokens { - self.tokens + pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { + Err(self.error(message)) } -} -fn concat_recursive(arguments: TokenStream) -> String { - fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { - for token_tree in arguments { - match token_tree { - TokenTree::Literal(literal) => { - let lit: Lit = parse_str(&literal.to_string()).expect( - "All proc_macro2::Literal values should be decodable as a syn::Lit", - ); - match lit { - Lit::Str(lit_str) => output.push_str(&lit_str.value()), - Lit::Char(lit_char) => output.push(lit_char.value()), - _ => { - output.push_str(&literal.to_string()); - } - } - } - TokenTree::Group(group) => match group.delimiter() { - Delimiter::Parenthesis => { - output.push('('); - concat_recursive_internal(output, group.stream()); - output.push(')'); - } - Delimiter::Brace => { - output.push('{'); - concat_recursive_internal(output, group.stream()); - output.push('}'); - } - Delimiter::Bracket => { - output.push('['); - concat_recursive_internal(output, group.stream()); - output.push(']'); - } - Delimiter::None => { - concat_recursive_internal(output, group.stream()); - } - }, - TokenTree::Punct(punct) => { - output.push(punct.as_char()); - } - TokenTree::Ident(ident) => output.push_str(&ident.to_string()), - } + /// Expects the remaining arguments to be non-empty + pub(crate) fn interpret_remaining_arguments(&mut self, interpreter: &mut Interpreter, substitution_mode: SubstitutionMode) -> Result { + if self.argument_tokens.is_empty() { + // This is simply for clarity / to make empty arguments explicit. + return self.err("Arguments were empty. Use [!empty!] if you want to use an empty token stream."); } + interpreter.interpret_tokens(&mut self.argument_tokens, substitution_mode) } - let mut output = String::new(); - concat_recursive_internal(&mut output, arguments); - output + pub(crate) fn argument_tokens(&mut self) -> &mut Tokens { + &mut self.argument_tokens + } + + pub(crate) fn into_argument_tokens(self) -> Tokens { + self.argument_tokens + } } + +impl HasSpanRange for Command { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} \ No newline at end of file diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index ac6fae22..e59e3b9c 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -24,6 +24,39 @@ fn parse_ident(value: &str, span: Span) -> Result { Ok(ident) } +fn concat_into_string( + interpreter: &mut Interpreter, + mut command: Command, + conversion_fn: impl Fn(&str) -> String, +) -> Result { + let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let concatenated = concat_recursive(interpreted); + let string_literal = string_literal(&conversion_fn(&concatenated), command.span()); + Ok(TokenStream::from(TokenTree::Literal(string_literal))) +} + +fn concat_into_ident( + interpreter: &mut Interpreter, + mut command: Command, + conversion_fn: impl Fn(&str) -> String, +) -> Result { + let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let concatenated = concat_recursive(interpreted); + let ident = parse_ident(&conversion_fn(&concatenated), command.span())?; + Ok(TokenStream::from(TokenTree::Ident(ident))) +} + +fn concat_into_literal( + interpreter: &mut Interpreter, + mut command: Command, + conversion_fn: impl Fn(&str) -> String, +) -> Result { + let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let concatenated = concat_recursive(interpreted); + let literal = parse_literal(&conversion_fn(&concatenated), command.span())?; + Ok(TokenStream::from(TokenTree::Literal(literal))) +} + //======================================= // Concatenating type-conversion commands //======================================= @@ -35,12 +68,9 @@ impl CommandDefinition for StringCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let string_literal = string_literal(&interpreted, command_span); - Ok(TokenStream::from(TokenTree::Literal(string_literal))) + concat_into_string(interpreter, command, |s| s.to_string()) } } @@ -51,12 +81,9 @@ impl CommandDefinition for IdentCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let parsed_ident = parse_ident(&interpreted, command_span)?; - Ok(TokenStream::from(TokenTree::Ident(parsed_ident))) + concat_into_ident(interpreter, command, |s| s.to_string()) } } @@ -67,13 +94,9 @@ impl CommandDefinition for IdentCamelCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let upper_camel_cased = to_upper_camel_case(&interpreted); - let parsed_ident = parse_ident(&upper_camel_cased, command_span)?; - Ok(TokenStream::from(TokenTree::Ident(parsed_ident))) + concat_into_ident(interpreter, command, to_upper_camel_case) } } @@ -84,13 +107,9 @@ impl CommandDefinition for IdentSnakeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let lower_snake_cased = to_lower_snake_case(&interpreted); - let parsed_ident = parse_ident(&lower_snake_cased, command_span)?; - Ok(TokenStream::from(TokenTree::Ident(parsed_ident))) + concat_into_ident(interpreter, command, to_lower_snake_case) } } @@ -101,13 +120,9 @@ impl CommandDefinition for IdentUpperSnakeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let upper_snake_cased = to_upper_snake_case(&interpreted); - let parsed_ident = parse_ident(&upper_snake_cased, command_span)?; - Ok(TokenStream::from(TokenTree::Ident(parsed_ident))) + concat_into_ident(interpreter, command, to_upper_snake_case) } } @@ -118,12 +133,9 @@ impl CommandDefinition for LiteralCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let parsed_literal = parse_literal(&interpreted, command_span)?; - Ok(TokenStream::from(TokenTree::Literal(parsed_literal))) + concat_into_literal(interpreter, command, |s| s.to_string()) } } @@ -131,17 +143,6 @@ impl CommandDefinition for LiteralCommand { // String conversion commands //=========================== -fn concat_string_and_convert( - interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, - conversion_fn: impl Fn(&str) -> String, -) -> Result { - let interpreted = argument.interpret_and_concat_to_string(interpreter)?; - let string_literal = string_literal(&conversion_fn(&interpreted), command_span); - Ok(TokenStream::from(TokenTree::Literal(string_literal))) -} - pub(crate) struct UpperCommand; impl CommandDefinition for UpperCommand { @@ -149,10 +150,9 @@ impl CommandDefinition for UpperCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, to_uppercase) + concat_into_string(interpreter, command, to_uppercase) } } @@ -163,10 +163,9 @@ impl CommandDefinition for LowerCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, to_lowercase) + concat_into_string(interpreter, command, to_lowercase) } } @@ -177,11 +176,10 @@ impl CommandDefinition for SnakeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { // Lower snake case is the more common casing in Rust, so default to that - LowerSnakeCommand::execute(interpreter, argument, command_span) + LowerSnakeCommand::execute(interpreter, command) } } @@ -192,10 +190,9 @@ impl CommandDefinition for LowerSnakeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, to_lower_snake_case) + concat_into_string(interpreter, command, to_lower_snake_case) } } @@ -206,10 +203,9 @@ impl CommandDefinition for UpperSnakeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, to_upper_snake_case) + concat_into_string(interpreter, command, to_upper_snake_case) } } @@ -220,12 +216,11 @@ impl CommandDefinition for KebabCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { // Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) // It can always be combined with other casing to get other versions - concat_string_and_convert(interpreter, argument, command_span, to_lower_kebab_case) + concat_into_string(interpreter, command, to_lower_kebab_case) } } @@ -236,11 +231,10 @@ impl CommandDefinition for CamelCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { // Upper camel case is the more common casing in Rust, so default to that - UpperCamelCommand::execute(interpreter, argument, command_span) + UpperCamelCommand::execute(interpreter, command) } } @@ -251,10 +245,9 @@ impl CommandDefinition for LowerCamelCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, to_lower_camel_case) + concat_into_string(interpreter, command, to_lower_camel_case) } } @@ -265,10 +258,9 @@ impl CommandDefinition for UpperCamelCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, to_upper_camel_case) + concat_into_string(interpreter, command, to_upper_camel_case) } } @@ -279,10 +271,9 @@ impl CommandDefinition for CapitalizeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, capitalize) + concat_into_string(interpreter, command, capitalize) } } @@ -293,10 +284,9 @@ impl CommandDefinition for DecapitalizeCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, decapitalize) + concat_into_string(interpreter, command, decapitalize) } } @@ -307,10 +297,9 @@ impl CommandDefinition for TitleCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert(interpreter, argument, command_span, title_case) + concat_into_string(interpreter, command, title_case) } } @@ -321,14 +310,61 @@ impl CommandDefinition for InsertSpacesCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + command: Command, ) -> Result { - concat_string_and_convert( + concat_into_string( interpreter, - argument, - command_span, + command, insert_spaces_between_words, ) } } + +fn concat_recursive(arguments: TokenStream) -> String { + fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { + for token_tree in arguments { + match token_tree { + TokenTree::Literal(literal) => { + let lit: Lit = parse_str(&literal.to_string()).expect( + "All proc_macro2::Literal values should be decodable as a syn::Lit", + ); + match lit { + Lit::Str(lit_str) => output.push_str(&lit_str.value()), + Lit::Char(lit_char) => output.push(lit_char.value()), + _ => { + output.push_str(&literal.to_string()); + } + } + } + TokenTree::Group(group) => match group.delimiter() { + Delimiter::Parenthesis => { + output.push('('); + concat_recursive_internal(output, group.stream()); + output.push(')'); + } + Delimiter::Brace => { + output.push('{'); + concat_recursive_internal(output, group.stream()); + output.push('}'); + } + Delimiter::Bracket => { + output.push('['); + concat_recursive_internal(output, group.stream()); + output.push(']'); + } + Delimiter::None => { + concat_recursive_internal(output, group.stream()); + } + }, + TokenTree::Punct(punct) => { + output.push(punct.as_char()); + } + TokenTree::Ident(ident) => output.push_str(&ident.to_string()), + } + } + } + + let mut output = String::new(); + concat_recursive_internal(&mut output, arguments); + output +} diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index c0be06df..e5dd3edf 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -7,17 +7,16 @@ impl CommandDefinition for IfCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + mut command: Command, ) -> Result { - let parsed = match parse_if_statement(&mut argument.tokens()) { + let parsed = match parse_if_statement(command.argument_tokens()) { Some(parsed) => parsed, None => { - return command_span.span_range().err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); + return command.err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); } }; - let interpreted_condition = interpreter.interpret_item(parsed.condition)?; + let interpreted_condition = interpreter.interpret_item(parsed.condition, SubstitutionMode::expression())?; let evaluated_condition = evaluate_expression( interpreted_condition, ExpressionParsingMode::BeforeCurlyBraces, @@ -26,9 +25,9 @@ impl CommandDefinition for IfCommand { .value(); if evaluated_condition { - interpreter.interpret_token_stream(parsed.true_code) + interpreter.interpret_token_stream(parsed.true_code, SubstitutionMode::token_stream()) } else if let Some(false_code) = parsed.false_code { - interpreter.interpret_token_stream(false_code) + interpreter.interpret_token_stream(false_code, SubstitutionMode::token_stream()) } else { Ok(TokenStream::new()) } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 14bc2523..05c891e9 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -7,19 +7,16 @@ impl CommandDefinition for SetCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + mut command: Command, ) -> Result { - let mut argument_tokens = argument.tokens(); - let variable_name = match parse_variable_set(&mut argument_tokens) { + let variable_name = match parse_variable_set(command.argument_tokens()) { Some(variable) => variable.variable_name().to_string(), None => { - return command_span - .err("A set call is expected to start with `#variable_name = ..`"); + return command.err("A set call is expected to start with `#variable_name = ..`"); } }; - let result_tokens = interpreter.interpret_tokens(argument_tokens)?; + let result_tokens = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; interpreter.set_variable(variable_name, result_tokens); Ok(TokenStream::new()) @@ -39,10 +36,9 @@ impl CommandDefinition for RawCommand { fn execute( _interpreter: &mut Interpreter, - argument: CommandArgumentStream, - _command_span: Span, + command: Command, ) -> Result { - Ok(argument.tokens().into_token_stream()) + Ok(command.into_argument_tokens().into_token_stream()) } } @@ -51,7 +47,7 @@ pub(crate) struct IgnoreCommand; impl CommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; - fn execute(_: &mut Interpreter, _: CommandArgumentStream, _: Span) -> Result { + fn execute(_: &mut Interpreter, _: Command) -> Result { Ok(TokenStream::new()) } } diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index b59b9dc9..fafa4d42 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -7,10 +7,9 @@ impl CommandDefinition for EvaluateCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - _command_span: Span, + mut command: Command, ) -> Result { - let token_stream = argument.interpret(interpreter)?; + let token_stream = command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?; Ok(evaluate_expression(token_stream, ExpressionParsingMode::Standard)?.into_token_stream()) } } @@ -22,13 +21,11 @@ impl CommandDefinition for IncrementCommand { fn execute( interpreter: &mut Interpreter, - argument: CommandArgumentStream, - command_span: Span, + mut command: Command, ) -> Result { let error_message = "Expected [!increment! #variable]"; - let mut tokens = argument.tokens(); - let variable = tokens.next_item_as_variable(error_message)?; - tokens.assert_end(error_message)?; + let variable = command.argument_tokens().next_item_as_variable(error_message)?; + command.argument_tokens().assert_end(error_message)?; let variable_contents = variable.execute_substitution(interpreter)?; let evaluated_integer = evaluate_expression(variable_contents, ExpressionParsingMode::Standard)? @@ -36,7 +33,7 @@ impl CommandDefinition for IncrementCommand { interpreter.set_variable( variable.variable_name().to_string(), evaluated_integer - .increment(command_span.span_range())? + .increment(command.span_range())? .to_token_stream(), ); Ok(TokenStream::new()) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index fd72142e..943474f0 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,12 +2,14 @@ mod concat_commands; mod control_flow_commands; mod core_commands; mod expression_commands; +mod token_commands; use crate::internal_prelude::*; use concat_commands::*; use control_flow_commands::*; use core_commands::*; use expression_commands::*; +use token_commands::*; define_commands! { pub(crate) enum CommandKind { @@ -45,5 +47,11 @@ define_commands! { // Control flow commands IfCommand, + + // Token Commands + EmptyCommand, + IsEmptyCommand, + LengthCommand, + GroupCommand, } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs new file mode 100644 index 00000000..a939ddec --- /dev/null +++ b/src/commands/token_commands.rs @@ -0,0 +1,63 @@ +use crate::internal_prelude::*; + +pub(crate) struct EmptyCommand; + +impl CommandDefinition for EmptyCommand { + const COMMAND_NAME: &'static str = "empty"; + + fn execute( + _interpreter: &mut Interpreter, + command: Command, + ) -> Result { + command.into_argument_tokens().assert_end("The !empty! command does not take any arguments")?; + Ok(TokenStream::new()) + } +} + +pub(crate) struct IsEmptyCommand; + +impl CommandDefinition for IsEmptyCommand { + const COMMAND_NAME: &'static str = "is_empty"; + + fn execute( + interpreter: &mut Interpreter, + mut command: Command, + ) -> Result { + let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + Ok(TokenTree::bool(interpreted.is_empty(), command.span()).into()) + } +} + +pub(crate) struct LengthCommand; + +impl CommandDefinition for LengthCommand { + const COMMAND_NAME: &'static str = "length"; + + fn execute( + interpreter: &mut Interpreter, + mut command: Command, + ) -> Result { + let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let stream_length = interpreted.into_iter().count(); + Ok(TokenTree::Literal(Literal::usize_unsuffixed(stream_length).with_span(command.span())).into()) + } +} + +pub(crate) struct GroupCommand; + +impl CommandDefinition for GroupCommand { + const COMMAND_NAME: &'static str = "group"; + + fn execute( + interpreter: &mut Interpreter, + mut command: Command, + ) -> Result { + let mut output = TokenStream::new(); + output.push_new_group( + command.span_range(), + Delimiter::None, + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?, + ); + Ok(output) + } +} \ No newline at end of file diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 102b619c..62a8b1f3 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -11,6 +11,16 @@ pub(crate) use crate::expressions::*; pub(crate) use crate::interpreter::*; pub(crate) use crate::string_conversion::*; +pub(crate) trait TokenTreeExt: Sized { + fn bool(value: bool, span: Span) -> Self; +} + +impl TokenTreeExt for TokenTree { + fn bool(value: bool, span: Span) -> Self { + TokenTree::Ident(Ident::new(&value.to_string(), span)) + } +} + pub(crate) trait TokenStreamExt: Sized { fn push_token_tree(&mut self, token_tree: TokenTree); fn push_new_group( diff --git a/src/interpreter.rs b/src/interpreter.rs index b8183470..46da4de0 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1,7 +1,10 @@ use crate::internal_prelude::*; pub(crate) fn interpret(token_stream: TokenStream) -> Result { - Interpreter::new().interpret_tokens(Tokens::new(token_stream)) + Interpreter::new().interpret_tokens( + &mut Tokens::new(token_stream), + SubstitutionMode::token_stream(), + ) } pub(crate) struct Interpreter { @@ -26,27 +29,28 @@ impl Interpreter { pub(crate) fn interpret_token_stream( &mut self, token_stream: TokenStream, + substitution_mode: SubstitutionMode, ) -> Result { - self.interpret_tokens(Tokens::new(token_stream)) + self.interpret_tokens(&mut Tokens::new(token_stream), substitution_mode) } - pub(crate) fn interpret_item(&mut self, item: NextItem) -> Result { + pub(crate) fn interpret_item(&mut self, item: NextItem, substitution_mode: SubstitutionMode) -> Result { let mut expanded = TokenStream::new(); - self.interpret_next_item(item, &mut expanded)?; + self.interpret_next_item(item, substitution_mode, &mut expanded)?; Ok(expanded) } - pub(crate) fn interpret_tokens(&mut self, mut source_tokens: Tokens) -> Result { + pub(crate) fn interpret_tokens(&mut self, source_tokens: &mut Tokens, substitution_mode: SubstitutionMode) -> Result { let mut expanded = TokenStream::new(); loop { match source_tokens.next_item()? { - Some(next_item) => self.interpret_next_item(next_item, &mut expanded)?, + Some(next_item) => self.interpret_next_item(next_item, substitution_mode, &mut expanded)?, None => return Ok(expanded), } } } - fn interpret_next_item(&mut self, next_item: NextItem, output: &mut TokenStream) -> Result<()> { + fn interpret_next_item(&mut self, next_item: NextItem, substitution_mode: SubstitutionMode, output: &mut TokenStream) -> Result<()> { // We wrap command/variable substitutions in a transparent group so that they // can be treated as a single item in other commands. // e.g. if #x = 1 + 1, then [!math! #x * #x] should be 4. @@ -61,23 +65,20 @@ impl Interpreter { output.push_new_group( group.span_range(), group.delimiter(), - self.interpret_tokens(Tokens::new(group.stream()))?, + self.interpret_tokens(&mut Tokens::new(group.stream()), substitution_mode)?, ); } - NextItem::Variable(variable_substitution) => { - // We wrap substituted variables in a transparent group so that - // they can be used collectively in future expressions, so that - // e.g. if #x = 1 + 1 then #x * #x = 4 rather than 3 - output.push_new_group( - variable_substitution.span_range(), - Delimiter::None, - variable_substitution.execute_substitution(self)?, + NextItem::Variable(variable) => { + substitution_mode.apply( + output, + variable.span_range(), + variable.execute_substitution(self)?, ); } NextItem::CommandInvocation(command_invocation) => { - output.push_new_group( + substitution_mode.apply( + output, command_invocation.span_range(), - Delimiter::None, command_invocation.execute(self)?, ); } @@ -86,6 +87,51 @@ impl Interpreter { } } +/// How to output `#variables` and `[!commands!]` into the output stream +#[derive(Clone, Copy)] +pub(crate) struct SubstitutionMode(SubstitutionModeInternal); + +#[derive(Clone, Copy)] +enum SubstitutionModeInternal { + /// The tokens are just output as they are to the token stream. + /// + /// This is the default, and should typically be used by commands which + /// deal with flattened token streams. + Extend, + /// The tokens are grouped into a group. + /// + /// This should be used when calculating expressions. + /// + /// We wrap substituted variables in a transparent group so that + /// they can be used collectively in future expressions, so that + /// e.g. if `#x = 1 + 1` then `#x * #x = 4` rather than `3`. + Group(Delimiter), +} + +impl SubstitutionMode { + /// When creating a token stream, substitutions extend the token stream output. + pub(crate) fn token_stream() -> Self { + Self(SubstitutionModeInternal::Extend) + } + + /// When creating a token stream for an expression, substitutions are wrapped + /// in a transparent group. + pub(crate) fn expression() -> Self { + Self(SubstitutionModeInternal::Group(Delimiter::None)) + } + + fn apply(self, tokens: &mut TokenStream, span_range: SpanRange, substitution: TokenStream) { + match self.0 { + SubstitutionModeInternal::Extend => tokens.extend(substitution), + SubstitutionModeInternal::Group(delimiter) => tokens.push_new_group( + span_range, + delimiter, + substitution, + ), + } + } +} + pub(crate) struct Tokens(iter::Peekable<::IntoIter>); impl Tokens { @@ -122,8 +168,12 @@ impl Tokens { } } + pub(crate) fn is_empty(&mut self) -> bool { + self.peek().is_none() + } + pub(crate) fn check_end(&mut self) -> Option<()> { - if self.peek().is_none() { + if self.is_empty() { Some(()) } else { None @@ -224,7 +274,7 @@ fn parse_command_invocation(group: &Group) -> Result> // We have now checked enough that we're confident the user is pretty intentionally using // the call convention. Any issues we hit from this point will be a helpful compiler error. match consume_command_end(&command_ident, &mut remaining_tokens) { - Some(command_kind) => Ok(Some(CommandInvocation::new(command_kind, group, remaining_tokens))), + Some(command_kind) => Ok(Some(CommandInvocation::new(command_ident.clone(), command_kind, group, remaining_tokens))), None => Err(command_ident.span().error( format!( "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", diff --git a/tests/control_flow.rs b/tests/control_flow.rs index cb7250c4..13df7987 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -7,12 +7,21 @@ macro_rules! assert_preinterpret_eq { } #[test] -fn test_basic_evaluate_works() { +fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); assert_preinterpret_eq!({ [!set! #x = 1 == 2] - [!if! (#x) { "YES" } !else! { "NO" }] + [!if! #x { "YES" } !else! { "NO" }] }, "NO"); + assert_preinterpret_eq!({ + [!set! #x = 1] + [!set! #y = 2] + [!if! (#x == #y) { "YES" } !else! { "NO" }] + }, "NO"); + assert_preinterpret_eq!({ + 0 + [!if! true { + 1 }] + }, 1); assert_preinterpret_eq!({ 0 [!if! false { + 1 }] diff --git a/tests/tokens.rs b/tests/tokens.rs new file mode 100644 index 00000000..ff043261 --- /dev/null +++ b/tests/tokens.rs @@ -0,0 +1,47 @@ +use preinterpret::preinterpret; + +macro_rules! assert_preinterpret_eq { + ($input:tt, $($output:tt)*) => { + assert_eq!(preinterpret!($input), $($output)*); + }; +} + +#[test] +fn test_empty_and_is_empty() { + assert_preinterpret_eq!({ + [!empty!] "hello" [!empty!] [!empty!] + }, "hello"); + assert_preinterpret_eq!([!is_empty! [!empty!]], true); + assert_preinterpret_eq!([!is_empty! [!empty!] [!empty!]], true); + assert_preinterpret_eq!([!is_empty! Not Empty], false); + assert_preinterpret_eq!({ + [!set! #x = [!empty!]] + [!is_empty! #x] + }, true); + assert_preinterpret_eq!({ + [!set! #x = [!empty!]] + [!set! #x = #x is no longer empty] + [!is_empty! #x] + }, false); +} + +#[test] +fn test_length_and_group() { + assert_preinterpret_eq!({ + [!length! "hello" World] + }, 2); + assert_preinterpret_eq!({ + [!length! ("hello" World)] + }, 1); + assert_preinterpret_eq!({ + [!length! [!group! "hello" World]] + }, 1); + assert_preinterpret_eq!({ + [!set! #x = Hello "World" (1 2 3 4 5)] + [!length! #x] + }, 3); + assert_preinterpret_eq!({ + [!set! #x = Hello "World" (1 2 3 4 5)] + [!length! [!group! #x]] + }, 1); +} From b57a6886d648a3c176ea7888d75b248eafa458a1 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 1 Jan 2025 20:43:31 +0000 Subject: [PATCH 006/126] WIP: Style fix --- src/command.rs | 44 ++++++----- src/commands/concat_commands.rs | 110 ++++++-------------------- src/commands/control_flow_commands.rs | 8 +- src/commands/core_commands.rs | 13 +-- src/commands/expression_commands.rs | 17 ++-- src/commands/token_commands.rs | 37 ++++----- src/interpreter.rs | 31 +++++--- tests/control_flow.rs | 1 + tests/tokens.rs | 8 +- 9 files changed, 105 insertions(+), 164 deletions(-) diff --git a/src/command.rs b/src/command.rs index dcb467ff..88b40e22 100644 --- a/src/command.rs +++ b/src/command.rs @@ -3,10 +3,7 @@ use crate::internal_prelude::*; pub(crate) trait CommandDefinition { const COMMAND_NAME: &'static str; - fn execute( - interpreter: &mut Interpreter, - argument: Command, - ) -> Result; + fn execute(interpreter: &mut Interpreter, argument: Command) -> Result; } macro_rules! define_commands { @@ -59,20 +56,20 @@ pub(crate) struct CommandInvocation { } impl CommandInvocation { - pub(crate) fn new(command_ident: Ident, command_kind: CommandKind, group: &Group, argument_tokens: Tokens) -> Self { + pub(crate) fn new( + command_ident: Ident, + command_kind: CommandKind, + group: &Group, + argument_tokens: Tokens, + ) -> Self { Self { command_kind, - command: Command::new( - command_ident, - group.span(), - argument_tokens, - ), + command: Command::new(command_ident, group.span(), argument_tokens), } } pub(crate) fn execute(self, interpreter: &mut Interpreter) -> Result { - self.command_kind - .execute(interpreter, self.command) + self.command_kind.execute(interpreter, self.command) } } @@ -103,10 +100,7 @@ impl Variable { &self, interpreter: &mut Interpreter, ) -> Result { - let Variable { - variable_name, - .. - } = self; + let Variable { variable_name, .. } = self; match interpreter.get_variable(&variable_name.to_string()) { Some(variable_value) => Ok(variable_value.clone()), None => { @@ -142,7 +136,11 @@ pub(crate) struct Command { impl Command { fn new(command_ident: Ident, command_span: Span, argument_tokens: Tokens) -> Self { - Self { command_ident, command_span, argument_tokens } + Self { + command_ident, + command_span, + argument_tokens, + } } #[allow(unused)] // Likely useful in future @@ -163,10 +161,16 @@ impl Command { } /// Expects the remaining arguments to be non-empty - pub(crate) fn interpret_remaining_arguments(&mut self, interpreter: &mut Interpreter, substitution_mode: SubstitutionMode) -> Result { + pub(crate) fn interpret_remaining_arguments( + &mut self, + interpreter: &mut Interpreter, + substitution_mode: SubstitutionMode, + ) -> Result { if self.argument_tokens.is_empty() { // This is simply for clarity / to make empty arguments explicit. - return self.err("Arguments were empty. Use [!empty!] if you want to use an empty token stream."); + return self.err( + "Arguments were empty. Use [!empty!] if you want to use an empty token stream.", + ); } interpreter.interpret_tokens(&mut self.argument_tokens, substitution_mode) } @@ -184,4 +188,4 @@ impl HasSpanRange for Command { fn span_range(&self) -> SpanRange { self.span().span_range() } -} \ No newline at end of file +} diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index e59e3b9c..9c8706c3 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -29,7 +29,8 @@ fn concat_into_string( mut command: Command, conversion_fn: impl Fn(&str) -> String, ) -> Result { - let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let interpreted = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; let concatenated = concat_recursive(interpreted); let string_literal = string_literal(&conversion_fn(&concatenated), command.span()); Ok(TokenStream::from(TokenTree::Literal(string_literal))) @@ -40,7 +41,8 @@ fn concat_into_ident( mut command: Command, conversion_fn: impl Fn(&str) -> String, ) -> Result { - let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let interpreted = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; let concatenated = concat_recursive(interpreted); let ident = parse_ident(&conversion_fn(&concatenated), command.span())?; Ok(TokenStream::from(TokenTree::Ident(ident))) @@ -51,7 +53,8 @@ fn concat_into_literal( mut command: Command, conversion_fn: impl Fn(&str) -> String, ) -> Result { - let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let interpreted = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; let concatenated = concat_recursive(interpreted); let literal = parse_literal(&conversion_fn(&concatenated), command.span())?; Ok(TokenStream::from(TokenTree::Literal(literal))) @@ -66,10 +69,7 @@ pub(crate) struct StringCommand; impl CommandDefinition for StringCommand { const COMMAND_NAME: &'static str = "string"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, |s| s.to_string()) } } @@ -79,10 +79,7 @@ pub(crate) struct IdentCommand; impl CommandDefinition for IdentCommand { const COMMAND_NAME: &'static str = "ident"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, |s| s.to_string()) } } @@ -92,10 +89,7 @@ pub(crate) struct IdentCamelCommand; impl CommandDefinition for IdentCamelCommand { const COMMAND_NAME: &'static str = "ident_camel"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, to_upper_camel_case) } } @@ -105,10 +99,7 @@ pub(crate) struct IdentSnakeCommand; impl CommandDefinition for IdentSnakeCommand { const COMMAND_NAME: &'static str = "ident_snake"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, to_lower_snake_case) } } @@ -118,10 +109,7 @@ pub(crate) struct IdentUpperSnakeCommand; impl CommandDefinition for IdentUpperSnakeCommand { const COMMAND_NAME: &'static str = "ident_upper_snake"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, to_upper_snake_case) } } @@ -131,10 +119,7 @@ pub(crate) struct LiteralCommand; impl CommandDefinition for LiteralCommand { const COMMAND_NAME: &'static str = "literal"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_literal(interpreter, command, |s| s.to_string()) } } @@ -148,10 +133,7 @@ pub(crate) struct UpperCommand; impl CommandDefinition for UpperCommand { const COMMAND_NAME: &'static str = "upper"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_uppercase) } } @@ -161,10 +143,7 @@ pub(crate) struct LowerCommand; impl CommandDefinition for LowerCommand { const COMMAND_NAME: &'static str = "lower"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_lowercase) } } @@ -174,10 +153,7 @@ pub(crate) struct SnakeCommand; impl CommandDefinition for SnakeCommand { const COMMAND_NAME: &'static str = "snake"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { // Lower snake case is the more common casing in Rust, so default to that LowerSnakeCommand::execute(interpreter, command) } @@ -188,10 +164,7 @@ pub(crate) struct LowerSnakeCommand; impl CommandDefinition for LowerSnakeCommand { const COMMAND_NAME: &'static str = "lower_snake"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_lower_snake_case) } } @@ -201,10 +174,7 @@ pub(crate) struct UpperSnakeCommand; impl CommandDefinition for UpperSnakeCommand { const COMMAND_NAME: &'static str = "upper_snake"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_upper_snake_case) } } @@ -214,10 +184,7 @@ pub(crate) struct KebabCommand; impl CommandDefinition for KebabCommand { const COMMAND_NAME: &'static str = "kebab"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { // Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) // It can always be combined with other casing to get other versions concat_into_string(interpreter, command, to_lower_kebab_case) @@ -229,10 +196,7 @@ pub(crate) struct CamelCommand; impl CommandDefinition for CamelCommand { const COMMAND_NAME: &'static str = "camel"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { // Upper camel case is the more common casing in Rust, so default to that UpperCamelCommand::execute(interpreter, command) } @@ -243,10 +207,7 @@ pub(crate) struct LowerCamelCommand; impl CommandDefinition for LowerCamelCommand { const COMMAND_NAME: &'static str = "lower_camel"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_lower_camel_case) } } @@ -256,10 +217,7 @@ pub(crate) struct UpperCamelCommand; impl CommandDefinition for UpperCamelCommand { const COMMAND_NAME: &'static str = "upper_camel"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_upper_camel_case) } } @@ -269,10 +227,7 @@ pub(crate) struct CapitalizeCommand; impl CommandDefinition for CapitalizeCommand { const COMMAND_NAME: &'static str = "capitalize"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, capitalize) } } @@ -282,10 +237,7 @@ pub(crate) struct DecapitalizeCommand; impl CommandDefinition for DecapitalizeCommand { const COMMAND_NAME: &'static str = "decapitalize"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, decapitalize) } } @@ -295,10 +247,7 @@ pub(crate) struct TitleCommand; impl CommandDefinition for TitleCommand { const COMMAND_NAME: &'static str = "title"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, title_case) } } @@ -308,15 +257,8 @@ pub(crate) struct InsertSpacesCommand; impl CommandDefinition for InsertSpacesCommand { const COMMAND_NAME: &'static str = "insert_spaces"; - fn execute( - interpreter: &mut Interpreter, - command: Command, - ) -> Result { - concat_into_string( - interpreter, - command, - insert_spaces_between_words, - ) + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + concat_into_string(interpreter, command, insert_spaces_between_words) } } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index e5dd3edf..1437cc13 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -5,10 +5,7 @@ pub(crate) struct IfCommand; impl CommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let parsed = match parse_if_statement(command.argument_tokens()) { Some(parsed) => parsed, None => { @@ -16,7 +13,8 @@ impl CommandDefinition for IfCommand { } }; - let interpreted_condition = interpreter.interpret_item(parsed.condition, SubstitutionMode::expression())?; + let interpreted_condition = + interpreter.interpret_item(parsed.condition, SubstitutionMode::expression())?; let evaluated_condition = evaluate_expression( interpreted_condition, ExpressionParsingMode::BeforeCurlyBraces, diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 05c891e9..3bb8f563 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -5,10 +5,7 @@ pub(crate) struct SetCommand; impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let variable_name = match parse_variable_set(command.argument_tokens()) { Some(variable) => variable.variable_name().to_string(), None => { @@ -16,7 +13,8 @@ impl CommandDefinition for SetCommand { } }; - let result_tokens = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let result_tokens = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; interpreter.set_variable(variable_name, result_tokens); Ok(TokenStream::new()) @@ -34,10 +32,7 @@ pub(crate) struct RawCommand; impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - fn execute( - _interpreter: &mut Interpreter, - command: Command, - ) -> Result { + fn execute(_interpreter: &mut Interpreter, command: Command) -> Result { Ok(command.into_argument_tokens().into_token_stream()) } } diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index fafa4d42..27b3211d 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -5,11 +5,9 @@ pub(crate) struct EvaluateCommand; impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { - let token_stream = command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?; + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let token_stream = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?; Ok(evaluate_expression(token_stream, ExpressionParsingMode::Standard)?.into_token_stream()) } } @@ -19,12 +17,11 @@ pub(crate) struct IncrementCommand; impl CommandDefinition for IncrementCommand { const COMMAND_NAME: &'static str = "increment"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let error_message = "Expected [!increment! #variable]"; - let variable = command.argument_tokens().next_item_as_variable(error_message)?; + let variable = command + .argument_tokens() + .next_item_as_variable(error_message)?; command.argument_tokens().assert_end(error_message)?; let variable_contents = variable.execute_substitution(interpreter)?; let evaluated_integer = diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index a939ddec..6e6908b3 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -5,11 +5,10 @@ pub(crate) struct EmptyCommand; impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; - fn execute( - _interpreter: &mut Interpreter, - command: Command, - ) -> Result { - command.into_argument_tokens().assert_end("The !empty! command does not take any arguments")?; + fn execute(_interpreter: &mut Interpreter, command: Command) -> Result { + command + .into_argument_tokens() + .assert_end("The !empty! command does not take any arguments")?; Ok(TokenStream::new()) } } @@ -19,11 +18,9 @@ pub(crate) struct IsEmptyCommand; impl CommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { - let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let interpreted = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; Ok(TokenTree::bool(interpreted.is_empty(), command.span()).into()) } } @@ -33,13 +30,14 @@ pub(crate) struct LengthCommand; impl CommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { - let interpreted = command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let interpreted = + command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; let stream_length = interpreted.into_iter().count(); - Ok(TokenTree::Literal(Literal::usize_unsuffixed(stream_length).with_span(command.span())).into()) + Ok( + TokenTree::Literal(Literal::usize_unsuffixed(stream_length).with_span(command.span())) + .into(), + ) } } @@ -48,10 +46,7 @@ pub(crate) struct GroupCommand; impl CommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - fn execute( - interpreter: &mut Interpreter, - mut command: Command, - ) -> Result { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let mut output = TokenStream::new(); output.push_new_group( command.span_range(), @@ -60,4 +55,4 @@ impl CommandDefinition for GroupCommand { ); Ok(output) } -} \ No newline at end of file +} diff --git a/src/interpreter.rs b/src/interpreter.rs index 46da4de0..b07ad206 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -34,23 +34,38 @@ impl Interpreter { self.interpret_tokens(&mut Tokens::new(token_stream), substitution_mode) } - pub(crate) fn interpret_item(&mut self, item: NextItem, substitution_mode: SubstitutionMode) -> Result { + pub(crate) fn interpret_item( + &mut self, + item: NextItem, + substitution_mode: SubstitutionMode, + ) -> Result { let mut expanded = TokenStream::new(); self.interpret_next_item(item, substitution_mode, &mut expanded)?; Ok(expanded) } - pub(crate) fn interpret_tokens(&mut self, source_tokens: &mut Tokens, substitution_mode: SubstitutionMode) -> Result { + pub(crate) fn interpret_tokens( + &mut self, + source_tokens: &mut Tokens, + substitution_mode: SubstitutionMode, + ) -> Result { let mut expanded = TokenStream::new(); loop { match source_tokens.next_item()? { - Some(next_item) => self.interpret_next_item(next_item, substitution_mode, &mut expanded)?, + Some(next_item) => { + self.interpret_next_item(next_item, substitution_mode, &mut expanded)? + } None => return Ok(expanded), } } } - fn interpret_next_item(&mut self, next_item: NextItem, substitution_mode: SubstitutionMode, output: &mut TokenStream) -> Result<()> { + fn interpret_next_item( + &mut self, + next_item: NextItem, + substitution_mode: SubstitutionMode, + output: &mut TokenStream, + ) -> Result<()> { // We wrap command/variable substitutions in a transparent group so that they // can be treated as a single item in other commands. // e.g. if #x = 1 + 1, then [!math! #x * #x] should be 4. @@ -123,11 +138,9 @@ impl SubstitutionMode { fn apply(self, tokens: &mut TokenStream, span_range: SpanRange, substitution: TokenStream) { match self.0 { SubstitutionModeInternal::Extend => tokens.extend(substitution), - SubstitutionModeInternal::Group(delimiter) => tokens.push_new_group( - span_range, - delimiter, - substitution, - ), + SubstitutionModeInternal::Group(delimiter) => { + tokens.push_new_group(span_range, delimiter, substitution) + } } } } diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 13df7987..6e657cdd 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -7,6 +7,7 @@ macro_rules! assert_preinterpret_eq { } #[test] +#[allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); assert_preinterpret_eq!({ diff --git a/tests/tokens.rs b/tests/tokens.rs index ff043261..f3f0737f 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -30,12 +30,8 @@ fn test_length_and_group() { assert_preinterpret_eq!({ [!length! "hello" World] }, 2); - assert_preinterpret_eq!({ - [!length! ("hello" World)] - }, 1); - assert_preinterpret_eq!({ - [!length! [!group! "hello" World]] - }, 1); + assert_preinterpret_eq!({ [!length! ("hello" World)] }, 1); + assert_preinterpret_eq!({ [!length! [!group! "hello" World]] }, 1); assert_preinterpret_eq!({ [!set! #x = Hello "World" (1 2 3 4 5)] [!length! #x] From befb78de4af65d5a557376d2d1fbbd647fa46153 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 2 Jan 2025 20:24:15 +0000 Subject: [PATCH 007/126] WIP: Add `while` and `assign` commands --- CHANGELOG.md | 13 ++--- src/command.rs | 75 +++++++++++++++++++------- src/commands/control_flow_commands.rs | 56 +++++++++++++++++++ src/commands/expression_commands.rs | 64 +++++++++++++++++++++- src/commands/mod.rs | 2 + src/commands/token_commands.rs | 4 +- src/internal_prelude.rs | 16 +++--- src/interpreter.rs | 78 ++++++++++++++++++++++----- tests/control_flow.rs | 17 +++++- tests/{evaluate.rs => expressions.rs} | 13 +++++ 10 files changed, 290 insertions(+), 48 deletions(-) rename tests/{evaluate.rs => expressions.rs} (90%) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa5c02f9..49059a2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,24 +9,25 @@ * `[!increment! ...]` * Control flow commands: * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` + * `[!while! cond {}]` * Token-stream utility commands: * `[!empty!]` * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. + * Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). ### To come -* Use `[!let! #x = 12]` instead of `[!set! ..]`. -* Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). -* `[!while! cond {}]` -* `[!for! #x in [#y] {}]`... and make it so that whether commands or variable substitutions get replaced by groups depends on the interpreter context (likely only expressions should use groups) +* ? Use `[!let! #x = 12]` instead of `[!set! ..]`. +* `[!for! #x in [#y] {}]` * `[!range! 0..5]` * `[!error! "message" token stream for span]` * Remove `[!increment! ...]` and replace with `[!assign! #x += 1]` +* Reconfiguring iteration limit * Support `!else if!` in `!if!` -* Token stream manipulation... and make it performant - * Extend token stream +* Token stream manipulation... and make it performant (maybe by storing either TokenStream for extension; or `ParseStream` for consumption) + * Extend token stream `[!extend! #x += ...]` * Consume from start of token stream * Support string & char literals (for comparisons & casts) in expressions * Add more tests diff --git a/src/command.rs b/src/command.rs index 88b40e22..23ca201c 100644 --- a/src/command.rs +++ b/src/command.rs @@ -15,6 +15,7 @@ macro_rules! define_commands { } ) => { #[allow(clippy::enum_variant_names)] + #[derive(Clone, Copy)] pub(crate) enum $enum_name { $( $command, @@ -50,6 +51,7 @@ macro_rules! define_commands { } pub(crate) use define_commands; +#[derive(Clone)] pub(crate) struct CommandInvocation { command_kind: CommandKind, command: Command, @@ -79,6 +81,7 @@ impl HasSpanRange for CommandInvocation { } } +#[derive(Clone)] pub(crate) struct Variable { marker: Punct, // # variable_name: Ident, @@ -92,28 +95,63 @@ impl Variable { } } - pub(crate) fn variable_name(&self) -> &Ident { - &self.variable_name + pub(crate) fn variable_name(&self) -> String { + self.variable_name.to_string() } - pub(crate) fn execute_substitution( + pub(crate) fn set<'i>( &self, - interpreter: &mut Interpreter, - ) -> Result { - let Variable { variable_name, .. } = self; - match interpreter.get_variable(&variable_name.to_string()) { - Some(variable_value) => Ok(variable_value.clone()), - None => { - self.span_range().err( - format!( - "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self, - self, - ), - ) - } + interpreter: &'i mut Interpreter, + value: TokenStream, + ) { + interpreter.set_variable(self.variable_name(), value); + } + + pub(crate) fn read_substitution<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Result<&'i TokenStream> { + self.read_or_else( + interpreter, + || format!( + "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", + self, + self, + ) + ) + } + + pub(crate) fn read_required<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Result<&'i TokenStream> { + self.read_or_else( + interpreter, + || format!( + "The variable {} wasn't set.", + self, + ) + ) + } + + pub(crate) fn read_or_else<'i>( + &self, + interpreter: &'i Interpreter, + create_error: impl FnOnce() -> String, + ) -> Result<&'i TokenStream> { + match self.read_option(interpreter) { + Some(token_stream) => Ok(token_stream), + None => self.span_range().err(create_error()), } } + + fn read_option<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Option<&'i TokenStream> { + let Variable { variable_name, .. } = self; + interpreter.get_variable(&variable_name.to_string()) + } } impl HasSpanRange for Variable { @@ -128,6 +166,7 @@ impl core::fmt::Display for Variable { } } +#[derive(Clone)] pub(crate) struct Command { command_ident: Ident, command_span: Span, @@ -156,7 +195,7 @@ impl Command { self.command_span.error(message) } - pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { + pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { Err(self.error(message)) } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 1437cc13..92e510ab 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -59,3 +59,59 @@ fn parse_if_statement(tokens: &mut Tokens) -> Option { false_code, }) } + +pub(crate) struct WhileCommand; + +impl CommandDefinition for WhileCommand { + const COMMAND_NAME: &'static str = "while"; + + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let parsed = match parse_while_statement(command.argument_tokens()) { + Some(parsed) => parsed, + None => { + return command.err("Expected [!while! (condition) { code }]"); + } + }; + + let mut output = TokenStream::new(); + let mut iteration_count = 0; + loop { + let interpreted_condition = + interpreter.interpret_item(parsed.condition.clone(), SubstitutionMode::expression())?; + let evaluated_condition = evaluate_expression( + interpreted_condition, + ExpressionParsingMode::BeforeCurlyBraces, + )? + .expect_bool("An if condition must evaluate to a boolean")? + .value(); + + iteration_count += 1; + if !evaluated_condition { + break; + } + interpreter.config().check_iteration_count(&command, iteration_count)?; + interpreter.interpret_token_stream_into( + parsed.code.clone(), + SubstitutionMode::token_stream(), + &mut output, + )?; + } + + Ok(output) + } +} + +struct WhileStatement { + condition: NextItem, + code: TokenStream, +} + +fn parse_while_statement(tokens: &mut Tokens) -> Option { + let condition = tokens.next_item().ok()??; + let code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); + tokens.check_end()?; + Some(WhileStatement { + condition, + code, + }) +} \ No newline at end of file diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 27b3211d..94a1d1ae 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -12,6 +12,66 @@ impl CommandDefinition for EvaluateCommand { } } +pub(crate) struct AssignCommand; + +impl CommandDefinition for AssignCommand { + const COMMAND_NAME: &'static str = "assign"; + + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let AssignStatementStart { + variable, + operator, + } = AssignStatementStart::parse(command.argument_tokens()) + .ok_or_else(|| command.error("Expected [!assign! #variable += ...] for + or some other operator supported in an expression"))?; + + let mut expression_tokens = TokenStream::new(); + expression_tokens.push_new_group( + // TODO: Replace with `variable.read_into(tokens, substitution_mode)` + // TODO: Replace most methods on interpeter with e.g. + // command.interpret_into, next_item.interpet_into, etc. + // And also create an Expression struct + // TODO: Fix Expression to not need different parsing modes, + // and to be parsed from the full token stream or until braces { .. } + // or as a single item + variable.read_required(interpreter)?.clone(), + Delimiter::None, + variable.span_range(), + ); + expression_tokens.push_token_tree(operator.into()); + expression_tokens.push_new_group( + command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?, + Delimiter::None, + command.span_range(), + ); + + let output = evaluate_expression(expression_tokens, ExpressionParsingMode::Standard)?.into_token_stream(); + variable.set(interpreter, output); + + Ok(TokenStream::new()) + } +} + +struct AssignStatementStart { + variable: Variable, + operator: Punct, +} + +impl AssignStatementStart { + fn parse(tokens: &mut Tokens) -> Option { + let variable = tokens.next_item_as_variable("").ok()?; + let operator = tokens.next_as_punct()?; + match operator.as_char() { + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} + _ => return None, + } + tokens.next_as_punct_matching('=')?; + Some(AssignStatementStart { + variable, + operator, + }) + } +} + pub(crate) struct IncrementCommand; impl CommandDefinition for IncrementCommand { @@ -23,9 +83,9 @@ impl CommandDefinition for IncrementCommand { .argument_tokens() .next_item_as_variable(error_message)?; command.argument_tokens().assert_end(error_message)?; - let variable_contents = variable.execute_substitution(interpreter)?; + let variable_contents = variable.read_substitution(interpreter)?; let evaluated_integer = - evaluate_expression(variable_contents, ExpressionParsingMode::Standard)? + evaluate_expression(variable_contents.clone(), ExpressionParsingMode::Standard)? .expect_integer(&format!("Expected {variable} to evaluate to an integer"))?; interpreter.set_variable( variable.variable_name().to_string(), diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 943474f0..ade9f90d 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -43,10 +43,12 @@ define_commands! { // Expression Commands EvaluateCommand, + AssignCommand, IncrementCommand, // Control flow commands IfCommand, + WhileCommand, // Token Commands EmptyCommand, diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 6e6908b3..76559caf 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -49,9 +49,9 @@ impl CommandDefinition for GroupCommand { fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let mut output = TokenStream::new(); output.push_new_group( - command.span_range(), - Delimiter::None, command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?, + Delimiter::None, + command.span_range(), ); Ok(output) } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 62a8b1f3..88ab0622 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -13,21 +13,26 @@ pub(crate) use crate::string_conversion::*; pub(crate) trait TokenTreeExt: Sized { fn bool(value: bool, span: Span) -> Self; + fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; } impl TokenTreeExt for TokenTree { fn bool(value: bool, span: Span) -> Self { TokenTree::Ident(Ident::new(&value.to_string(), span)) } + + fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { + TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) + } } pub(crate) trait TokenStreamExt: Sized { fn push_token_tree(&mut self, token_tree: TokenTree); fn push_new_group( &mut self, - span_range: SpanRange, - delimiter: Delimiter, inner_tokens: TokenStream, + delimiter: Delimiter, + span_range: SpanRange, ); } @@ -38,12 +43,11 @@ impl TokenStreamExt for TokenStream { fn push_new_group( &mut self, - span_range: SpanRange, - delimiter: Delimiter, inner_tokens: TokenStream, + delimiter: Delimiter, + span_range: SpanRange, ) { - let group = Group::new(delimiter, inner_tokens).with_span(span_range.span()); - self.push_token_tree(TokenTree::Group(group)); + self.push_token_tree(TokenTree::group(inner_tokens, delimiter, span_range.span())); } } diff --git a/src/interpreter.rs b/src/interpreter.rs index b07ad206..0f6f6888 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -8,12 +8,14 @@ pub(crate) fn interpret(token_stream: TokenStream) -> Result { } pub(crate) struct Interpreter { + config: InterpreterConfig, variables: HashMap, } impl Interpreter { pub(crate) fn new() -> Self { Self { + config: Default::default(), variables: Default::default(), } } @@ -26,6 +28,10 @@ impl Interpreter { self.variables.get(name) } + pub(crate) fn config(&self) -> &InterpreterConfig { + &self.config + } + pub(crate) fn interpret_token_stream( &mut self, token_stream: TokenStream, @@ -34,13 +40,22 @@ impl Interpreter { self.interpret_tokens(&mut Tokens::new(token_stream), substitution_mode) } + pub(crate) fn interpret_token_stream_into( + &mut self, + token_stream: TokenStream, + substitution_mode: SubstitutionMode, + output: &mut TokenStream, + ) -> Result<()> { + self.interpret_tokens_into(&mut Tokens::new(token_stream), substitution_mode, output) + } + pub(crate) fn interpret_item( &mut self, item: NextItem, substitution_mode: SubstitutionMode, ) -> Result { let mut expanded = TokenStream::new(); - self.interpret_next_item(item, substitution_mode, &mut expanded)?; + self.interpret_next_item_into(item, substitution_mode, &mut expanded)?; Ok(expanded) } @@ -50,27 +65,32 @@ impl Interpreter { substitution_mode: SubstitutionMode, ) -> Result { let mut expanded = TokenStream::new(); + self.interpret_tokens_into(source_tokens, substitution_mode, &mut expanded)?; + Ok(expanded) + } + + pub(crate) fn interpret_tokens_into( + &mut self, + source_tokens: &mut Tokens, + substitution_mode: SubstitutionMode, + output: &mut TokenStream, + ) -> Result<()> { loop { match source_tokens.next_item()? { Some(next_item) => { - self.interpret_next_item(next_item, substitution_mode, &mut expanded)? + self.interpret_next_item_into(next_item, substitution_mode, output)? } - None => return Ok(expanded), + None => return Ok(()), } } } - fn interpret_next_item( + fn interpret_next_item_into( &mut self, next_item: NextItem, substitution_mode: SubstitutionMode, output: &mut TokenStream, ) -> Result<()> { - // We wrap command/variable substitutions in a transparent group so that they - // can be treated as a single item in other commands. - // e.g. if #x = 1 + 1, then [!math! #x * #x] should be 4. - // Note that such groups are ignored in the macro output, due to this - // issue in rustc: https://github.com/rust-lang/rust/issues/67062 match next_item { NextItem::Leaf(token_tree) => { output.push_token_tree(token_tree); @@ -78,16 +98,16 @@ impl Interpreter { NextItem::Group(group) => { // If it's a group, run interpret on its contents recursively. output.push_new_group( - group.span_range(), - group.delimiter(), self.interpret_tokens(&mut Tokens::new(group.stream()), substitution_mode)?, + group.delimiter(), + group.span_range(), ); } NextItem::Variable(variable) => { substitution_mode.apply( output, variable.span_range(), - variable.execute_substitution(self)?, + variable.read_substitution(self)?.clone(), ); } NextItem::CommandInvocation(command_invocation) => { @@ -102,6 +122,29 @@ impl Interpreter { } } +pub(crate) struct InterpreterConfig { + iteration_limit: Option, +} + +impl Default for InterpreterConfig { + fn default() -> Self { + Self { + iteration_limit: Some(10000), + } + } +} + +impl InterpreterConfig { + pub(crate) fn check_iteration_count(&self, command: &Command, count: usize) -> Result<()> { + if let Some(limit) = self.iteration_limit { + if count > limit { + return command.err(format!("Iteration limit of {} exceeded", limit)); + } + } + Ok(()) + } +} + /// How to output `#variables` and `[!commands!]` into the output stream #[derive(Clone, Copy)] pub(crate) struct SubstitutionMode(SubstitutionModeInternal); @@ -139,12 +182,13 @@ impl SubstitutionMode { match self.0 { SubstitutionModeInternal::Extend => tokens.extend(substitution), SubstitutionModeInternal::Group(delimiter) => { - tokens.push_new_group(span_range, delimiter, substitution) + tokens.push_new_group(substitution, delimiter, span_range) } } } } +#[derive(Clone)] pub(crate) struct Tokens(iter::Peekable<::IntoIter>); impl Tokens { @@ -167,6 +211,13 @@ impl Tokens { } } + pub(crate) fn next_as_punct(&mut self) -> Option { + match self.next() { + Some(TokenTree::Punct(punct)) => Some(punct), + _ => None, + } + } + pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { match self.next() { Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), @@ -242,6 +293,7 @@ impl Tokens { } } +#[derive(Clone)] pub(crate) enum NextItem { CommandInvocation(CommandInvocation), Variable(Variable), diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 6e657cdd..2221b283 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -1,3 +1,5 @@ +#![allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 + use preinterpret::preinterpret; macro_rules! assert_preinterpret_eq { @@ -7,7 +9,6 @@ macro_rules! assert_preinterpret_eq { } #[test] -#[allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); assert_preinterpret_eq!({ @@ -28,3 +29,17 @@ fn test_if() { [!if! false { + 1 }] }, 0); } + +#[test] +fn test_while() { + assert_preinterpret_eq!({ + [!set! #x = 0] + [!while! (#x < 5) { [!increment! #x] }] + #x + }, 5); +} + +// TODO: Check compilation error for: +// assert_preinterpret_eq!({ +// [!while! true {}] +// }, 5); \ No newline at end of file diff --git a/tests/evaluate.rs b/tests/expressions.rs similarity index 90% rename from tests/evaluate.rs rename to tests/expressions.rs index 4ecc32e2..9f04f256 100644 --- a/tests/evaluate.rs +++ b/tests/expressions.rs @@ -53,5 +53,18 @@ fn increment_works() { ); } +#[test] +fn assign_works() { + assert_preinterpret_eq!( + { + [!set! #x = 8 + 2] // 10 + [!assign! #x /= 1 + 1] // 5 + [!assign! #x += 2 + #x] // 12 + #x + }, + 12 + ); +} + // TODO - Add failing tests for these: // assert_preinterpret_eq!([!evaluate! !!(!!({true}))], true); From af5aaf02daf6cdc13df42a44660fb41345fc2676 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 3 Jan 2025 13:24:31 +0000 Subject: [PATCH 008/126] feat: Add `!assign!` and remove `!increment!` --- CHANGELOG.md | 3 +-- README.md | 12 ++++++++++-- src/commands/expression_commands.rs | 25 ------------------------- src/commands/mod.rs | 1 - src/expressions/integer.rs | 19 ------------------- tests/control_flow.rs | 2 +- tests/expressions.rs | 13 ------------- 7 files changed, 12 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49059a2e..a5afb2ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * Expression commands: * `[!evaluate! ...]` - * `[!increment! ...]` + * `[!assign! #x += ...]` for `+` and other supported operators * Control flow commands: * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` * `[!while! cond {}]` @@ -23,7 +23,6 @@ * `[!for! #x in [#y] {}]` * `[!range! 0..5]` * `[!error! "message" token stream for span]` -* Remove `[!increment! ...]` and replace with `[!assign! #x += 1]` * Reconfiguring iteration limit * Support `!else if!` in `!if!` * Token stream manipulation... and make it performant (maybe by storing either TokenStream for extension; or `ParseStream` for consumption) diff --git a/README.md b/README.md index 6c44bfd2..89494c57 100644 --- a/README.md +++ b/README.md @@ -445,20 +445,28 @@ Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. +### Possible extension: Loop, Break + +These aren't the highest priority, as they can be simulated with `if` statements inside a `while` loop: + +* `[!loop! { ... }]` for `[!while! true { ... }]` +* `[!break!]` to break the inner-most loop + ### Possible extension: Goto -_This probably isn't needed_. +_This probably isn't needed, if we have `while` and `for`_. * `[!label! loop_start]` - defines a label which can be returned to. Effectively, it takes a clones of the remaining token stream after the label in the interpreter. * `[!goto! loop_start]` - jumps to the last execution of `[!label! loop_start]`. It unrolls the preinterpret stack (dropping all unwritten token streams) until it finds a stackframe in which the interpreter has the defined label, and continues the token stream from there. ```rust,ignore // Hypothetical future syntax - not yet implemented! +// For now you can use a `[!while! #i <= 100 { ... }]` instead preinterpret::preinterpret!{ [!set! #i = 0] [!label! loop] const [!ident! AB #i]: u8 = 0; - [!increment! #i] + [!assign! #i += 1] [!if! (#i <= 100) { [!goto! loop] }] } ``` diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 94a1d1ae..89210d8f 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -71,28 +71,3 @@ impl AssignStatementStart { }) } } - -pub(crate) struct IncrementCommand; - -impl CommandDefinition for IncrementCommand { - const COMMAND_NAME: &'static str = "increment"; - - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let error_message = "Expected [!increment! #variable]"; - let variable = command - .argument_tokens() - .next_item_as_variable(error_message)?; - command.argument_tokens().assert_end(error_message)?; - let variable_contents = variable.read_substitution(interpreter)?; - let evaluated_integer = - evaluate_expression(variable_contents.clone(), ExpressionParsingMode::Standard)? - .expect_integer(&format!("Expected {variable} to evaluate to an integer"))?; - interpreter.set_variable( - variable.variable_name().to_string(), - evaluated_integer - .increment(command.span_range())? - .to_token_stream(), - ); - Ok(TokenStream::new()) - } -} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index ade9f90d..e2d89695 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -44,7 +44,6 @@ define_commands! { // Expression Commands EvaluateCommand, AssignCommand, - IncrementCommand, // Control flow commands IfCommand, diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 101a9016..17ee5a9e 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -85,25 +85,6 @@ impl EvaluationInteger { } } } - - pub(crate) fn increment(self, operator_span: SpanRange) -> Result { - let span_for_output = self.source_span; - EvaluationOutput::Value(EvaluationValue::Integer(self)) - .expect_value_pair( - PairedBinaryOperator::Addition, - EvaluationOutput::Value(EvaluationValue::Integer(EvaluationInteger::new( - EvaluationIntegerValue::Untyped(UntypedInteger::from_fallback(1)), - operator_span, - ))), - operator_span, - )? - .handle_paired_binary_operation(BinaryOperation { - span_for_output, - operator_span, - operator: BinaryOperator::Paired(PairedBinaryOperator::Addition), - })? - .expect_integer("Integer should be created by summing too integers") - } } impl quote::ToTokens for EvaluationInteger { diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 2221b283..67b1306f 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -34,7 +34,7 @@ fn test_if() { fn test_while() { assert_preinterpret_eq!({ [!set! #x = 0] - [!while! (#x < 5) { [!increment! #x] }] + [!while! (#x < 5) { [!assign! #x += 1] }] #x }, 5); } diff --git a/tests/expressions.rs b/tests/expressions.rs index 9f04f256..20b253bb 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -40,19 +40,6 @@ fn test_basic_evaluate_works() { ); } -#[test] -fn increment_works() { - assert_preinterpret_eq!( - { - [!set! #x = 2 + 2] - [!increment! #x] - [!increment! #x] - #x - }, - 6 - ); -} - #[test] fn assign_works() { assert_preinterpret_eq!( From 55320463bbb552764b6e8d06aa351977abd32d31 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 4 Jan 2025 20:58:54 +0000 Subject: [PATCH 009/126] WIP: Refactor to separate stream types --- CHANGELOG.md | 2 + src/command.rs | 230 -------------- src/commands/concat_commands.rs | 63 ++-- src/commands/control_flow_commands.rs | 53 ++-- src/commands/core_commands.rs | 17 +- src/commands/expression_commands.rs | 38 +-- src/commands/token_commands.rs | 32 +- src/expressions/evaluation_tree.rs | 6 +- src/expressions/expression_stream.rs | 46 +++ src/expressions/mod.rs | 34 +-- src/internal_prelude.rs | 202 +------------ src/interpretation/command.rs | 147 +++++++++ src/interpretation/interpreted_stream.rs | 65 ++++ src/interpretation/interpreter.rs | 50 ++++ src/interpretation/mod.rs | 13 + src/interpretation/next_item.rs | 58 ++++ src/interpretation/tokens.rs | 193 ++++++++++++ src/interpretation/variable.rs | 88 ++++++ src/interpreter.rs | 366 ----------------------- src/lib.rs | 9 +- src/traits.rs | 222 ++++++++++++++ 21 files changed, 987 insertions(+), 947 deletions(-) delete mode 100644 src/command.rs create mode 100644 src/expressions/expression_stream.rs create mode 100644 src/interpretation/command.rs create mode 100644 src/interpretation/interpreted_stream.rs create mode 100644 src/interpretation/interpreter.rs create mode 100644 src/interpretation/mod.rs create mode 100644 src/interpretation/next_item.rs create mode 100644 src/interpretation/tokens.rs create mode 100644 src/interpretation/variable.rs delete mode 100644 src/interpreter.rs create mode 100644 src/traits.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a5afb2ab..f361c753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ ### To come * ? Use `[!let! #x = 12]` instead of `[!set! ..]`. +* Fix `if` and `while` to read expression until braces +* Refactor command parsing/execution * `[!for! #x in [#y] {}]` * `[!range! 0..5]` * `[!error! "message" token stream for span]` diff --git a/src/command.rs b/src/command.rs deleted file mode 100644 index 23ca201c..00000000 --- a/src/command.rs +++ /dev/null @@ -1,230 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) trait CommandDefinition { - const COMMAND_NAME: &'static str; - - fn execute(interpreter: &mut Interpreter, argument: Command) -> Result; -} - -macro_rules! define_commands { - ( - pub(crate) enum $enum_name:ident { - $( - $command:ident, - )* - } - ) => { - #[allow(clippy::enum_variant_names)] - #[derive(Clone, Copy)] - pub(crate) enum $enum_name { - $( - $command, - )* - } - - impl $enum_name { - pub(crate) fn execute(self, interpreter: &mut Interpreter, command: Command) -> Result { - match self { - $( - Self::$command => $command::execute(interpreter, command), - )* - } - } - - pub(crate) fn attempt_parse(ident: &Ident) -> Option { - Some(match ident.to_string().as_ref() { - $( - <$command as CommandDefinition>::COMMAND_NAME => Self::$command, - )* - _ => return None, - }) - } - - const ALL_KIND_NAMES: &'static [&'static str] = &[$($command::COMMAND_NAME,)*]; - - pub(crate) fn list_all() -> String { - // TODO improve to add an "and" at the end - Self::ALL_KIND_NAMES.join(", ") - } - } - }; -} -pub(crate) use define_commands; - -#[derive(Clone)] -pub(crate) struct CommandInvocation { - command_kind: CommandKind, - command: Command, -} - -impl CommandInvocation { - pub(crate) fn new( - command_ident: Ident, - command_kind: CommandKind, - group: &Group, - argument_tokens: Tokens, - ) -> Self { - Self { - command_kind, - command: Command::new(command_ident, group.span(), argument_tokens), - } - } - - pub(crate) fn execute(self, interpreter: &mut Interpreter) -> Result { - self.command_kind.execute(interpreter, self.command) - } -} - -impl HasSpanRange for CommandInvocation { - fn span_range(&self) -> SpanRange { - self.command.span_range() - } -} - -#[derive(Clone)] -pub(crate) struct Variable { - marker: Punct, // # - variable_name: Ident, -} - -impl Variable { - pub(crate) fn new(marker: Punct, variable_name: Ident) -> Self { - Self { - marker, - variable_name, - } - } - - pub(crate) fn variable_name(&self) -> String { - self.variable_name.to_string() - } - - pub(crate) fn set<'i>( - &self, - interpreter: &'i mut Interpreter, - value: TokenStream, - ) { - interpreter.set_variable(self.variable_name(), value); - } - - pub(crate) fn read_substitution<'i>( - &self, - interpreter: &'i Interpreter, - ) -> Result<&'i TokenStream> { - self.read_or_else( - interpreter, - || format!( - "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self, - self, - ) - ) - } - - pub(crate) fn read_required<'i>( - &self, - interpreter: &'i Interpreter, - ) -> Result<&'i TokenStream> { - self.read_or_else( - interpreter, - || format!( - "The variable {} wasn't set.", - self, - ) - ) - } - - pub(crate) fn read_or_else<'i>( - &self, - interpreter: &'i Interpreter, - create_error: impl FnOnce() -> String, - ) -> Result<&'i TokenStream> { - match self.read_option(interpreter) { - Some(token_stream) => Ok(token_stream), - None => self.span_range().err(create_error()), - } - } - - fn read_option<'i>( - &self, - interpreter: &'i Interpreter, - ) -> Option<&'i TokenStream> { - let Variable { variable_name, .. } = self; - interpreter.get_variable(&variable_name.to_string()) - } -} - -impl HasSpanRange for Variable { - fn span_range(&self) -> SpanRange { - SpanRange::new_between(self.marker.span(), self.variable_name.span()) - } -} - -impl core::fmt::Display for Variable { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "{}{}", self.marker.as_char(), self.variable_name) - } -} - -#[derive(Clone)] -pub(crate) struct Command { - command_ident: Ident, - command_span: Span, - argument_tokens: Tokens, -} - -impl Command { - fn new(command_ident: Ident, command_span: Span, argument_tokens: Tokens) -> Self { - Self { - command_ident, - command_span, - argument_tokens, - } - } - - #[allow(unused)] // Likely useful in future - pub(crate) fn ident_span(&self) -> Span { - self.command_ident.span() - } - - pub(crate) fn span(&self) -> Span { - self.command_span - } - - pub(crate) fn error(&self, message: impl core::fmt::Display) -> syn::Error { - self.command_span.error(message) - } - - pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { - Err(self.error(message)) - } - - /// Expects the remaining arguments to be non-empty - pub(crate) fn interpret_remaining_arguments( - &mut self, - interpreter: &mut Interpreter, - substitution_mode: SubstitutionMode, - ) -> Result { - if self.argument_tokens.is_empty() { - // This is simply for clarity / to make empty arguments explicit. - return self.err( - "Arguments were empty. Use [!empty!] if you want to use an empty token stream.", - ); - } - interpreter.interpret_tokens(&mut self.argument_tokens, substitution_mode) - } - - pub(crate) fn argument_tokens(&mut self) -> &mut Tokens { - &mut self.argument_tokens - } - - pub(crate) fn into_argument_tokens(self) -> Tokens { - self.argument_tokens - } -} - -impl HasSpanRange for Command { - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 9c8706c3..125c7da3 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -28,36 +28,33 @@ fn concat_into_string( interpreter: &mut Interpreter, mut command: Command, conversion_fn: impl Fn(&str) -> String, -) -> Result { - let interpreted = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; +) -> Result { + let interpreted = command.arguments().interpret_as_tokens(interpreter)?; let concatenated = concat_recursive(interpreted); let string_literal = string_literal(&conversion_fn(&concatenated), command.span()); - Ok(TokenStream::from(TokenTree::Literal(string_literal))) + Ok(InterpretedStream::of_literal(string_literal)) } fn concat_into_ident( interpreter: &mut Interpreter, mut command: Command, conversion_fn: impl Fn(&str) -> String, -) -> Result { - let interpreted = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; +) -> Result { + let interpreted = command.arguments().interpret_as_tokens(interpreter)?; let concatenated = concat_recursive(interpreted); let ident = parse_ident(&conversion_fn(&concatenated), command.span())?; - Ok(TokenStream::from(TokenTree::Ident(ident))) + Ok(InterpretedStream::of_ident(ident)) } fn concat_into_literal( interpreter: &mut Interpreter, mut command: Command, conversion_fn: impl Fn(&str) -> String, -) -> Result { - let interpreted = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; +) -> Result { + let interpreted = command.arguments().interpret_as_tokens(interpreter)?; let concatenated = concat_recursive(interpreted); let literal = parse_literal(&conversion_fn(&concatenated), command.span())?; - Ok(TokenStream::from(TokenTree::Literal(literal))) + Ok(InterpretedStream::of_literal(literal)) } //======================================= @@ -69,7 +66,7 @@ pub(crate) struct StringCommand; impl CommandDefinition for StringCommand { const COMMAND_NAME: &'static str = "string"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, |s| s.to_string()) } } @@ -79,7 +76,7 @@ pub(crate) struct IdentCommand; impl CommandDefinition for IdentCommand { const COMMAND_NAME: &'static str = "ident"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, |s| s.to_string()) } } @@ -89,7 +86,7 @@ pub(crate) struct IdentCamelCommand; impl CommandDefinition for IdentCamelCommand { const COMMAND_NAME: &'static str = "ident_camel"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, to_upper_camel_case) } } @@ -99,7 +96,7 @@ pub(crate) struct IdentSnakeCommand; impl CommandDefinition for IdentSnakeCommand { const COMMAND_NAME: &'static str = "ident_snake"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, to_lower_snake_case) } } @@ -109,7 +106,7 @@ pub(crate) struct IdentUpperSnakeCommand; impl CommandDefinition for IdentUpperSnakeCommand { const COMMAND_NAME: &'static str = "ident_upper_snake"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_ident(interpreter, command, to_upper_snake_case) } } @@ -119,7 +116,7 @@ pub(crate) struct LiteralCommand; impl CommandDefinition for LiteralCommand { const COMMAND_NAME: &'static str = "literal"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_literal(interpreter, command, |s| s.to_string()) } } @@ -133,7 +130,7 @@ pub(crate) struct UpperCommand; impl CommandDefinition for UpperCommand { const COMMAND_NAME: &'static str = "upper"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_uppercase) } } @@ -143,7 +140,7 @@ pub(crate) struct LowerCommand; impl CommandDefinition for LowerCommand { const COMMAND_NAME: &'static str = "lower"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_lowercase) } } @@ -153,7 +150,7 @@ pub(crate) struct SnakeCommand; impl CommandDefinition for SnakeCommand { const COMMAND_NAME: &'static str = "snake"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { // Lower snake case is the more common casing in Rust, so default to that LowerSnakeCommand::execute(interpreter, command) } @@ -164,7 +161,7 @@ pub(crate) struct LowerSnakeCommand; impl CommandDefinition for LowerSnakeCommand { const COMMAND_NAME: &'static str = "lower_snake"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_lower_snake_case) } } @@ -174,7 +171,7 @@ pub(crate) struct UpperSnakeCommand; impl CommandDefinition for UpperSnakeCommand { const COMMAND_NAME: &'static str = "upper_snake"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_upper_snake_case) } } @@ -184,7 +181,7 @@ pub(crate) struct KebabCommand; impl CommandDefinition for KebabCommand { const COMMAND_NAME: &'static str = "kebab"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { // Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) // It can always be combined with other casing to get other versions concat_into_string(interpreter, command, to_lower_kebab_case) @@ -196,7 +193,7 @@ pub(crate) struct CamelCommand; impl CommandDefinition for CamelCommand { const COMMAND_NAME: &'static str = "camel"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { // Upper camel case is the more common casing in Rust, so default to that UpperCamelCommand::execute(interpreter, command) } @@ -207,7 +204,7 @@ pub(crate) struct LowerCamelCommand; impl CommandDefinition for LowerCamelCommand { const COMMAND_NAME: &'static str = "lower_camel"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_lower_camel_case) } } @@ -217,7 +214,7 @@ pub(crate) struct UpperCamelCommand; impl CommandDefinition for UpperCamelCommand { const COMMAND_NAME: &'static str = "upper_camel"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, to_upper_camel_case) } } @@ -227,7 +224,7 @@ pub(crate) struct CapitalizeCommand; impl CommandDefinition for CapitalizeCommand { const COMMAND_NAME: &'static str = "capitalize"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, capitalize) } } @@ -237,7 +234,7 @@ pub(crate) struct DecapitalizeCommand; impl CommandDefinition for DecapitalizeCommand { const COMMAND_NAME: &'static str = "decapitalize"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, decapitalize) } } @@ -247,7 +244,7 @@ pub(crate) struct TitleCommand; impl CommandDefinition for TitleCommand { const COMMAND_NAME: &'static str = "title"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, title_case) } } @@ -257,12 +254,12 @@ pub(crate) struct InsertSpacesCommand; impl CommandDefinition for InsertSpacesCommand { const COMMAND_NAME: &'static str = "insert_spaces"; - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, command: Command) -> Result { concat_into_string(interpreter, command, insert_spaces_between_words) } } -fn concat_recursive(arguments: TokenStream) -> String { +fn concat_recursive(arguments: InterpretedStream) -> String { fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { for token_tree in arguments { match token_tree { @@ -307,6 +304,6 @@ fn concat_recursive(arguments: TokenStream) -> String { } let mut output = String::new(); - concat_recursive_internal(&mut output, arguments); + concat_recursive_internal(&mut output, arguments.into_token_stream()); output } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 92e510ab..eda530dd 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -5,29 +5,27 @@ pub(crate) struct IfCommand; impl CommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let parsed = match parse_if_statement(command.argument_tokens()) { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let parsed = match parse_if_statement(command.arguments()) { Some(parsed) => parsed, None => { return command.err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); } }; - let interpreted_condition = - interpreter.interpret_item(parsed.condition, SubstitutionMode::expression())?; - let evaluated_condition = evaluate_expression( - interpreted_condition, - ExpressionParsingMode::BeforeCurlyBraces, - )? - .expect_bool("An if condition must evaluate to a boolean")? - .value(); + let evaluated_condition = parsed + .condition + .interpret_as_expression(interpreter)? + .evaluate()? + .expect_bool("An if condition must evaluate to a boolean")? + .value(); if evaluated_condition { - interpreter.interpret_token_stream(parsed.true_code, SubstitutionMode::token_stream()) + parsed.true_code.interpret_as_tokens(interpreter) } else if let Some(false_code) = parsed.false_code { - interpreter.interpret_token_stream(false_code, SubstitutionMode::token_stream()) + false_code.interpret_as_tokens(interpreter) } else { - Ok(TokenStream::new()) + Ok(InterpretedStream::new()) } } } @@ -65,34 +63,33 @@ pub(crate) struct WhileCommand; impl CommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let parsed = match parse_while_statement(command.argument_tokens()) { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let parsed = match parse_while_statement(command.arguments()) { Some(parsed) => parsed, None => { return command.err("Expected [!while! (condition) { code }]"); } }; - let mut output = TokenStream::new(); + let mut output = InterpretedStream::new(); let mut iteration_count = 0; loop { - let interpreted_condition = - interpreter.interpret_item(parsed.condition.clone(), SubstitutionMode::expression())?; - let evaluated_condition = evaluate_expression( - interpreted_condition, - ExpressionParsingMode::BeforeCurlyBraces, - )? - .expect_bool("An if condition must evaluate to a boolean")? - .value(); + let evaluated_condition = parsed + .condition + .clone() + .interpret_as_expression(interpreter)? + .evaluate()? + .expect_bool("An if condition must evaluate to a boolean")? + .value(); - iteration_count += 1; if !evaluated_condition { break; } + + iteration_count += 1; interpreter.config().check_iteration_count(&command, iteration_count)?; - interpreter.interpret_token_stream_into( - parsed.code.clone(), - SubstitutionMode::token_stream(), + parsed.code.clone().interpret_as_tokens_into( + interpreter, &mut output, )?; } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 3bb8f563..d9c41040 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -5,19 +5,18 @@ pub(crate) struct SetCommand; impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let variable_name = match parse_variable_set(command.argument_tokens()) { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let variable_name = match parse_variable_set(command.arguments()) { Some(variable) => variable.variable_name().to_string(), None => { return command.err("A set call is expected to start with `#variable_name = ..`"); } }; - let result_tokens = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + let result_tokens = command.arguments().interpret_as_tokens(interpreter)?; interpreter.set_variable(variable_name, result_tokens); - Ok(TokenStream::new()) + Ok(InterpretedStream::new()) } } @@ -32,8 +31,8 @@ pub(crate) struct RawCommand; impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - fn execute(_interpreter: &mut Interpreter, command: Command) -> Result { - Ok(command.into_argument_tokens().into_token_stream()) + fn execute(_interpreter: &mut Interpreter, mut command: Command) -> Result { + Ok(InterpretedStream::raw(command.arguments().into_token_stream())) } } @@ -42,7 +41,7 @@ pub(crate) struct IgnoreCommand; impl CommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; - fn execute(_: &mut Interpreter, _: Command) -> Result { - Ok(TokenStream::new()) + fn execute(_: &mut Interpreter, _: Command) -> Result { + Ok(InterpretedStream::new()) } } diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 89210d8f..040df886 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -5,10 +5,9 @@ pub(crate) struct EvaluateCommand; impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let token_stream = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?; - Ok(evaluate_expression(token_stream, ExpressionParsingMode::Standard)?.into_token_stream()) + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let expression = command.arguments().interpret_as_expression(interpreter)?; + Ok(expression.evaluate()?.into_interpreted_stream()) } } @@ -17,37 +16,22 @@ pub(crate) struct AssignCommand; impl CommandDefinition for AssignCommand { const COMMAND_NAME: &'static str = "assign"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { let AssignStatementStart { variable, operator, - } = AssignStatementStart::parse(command.argument_tokens()) + } = AssignStatementStart::parse(command.arguments()) .ok_or_else(|| command.error("Expected [!assign! #variable += ...] for + or some other operator supported in an expression"))?; - let mut expression_tokens = TokenStream::new(); - expression_tokens.push_new_group( - // TODO: Replace with `variable.read_into(tokens, substitution_mode)` - // TODO: Replace most methods on interpeter with e.g. - // command.interpret_into, next_item.interpet_into, etc. - // And also create an Expression struct - // TODO: Fix Expression to not need different parsing modes, - // and to be parsed from the full token stream or until braces { .. } - // or as a single item - variable.read_required(interpreter)?.clone(), - Delimiter::None, - variable.span_range(), - ); - expression_tokens.push_token_tree(operator.into()); - expression_tokens.push_new_group( - command.interpret_remaining_arguments(interpreter, SubstitutionMode::expression())?, - Delimiter::None, - command.span_range(), - ); + let mut expression_stream = ExpressionStream::new(); + variable.interpret_as_expression_into(interpreter, &mut expression_stream)?; + operator.into_token_stream().interpret_as_expression_into(interpreter, &mut expression_stream)?; + command.arguments().interpret_as_expression_into(interpreter, &mut expression_stream)?; - let output = evaluate_expression(expression_tokens, ExpressionParsingMode::Standard)?.into_token_stream(); + let output = expression_stream.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output); - Ok(TokenStream::new()) + Ok(InterpretedStream::new()) } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 76559caf..b67271e7 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -5,11 +5,9 @@ pub(crate) struct EmptyCommand; impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; - fn execute(_interpreter: &mut Interpreter, command: Command) -> Result { - command - .into_argument_tokens() - .assert_end("The !empty! command does not take any arguments")?; - Ok(TokenStream::new()) + fn execute(_interpreter: &mut Interpreter, mut command: Command) -> Result { + command.arguments().assert_end("The !empty! command does not take any arguments")?; + Ok(InterpretedStream::new()) } } @@ -18,9 +16,8 @@ pub(crate) struct IsEmptyCommand; impl CommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let interpreted = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let interpreted = command.arguments().interpret_as_tokens(interpreter)?; Ok(TokenTree::bool(interpreted.is_empty(), command.span()).into()) } } @@ -30,14 +27,11 @@ pub(crate) struct LengthCommand; impl CommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let interpreted = - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?; - let stream_length = interpreted.into_iter().count(); - Ok( - TokenTree::Literal(Literal::usize_unsuffixed(stream_length).with_span(command.span())) - .into(), - ) + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let interpreted = command.arguments().interpret_as_tokens(interpreter)?; + let stream_length = interpreted.into_token_stream().into_iter().count(); + let length_literal = Literal::usize_unsuffixed(stream_length).with_span(command.span()); + Ok(InterpretedStream::of_literal(length_literal)) } } @@ -46,10 +40,10 @@ pub(crate) struct GroupCommand; impl CommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let mut output = TokenStream::new(); + fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + let mut output = InterpretedStream::new(); output.push_new_group( - command.interpret_remaining_arguments(interpreter, SubstitutionMode::token_stream())?, + command.arguments().interpret_as_tokens(interpreter)?, Delimiter::None, command.span_range(), ); diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index 4602cb2e..088f41bb 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -429,9 +429,13 @@ impl EvaluationOutput { pub(super) fn into_value(self) -> EvaluationValue { match self { - Self::Value(literal) => literal, + Self::Value(value) => value, } } + + pub(crate) fn into_interpreted_stream(self) -> InterpretedStream { + InterpretedStream::raw(self.into_token_stream()) + } } impl From for EvaluationOutput { diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs new file mode 100644 index 00000000..0a7ce27d --- /dev/null +++ b/src/expressions/expression_stream.rs @@ -0,0 +1,46 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionStream { + interpreted_stream: InterpretedStream, +} + +impl ExpressionStream { + pub(crate) fn new() -> Self { + Self { + interpreted_stream: InterpretedStream::new(), + } + } + + pub(crate) fn push_raw_token_tree(&mut self, token_tree: TokenTree) { + self.interpreted_stream.push_raw_token_tree(token_tree); + } + + pub(crate) fn push_interpreted_group(&mut self, contents: InterpretedStream, span_range: SpanRange) { + self.interpreted_stream.push_new_group(contents, Delimiter::None, span_range); + } + + pub(crate) fn push_expression_group(&mut self, contents: Self, delimiter: Delimiter, span_range: SpanRange) { + self.interpreted_stream.push_new_group(contents.interpreted_stream, delimiter, span_range); + } + + pub(crate) fn evaluate(self) -> Result { + use syn::parse::{Parse, Parser}; + + // Parsing into a rust expression is overkill here. + // + // In future we could choose to implement a subset of the grammar which we actually can use/need. + // + // That said, it's useful for now for two reasons: + // * Aligning with rust syntax + // * Saving implementation work, particularly given that syn is likely pulled into most code bases anyway + // + // Because of the kind of expressions we're parsing (i.e. no {} allowed), + // we can get by with parsing it as `Expr::parse` rather than with + // `Expr::parse_without_eager_brace` or `Expr::parse_with_earlier_boundary_rule`. + let expression = Expr::parse.parse2(self.interpreted_stream.into_token_stream())?; + + EvaluationTree::build_from(&expression)? + .evaluate() + } +} diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 7f8b5da8..e5f9c086 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -1,10 +1,12 @@ mod boolean; mod evaluation_tree; +mod expression_stream; mod float; mod integer; mod operations; mod value; +// Marked as use for expression sub-modules to use with a `use super::*` statement use crate::internal_prelude::*; use boolean::*; use evaluation_tree::*; @@ -13,34 +15,4 @@ use integer::*; use operations::*; use value::*; -#[allow(unused)] -pub(crate) enum ExpressionParsingMode { - Standard, - BeforeCurlyBraces, - StartOfStatement, -} - -pub(crate) fn evaluate_expression( - token_stream: TokenStream, - mode: ExpressionParsingMode, -) -> Result { - use syn::parse::{Parse, Parser}; - - // Parsing into a rust expression is overkill here. - // In future we could choose to implement a subset of the grammar which we actually can use/need. - // - // That said, it's useful for now for two reasons: - // * Aligning with rust syntax - // * Saving implementation work, particularly given that syn is likely pulled into most code bases anyway - let expression = match mode { - ExpressionParsingMode::Standard => Expr::parse.parse2(token_stream)?, - ExpressionParsingMode::BeforeCurlyBraces => { - (Expr::parse_without_eager_brace).parse2(token_stream)? - } - ExpressionParsingMode::StartOfStatement => { - (Expr::parse_with_earlier_boundary_rule).parse2(token_stream)? - } - }; - - EvaluationTree::build_from(&expression)?.evaluate() -} +pub(crate) use expression_stream::*; diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 88ab0622..e5d68c51 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -1,209 +1,11 @@ pub(crate) use core::iter; -use extra::DelimSpan; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, UnOp}; -pub(crate) use crate::command::*; pub(crate) use crate::commands::*; pub(crate) use crate::expressions::*; -pub(crate) use crate::interpreter::*; +pub(crate) use crate::traits::*; +pub(crate) use crate::interpretation::*; pub(crate) use crate::string_conversion::*; - -pub(crate) trait TokenTreeExt: Sized { - fn bool(value: bool, span: Span) -> Self; - fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; -} - -impl TokenTreeExt for TokenTree { - fn bool(value: bool, span: Span) -> Self { - TokenTree::Ident(Ident::new(&value.to_string(), span)) - } - - fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { - TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) - } -} - -pub(crate) trait TokenStreamExt: Sized { - fn push_token_tree(&mut self, token_tree: TokenTree); - fn push_new_group( - &mut self, - inner_tokens: TokenStream, - delimiter: Delimiter, - span_range: SpanRange, - ); -} - -impl TokenStreamExt for TokenStream { - fn push_token_tree(&mut self, token_tree: TokenTree) { - self.extend(iter::once(token_tree)); - } - - fn push_new_group( - &mut self, - inner_tokens: TokenStream, - delimiter: Delimiter, - span_range: SpanRange, - ) { - self.push_token_tree(TokenTree::group(inner_tokens, delimiter, span_range.span())); - } -} - -pub(crate) trait SpanErrorExt: Sized { - fn err(self, message: impl std::fmt::Display) -> syn::Result { - Err(self.error(message)) - } - - fn error(self, message: impl std::fmt::Display) -> syn::Error; -} - -impl SpanErrorExt for Span { - fn error(self, message: impl std::fmt::Display) -> syn::Error { - syn::Error::new(self, message) - } -} - -impl SpanErrorExt for SpanRange { - fn error(self, message: impl std::fmt::Display) -> syn::Error { - syn::Error::new_spanned(self, message) - } -} - -pub(crate) trait WithSpanExt { - fn with_span(self, span: Span) -> Self; -} - -impl WithSpanExt for Literal { - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -impl WithSpanExt for Punct { - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -impl WithSpanExt for Group { - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -pub(crate) trait HasSpanRange { - fn span_range(&self) -> SpanRange; -} - -/// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] -/// and falls back to the span of the first token when not available. -/// -/// Instead, [`syn::Error`] uses a trick involving a span range. This effectively -/// allows capturing this trick when we're not immediately creating an error. -/// -/// When [`proc_macro::Span::join`] is stabilised and [`syn::spanned`] works, -/// we can swap [`SpanRange`] contents for [`Span`] (or even remove it and [`HasSpanRange`]). -#[derive(Copy, Clone)] -pub(crate) struct SpanRange { - start: Span, - end: Span, -} - -#[allow(unused)] -impl SpanRange { - pub(crate) fn new_between(start: Span, end: Span) -> Self { - Self { start, end } - } - - /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) - /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) - pub(crate) fn span(&self) -> Span { - ::span(self) - } - - pub(crate) fn start(&self) -> Span { - self.start - } - - pub(crate) fn end(&self) -> Span { - self.end - } - - pub(crate) fn replace_start(self, start: Span) -> Self { - Self { start, ..self } - } - - pub(crate) fn replace_end(self, end: Span) -> Self { - Self { end, ..self } - } -} - -// This is implemented so we can create an error from it using `Error::new_spanned(..)` -impl ToTokens for SpanRange { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.extend([ - TokenTree::Punct(Punct::new('<', Spacing::Alone).with_span(self.start)), - TokenTree::Punct(Punct::new('>', Spacing::Alone).with_span(self.end)), - ]); - } -} - -impl HasSpanRange for Span { - fn span_range(&self) -> SpanRange { - SpanRange::new_between(*self, *self) - } -} - -impl HasSpanRange for TokenTree { - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} - -impl HasSpanRange for Group { - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} - -impl HasSpanRange for DelimSpan { - fn span_range(&self) -> SpanRange { - SpanRange::new_between(self.open(), self.close()) - } -} - -impl HasSpanRange for T { - fn span_range(&self) -> SpanRange { - let mut iter = self.into_token_stream().into_iter(); - let start = iter.next().map_or_else(Span::call_site, |t| t.span()); - let end = iter.last().map_or(start, |t| t.span()); - SpanRange { start, end } - } -} - -/// This should only be used for syn built-ins or when there isn't a better -/// span range available -trait AutoSpanRange {} - -macro_rules! impl_auto_span_range { - ($($ty:ty),* $(,)?) => { - $( - impl AutoSpanRange for $ty {} - )* - }; -} - -impl_auto_span_range! { - syn::Expr, - syn::ExprBinary, - syn::ExprUnary, - syn::BinOp, - syn::UnOp, - syn::Type, - syn::TypePath, -} diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs new file mode 100644 index 00000000..ac2b7099 --- /dev/null +++ b/src/interpretation/command.rs @@ -0,0 +1,147 @@ +use crate::internal_prelude::*; + +pub(crate) trait CommandDefinition { + const COMMAND_NAME: &'static str; + + fn execute(interpreter: &mut Interpreter, command: Command) -> Result; +} + +macro_rules! define_commands { + ( + pub(crate) enum $enum_name:ident { + $( + $command:ident, + )* + } + ) => { + #[allow(clippy::enum_variant_names)] + #[derive(Clone, Copy)] + pub(crate) enum $enum_name { + $( + $command, + )* + } + + impl $enum_name { + pub(crate) fn execute(self, interpreter: &mut Interpreter, command: Command) -> Result { + match self { + $( + Self::$command => $command::execute(interpreter, command), + )* + } + } + + pub(crate) fn attempt_parse(ident: &Ident) -> Option { + Some(match ident.to_string().as_ref() { + $( + <$command as CommandDefinition>::COMMAND_NAME => Self::$command, + )* + _ => return None, + }) + } + + const ALL_KIND_NAMES: &'static [&'static str] = &[$($command::COMMAND_NAME,)*]; + + pub(crate) fn list_all() -> String { + // TODO improve to add an "and" at the end + Self::ALL_KIND_NAMES.join(", ") + } + } + }; +} +pub(crate) use define_commands; + +#[derive(Clone)] +pub(crate) struct CommandInvocation { + command_kind: CommandKind, + command: Command, +} + +impl CommandInvocation { + pub(crate) fn new( + command_ident: Ident, + command_kind: CommandKind, + group: &Group, + argument_tokens: Tokens, + ) -> Self { + Self { + command_kind, + command: Command::new(command_ident, group.span(), argument_tokens), + } + } + + fn execute_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + let substitution = self.command_kind.execute(interpreter, self.command)?; + output.extend(substitution); + Ok(()) + } +} + +impl HasSpanRange for CommandInvocation { + fn span_range(&self) -> SpanRange { + self.command.span_range() + } +} + +impl Interpret for CommandInvocation { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + self.execute_into(interpreter, output) + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + let span_range = self.span_range(); + expression_stream.push_interpreted_group( + self.interpret_as_tokens(interpreter)?, + span_range, + ); + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct Command { + command_ident: Ident, + command_span: Span, + argument_tokens: Tokens, +} + +impl Command { + fn new(command_ident: Ident, command_span: Span, argument_tokens: Tokens) -> Self { + Self { + command_ident, + command_span, + argument_tokens, + } + } + + #[allow(unused)] // Likely useful in future + pub(crate) fn ident_span(&self) -> Span { + self.command_ident.span() + } + + pub(crate) fn span(&self) -> Span { + self.command_span + } + + pub(crate) fn error(&self, message: impl core::fmt::Display) -> syn::Error { + self.command_span.error(message) + } + + pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { + Err(self.error(message)) + } + + pub(crate) fn arguments(&mut self) -> &mut Tokens { + &mut self.argument_tokens + } +} + +impl HasSpanRange for Command { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs new file mode 100644 index 00000000..f7ab9954 --- /dev/null +++ b/src/interpretation/interpreted_stream.rs @@ -0,0 +1,65 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct InterpretedStream { + token_stream: TokenStream, +} + +impl InterpretedStream { + pub(crate) fn new() -> Self { + Self { + token_stream: TokenStream::new(), + } + } + + pub(crate) fn raw(token_stream: TokenStream) -> Self { + Self { token_stream } + } + + pub(crate) fn of_ident(ident: Ident) -> Self { + TokenTree::Ident(ident).into() + } + + pub(crate) fn of_literal(literal: Literal) -> Self { + TokenTree::Literal(literal).into() + } + + pub(crate) fn extend(&mut self, interpreted_stream: InterpretedStream) { + self.token_stream.extend(interpreted_stream.token_stream); + } + + pub(crate) fn push_raw_token_tree(&mut self, token_tree: TokenTree) { + self.token_stream.extend(iter::once(token_tree)); + } + + pub(crate) fn push_new_group( + &mut self, + inner_tokens: InterpretedStream, + delimiter: Delimiter, + span_range: SpanRange, + ) { + self.push_raw_token_tree(TokenTree::group(inner_tokens.token_stream, delimiter, span_range.span())); + } + + pub(crate) fn is_empty(&self) -> bool { + self.token_stream.is_empty() + } + + pub(crate) fn into_token_stream(self) -> TokenStream { + self.token_stream + } +} + +impl From for InterpretedStream { + fn from(value: TokenTree) -> Self { + InterpretedStream { + token_stream: value.into(), + } + } +} + +impl HasSpanRange for InterpretedStream { + fn span_range(&self) -> SpanRange { + self.token_stream.span_range() + } +} \ No newline at end of file diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs new file mode 100644 index 00000000..3758e809 --- /dev/null +++ b/src/interpretation/interpreter.rs @@ -0,0 +1,50 @@ +use crate::internal_prelude::*; + +pub(crate) struct Interpreter { + config: InterpreterConfig, + variables: HashMap, +} + +impl Interpreter { + pub(crate) fn new() -> Self { + Self { + config: Default::default(), + variables: Default::default(), + } + } + + pub(crate) fn set_variable(&mut self, name: String, tokens: InterpretedStream) { + self.variables.insert(name, tokens); + } + + pub(crate) fn get_variable(&self, name: &str) -> Option<&InterpretedStream> { + self.variables.get(name) + } + + pub(crate) fn config(&self) -> &InterpreterConfig { + &self.config + } +} + +pub(crate) struct InterpreterConfig { + iteration_limit: Option, +} + +impl Default for InterpreterConfig { + fn default() -> Self { + Self { + iteration_limit: Some(10000), + } + } +} + +impl InterpreterConfig { + pub(crate) fn check_iteration_count(&self, command: &Command, count: usize) -> Result<()> { + if let Some(limit) = self.iteration_limit { + if count > limit { + return command.err(format!("Iteration limit of {} exceeded", limit)); + } + } + Ok(()) + } +} diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs new file mode 100644 index 00000000..d5bc412b --- /dev/null +++ b/src/interpretation/mod.rs @@ -0,0 +1,13 @@ +mod command; +mod tokens; +mod next_item; +mod interpreted_stream; +mod interpreter; +mod variable; + +pub(crate) use command::*; +pub(crate) use tokens::*; +pub(crate) use next_item::*; +pub(crate) use interpreted_stream::*; +pub(crate) use interpreter::*; +pub(crate) use variable::*; \ No newline at end of file diff --git a/src/interpretation/next_item.rs b/src/interpretation/next_item.rs new file mode 100644 index 00000000..3d62b538 --- /dev/null +++ b/src/interpretation/next_item.rs @@ -0,0 +1,58 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) enum NextItem { + CommandInvocation(CommandInvocation), + Variable(Variable), + Group(Group), + Leaf(TokenTree), +} + +impl Interpret for NextItem { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + match self { + NextItem::Leaf(token_tree) => { + output.push_raw_token_tree(token_tree); + } + NextItem::Group(group) => { + group.interpret_as_tokens_into(interpreter, output)?; + } + NextItem::Variable(variable) => { + variable.interpret_as_tokens_into(interpreter, output)?; + } + NextItem::CommandInvocation(command_invocation) => { + command_invocation.interpret_as_tokens_into(interpreter, output)?; + } + } + Ok(()) + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + match self { + NextItem::Leaf(token_tree) => { + expression_stream.push_raw_token_tree(token_tree); + } + NextItem::Group(group) => { + group.interpret_as_expression_into(interpreter, expression_stream)?; + } + NextItem::Variable(variable) => { + variable.interpret_as_expression_into(interpreter, expression_stream)?; + } + NextItem::CommandInvocation(command_invocation) => { + command_invocation.interpret_as_expression_into(interpreter, expression_stream)?; + } + } + Ok(()) + } +} + +impl HasSpanRange for NextItem { + fn span_range(&self) -> SpanRange { + match self { + NextItem::CommandInvocation(command_invocation) => command_invocation.span_range(), + NextItem::Variable(variable_substitution) => variable_substitution.span_range(), + NextItem::Group(group) => group.span_range(), + NextItem::Leaf(token_tree) => token_tree.span_range(), + } + } +} diff --git a/src/interpretation/tokens.rs b/src/interpretation/tokens.rs new file mode 100644 index 00000000..e89ea084 --- /dev/null +++ b/src/interpretation/tokens.rs @@ -0,0 +1,193 @@ +use crate::internal_prelude::*; + +/// An analogue to [`syn::parse::ParseStream`]. +/// +/// In future, perhaps we should use it. +#[derive(Clone)] +pub(crate) struct Tokens(iter::Peekable<::IntoIter>, SpanRange); + +impl Tokens { + pub(crate) fn new(tokens: TokenStream, span_range: SpanRange) -> Self { + Self(tokens.into_iter().peekable(), span_range) + } + + pub(crate) fn peek(&mut self) -> Option<&TokenTree> { + self.0.peek() + } + + pub(crate) fn next(&mut self) -> Option { + self.0.next() + } + + pub(crate) fn next_as_ident(&mut self) -> Option { + match self.next() { + Some(TokenTree::Ident(ident)) => Some(ident), + _ => None, + } + } + + pub(crate) fn next_as_punct(&mut self) -> Option { + match self.next() { + Some(TokenTree::Punct(punct)) => Some(punct), + _ => None, + } + } + + pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { + match self.next() { + Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), + _ => None, + } + } + + pub(crate) fn next_as_kinded_group(&mut self, delimiter: Delimiter) -> Option { + match self.next() { + Some(TokenTree::Group(group)) if group.delimiter() == delimiter => Some(group), + _ => None, + } + } + + pub(crate) fn is_empty(&mut self) -> bool { + self.peek().is_none() + } + + pub(crate) fn check_end(&mut self) -> Option<()> { + if self.is_empty() { + Some(()) + } else { + None + } + } + + pub(crate) fn assert_end(&mut self, error_message: &'static str) -> Result<()> { + match self.next() { + Some(token) => token.span_range().err(error_message), + None => Ok(()), + } + } + + pub(crate) fn next_item(&mut self) -> Result> { + let next = match self.next() { + Some(next) => next, + None => return Ok(None), + }; + Ok(Some(match next { + TokenTree::Group(group) => { + if let Some(command_invocation) = parse_command_invocation(&group)? { + NextItem::CommandInvocation(command_invocation) + } else { + NextItem::Group(group) + } + } + TokenTree::Punct(punct) => { + if let Some(variable_substitution) = + parse_only_if_variable_substitution(&punct, self) + { + NextItem::Variable(variable_substitution) + } else { + NextItem::Leaf(TokenTree::Punct(punct)) + } + } + leaf => NextItem::Leaf(leaf), + })) + } + + pub(crate) fn next_item_as_variable( + &mut self, + error_message: &'static str, + ) -> Result { + match self.next_item()? { + Some(NextItem::Variable(variable_substitution)) => Ok(variable_substitution), + Some(item) => item.span_range().err(error_message), + None => Span::call_site().span_range().err(error_message), + } + } + + pub(crate) fn into_token_stream(&mut self) -> TokenStream { + core::mem::replace(&mut self.0, TokenStream::new().into_iter().peekable()).collect() + } +} + +impl<'a> Interpret for &'a mut Tokens { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + loop { + match self.next_item()? { + Some(next_item) => { + next_item.interpret_as_tokens_into(interpreter, output)? + } + None => return Ok(()), + } + } + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + let mut inner_expression_stream = ExpressionStream::new(); + loop { + match self.next_item()? { + Some(next_item) => { + next_item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)? + } + None => break, + } + } + expression_stream.push_expression_group( + inner_expression_stream, + Delimiter::None, + self.1, + ); + Ok(()) + } +} + +fn parse_command_invocation(group: &Group) -> Result> { + fn consume_command_start(group: &Group) -> Option<(Ident, Tokens)> { + if group.delimiter() != Delimiter::Bracket { + return None; + } + let mut tokens = Tokens::new(group.stream(), group.span_range()); + tokens.next_as_punct_matching('!')?; + let ident = tokens.next_as_ident()?; + Some((ident, tokens)) + } + + fn consume_command_end(command_ident: &Ident, tokens: &mut Tokens) -> Option { + let command_kind = CommandKind::attempt_parse(command_ident)?; + tokens.next_as_punct_matching('!')?; + Some(command_kind) + } + + // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, + // so return `Ok(None)` + let (command_ident, mut remaining_tokens) = match consume_command_start(group) { + Some(command_start) => command_start, + None => return Ok(None), + }; + + // We have now checked enough that we're confident the user is pretty intentionally using + // the call convention. Any issues we hit from this point will be a helpful compiler error. + match consume_command_end(&command_ident, &mut remaining_tokens) { + Some(command_kind) => Ok(Some(CommandInvocation::new(command_ident.clone(), command_kind, group, remaining_tokens))), + None => Err(command_ident.span().error( + format!( + "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", + CommandKind::list_all(), + command_ident, + ), + )), + } +} + +// We ensure we don't consume any tokens unless we have a variable substitution +fn parse_only_if_variable_substitution(punct: &Punct, tokens: &mut Tokens) -> Option { + if punct.as_char() != '#' { + return None; + } + match tokens.peek() { + Some(TokenTree::Ident(_)) => {} + _ => return None, + } + match tokens.next() { + Some(TokenTree::Ident(variable_name)) => Some(Variable::new(punct.clone(), variable_name)), + _ => unreachable!("We just peeked a token of this type"), + } +} diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs new file mode 100644 index 00000000..2fb75f1d --- /dev/null +++ b/src/interpretation/variable.rs @@ -0,0 +1,88 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct Variable { + marker: Punct, // # + variable_name: Ident, +} + +impl Variable { + pub(crate) fn new(marker: Punct, variable_name: Ident) -> Self { + Self { + marker, + variable_name, + } + } + + pub(crate) fn variable_name(&self) -> String { + self.variable_name.to_string() + } + + pub(crate) fn set<'i>( + &self, + interpreter: &'i mut Interpreter, + value: InterpretedStream, + ) { + interpreter.set_variable(self.variable_name(), value); + } + + fn substitute( + &self, + interpreter: &Interpreter, + ) -> Result { + Ok(self.read_or_else( + interpreter, + || format!( + "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", + self, + self, + ) + )?.clone()) + } + + fn read_or_else<'i>( + &self, + interpreter: &'i Interpreter, + create_error: impl FnOnce() -> String, + ) -> Result<&'i InterpretedStream> { + match self.read_option(interpreter) { + Some(token_stream) => Ok(token_stream), + None => self.span_range().err(create_error()), + } + } + + fn read_option<'i>( + &self, + interpreter: &'i Interpreter, + ) -> Option<&'i InterpretedStream> { + let Variable { variable_name, .. } = self; + interpreter.get_variable(&variable_name.to_string()) + } +} + +impl<'a> Interpret for &'a Variable { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + output.extend(self.substitute(interpreter)?); + Ok(()) + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + expression_stream.push_interpreted_group( + self.substitute(interpreter)?, + self.span_range(), + ); + Ok(()) + } +} + +impl HasSpanRange for Variable { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.marker.span(), self.variable_name.span()) + } +} + +impl core::fmt::Display for Variable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}{}", self.marker.as_char(), self.variable_name) + } +} diff --git a/src/interpreter.rs b/src/interpreter.rs deleted file mode 100644 index 0f6f6888..00000000 --- a/src/interpreter.rs +++ /dev/null @@ -1,366 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) fn interpret(token_stream: TokenStream) -> Result { - Interpreter::new().interpret_tokens( - &mut Tokens::new(token_stream), - SubstitutionMode::token_stream(), - ) -} - -pub(crate) struct Interpreter { - config: InterpreterConfig, - variables: HashMap, -} - -impl Interpreter { - pub(crate) fn new() -> Self { - Self { - config: Default::default(), - variables: Default::default(), - } - } - - pub(crate) fn set_variable(&mut self, name: String, tokens: TokenStream) { - self.variables.insert(name, tokens); - } - - pub(crate) fn get_variable(&self, name: &str) -> Option<&TokenStream> { - self.variables.get(name) - } - - pub(crate) fn config(&self) -> &InterpreterConfig { - &self.config - } - - pub(crate) fn interpret_token_stream( - &mut self, - token_stream: TokenStream, - substitution_mode: SubstitutionMode, - ) -> Result { - self.interpret_tokens(&mut Tokens::new(token_stream), substitution_mode) - } - - pub(crate) fn interpret_token_stream_into( - &mut self, - token_stream: TokenStream, - substitution_mode: SubstitutionMode, - output: &mut TokenStream, - ) -> Result<()> { - self.interpret_tokens_into(&mut Tokens::new(token_stream), substitution_mode, output) - } - - pub(crate) fn interpret_item( - &mut self, - item: NextItem, - substitution_mode: SubstitutionMode, - ) -> Result { - let mut expanded = TokenStream::new(); - self.interpret_next_item_into(item, substitution_mode, &mut expanded)?; - Ok(expanded) - } - - pub(crate) fn interpret_tokens( - &mut self, - source_tokens: &mut Tokens, - substitution_mode: SubstitutionMode, - ) -> Result { - let mut expanded = TokenStream::new(); - self.interpret_tokens_into(source_tokens, substitution_mode, &mut expanded)?; - Ok(expanded) - } - - pub(crate) fn interpret_tokens_into( - &mut self, - source_tokens: &mut Tokens, - substitution_mode: SubstitutionMode, - output: &mut TokenStream, - ) -> Result<()> { - loop { - match source_tokens.next_item()? { - Some(next_item) => { - self.interpret_next_item_into(next_item, substitution_mode, output)? - } - None => return Ok(()), - } - } - } - - fn interpret_next_item_into( - &mut self, - next_item: NextItem, - substitution_mode: SubstitutionMode, - output: &mut TokenStream, - ) -> Result<()> { - match next_item { - NextItem::Leaf(token_tree) => { - output.push_token_tree(token_tree); - } - NextItem::Group(group) => { - // If it's a group, run interpret on its contents recursively. - output.push_new_group( - self.interpret_tokens(&mut Tokens::new(group.stream()), substitution_mode)?, - group.delimiter(), - group.span_range(), - ); - } - NextItem::Variable(variable) => { - substitution_mode.apply( - output, - variable.span_range(), - variable.read_substitution(self)?.clone(), - ); - } - NextItem::CommandInvocation(command_invocation) => { - substitution_mode.apply( - output, - command_invocation.span_range(), - command_invocation.execute(self)?, - ); - } - } - Ok(()) - } -} - -pub(crate) struct InterpreterConfig { - iteration_limit: Option, -} - -impl Default for InterpreterConfig { - fn default() -> Self { - Self { - iteration_limit: Some(10000), - } - } -} - -impl InterpreterConfig { - pub(crate) fn check_iteration_count(&self, command: &Command, count: usize) -> Result<()> { - if let Some(limit) = self.iteration_limit { - if count > limit { - return command.err(format!("Iteration limit of {} exceeded", limit)); - } - } - Ok(()) - } -} - -/// How to output `#variables` and `[!commands!]` into the output stream -#[derive(Clone, Copy)] -pub(crate) struct SubstitutionMode(SubstitutionModeInternal); - -#[derive(Clone, Copy)] -enum SubstitutionModeInternal { - /// The tokens are just output as they are to the token stream. - /// - /// This is the default, and should typically be used by commands which - /// deal with flattened token streams. - Extend, - /// The tokens are grouped into a group. - /// - /// This should be used when calculating expressions. - /// - /// We wrap substituted variables in a transparent group so that - /// they can be used collectively in future expressions, so that - /// e.g. if `#x = 1 + 1` then `#x * #x = 4` rather than `3`. - Group(Delimiter), -} - -impl SubstitutionMode { - /// When creating a token stream, substitutions extend the token stream output. - pub(crate) fn token_stream() -> Self { - Self(SubstitutionModeInternal::Extend) - } - - /// When creating a token stream for an expression, substitutions are wrapped - /// in a transparent group. - pub(crate) fn expression() -> Self { - Self(SubstitutionModeInternal::Group(Delimiter::None)) - } - - fn apply(self, tokens: &mut TokenStream, span_range: SpanRange, substitution: TokenStream) { - match self.0 { - SubstitutionModeInternal::Extend => tokens.extend(substitution), - SubstitutionModeInternal::Group(delimiter) => { - tokens.push_new_group(substitution, delimiter, span_range) - } - } - } -} - -#[derive(Clone)] -pub(crate) struct Tokens(iter::Peekable<::IntoIter>); - -impl Tokens { - pub(crate) fn new(tokens: TokenStream) -> Self { - Self(tokens.into_iter().peekable()) - } - - pub(crate) fn peek(&mut self) -> Option<&TokenTree> { - self.0.peek() - } - - pub(crate) fn next(&mut self) -> Option { - self.0.next() - } - - pub(crate) fn next_as_ident(&mut self) -> Option { - match self.next() { - Some(TokenTree::Ident(ident)) => Some(ident), - _ => None, - } - } - - pub(crate) fn next_as_punct(&mut self) -> Option { - match self.next() { - Some(TokenTree::Punct(punct)) => Some(punct), - _ => None, - } - } - - pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { - match self.next() { - Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), - _ => None, - } - } - - pub(crate) fn next_as_kinded_group(&mut self, delimiter: Delimiter) -> Option { - match self.next() { - Some(TokenTree::Group(group)) if group.delimiter() == delimiter => Some(group), - _ => None, - } - } - - pub(crate) fn is_empty(&mut self) -> bool { - self.peek().is_none() - } - - pub(crate) fn check_end(&mut self) -> Option<()> { - if self.is_empty() { - Some(()) - } else { - None - } - } - - pub(crate) fn assert_end(&mut self, error_message: &'static str) -> Result<()> { - match self.next() { - Some(token) => token.span_range().err(error_message), - None => Ok(()), - } - } - - pub(crate) fn next_item(&mut self) -> Result> { - let next = match self.next() { - Some(next) => next, - None => return Ok(None), - }; - Ok(Some(match next { - TokenTree::Group(group) => { - if let Some(command_invocation) = parse_command_invocation(&group)? { - NextItem::CommandInvocation(command_invocation) - } else { - NextItem::Group(group) - } - } - TokenTree::Punct(punct) => { - if let Some(variable_substitution) = - parse_only_if_variable_substitution(&punct, self) - { - NextItem::Variable(variable_substitution) - } else { - NextItem::Leaf(TokenTree::Punct(punct)) - } - } - leaf => NextItem::Leaf(leaf), - })) - } - - pub(crate) fn next_item_as_variable( - &mut self, - error_message: &'static str, - ) -> Result { - match self.next_item()? { - Some(NextItem::Variable(variable_substitution)) => Ok(variable_substitution), - Some(item) => item.span_range().err(error_message), - None => Span::call_site().span_range().err(error_message), - } - } - - pub(crate) fn into_token_stream(self) -> TokenStream { - self.0.collect() - } -} - -#[derive(Clone)] -pub(crate) enum NextItem { - CommandInvocation(CommandInvocation), - Variable(Variable), - Group(Group), - Leaf(TokenTree), -} - -impl HasSpanRange for NextItem { - fn span_range(&self) -> SpanRange { - match self { - NextItem::CommandInvocation(command_invocation) => command_invocation.span_range(), - NextItem::Variable(variable_substitution) => variable_substitution.span_range(), - NextItem::Group(group) => group.span_range(), - NextItem::Leaf(token_tree) => token_tree.span_range(), - } - } -} - -fn parse_command_invocation(group: &Group) -> Result> { - fn consume_command_start(group: &Group) -> Option<(Ident, Tokens)> { - if group.delimiter() != Delimiter::Bracket { - return None; - } - let mut tokens = Tokens::new(group.stream()); - tokens.next_as_punct_matching('!')?; - let ident = tokens.next_as_ident()?; - Some((ident, tokens)) - } - - fn consume_command_end(command_ident: &Ident, tokens: &mut Tokens) -> Option { - let command_kind = CommandKind::attempt_parse(command_ident)?; - tokens.next_as_punct_matching('!')?; - Some(command_kind) - } - - // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, - // so return `Ok(None)` - let (command_ident, mut remaining_tokens) = match consume_command_start(group) { - Some(command_start) => command_start, - None => return Ok(None), - }; - - // We have now checked enough that we're confident the user is pretty intentionally using - // the call convention. Any issues we hit from this point will be a helpful compiler error. - match consume_command_end(&command_ident, &mut remaining_tokens) { - Some(command_kind) => Ok(Some(CommandInvocation::new(command_ident.clone(), command_kind, group, remaining_tokens))), - None => Err(command_ident.span().error( - format!( - "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", - CommandKind::list_all(), - command_ident, - ), - )), - } -} - -// We ensure we don't consume any tokens unless we have a variable substitution -fn parse_only_if_variable_substitution(punct: &Punct, tokens: &mut Tokens) -> Option { - if punct.as_char() != '#' { - return None; - } - match tokens.peek() { - Some(TokenTree::Ident(_)) => {} - _ => return None, - } - match tokens.next() { - Some(TokenTree::Ident(variable_name)) => Some(Variable::new(punct.clone(), variable_name)), - _ => unreachable!("We just peeked a token of this type"), - } -} diff --git a/src/lib.rs b/src/lib.rs index 96dce47c..8bb63a45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -510,11 +510,11 @@ //! //! Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in this crate by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. //! -mod command; mod commands; mod expressions; mod internal_prelude; -mod interpreter; +mod interpretation; +mod traits; mod string_conversion; use internal_prelude::*; @@ -538,7 +538,10 @@ use internal_prelude::*; /// See the [crate-level documentation](crate) for full details. #[proc_macro] pub fn preinterpret(token_stream: proc_macro::TokenStream) -> proc_macro::TokenStream { - interpret(proc_macro2::TokenStream::from(token_stream)) + let mut interpreter = Interpreter::new(); + proc_macro2::TokenStream::from(token_stream) + .interpret_as_tokens(&mut interpreter) + .map(InterpretedStream::into_token_stream) .unwrap_or_else(|err| err.to_compile_error()) .into() } diff --git a/src/traits.rs b/src/traits.rs new file mode 100644 index 00000000..de11c99c --- /dev/null +++ b/src/traits.rs @@ -0,0 +1,222 @@ +use crate::internal_prelude::*; + +pub(crate) trait Interpret: Sized { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()>; + fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { + let mut output = InterpretedStream::new(); + self.interpret_as_tokens_into(interpreter, &mut output)?; + Ok(output) + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()>; + fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { + let mut output = ExpressionStream::new(); + self.interpret_as_expression_into(interpreter, &mut output)?; + Ok(output) + } +} + +pub(crate) trait TokenTreeExt: Sized { + fn bool(value: bool, span: Span) -> Self; + fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; +} + +impl TokenTreeExt for TokenTree { + fn bool(value: bool, span: Span) -> Self { + TokenTree::Ident(Ident::new(&value.to_string(), span)) + } + + fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { + TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) + } +} + +impl Interpret for TokenStream { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + let span_range = self.span_range(); + Tokens::new(self, span_range).interpret_as_tokens_into(interpreter, output) + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + let span_range = self.span_range(); + Tokens::new(self, span_range).interpret_as_expression_into(interpreter, expression_stream) + } +} + +impl Interpret for Group { + fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + output.push_new_group( + self.stream().interpret_as_tokens(interpreter)?, + self.delimiter(), + self.span_range(), + ); + Ok(()) + } + + fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + expression_stream.push_expression_group( + self.stream().interpret_as_expression(interpreter)?, + self.delimiter(), + self.span_range(), + ); + Ok(()) + } +} + +pub(crate) trait SpanErrorExt: Sized { + fn err(self, message: impl std::fmt::Display) -> syn::Result { + Err(self.error(message)) + } + + fn error(self, message: impl std::fmt::Display) -> syn::Error; +} + +impl SpanErrorExt for Span { + fn error(self, message: impl std::fmt::Display) -> syn::Error { + syn::Error::new(self, message) + } +} + +impl SpanErrorExt for SpanRange { + fn error(self, message: impl std::fmt::Display) -> syn::Error { + syn::Error::new_spanned(self, message) + } +} + +pub(crate) trait WithSpanExt { + fn with_span(self, span: Span) -> Self; +} + +impl WithSpanExt for Literal { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +impl WithSpanExt for Punct { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +impl WithSpanExt for Group { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +pub(crate) trait HasSpanRange { + fn span_range(&self) -> SpanRange; +} + +/// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] +/// and falls back to the span of the first token when not available. +/// +/// Instead, [`syn::Error`] uses a trick involving a span range. This effectively +/// allows capturing this trick when we're not immediately creating an error. +/// +/// When [`proc_macro::Span::join`] is stabilised and [`syn::spanned`] works, +/// we can swap [`SpanRange`] contents for [`Span`] (or even remove it and [`HasSpanRange`]). +#[derive(Copy, Clone)] +pub(crate) struct SpanRange { + start: Span, + end: Span, +} + +#[allow(unused)] +impl SpanRange { + pub(crate) fn new_between(start: Span, end: Span) -> Self { + Self { start, end } + } + + /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) + /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) + pub(crate) fn span(&self) -> Span { + ::span(self) + } + + pub(crate) fn start(&self) -> Span { + self.start + } + + pub(crate) fn end(&self) -> Span { + self.end + } + + pub(crate) fn replace_start(self, start: Span) -> Self { + Self { start, ..self } + } + + pub(crate) fn replace_end(self, end: Span) -> Self { + Self { end, ..self } + } +} + +// This is implemented so we can create an error from it using `Error::new_spanned(..)` +impl ToTokens for SpanRange { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend([ + TokenTree::Punct(Punct::new('<', Spacing::Alone).with_span(self.start)), + TokenTree::Punct(Punct::new('>', Spacing::Alone).with_span(self.end)), + ]); + } +} + +impl HasSpanRange for Span { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(*self, *self) + } +} + +impl HasSpanRange for TokenTree { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl HasSpanRange for Group { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl HasSpanRange for proc_macro2::extra::DelimSpan { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.open(), self.close()) + } +} + +impl HasSpanRange for T { + fn span_range(&self) -> SpanRange { + let mut iter = self.into_token_stream().into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + SpanRange { start, end } + } +} + +/// This should only be used for syn built-ins or when there isn't a better +/// span range available +trait AutoSpanRange {} + +macro_rules! impl_auto_span_range { + ($($ty:ty),* $(,)?) => { + $( + impl AutoSpanRange for $ty {} + )* + }; +} + +impl_auto_span_range! { + TokenStream, + syn::Expr, + syn::ExprBinary, + syn::ExprUnary, + syn::BinOp, + syn::UnOp, + syn::Type, + syn::TypePath, +} From f77f7e29c1ddf219cb650bfe216cbc554005c593 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 4 Jan 2025 21:02:23 +0000 Subject: [PATCH 010/126] chore: Fix style --- src/commands/control_flow_commands.rs | 19 +++++----- src/commands/core_commands.rs | 4 ++- src/commands/expression_commands.rs | 13 +++---- src/commands/token_commands.rs | 4 ++- src/expressions/expression_stream.rs | 22 ++++++++---- src/internal_prelude.rs | 2 +- src/interpretation/command.rs | 18 ++++++---- src/interpretation/interpreted_stream.rs | 8 +++-- src/interpretation/mod.rs | 10 +++--- src/interpretation/next_item.rs | 12 +++++-- src/interpretation/tokens.rs | 46 ++++++++++++------------ src/interpretation/variable.rs | 33 ++++++++--------- src/lib.rs | 2 +- src/traits.rs | 36 +++++++++++++++---- tests/control_flow.rs | 2 +- 15 files changed, 140 insertions(+), 91 deletions(-) diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index eda530dd..f73716bf 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -87,11 +87,13 @@ impl CommandDefinition for WhileCommand { } iteration_count += 1; - interpreter.config().check_iteration_count(&command, iteration_count)?; - parsed.code.clone().interpret_as_tokens_into( - interpreter, - &mut output, - )?; + interpreter + .config() + .check_iteration_count(&command, iteration_count)?; + parsed + .code + .clone() + .interpret_as_tokens_into(interpreter, &mut output)?; } Ok(output) @@ -107,8 +109,5 @@ fn parse_while_statement(tokens: &mut Tokens) -> Option { let condition = tokens.next_item().ok()??; let code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); tokens.check_end()?; - Some(WhileStatement { - condition, - code, - }) -} \ No newline at end of file + Some(WhileStatement { condition, code }) +} diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index d9c41040..b2692f56 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -32,7 +32,9 @@ impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; fn execute(_interpreter: &mut Interpreter, mut command: Command) -> Result { - Ok(InterpretedStream::raw(command.arguments().into_token_stream())) + Ok(InterpretedStream::raw( + command.arguments().read_all_as_token_stream(), + )) } } diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 040df886..e3fac3d3 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -25,8 +25,12 @@ impl CommandDefinition for AssignCommand { let mut expression_stream = ExpressionStream::new(); variable.interpret_as_expression_into(interpreter, &mut expression_stream)?; - operator.into_token_stream().interpret_as_expression_into(interpreter, &mut expression_stream)?; - command.arguments().interpret_as_expression_into(interpreter, &mut expression_stream)?; + operator + .into_token_stream() + .interpret_as_expression_into(interpreter, &mut expression_stream)?; + command + .arguments() + .interpret_as_expression_into(interpreter, &mut expression_stream)?; let output = expression_stream.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output); @@ -49,9 +53,6 @@ impl AssignStatementStart { _ => return None, } tokens.next_as_punct_matching('=')?; - Some(AssignStatementStart { - variable, - operator, - }) + Some(AssignStatementStart { variable, operator }) } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index b67271e7..a834db2d 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -6,7 +6,9 @@ impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; fn execute(_interpreter: &mut Interpreter, mut command: Command) -> Result { - command.arguments().assert_end("The !empty! command does not take any arguments")?; + command + .arguments() + .assert_end("The !empty! command does not take any arguments")?; Ok(InterpretedStream::new()) } } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 0a7ce27d..1d23e7cb 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -16,12 +16,23 @@ impl ExpressionStream { self.interpreted_stream.push_raw_token_tree(token_tree); } - pub(crate) fn push_interpreted_group(&mut self, contents: InterpretedStream, span_range: SpanRange) { - self.interpreted_stream.push_new_group(contents, Delimiter::None, span_range); + pub(crate) fn push_interpreted_group( + &mut self, + contents: InterpretedStream, + span_range: SpanRange, + ) { + self.interpreted_stream + .push_new_group(contents, Delimiter::None, span_range); } - pub(crate) fn push_expression_group(&mut self, contents: Self, delimiter: Delimiter, span_range: SpanRange) { - self.interpreted_stream.push_new_group(contents.interpreted_stream, delimiter, span_range); + pub(crate) fn push_expression_group( + &mut self, + contents: Self, + delimiter: Delimiter, + span_range: SpanRange, + ) { + self.interpreted_stream + .push_new_group(contents.interpreted_stream, delimiter, span_range); } pub(crate) fn evaluate(self) -> Result { @@ -40,7 +51,6 @@ impl ExpressionStream { // `Expr::parse_without_eager_brace` or `Expr::parse_with_earlier_boundary_rule`. let expression = Expr::parse.parse2(self.interpreted_stream.into_token_stream())?; - EvaluationTree::build_from(&expression)? - .evaluate() + EvaluationTree::build_from(&expression)?.evaluate() } } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index e5d68c51..8078b557 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -6,6 +6,6 @@ pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, R pub(crate) use crate::commands::*; pub(crate) use crate::expressions::*; -pub(crate) use crate::traits::*; pub(crate) use crate::interpretation::*; pub(crate) use crate::string_conversion::*; +pub(crate) use crate::traits::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index ac2b7099..6fed8b1b 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -88,16 +88,22 @@ impl HasSpanRange for CommandInvocation { } impl Interpret for CommandInvocation { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { self.execute_into(interpreter, output) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { let span_range = self.span_range(); - expression_stream.push_interpreted_group( - self.interpret_as_tokens(interpreter)?, - span_range, - ); + expression_stream + .push_interpreted_group(self.interpret_as_tokens(interpreter)?, span_range); Ok(()) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index f7ab9954..6ef56697 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -38,7 +38,11 @@ impl InterpretedStream { delimiter: Delimiter, span_range: SpanRange, ) { - self.push_raw_token_tree(TokenTree::group(inner_tokens.token_stream, delimiter, span_range.span())); + self.push_raw_token_tree(TokenTree::group( + inner_tokens.token_stream, + delimiter, + span_range.span(), + )); } pub(crate) fn is_empty(&self) -> bool { @@ -62,4 +66,4 @@ impl HasSpanRange for InterpretedStream { fn span_range(&self) -> SpanRange { self.token_stream.span_range() } -} \ No newline at end of file +} diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index d5bc412b..4b3531db 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,13 +1,13 @@ mod command; -mod tokens; -mod next_item; mod interpreted_stream; mod interpreter; +mod next_item; +mod tokens; mod variable; pub(crate) use command::*; -pub(crate) use tokens::*; -pub(crate) use next_item::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; -pub(crate) use variable::*; \ No newline at end of file +pub(crate) use next_item::*; +pub(crate) use tokens::*; +pub(crate) use variable::*; diff --git a/src/interpretation/next_item.rs b/src/interpretation/next_item.rs index 3d62b538..58f225f9 100644 --- a/src/interpretation/next_item.rs +++ b/src/interpretation/next_item.rs @@ -9,7 +9,11 @@ pub(crate) enum NextItem { } impl Interpret for NextItem { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { match self { NextItem::Leaf(token_tree) => { output.push_raw_token_tree(token_tree); @@ -27,7 +31,11 @@ impl Interpret for NextItem { Ok(()) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { match self { NextItem::Leaf(token_tree) => { expression_stream.push_raw_token_tree(token_tree); diff --git a/src/interpretation/tokens.rs b/src/interpretation/tokens.rs index e89ea084..1fdba9ba 100644 --- a/src/interpretation/tokens.rs +++ b/src/interpretation/tokens.rs @@ -1,10 +1,13 @@ use crate::internal_prelude::*; /// An analogue to [`syn::parse::ParseStream`]. -/// +/// /// In future, perhaps we should use it. #[derive(Clone)] -pub(crate) struct Tokens(iter::Peekable<::IntoIter>, SpanRange); +pub(crate) struct Tokens( + iter::Peekable<::IntoIter>, + SpanRange, +); impl Tokens { pub(crate) fn new(tokens: TokenStream, span_range: SpanRange) -> Self { @@ -103,38 +106,33 @@ impl Tokens { } } - pub(crate) fn into_token_stream(&mut self) -> TokenStream { + pub(crate) fn read_all_as_token_stream(&mut self) -> TokenStream { core::mem::replace(&mut self.0, TokenStream::new().into_iter().peekable()).collect() } } impl<'a> Interpret for &'a mut Tokens { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { - loop { - match self.next_item()? { - Some(next_item) => { - next_item.interpret_as_tokens_into(interpreter, output)? - } - None => return Ok(()), - } + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + while let Some(next_item) = self.next_item()? { + next_item.interpret_as_tokens_into(interpreter, output)?; } + Ok(()) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { let mut inner_expression_stream = ExpressionStream::new(); - loop { - match self.next_item()? { - Some(next_item) => { - next_item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)? - } - None => break, - } + while let Some(next_item) = self.next_item()? { + next_item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)?; } - expression_stream.push_expression_group( - inner_expression_stream, - Delimiter::None, - self.1, - ); + expression_stream.push_expression_group(inner_expression_stream, Delimiter::None, self.1); Ok(()) } } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 2fb75f1d..d61aac97 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -18,18 +18,11 @@ impl Variable { self.variable_name.to_string() } - pub(crate) fn set<'i>( - &self, - interpreter: &'i mut Interpreter, - value: InterpretedStream, - ) { + pub(crate) fn set(&self, interpreter: &mut Interpreter, value: InterpretedStream) { interpreter.set_variable(self.variable_name(), value); } - fn substitute( - &self, - interpreter: &Interpreter, - ) -> Result { + fn substitute(&self, interpreter: &Interpreter) -> Result { Ok(self.read_or_else( interpreter, || format!( @@ -51,26 +44,28 @@ impl Variable { } } - fn read_option<'i>( - &self, - interpreter: &'i Interpreter, - ) -> Option<&'i InterpretedStream> { + fn read_option<'i>(&self, interpreter: &'i Interpreter) -> Option<&'i InterpretedStream> { let Variable { variable_name, .. } = self; interpreter.get_variable(&variable_name.to_string()) } } impl<'a> Interpret for &'a Variable { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { output.extend(self.substitute(interpreter)?); Ok(()) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { - expression_stream.push_interpreted_group( - self.substitute(interpreter)?, - self.span_range(), - ); + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { + expression_stream.push_interpreted_group(self.substitute(interpreter)?, self.span_range()); Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index 8bb63a45..ff7ce23a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -514,8 +514,8 @@ mod commands; mod expressions; mod internal_prelude; mod interpretation; -mod traits; mod string_conversion; +mod traits; use internal_prelude::*; diff --git a/src/traits.rs b/src/traits.rs index de11c99c..f5da6c94 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,14 +1,22 @@ use crate::internal_prelude::*; pub(crate) trait Interpret: Sized { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()>; + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()>; fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(); self.interpret_as_tokens_into(interpreter, &mut output)?; Ok(output) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()>; + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()>; fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { let mut output = ExpressionStream::new(); self.interpret_as_expression_into(interpreter, &mut output)?; @@ -32,19 +40,31 @@ impl TokenTreeExt for TokenTree { } impl Interpret for TokenStream { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { let span_range = self.span_range(); Tokens::new(self, span_range).interpret_as_tokens_into(interpreter, output) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { let span_range = self.span_range(); Tokens::new(self, span_range).interpret_as_expression_into(interpreter, expression_stream) } } impl Interpret for Group { - fn interpret_as_tokens_into(self, interpreter: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { output.push_new_group( self.stream().interpret_as_tokens(interpreter)?, self.delimiter(), @@ -53,7 +73,11 @@ impl Interpret for Group { Ok(()) } - fn interpret_as_expression_into(self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream) -> Result<()> { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { expression_stream.push_expression_group( self.stream().interpret_as_expression(interpreter)?, self.delimiter(), diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 67b1306f..582e2572 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -42,4 +42,4 @@ fn test_while() { // TODO: Check compilation error for: // assert_preinterpret_eq!({ // [!while! true {}] -// }, 5); \ No newline at end of file +// }, 5); From 542b71ee98ab5fffade8c6405b72025b168821c6 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 4 Jan 2025 22:13:21 +0000 Subject: [PATCH 011/126] wip: Minor refactors --- Cargo.toml | 2 +- src/commands/control_flow_commands.rs | 4 +-- src/commands/core_commands.rs | 2 +- src/commands/expression_commands.rs | 2 +- src/interpretation/command.rs | 12 ++++--- src/interpretation/tokens.rs | 49 ++++++++++++++++----------- src/traits.rs | 5 +-- 7 files changed, 46 insertions(+), 30 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 314a15ef..7213cd89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,5 +20,5 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0" } -syn = { version = "2.0", default-features = false, features = ["full", "parsing", "derive", "printing"] } +syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing"] } quote = { version = "1.0", default-features = false } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index f73716bf..77dd4c5d 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -36,7 +36,7 @@ struct IfStatement { false_code: Option, } -fn parse_if_statement(tokens: &mut Tokens) -> Option { +fn parse_if_statement(tokens: &mut InterpreterParseStream) -> Option { let condition = tokens.next_item().ok()??; let true_code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); let false_code = if tokens.peek().is_some() { @@ -105,7 +105,7 @@ struct WhileStatement { code: TokenStream, } -fn parse_while_statement(tokens: &mut Tokens) -> Option { +fn parse_while_statement(tokens: &mut InterpreterParseStream) -> Option { let condition = tokens.next_item().ok()??; let code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); tokens.check_end()?; diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index b2692f56..c6824c33 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -20,7 +20,7 @@ impl CommandDefinition for SetCommand { } } -pub(crate) fn parse_variable_set(tokens: &mut Tokens) -> Option { +pub(crate) fn parse_variable_set(tokens: &mut InterpreterParseStream) -> Option { let variable = tokens.next_item_as_variable("").ok()?; tokens.next_as_punct_matching('=')?; Some(variable) diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index e3fac3d3..6cdb1fde 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -45,7 +45,7 @@ struct AssignStatementStart { } impl AssignStatementStart { - fn parse(tokens: &mut Tokens) -> Option { + fn parse(tokens: &mut InterpreterParseStream) -> Option { let variable = tokens.next_item_as_variable("").ok()?; let operator = tokens.next_as_punct()?; match operator.as_char() { diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 6fed8b1b..9131e563 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -62,7 +62,7 @@ impl CommandInvocation { command_ident: Ident, command_kind: CommandKind, group: &Group, - argument_tokens: Tokens, + argument_tokens: InterpreterParseStream, ) -> Self { Self { command_kind, @@ -112,11 +112,15 @@ impl Interpret for CommandInvocation { pub(crate) struct Command { command_ident: Ident, command_span: Span, - argument_tokens: Tokens, + argument_tokens: InterpreterParseStream, } impl Command { - fn new(command_ident: Ident, command_span: Span, argument_tokens: Tokens) -> Self { + fn new( + command_ident: Ident, + command_span: Span, + argument_tokens: InterpreterParseStream, + ) -> Self { Self { command_ident, command_span, @@ -141,7 +145,7 @@ impl Command { Err(self.error(message)) } - pub(crate) fn arguments(&mut self) -> &mut Tokens { + pub(crate) fn arguments(&mut self) -> &mut InterpreterParseStream { &mut self.argument_tokens } } diff --git a/src/interpretation/tokens.rs b/src/interpretation/tokens.rs index 1fdba9ba..c6415dc6 100644 --- a/src/interpretation/tokens.rs +++ b/src/interpretation/tokens.rs @@ -1,25 +1,26 @@ use crate::internal_prelude::*; -/// An analogue to [`syn::parse::ParseStream`]. -/// -/// In future, perhaps we should use it. #[derive(Clone)] -pub(crate) struct Tokens( - iter::Peekable<::IntoIter>, - SpanRange, -); +pub(crate) struct InterpreterParseStream { + // In future, we should consider making this a `syn::Cursor`... + tokens: iter::Peekable<::IntoIter>, + span_range: SpanRange, +} -impl Tokens { - pub(crate) fn new(tokens: TokenStream, span_range: SpanRange) -> Self { - Self(tokens.into_iter().peekable(), span_range) +impl InterpreterParseStream { + pub(crate) fn new(token_stream: TokenStream, span_range: SpanRange) -> Self { + Self { + tokens: token_stream.into_iter().peekable(), + span_range, + } } pub(crate) fn peek(&mut self) -> Option<&TokenTree> { - self.0.peek() + self.tokens.peek() } pub(crate) fn next(&mut self) -> Option { - self.0.next() + self.tokens.next() } pub(crate) fn next_as_ident(&mut self) -> Option { @@ -107,11 +108,11 @@ impl Tokens { } pub(crate) fn read_all_as_token_stream(&mut self) -> TokenStream { - core::mem::replace(&mut self.0, TokenStream::new().into_iter().peekable()).collect() + core::mem::replace(&mut self.tokens, TokenStream::new().into_iter().peekable()).collect() } } -impl<'a> Interpret for &'a mut Tokens { +impl<'a> Interpret for &'a mut InterpreterParseStream { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, @@ -132,23 +133,30 @@ impl<'a> Interpret for &'a mut Tokens { while let Some(next_item) = self.next_item()? { next_item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)?; } - expression_stream.push_expression_group(inner_expression_stream, Delimiter::None, self.1); + expression_stream.push_expression_group( + inner_expression_stream, + Delimiter::None, + self.span_range, + ); Ok(()) } } fn parse_command_invocation(group: &Group) -> Result> { - fn consume_command_start(group: &Group) -> Option<(Ident, Tokens)> { + fn consume_command_start(group: &Group) -> Option<(Ident, InterpreterParseStream)> { if group.delimiter() != Delimiter::Bracket { return None; } - let mut tokens = Tokens::new(group.stream(), group.span_range()); + let mut tokens = InterpreterParseStream::new(group.stream(), group.span_range()); tokens.next_as_punct_matching('!')?; let ident = tokens.next_as_ident()?; Some((ident, tokens)) } - fn consume_command_end(command_ident: &Ident, tokens: &mut Tokens) -> Option { + fn consume_command_end( + command_ident: &Ident, + tokens: &mut InterpreterParseStream, + ) -> Option { let command_kind = CommandKind::attempt_parse(command_ident)?; tokens.next_as_punct_matching('!')?; Some(command_kind) @@ -176,7 +184,10 @@ fn parse_command_invocation(group: &Group) -> Result> } // We ensure we don't consume any tokens unless we have a variable substitution -fn parse_only_if_variable_substitution(punct: &Punct, tokens: &mut Tokens) -> Option { +fn parse_only_if_variable_substitution( + punct: &Punct, + tokens: &mut InterpreterParseStream, +) -> Option { if punct.as_char() != '#' { return None; } diff --git a/src/traits.rs b/src/traits.rs index f5da6c94..7a8f53de 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -46,7 +46,7 @@ impl Interpret for TokenStream { output: &mut InterpretedStream, ) -> Result<()> { let span_range = self.span_range(); - Tokens::new(self, span_range).interpret_as_tokens_into(interpreter, output) + InterpreterParseStream::new(self, span_range).interpret_as_tokens_into(interpreter, output) } fn interpret_as_expression_into( @@ -55,7 +55,8 @@ impl Interpret for TokenStream { expression_stream: &mut ExpressionStream, ) -> Result<()> { let span_range = self.span_range(); - Tokens::new(self, span_range).interpret_as_expression_into(interpreter, expression_stream) + InterpreterParseStream::new(self, span_range) + .interpret_as_expression_into(interpreter, expression_stream) } } From f2d9baaecce98372018a3875b13c3ecda688c001 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 4 Jan 2025 22:36:45 +0000 Subject: [PATCH 012/126] WIP: Add some notes/docs --- ...{tokens.rs => interpreter_parse_stream.rs} | 28 ++++++++++++++++++- src/interpretation/mod.rs | 4 +-- 2 files changed, 29 insertions(+), 3 deletions(-) rename src/interpretation/{tokens.rs => interpreter_parse_stream.rs} (84%) diff --git a/src/interpretation/tokens.rs b/src/interpretation/interpreter_parse_stream.rs similarity index 84% rename from src/interpretation/tokens.rs rename to src/interpretation/interpreter_parse_stream.rs index c6415dc6..ee12e24a 100644 --- a/src/interpretation/tokens.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -1,8 +1,34 @@ use crate::internal_prelude::*; +// =============================================== +// How syn features fits with preinterpret parsing +// =============================================== +// +// I spent quite a while considering whether this could be wrapping a +// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`, instead +// of a custom Peekable +// +// I came to the conclusion it doesn't make much sense for now, but +// could be explored in future: +// +// * ParseBuffer / ParseStream is powerful but restrictive +// > We currently have an Interpreter with us as we parse, which the `Parse` +// trait doesn't allow. +// > We could consider splitting into a two-pass Parse/Interpret cycle, +// but it might be hard to reason about and would be a big change +// * Cursor needs to reference into some TokenBuffer +// > We would convert the input TokenStream into a TokenBuffer and +// Cursor into that +// > This could work, assuming we don't have a need to parse intermediate +// outputs +// +// In terms of future features: +// * We may need to store special TokenBuffer / parsable #VARIABLES +// * For parse/destructuring operations, we may need to temporarily +// create parse streams in scope of a command execution + #[derive(Clone)] pub(crate) struct InterpreterParseStream { - // In future, we should consider making this a `syn::Cursor`... tokens: iter::Peekable<::IntoIter>, span_range: SpanRange, } diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 4b3531db..c28e7ffd 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,13 +1,13 @@ mod command; mod interpreted_stream; mod interpreter; +mod interpreter_parse_stream; mod next_item; -mod tokens; mod variable; pub(crate) use command::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; +pub(crate) use interpreter_parse_stream::*; pub(crate) use next_item::*; -pub(crate) use tokens::*; pub(crate) use variable::*; From 08b3e7ce43c372a9ec31b0ce001179ef8fd8a165 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 7 Jan 2025 01:47:48 +0000 Subject: [PATCH 013/126] WIP: Write down thoughts/plans --- CHANGELOG.md | 61 ++++++++++++++++--- README.md | 2 +- .../interpreter_parse_stream.rs | 20 ++++-- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f361c753..6b60b6ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,22 +19,63 @@ ### To come -* ? Use `[!let! #x = 12]` instead of `[!set! ..]`. +* ? Use `[!let! #x = 12]` instead of `[!set! ...]` + * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces -* Refactor command parsing/execution -* `[!for! #x in [#y] {}]` -* `[!range! 0..5]` -* `[!error! "message" token stream for span]` -* Reconfiguring iteration limit -* Support `!else if!` in `!if!` -* Token stream manipulation... and make it performant (maybe by storing either TokenStream for extension; or `ParseStream` for consumption) - * Extend token stream `[!extend! #x += ...]` - * Consume from start of token stream * Support string & char literals (for comparisons & casts) in expressions * Add more tests * e.g. for various expressions * e.g. for long sums * Add compile failure tests +* Refactor command parsing/execution +* `[!range! 0..5]` outputs `[0 1 2 3 4]` +* `[!error! "message" token stream for span]` +* Reconfiguring iteration limit +* Support `!else if!` in `!if!` +* `[!extend! #x += ...]` to make such actions more performant +* Support `!for!` so we can use it for simple generation scenarios without needing macros at all: + * Complexities: + * Parsing `,` + * When parsing `Punctuated` in syn, it typically needs to know how to parse a full X (e.g. X of an item) + * But ideally we want to defer the parsing of the next level... + * This means we may need to instead do something naive for now, e.g. split on `,` in the token stream + * Iterating over already parsed structure; e.g. if we parse a struct and want to iterate over the contents of the path of the type of the first field? + * Do we store such structured variables? + * Option 1 - Simple. For consumes 1 token tree per iteration. + * This means that we may need to pre-process the stream... + * Examples: + * `[!for! #x in [Hello World] {}]` + * Questions: + * How does this extend to parsing scenarios, such as a punctuated `,`? + * It doesn't explicitly... + * But `[!for! #x in [!split! [Hello, World,] on ,]]` could work, if `[!split!]` outputs transparent groups, and discards empty final items + * How does this handle zipping / unzipping and un-grouping, and/or parsing a more complicated group? + * Proposal: The parsing operates over the content of a single token tree, possibly via explicitly ungrouping. + * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` + * To get this to work, we'd need to: + * Make it so that the `#x` binding consumes a single token tree, and auto-expands zero or more transparent group/s + * (Incidentally this is also how the syn Cursor type iteration works, roughly) + * And possibly have `#..x` consume the remainder of the stream?? + * Make it so that `[!set! #x = ...]` wraps the `...` in a transparent group so it's consistent. + * And we can support basic parsing, via: + * Various groupings or `[!GROUP!]` for a transparent group + * Consuming various explicit tokens + * `[!OPTIONAL! ...]` + * Option 2 - we specify some stream-processor around each value + * `[!for! [!EACH! #x] in [Hello World]]` + * `[!for! [!SPLIT! #x,] in [Hello, World,]]` + * Even though this might be more performant, I'm not too much of a fan of this, as it's hard to understand + * Perhaps we can leave it to the `[!while_parse! #x[!OPTIONAL! ,] from #X]` style commands? +* `[!split!]` and `[!split_no_trailing!]` +* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` +* Basic place parsing + * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! + * Explicit Punct, Idents, Literals + * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings + * `[!OPTIONAL! ...]` + * Groups or `[!GROUP! ...]` + * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` +* `[!match!]` * Work on book * Including documenting expressions diff --git a/README.md b/README.md index 89494c57..cebe90dd 100644 --- a/README.md +++ b/README.md @@ -427,7 +427,7 @@ We could support a piped calling convention, such as the `[!pipe! ...]` special Do some testing and ensure there are tools to avoid `N^2` performance when doing token manipulation, e.g. with: -* `[!append! #stream += new tokens...]` +* `[!extend! #stream += new tokens...]` * `[!consume_from! #stream #x]` where `#x` is read as the first token tree from `#stream` * `[!consume_from! #stream ()]` where the parser is read greedily from `#stream` diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index ee12e24a..163a09fc 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -12,20 +12,28 @@ use crate::internal_prelude::*; // could be explored in future: // // * ParseBuffer / ParseStream is powerful but restrictive +// > Due to how it works, we'd need to use it to parse the initial input +// in a single initial pass. // > We currently have an Interpreter with us as we parse, which the `Parse` // trait doesn't allow. -// > We could consider splitting into a two-pass Parse/Interpret cycle, -// but it might be hard to reason about and would be a big change +// > We could consider splitting into a two-pass approach, where we start +// with a Parse step, and then we interpret after, but it would be quite +// a big change internally // * Cursor needs to reference into some TokenBuffer // > We would convert the input TokenStream into a TokenBuffer and // Cursor into that -// > This could work, assuming we don't have a need to parse intermediate -// outputs +// > But this can't be converted into a ParseBuffer outside of the syn crate, +// and so it doesn't get much benefit +// +// Either of these approaches appear disjoint from the parse/destructuring +// operations... // -// In terms of future features: -// * We may need to store special TokenBuffer / parsable #VARIABLES // * For parse/destructuring operations, we may need to temporarily // create parse streams in scope of a command execution +// * For #VARIABLES which can be incrementally consumed / parsed, +// it would be nice to be able to store a Cursor into a TokenBuffer, +// but annoyingly it isn't possible to convert this to a ParseStream +// outside of syn. #[derive(Clone)] pub(crate) struct InterpreterParseStream { From 9c48d1b97ee4b56d76d43254c62823861b3f50b7 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 9 Jan 2025 02:43:04 +0000 Subject: [PATCH 014/126] refactor: Have a separate parse step and interpret step --- CHANGELOG.md | 12 +- src/commands/concat_commands.rs | 287 +++++------------- src/commands/control_flow_commands.rs | 120 ++++---- src/commands/core_commands.rs | 68 +++-- src/commands/expression_commands.rs | 84 +++-- src/commands/mod.rs | 76 +++-- src/commands/token_commands.rs | 81 ++++- src/expressions/expression_stream.rs | 20 +- src/interpretation/command.rs | 193 +++++++----- src/interpretation/interpretation_stream.rs | 121 ++++++++ src/interpretation/interpreted_stream.rs | 16 +- src/interpretation/interpreter.rs | 4 +- .../interpreter_parse_stream.rs | 203 ++++--------- src/interpretation/mod.rs | 2 + src/interpretation/next_item.rs | 70 +++-- src/interpretation/variable.rs | 21 +- src/lib.rs | 12 +- src/traits.rs | 70 +---- 18 files changed, 757 insertions(+), 703 deletions(-) create mode 100644 src/interpretation/interpretation_stream.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b60b6ab..29687e5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,15 @@ * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. - * Disallow `[!let! #x =]` and require `[!let! #x = [!empty!]]` (give a good error message). + +I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. ### To come +* Update big comment in parse stream file +* Grouping... Proposal: + * #x is a group, #..x is flattened + * Whether a command is flattened or not is dependent on the command * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces @@ -27,7 +32,6 @@ * e.g. for various expressions * e.g. for long sums * Add compile failure tests -* Refactor command parsing/execution * `[!range! 0..5]` outputs `[0 1 2 3 4]` * `[!error! "message" token stream for span]` * Reconfiguring iteration limit @@ -55,7 +59,7 @@ * To get this to work, we'd need to: * Make it so that the `#x` binding consumes a single token tree, and auto-expands zero or more transparent group/s * (Incidentally this is also how the syn Cursor type iteration works, roughly) - * And possibly have `#..x` consume the remainder of the stream?? + * And possibly have `#..x` consumes the remainder of the stream?? * Make it so that `[!set! #x = ...]` wraps the `...` in a transparent group so it's consistent. * And we can support basic parsing, via: * Various groupings or `[!GROUP!]` for a transparent group @@ -75,7 +79,7 @@ * `[!OPTIONAL! ...]` * Groups or `[!GROUP! ...]` * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` -* `[!match!]` +* `[!match!]` (with `#..x` as a catch-all) * Work on book * Including documenting expressions diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 125c7da3..fabe95cb 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -25,239 +25,38 @@ fn parse_ident(value: &str, span: Span) -> Result { } fn concat_into_string( + input: InterpretationStream, interpreter: &mut Interpreter, - mut command: Command, conversion_fn: impl Fn(&str) -> String, ) -> Result { - let interpreted = command.arguments().interpret_as_tokens(interpreter)?; - let concatenated = concat_recursive(interpreted); - let string_literal = string_literal(&conversion_fn(&concatenated), command.span()); + let output_span = input.span(); + let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); + let string_literal = string_literal(&conversion_fn(&concatenated), output_span); Ok(InterpretedStream::of_literal(string_literal)) } fn concat_into_ident( + input: InterpretationStream, interpreter: &mut Interpreter, - mut command: Command, conversion_fn: impl Fn(&str) -> String, ) -> Result { - let interpreted = command.arguments().interpret_as_tokens(interpreter)?; - let concatenated = concat_recursive(interpreted); - let ident = parse_ident(&conversion_fn(&concatenated), command.span())?; + let output_span = input.span(); + let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); + let ident = parse_ident(&conversion_fn(&concatenated), output_span)?; Ok(InterpretedStream::of_ident(ident)) } fn concat_into_literal( + input: InterpretationStream, interpreter: &mut Interpreter, - mut command: Command, conversion_fn: impl Fn(&str) -> String, ) -> Result { - let interpreted = command.arguments().interpret_as_tokens(interpreter)?; - let concatenated = concat_recursive(interpreted); - let literal = parse_literal(&conversion_fn(&concatenated), command.span())?; + let output_span = input.span(); + let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); + let literal = parse_literal(&conversion_fn(&concatenated), output_span)?; Ok(InterpretedStream::of_literal(literal)) } -//======================================= -// Concatenating type-conversion commands -//======================================= - -pub(crate) struct StringCommand; - -impl CommandDefinition for StringCommand { - const COMMAND_NAME: &'static str = "string"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, |s| s.to_string()) - } -} - -pub(crate) struct IdentCommand; - -impl CommandDefinition for IdentCommand { - const COMMAND_NAME: &'static str = "ident"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_ident(interpreter, command, |s| s.to_string()) - } -} - -pub(crate) struct IdentCamelCommand; - -impl CommandDefinition for IdentCamelCommand { - const COMMAND_NAME: &'static str = "ident_camel"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_ident(interpreter, command, to_upper_camel_case) - } -} - -pub(crate) struct IdentSnakeCommand; - -impl CommandDefinition for IdentSnakeCommand { - const COMMAND_NAME: &'static str = "ident_snake"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_ident(interpreter, command, to_lower_snake_case) - } -} - -pub(crate) struct IdentUpperSnakeCommand; - -impl CommandDefinition for IdentUpperSnakeCommand { - const COMMAND_NAME: &'static str = "ident_upper_snake"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_ident(interpreter, command, to_upper_snake_case) - } -} - -pub(crate) struct LiteralCommand; - -impl CommandDefinition for LiteralCommand { - const COMMAND_NAME: &'static str = "literal"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_literal(interpreter, command, |s| s.to_string()) - } -} - -//=========================== -// String conversion commands -//=========================== - -pub(crate) struct UpperCommand; - -impl CommandDefinition for UpperCommand { - const COMMAND_NAME: &'static str = "upper"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, to_uppercase) - } -} - -pub(crate) struct LowerCommand; - -impl CommandDefinition for LowerCommand { - const COMMAND_NAME: &'static str = "lower"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, to_lowercase) - } -} - -pub(crate) struct SnakeCommand; - -impl CommandDefinition for SnakeCommand { - const COMMAND_NAME: &'static str = "snake"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - // Lower snake case is the more common casing in Rust, so default to that - LowerSnakeCommand::execute(interpreter, command) - } -} - -pub(crate) struct LowerSnakeCommand; - -impl CommandDefinition for LowerSnakeCommand { - const COMMAND_NAME: &'static str = "lower_snake"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, to_lower_snake_case) - } -} - -pub(crate) struct UpperSnakeCommand; - -impl CommandDefinition for UpperSnakeCommand { - const COMMAND_NAME: &'static str = "upper_snake"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, to_upper_snake_case) - } -} - -pub(crate) struct KebabCommand; - -impl CommandDefinition for KebabCommand { - const COMMAND_NAME: &'static str = "kebab"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - // Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) - // It can always be combined with other casing to get other versions - concat_into_string(interpreter, command, to_lower_kebab_case) - } -} - -pub(crate) struct CamelCommand; - -impl CommandDefinition for CamelCommand { - const COMMAND_NAME: &'static str = "camel"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - // Upper camel case is the more common casing in Rust, so default to that - UpperCamelCommand::execute(interpreter, command) - } -} - -pub(crate) struct LowerCamelCommand; - -impl CommandDefinition for LowerCamelCommand { - const COMMAND_NAME: &'static str = "lower_camel"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, to_lower_camel_case) - } -} - -pub(crate) struct UpperCamelCommand; - -impl CommandDefinition for UpperCamelCommand { - const COMMAND_NAME: &'static str = "upper_camel"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, to_upper_camel_case) - } -} - -pub(crate) struct CapitalizeCommand; - -impl CommandDefinition for CapitalizeCommand { - const COMMAND_NAME: &'static str = "capitalize"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, capitalize) - } -} - -pub(crate) struct DecapitalizeCommand; - -impl CommandDefinition for DecapitalizeCommand { - const COMMAND_NAME: &'static str = "decapitalize"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, decapitalize) - } -} - -pub(crate) struct TitleCommand; - -impl CommandDefinition for TitleCommand { - const COMMAND_NAME: &'static str = "title"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, title_case) - } -} - -pub(crate) struct InsertSpacesCommand; - -impl CommandDefinition for InsertSpacesCommand { - const COMMAND_NAME: &'static str = "insert_spaces"; - - fn execute(interpreter: &mut Interpreter, command: Command) -> Result { - concat_into_string(interpreter, command, insert_spaces_between_words) - } -} fn concat_recursive(arguments: InterpretedStream) -> String { fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { @@ -307,3 +106,65 @@ fn concat_recursive(arguments: InterpretedStream) -> String { concat_recursive_internal(&mut output, arguments.into_token_stream()); output } + +macro_rules! define_concat_command { + ( + $command_name:literal => $command:ident: $output_fn:ident($conversion_fn:expr) + ) => { + #[derive(Clone)] + pub(crate) struct $command { + arguments: InterpretationStream, + } + + impl CommandDefinition for $command { + const COMMAND_NAME: &'static str = $command_name; + + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) + } + } + + impl CommandInvocation for $command { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + $output_fn(self.arguments, interpreter, $conversion_fn) + } + } + } +} + +//======================================= +// Concatenating type-conversion commands +//======================================= + +define_concat_command!("string" => StringCommand: concat_into_string(|s| s.to_string())); +define_concat_command!("ident" => IdentCommand: concat_into_ident(|s| s.to_string())); +define_concat_command!("ident_camel" => IdentCamelCommand: concat_into_ident(to_upper_camel_case)); +define_concat_command!("ident_snake" => IdentSnakeCommand: concat_into_ident(to_lower_snake_case)); +define_concat_command!("ident_upper_snake" => IdentUpperSnakeCommand: concat_into_ident(to_upper_snake_case)); +define_concat_command!("literal" => LiteralCommand: concat_into_literal(|s| s.to_string())); + +//=========================== +// String conversion commands +//=========================== + +define_concat_command!("upper" => UpperCommand: concat_into_string(to_uppercase)); +define_concat_command!("lower" => LowerCommand: concat_into_string(to_lowercase)); +// Snake case is typically lower snake case in Rust, so default to that +define_concat_command!("snake" => SnakeCommand: concat_into_string(to_lower_snake_case)); +define_concat_command!("lower_snake" => LowerSnakeCommand: concat_into_string(to_lower_snake_case)); +define_concat_command!("upper_snake" => UpperSnakeCommand: concat_into_string(to_upper_snake_case)); +// Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) +// It can always be combined with other casing to get other versions +define_concat_command!("kebab" => KebabCommand: concat_into_string(to_lower_kebab_case)); +// Upper camel case is the more common casing in Rust, so default to that +define_concat_command!("camel" => CamelCommand: concat_into_string(to_upper_camel_case)); +define_concat_command!("lower_camel" => LowerCamelCommand: concat_into_string(to_lower_camel_case)); +define_concat_command!("upper_camel" => UpperCamelCommand: concat_into_string(to_upper_camel_case)); +define_concat_command!("capitalize" => CapitalizeCommand: concat_into_string(capitalize)); +define_concat_command!("decapitalize" => DecapitalizeCommand: concat_into_string(decapitalize)); +define_concat_command!("title" => TitleCommand: concat_into_string(title_case)); +define_concat_command!("insert_spaces" => InsertSpacesCommand: concat_into_string(insert_spaces_between_words)); diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 77dd4c5d..a13c8232 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -1,19 +1,43 @@ use crate::internal_prelude::*; -pub(crate) struct IfCommand; +#[derive(Clone)] +pub(crate) struct IfCommand { + condition: NextItem, + true_code: InterpretationStream, + false_code: Option, +} impl CommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let parsed = match parse_if_statement(command.arguments()) { - Some(parsed) => parsed, - None => { - return command.err("Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"); - } + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + static ERROR: &str = "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"; + + let condition = arguments.next_item(ERROR)?; + let true_code = arguments.next_as_kinded_group(Delimiter::Brace, ERROR)?.into_inner_stream(); + let false_code = if !arguments.is_empty() { + arguments.next_as_punct_matching('!', ERROR)?; + arguments.next_as_ident_matching("else", ERROR)?; + arguments.next_as_punct_matching('!', ERROR)?; + Some(arguments.next_as_kinded_group(Delimiter::Brace, ERROR)?.into_inner_stream()) + } else { + None }; + arguments.assert_end(ERROR)?; - let evaluated_condition = parsed + Ok(Self { + condition, + true_code, + false_code, + }) + } +} + +impl CommandInvocation for IfCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let evaluated_condition = self .condition .interpret_as_expression(interpreter)? .evaluate()? @@ -21,8 +45,8 @@ impl CommandDefinition for IfCommand { .value(); if evaluated_condition { - parsed.true_code.interpret_as_tokens(interpreter) - } else if let Some(false_code) = parsed.false_code { + self.true_code.interpret_as_tokens(interpreter) + } else if let Some(false_code) = self.false_code { false_code.interpret_as_tokens(interpreter) } else { Ok(InterpretedStream::new()) @@ -30,52 +54,37 @@ impl CommandDefinition for IfCommand { } } -struct IfStatement { +#[derive(Clone)] +pub(crate) struct WhileCommand { condition: NextItem, - true_code: TokenStream, - false_code: Option, -} - -fn parse_if_statement(tokens: &mut InterpreterParseStream) -> Option { - let condition = tokens.next_item().ok()??; - let true_code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); - let false_code = if tokens.peek().is_some() { - tokens.next_as_punct_matching('!')?; - let else_word = tokens.next_as_ident()?; - tokens.next_as_punct_matching('!')?; - if else_word != "else" { - return None; - } - Some(tokens.next_as_kinded_group(Delimiter::Brace)?.stream()) - } else { - None - }; - tokens.check_end()?; - Some(IfStatement { - condition, - true_code, - false_code, - }) + loop_code: InterpretationStream, } -pub(crate) struct WhileCommand; - impl CommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let parsed = match parse_while_statement(command.arguments()) { - Some(parsed) => parsed, - None => { - return command.err("Expected [!while! (condition) { code }]"); - } - }; + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + static ERROR: &str = "Expected [!while! (condition) { code }]"; + + let condition = arguments.next_item(ERROR)?; + let loop_code = arguments.next_as_kinded_group(Delimiter::Brace, ERROR)?.into_inner_stream(); + arguments.assert_end(ERROR)?; + + Ok(Self { + condition, + loop_code, + }) + } +} +impl CommandInvocation for WhileCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(); let mut iteration_count = 0; loop { - let evaluated_condition = parsed - .condition + let evaluated_condition = self.condition .clone() .interpret_as_expression(interpreter)? .evaluate()? @@ -89,25 +98,10 @@ impl CommandDefinition for WhileCommand { iteration_count += 1; interpreter .config() - .check_iteration_count(&command, iteration_count)?; - parsed - .code - .clone() - .interpret_as_tokens_into(interpreter, &mut output)?; + .check_iteration_count(&self.condition, iteration_count)?; + self.loop_code.clone().interpret_as_tokens_into(interpreter, &mut output)?; } Ok(output) } } - -struct WhileStatement { - condition: NextItem, - code: TokenStream, -} - -fn parse_while_statement(tokens: &mut InterpreterParseStream) -> Option { - let condition = tokens.next_item().ok()??; - let code = tokens.next_as_kinded_group(Delimiter::Brace)?.stream(); - tokens.check_end()?; - Some(WhileStatement { condition, code }) -} diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index c6824c33..82759db3 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -1,49 +1,75 @@ use crate::internal_prelude::*; -pub(crate) struct SetCommand; +#[derive(Clone)] +pub(crate) struct SetCommand { + variable: Variable, + #[allow(unused)] + equals: Punct, + arguments: InterpretationStream, +} impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let variable_name = match parse_variable_set(command.arguments()) { - Some(variable) => variable.variable_name().to_string(), - None => { - return command.err("A set call is expected to start with `#variable_name = ..`"); - } - }; + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + static ERROR: &str = "Expected [!set! #variable = ... ]"; + Ok(Self { + variable: arguments.next_as_variable(ERROR)?, + equals: arguments.next_as_punct_matching('=', ERROR)?, + arguments: arguments.parse_all_for_interpretation()?, + }) + } +} - let result_tokens = command.arguments().interpret_as_tokens(interpreter)?; - interpreter.set_variable(variable_name, result_tokens); +impl CommandInvocation for SetCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let result_tokens = self.arguments.interpret_as_tokens(interpreter)?; + self.variable.set(interpreter, result_tokens); Ok(InterpretedStream::new()) } } -pub(crate) fn parse_variable_set(tokens: &mut InterpreterParseStream) -> Option { - let variable = tokens.next_item_as_variable("").ok()?; - tokens.next_as_punct_matching('=')?; - Some(variable) +#[derive(Clone)] +pub(crate) struct RawCommand { + token_stream: TokenStream, } -pub(crate) struct RawCommand; - impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - fn execute(_interpreter: &mut Interpreter, mut command: Command) -> Result { - Ok(InterpretedStream::raw( - command.arguments().read_all_as_token_stream(), - )) + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + token_stream: arguments.read_all_as_raw_token_stream(), + }) } } +impl CommandInvocation for RawCommand { + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { + Ok(InterpretedStream::raw(self.token_stream)) + } +} + +#[derive(Clone)] pub(crate) struct IgnoreCommand; impl CommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; - fn execute(_: &mut Interpreter, _: Command) -> Result { + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; + + fn parse(_arguments: InterpreterParseStream) -> Result { + Ok(Self) + } +} + +impl CommandInvocation for IgnoreCommand { + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { Ok(InterpretedStream::new()) } } diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 6cdb1fde..bd53d953 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -1,36 +1,74 @@ use crate::internal_prelude::*; -pub(crate) struct EvaluateCommand; +#[derive(Clone)] +pub(crate) struct EvaluateCommand { + expression: InterpretationStream, +} impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let expression = command.arguments().interpret_as_expression(interpreter)?; + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + expression: arguments.parse_all_for_interpretation()?, + }) + } +} + +impl CommandInvocation for EvaluateCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let expression = self.expression.interpret_as_expression(interpreter)?; Ok(expression.evaluate()?.into_interpreted_stream()) } } -pub(crate) struct AssignCommand; +#[derive(Clone)] +pub(crate) struct AssignCommand { + variable: Variable, + operator: Punct, + #[allow(unused)] + equals: Punct, + expression: InterpretationStream, +} impl CommandDefinition for AssignCommand { const COMMAND_NAME: &'static str = "assign"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let AssignStatementStart { + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + static ERROR: &str = "Expected [!assign! #variable += ...] for + or some other operator supported in an expression"; + Ok(Self { + variable: arguments.next_as_variable(ERROR)?, + operator: { + let operator = arguments.next_as_punct(ERROR)?; + match operator.as_char() { + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} + _ => return operator.err("Expected one of + - * / % & | or ^"), + } + operator + }, + equals: arguments.next_as_punct_matching('=', ERROR)?, + expression: arguments.parse_all_for_interpretation()?, + }) + } +} + +impl CommandInvocation for AssignCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let Self { variable, operator, - } = AssignStatementStart::parse(command.arguments()) - .ok_or_else(|| command.error("Expected [!assign! #variable += ...] for + or some other operator supported in an expression"))?; + equals: _, + expression, + } = *self; let mut expression_stream = ExpressionStream::new(); variable.interpret_as_expression_into(interpreter, &mut expression_stream)?; - operator - .into_token_stream() - .interpret_as_expression_into(interpreter, &mut expression_stream)?; - command - .arguments() - .interpret_as_expression_into(interpreter, &mut expression_stream)?; + expression_stream.push_punct(operator); + expression.interpret_as_expression_into(interpreter, &mut expression_stream)?; let output = expression_stream.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output); @@ -38,21 +76,3 @@ impl CommandDefinition for AssignCommand { Ok(InterpretedStream::new()) } } - -struct AssignStatementStart { - variable: Variable, - operator: Punct, -} - -impl AssignStatementStart { - fn parse(tokens: &mut InterpreterParseStream) -> Option { - let variable = tokens.next_item_as_variable("").ok()?; - let operator = tokens.next_as_punct()?; - match operator.as_char() { - '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => return None, - } - tokens.next_as_punct_matching('=')?; - Some(AssignStatementStart { variable, operator }) - } -} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index e2d89695..6635c646 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -11,48 +11,46 @@ use core_commands::*; use expression_commands::*; use token_commands::*; -define_commands! { - pub(crate) enum CommandKind { - // Core Commands - SetCommand, - RawCommand, - IgnoreCommand, +define_command_kind! { + // Core Commands + SetCommand, + RawCommand, + IgnoreCommand, - // Concat & Type Convert Commands - StringCommand, - IdentCommand, - IdentCamelCommand, - IdentSnakeCommand, - IdentUpperSnakeCommand, - LiteralCommand, + // Concat & Type Convert Commands + StringCommand, + IdentCommand, + IdentCamelCommand, + IdentSnakeCommand, + IdentUpperSnakeCommand, + LiteralCommand, - // Concat & String Convert Commands - UpperCommand, - LowerCommand, - SnakeCommand, - LowerSnakeCommand, - UpperSnakeCommand, - CamelCommand, - LowerCamelCommand, - UpperCamelCommand, - KebabCommand, - CapitalizeCommand, - DecapitalizeCommand, - TitleCommand, - InsertSpacesCommand, + // Concat & String Convert Commands + UpperCommand, + LowerCommand, + SnakeCommand, + LowerSnakeCommand, + UpperSnakeCommand, + CamelCommand, + LowerCamelCommand, + UpperCamelCommand, + KebabCommand, + CapitalizeCommand, + DecapitalizeCommand, + TitleCommand, + InsertSpacesCommand, - // Expression Commands - EvaluateCommand, - AssignCommand, + // Expression Commands + EvaluateCommand, + AssignCommand, - // Control flow commands - IfCommand, - WhileCommand, + // Control flow commands + IfCommand, + WhileCommand, - // Token Commands - EmptyCommand, - IsEmptyCommand, - LengthCommand, - GroupCommand, - } + // Token Commands + EmptyCommand, + IsEmptyCommand, + LengthCommand, + GroupCommand, } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index a834db2d..527e0af8 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -1,53 +1,102 @@ use crate::internal_prelude::*; +#[derive(Clone)] pub(crate) struct EmptyCommand; impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; - fn execute(_interpreter: &mut Interpreter, mut command: Command) -> Result { - command - .arguments() - .assert_end("The !empty! command does not take any arguments")?; + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + arguments.assert_end("The !empty! command does not take any arguments")?; + Ok(Self) + } +} + +impl CommandInvocation for EmptyCommand { + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { Ok(InterpretedStream::new()) } } -pub(crate) struct IsEmptyCommand; +#[derive(Clone)] +pub(crate) struct IsEmptyCommand { + arguments: InterpretationStream, +} impl CommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let interpreted = command.arguments().interpret_as_tokens(interpreter)?; - Ok(TokenTree::bool(interpreted.is_empty(), command.span()).into()) + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) } } -pub(crate) struct LengthCommand; +impl CommandInvocation for IsEmptyCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let output_span = self.arguments.span_range().span(); + let interpreted = self.arguments.interpret_as_tokens(interpreter)?; + Ok(TokenTree::bool(interpreted.is_empty(), output_span).into()) + } +} + +#[derive(Clone)] +pub(crate) struct LengthCommand { + arguments: InterpretationStream, +} impl CommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { - let interpreted = command.arguments().interpret_as_tokens(interpreter)?; + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) + } +} + +impl CommandInvocation for LengthCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let output_span = self.arguments.span_range().span(); + let interpreted = self.arguments.interpret_as_tokens(interpreter)?; let stream_length = interpreted.into_token_stream().into_iter().count(); - let length_literal = Literal::usize_unsuffixed(stream_length).with_span(command.span()); + let length_literal = Literal::usize_unsuffixed(stream_length).with_span(output_span); Ok(InterpretedStream::of_literal(length_literal)) } } -pub(crate) struct GroupCommand; +#[derive(Clone)] +pub(crate) struct GroupCommand { + arguments: InterpretationStream, +} impl CommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - fn execute(interpreter: &mut Interpreter, mut command: Command) -> Result { + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) + } +} + +impl CommandInvocation for GroupCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(); + let span_range = self.arguments.span_range(); output.push_new_group( - command.arguments().interpret_as_tokens(interpreter)?, + self.arguments.interpret_as_tokens(interpreter)?, Delimiter::None, - command.span_range(), + span_range, ); Ok(output) } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 1d23e7cb..cb9edf0f 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -1,5 +1,12 @@ use super::*; +/// This abstraction is a bit ropey... +/// +/// Ideally we'd parse expressions at parse time, but that requires writing a custom parser for +/// a subset of the rust expression tree... +/// +/// Instead, to be lazy for now, we interpret the stream at intepretation time to substitute +/// in variables and commands, and then parse the resulting expression with syn. #[derive(Clone)] pub(crate) struct ExpressionStream { interpreted_stream: InterpretedStream, @@ -12,8 +19,17 @@ impl ExpressionStream { } } - pub(crate) fn push_raw_token_tree(&mut self, token_tree: TokenTree) { - self.interpreted_stream.push_raw_token_tree(token_tree); + pub(crate) fn push_literal(&mut self, literal: Literal) { + self.interpreted_stream.push_literal(literal); + } + + /// Only true and false make sense, but allow all here and catch others at evaluation time + pub(crate) fn push_ident(&mut self, ident: Ident) { + self.interpreted_stream.push_ident(ident); + } + + pub(crate) fn push_punct(&mut self, punct: Punct) { + self.interpreted_stream.push_punct(punct); } pub(crate) fn push_interpreted_group( diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 9131e563..ff53688f 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,37 +1,74 @@ use crate::internal_prelude::*; -pub(crate) trait CommandDefinition { +pub(crate) enum CommandOutputBehaviour { + EmptyStream, + #[allow(unused)] // Likely useful in future + GroupedStream, + AppendStream, + SingleToken, +} + +pub(crate) trait CommandDefinition: CommandInvocation + Clone { const COMMAND_NAME: &'static str; + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour; + + fn parse(arguments: InterpreterParseStream) -> Result; +} + +pub(crate) trait CommandInvocation { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; +} - fn execute(interpreter: &mut Interpreter, command: Command) -> Result; +pub(crate) trait ClonableCommandInvocation: CommandInvocation { + fn clone_box(&self) -> Box; } -macro_rules! define_commands { +impl ClonableCommandInvocation for C { + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +macro_rules! define_command_kind { ( - pub(crate) enum $enum_name:ident { - $( - $command:ident, - )* - } + $( + $command:ident, + )* ) => { #[allow(clippy::enum_variant_names)] #[derive(Clone, Copy)] - pub(crate) enum $enum_name { + pub(crate) enum CommandKind { $( $command, )* } - impl $enum_name { - pub(crate) fn execute(self, interpreter: &mut Interpreter, command: Command) -> Result { + impl CommandKind { + pub(crate) fn parse_invocation(&self, arguments: InterpreterParseStream) -> Result> { + Ok(match self { + $( + Self::$command => Box::new( + <$command as CommandDefinition>::parse(arguments)? + ), + )* + }) + } + + pub(crate) fn output_behaviour(&self) -> CommandOutputBehaviour { match self { $( - Self::$command => $command::execute(interpreter, command), + Self::$command => <$command as CommandDefinition>::OUTPUT_BEHAVIOUR, )* } } - pub(crate) fn attempt_parse(ident: &Ident) -> Option { + pub(crate) fn for_ident(ident: &Ident) -> Option { Some(match ident.to_string().as_ref() { $( <$command as CommandDefinition>::COMMAND_NAME => Self::$command, @@ -40,7 +77,7 @@ macro_rules! define_commands { }) } - const ALL_KIND_NAMES: &'static [&'static str] = &[$($command::COMMAND_NAME,)*]; + const ALL_KIND_NAMES: &'static [&'static str] = &[$(<$command as CommandDefinition>::COMMAND_NAME,)*]; pub(crate) fn list_all() -> String { // TODO improve to add an "and" at the end @@ -49,24 +86,61 @@ macro_rules! define_commands { } }; } -pub(crate) use define_commands; +pub(crate) use define_command_kind; #[derive(Clone)] -pub(crate) struct CommandInvocation { +pub(crate) struct Command { command_kind: CommandKind, - command: Command, + source_group_span_range: SpanRange, + invocation: Box, } -impl CommandInvocation { - pub(crate) fn new( - command_ident: Ident, - command_kind: CommandKind, - group: &Group, - argument_tokens: InterpreterParseStream, - ) -> Self { - Self { - command_kind, - command: Command::new(command_ident, group.span(), argument_tokens), +impl Command { + pub(super) fn attempt_parse_from_group(group: &Group) -> Result> { + fn matches_command_start(group: &Group) -> Option<(Ident, InterpreterParseStream)> { + if group.delimiter() != Delimiter::Bracket { + return None; + } + let mut tokens = InterpreterParseStream::new(group.stream(), group.span_range()); + tokens.next_as_punct_matching('!', "").ok()?; + let ident = tokens.next_as_ident("").ok()?; + Some((ident, tokens)) + } + + fn extract_command_data( + command_ident: &Ident, + parse_stream: &mut InterpreterParseStream, + ) -> Option { + let command_kind = CommandKind::for_ident(command_ident)?; + parse_stream.next_as_punct_matching('!', "").ok()?; + Some(command_kind) + } + + // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, + // so return `Ok(None)` + let (command_ident, mut parse_stream) = match matches_command_start(group) { + Some(command_start) => command_start, + None => return Ok(None), + }; + + // We have now checked enough that we're confident the user is pretty intentionally using + // the call convention. Any issues we hit from this point will be a helpful compiler error. + match extract_command_data(&command_ident, &mut parse_stream) { + Some(command_kind) => { + let invocation = command_kind.parse_invocation( parse_stream)?; + Ok(Some(Self { + command_kind, + source_group_span_range: group.span_range(), + invocation, + })) + }, + None => Err(command_ident.span().error( + format!( + "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", + CommandKind::list_all(), + command_ident, + ), + )), } } @@ -75,19 +149,26 @@ impl CommandInvocation { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - let substitution = self.command_kind.execute(interpreter, self.command)?; - output.extend(substitution); + let substitution = self.invocation.execute(interpreter)?; + match self.command_kind.output_behaviour() { + CommandOutputBehaviour::GroupedStream => { + output.push_new_group(substitution, Delimiter::None, self.source_group_span_range); + } + CommandOutputBehaviour::EmptyStream | CommandOutputBehaviour::SingleToken | CommandOutputBehaviour::AppendStream => { + output.extend(substitution); + } + } Ok(()) } } -impl HasSpanRange for CommandInvocation { +impl HasSpanRange for Command { fn span_range(&self) -> SpanRange { - self.command.span_range() + self.source_group_span_range } } -impl Interpret for CommandInvocation { +impl Interpret for Command { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, @@ -107,51 +188,3 @@ impl Interpret for CommandInvocation { Ok(()) } } - -#[derive(Clone)] -pub(crate) struct Command { - command_ident: Ident, - command_span: Span, - argument_tokens: InterpreterParseStream, -} - -impl Command { - fn new( - command_ident: Ident, - command_span: Span, - argument_tokens: InterpreterParseStream, - ) -> Self { - Self { - command_ident, - command_span, - argument_tokens, - } - } - - #[allow(unused)] // Likely useful in future - pub(crate) fn ident_span(&self) -> Span { - self.command_ident.span() - } - - pub(crate) fn span(&self) -> Span { - self.command_span - } - - pub(crate) fn error(&self, message: impl core::fmt::Display) -> syn::Error { - self.command_span.error(message) - } - - pub(crate) fn err(&self, message: impl core::fmt::Display) -> Result { - Err(self.error(message)) - } - - pub(crate) fn arguments(&mut self) -> &mut InterpreterParseStream { - &mut self.argument_tokens - } -} - -impl HasSpanRange for Command { - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs new file mode 100644 index 00000000..03ea9a9d --- /dev/null +++ b/src/interpretation/interpretation_stream.rs @@ -0,0 +1,121 @@ +use crate::internal_prelude::*; + +/// A parsed stream ready for interpretation +#[derive(Clone)] +pub(crate) struct InterpretationStream { + items: Vec, + span_range: SpanRange, +} + +impl InterpretationStream { + pub(crate) fn parse_from_token_stream(token_stream: TokenStream, span_range: SpanRange) -> Result { + InterpreterParseStream::new(token_stream, span_range).parse_all_for_interpretation() + } + + pub(crate) fn parse(parse_stream: &mut InterpreterParseStream, span_range: SpanRange) -> Result { + let mut items = Vec::new(); + while let Some(next_item) = NextItem::parse(parse_stream)? { + items.push(next_item); + } + Ok(Self { + items, + span_range, + }) + } +} + +impl<'a> Interpret for InterpretationStream { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + for item in self.items { + item.interpret_as_tokens_into(interpreter, output)?; + } + Ok(()) + } + + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { + let mut inner_expression_stream = ExpressionStream::new(); + for item in self.items { + item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)?; + } + expression_stream.push_expression_group( + inner_expression_stream, + Delimiter::None, + self.span_range, + ); + Ok(()) + } +} + +impl HasSpanRange for InterpretationStream { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +/// A parsed group ready for interpretation +#[derive(Clone)] +pub(crate) struct InterpretationGroup { + source_group: Group, + interpretation_stream: InterpretationStream, +} + +impl InterpretationGroup { + pub(super) fn parse(source_group: Group) -> Result { + let interpretation_stream = InterpreterParseStream::new(source_group.stream(), source_group.span_range()) + .parse_all_for_interpretation()?; + Ok(Self { + source_group, + interpretation_stream, + }) + } + + pub(crate) fn delimiter(&self) -> Delimiter { + self.source_group.delimiter() + } + + pub(crate) fn into_inner_stream(self) -> InterpretationStream { + self.interpretation_stream + } +} + +impl Interpret for InterpretationGroup { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + output.push_new_group( + self.interpretation_stream.interpret_as_tokens(interpreter)?, + self.source_group.delimiter(), + self.source_group.span_range(), + ); + Ok(()) + } + + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { + expression_stream.push_expression_group( + self.interpretation_stream.interpret_as_expression(interpreter)?, + self.source_group.delimiter(), + self.source_group.span_range(), + ); + Ok(()) + } +} + +impl HasSpanRange for InterpretationGroup { + fn span_range(&self) -> SpanRange { + self.source_group.span_range() + } +} diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 6ef56697..f95552d1 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -28,8 +28,16 @@ impl InterpretedStream { self.token_stream.extend(interpreted_stream.token_stream); } - pub(crate) fn push_raw_token_tree(&mut self, token_tree: TokenTree) { - self.token_stream.extend(iter::once(token_tree)); + pub(crate) fn push_literal(&mut self, literal: Literal) { + self.push_raw_token_tree(literal.into()); + } + + pub(crate) fn push_ident(&mut self, ident: Ident) { + self.push_raw_token_tree(ident.into()); + } + + pub(crate) fn push_punct(&mut self, punct: Punct) { + self.push_raw_token_tree(punct.into()); } pub(crate) fn push_new_group( @@ -45,6 +53,10 @@ impl InterpretedStream { )); } + fn push_raw_token_tree(&mut self, token_tree: TokenTree) { + self.token_stream.extend(iter::once(token_tree)); + } + pub(crate) fn is_empty(&self) -> bool { self.token_stream.is_empty() } diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 3758e809..ce9b6f50 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -39,10 +39,10 @@ impl Default for InterpreterConfig { } impl InterpreterConfig { - pub(crate) fn check_iteration_count(&self, command: &Command, count: usize) -> Result<()> { + pub(crate) fn check_iteration_count(&self, span_source: &impl HasSpanRange, count: usize) -> Result<()> { if let Some(limit) = self.iteration_limit { if count > limit { - return command.err(format!("Iteration limit of {} exceeded", limit)); + return span_source.err(format!("Iteration limit of {} exceeded", limit)); } } Ok(()) diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 163a09fc..45e9c9e2 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -38,199 +38,108 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct InterpreterParseStream { tokens: iter::Peekable<::IntoIter>, - span_range: SpanRange, + /// The span range of the original stream, before tokens were consumed + full_span_range: SpanRange, + /// The span of the last item consumed (or the full span range if no items have been consumed yet) + latest_item_span_range: SpanRange, } impl InterpreterParseStream { pub(crate) fn new(token_stream: TokenStream, span_range: SpanRange) -> Self { Self { tokens: token_stream.into_iter().peekable(), - span_range, + full_span_range: span_range, + latest_item_span_range: span_range, } } - pub(crate) fn peek(&mut self) -> Option<&TokenTree> { + pub(super) fn peek_token_tree(&mut self) -> Option<&TokenTree> { self.tokens.peek() } - pub(crate) fn next(&mut self) -> Option { + pub(super) fn next_token_tree_or_end(&mut self) -> Option { self.tokens.next() } - pub(crate) fn next_as_ident(&mut self) -> Option { - match self.next() { - Some(TokenTree::Ident(ident)) => Some(ident), - _ => None, - } + fn next_item_or_end(&mut self) -> Result> { + let next_item = NextItem::parse(self)?; + Ok(match next_item { + Some(next_item) => { + self.latest_item_span_range = next_item.span_range(); + Some(next_item) + }, + None => None, + }) } - pub(crate) fn next_as_punct(&mut self) -> Option { - match self.next() { - Some(TokenTree::Punct(punct)) => Some(punct), - _ => None, + pub(crate) fn next_item(&mut self, error_message: &'static str) -> Result { + match self.next_item_or_end()? { + Some(item) => Ok(item), + None => self.latest_item_span_range.err(format!("Unexpected end: {error_message}")), } } - pub(crate) fn next_as_punct_matching(&mut self, char: char) -> Option { - match self.next() { - Some(TokenTree::Punct(punct)) if punct.as_char() == char => Some(punct), - _ => None, + pub(crate) fn next_as_ident(&mut self, error_message: &'static str) -> Result { + match self.next_item(error_message)? { + NextItem::Ident(ident) => Ok(ident), + other => other.err(error_message), } } - pub(crate) fn next_as_kinded_group(&mut self, delimiter: Delimiter) -> Option { - match self.next() { - Some(TokenTree::Group(group)) if group.delimiter() == delimiter => Some(group), - _ => None, + pub(crate) fn next_as_ident_matching(&mut self, ident_name: &str, error_message: &'static str) -> Result { + match self.next_item(error_message)? { + NextItem::Ident(ident) if &ident.to_string() == ident_name => Ok(ident), + other => other.err(error_message), } } - pub(crate) fn is_empty(&mut self) -> bool { - self.peek().is_none() - } - - pub(crate) fn check_end(&mut self) -> Option<()> { - if self.is_empty() { - Some(()) - } else { - None + pub(crate) fn next_as_punct(&mut self, error_message: &'static str) -> Result { + match self.next_item(error_message)? { + NextItem::Punct(punct) => Ok(punct), + other => other.err(error_message), } } - pub(crate) fn assert_end(&mut self, error_message: &'static str) -> Result<()> { - match self.next() { - Some(token) => token.span_range().err(error_message), - None => Ok(()), + pub(crate) fn next_as_punct_matching(&mut self, char: char, error_message: &'static str) -> Result { + match self.next_item(error_message)? { + NextItem::Punct(punct) if punct.as_char() == char => Ok(punct), + other => other.err(error_message), } } - pub(crate) fn next_item(&mut self) -> Result> { - let next = match self.next() { - Some(next) => next, - None => return Ok(None), - }; - Ok(Some(match next { - TokenTree::Group(group) => { - if let Some(command_invocation) = parse_command_invocation(&group)? { - NextItem::CommandInvocation(command_invocation) - } else { - NextItem::Group(group) - } - } - TokenTree::Punct(punct) => { - if let Some(variable_substitution) = - parse_only_if_variable_substitution(&punct, self) - { - NextItem::Variable(variable_substitution) - } else { - NextItem::Leaf(TokenTree::Punct(punct)) - } - } - leaf => NextItem::Leaf(leaf), - })) + pub(crate) fn next_as_kinded_group(&mut self, delimiter: Delimiter, error_message: &'static str) -> Result { + match self.next_item(error_message)? { + NextItem::Group(group) if group.delimiter() == delimiter => Ok(group), + other => other.err(error_message), + } } - pub(crate) fn next_item_as_variable( + pub(crate) fn next_as_variable( &mut self, error_message: &'static str, ) -> Result { - match self.next_item()? { - Some(NextItem::Variable(variable_substitution)) => Ok(variable_substitution), - Some(item) => item.span_range().err(error_message), - None => Span::call_site().span_range().err(error_message), + match self.next_item(error_message)? { + NextItem::Variable(variable_substitution) => Ok(variable_substitution), + other => other.err(error_message), } } - pub(crate) fn read_all_as_token_stream(&mut self) -> TokenStream { - core::mem::replace(&mut self.tokens, TokenStream::new().into_iter().peekable()).collect() - } -} - -impl<'a> Interpret for &'a mut InterpreterParseStream { - fn interpret_as_tokens_into( - self, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()> { - while let Some(next_item) = self.next_item()? { - next_item.interpret_as_tokens_into(interpreter, output)?; - } - Ok(()) - } - - fn interpret_as_expression_into( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, - ) -> Result<()> { - let mut inner_expression_stream = ExpressionStream::new(); - while let Some(next_item) = self.next_item()? { - next_item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)?; - } - expression_stream.push_expression_group( - inner_expression_stream, - Delimiter::None, - self.span_range, - ); - Ok(()) + pub(crate) fn is_empty(&mut self) -> bool { + self.peek_token_tree().is_none() } -} -fn parse_command_invocation(group: &Group) -> Result> { - fn consume_command_start(group: &Group) -> Option<(Ident, InterpreterParseStream)> { - if group.delimiter() != Delimiter::Bracket { - return None; + pub(crate) fn assert_end(&mut self, error_message: &'static str) -> Result<()> { + match self.next_token_tree_or_end() { + Some(token) => token.span_range().err(error_message), + None => Ok(()), } - let mut tokens = InterpreterParseStream::new(group.stream(), group.span_range()); - tokens.next_as_punct_matching('!')?; - let ident = tokens.next_as_ident()?; - Some((ident, tokens)) } - fn consume_command_end( - command_ident: &Ident, - tokens: &mut InterpreterParseStream, - ) -> Option { - let command_kind = CommandKind::attempt_parse(command_ident)?; - tokens.next_as_punct_matching('!')?; - Some(command_kind) + pub(crate) fn parse_all_for_interpretation(&mut self) -> Result { + InterpretationStream::parse(self, self.full_span_range) } - // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, - // so return `Ok(None)` - let (command_ident, mut remaining_tokens) = match consume_command_start(group) { - Some(command_start) => command_start, - None => return Ok(None), - }; - - // We have now checked enough that we're confident the user is pretty intentionally using - // the call convention. Any issues we hit from this point will be a helpful compiler error. - match consume_command_end(&command_ident, &mut remaining_tokens) { - Some(command_kind) => Ok(Some(CommandInvocation::new(command_ident.clone(), command_kind, group, remaining_tokens))), - None => Err(command_ident.span().error( - format!( - "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", - CommandKind::list_all(), - command_ident, - ), - )), - } -} - -// We ensure we don't consume any tokens unless we have a variable substitution -fn parse_only_if_variable_substitution( - punct: &Punct, - tokens: &mut InterpreterParseStream, -) -> Option { - if punct.as_char() != '#' { - return None; - } - match tokens.peek() { - Some(TokenTree::Ident(_)) => {} - _ => return None, - } - match tokens.next() { - Some(TokenTree::Ident(variable_name)) => Some(Variable::new(punct.clone(), variable_name)), - _ => unreachable!("We just peeked a token of this type"), + pub(crate) fn read_all_as_raw_token_stream(&mut self) -> TokenStream { + core::mem::replace(&mut self.tokens, TokenStream::new().into_iter().peekable()).collect() } } diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index c28e7ffd..9e855686 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,4 +1,5 @@ mod command; +mod interpretation_stream; mod interpreted_stream; mod interpreter; mod interpreter_parse_stream; @@ -8,6 +9,7 @@ mod variable; pub(crate) use command::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; +pub(crate) use interpretation_stream::*; pub(crate) use interpreter_parse_stream::*; pub(crate) use next_item::*; pub(crate) use variable::*; diff --git a/src/interpretation/next_item.rs b/src/interpretation/next_item.rs index 58f225f9..240f1289 100644 --- a/src/interpretation/next_item.rs +++ b/src/interpretation/next_item.rs @@ -2,10 +2,40 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum NextItem { - CommandInvocation(CommandInvocation), + Command(Command), Variable(Variable), - Group(Group), - Leaf(TokenTree), + Group(InterpretationGroup), + Punct(Punct), + Ident(Ident), + Literal(Literal), +} + +impl NextItem { + pub(super) fn parse(parse_stream: &mut InterpreterParseStream) -> Result> { + let next = match parse_stream.next_token_tree_or_end() { + Some(next) => next, + None => return Ok(None), + }; + Ok(Some(match next { + TokenTree::Group(group) => { + if let Some(command) = Command::attempt_parse_from_group(&group)? { + NextItem::Command(command) + } else { + NextItem::Group(InterpretationGroup::parse(group)?) + } + } + TokenTree::Punct(punct) => { + if let Some(variable) = Variable::parse_consuming_only_if_match(&punct, parse_stream) + { + NextItem::Variable(variable) + } else { + NextItem::Punct(punct) + } + } + TokenTree::Ident(ident) => NextItem::Ident(ident), + TokenTree::Literal(literal) => NextItem::Literal(literal), + })) + } } impl Interpret for NextItem { @@ -15,18 +45,18 @@ impl Interpret for NextItem { output: &mut InterpretedStream, ) -> Result<()> { match self { - NextItem::Leaf(token_tree) => { - output.push_raw_token_tree(token_tree); - } - NextItem::Group(group) => { - group.interpret_as_tokens_into(interpreter, output)?; + NextItem::Command(command_invocation) => { + command_invocation.interpret_as_tokens_into(interpreter, output)?; } NextItem::Variable(variable) => { variable.interpret_as_tokens_into(interpreter, output)?; } - NextItem::CommandInvocation(command_invocation) => { - command_invocation.interpret_as_tokens_into(interpreter, output)?; + NextItem::Group(group) => { + group.interpret_as_tokens_into(interpreter, output)?; } + NextItem::Punct(punct) => output.push_punct(punct), + NextItem::Ident(ident) => output.push_ident(ident), + NextItem::Literal(literal) => output.push_literal(literal), } Ok(()) } @@ -37,18 +67,18 @@ impl Interpret for NextItem { expression_stream: &mut ExpressionStream, ) -> Result<()> { match self { - NextItem::Leaf(token_tree) => { - expression_stream.push_raw_token_tree(token_tree); - } - NextItem::Group(group) => { - group.interpret_as_expression_into(interpreter, expression_stream)?; + NextItem::Command(command_invocation) => { + command_invocation.interpret_as_expression_into(interpreter, expression_stream)?; } NextItem::Variable(variable) => { variable.interpret_as_expression_into(interpreter, expression_stream)?; } - NextItem::CommandInvocation(command_invocation) => { - command_invocation.interpret_as_expression_into(interpreter, expression_stream)?; + NextItem::Group(group) => { + group.interpret_as_expression_into(interpreter, expression_stream)?; } + NextItem::Punct(punct) => expression_stream.push_punct(punct), + NextItem::Ident(ident) => expression_stream.push_ident(ident), + NextItem::Literal(literal) => expression_stream.push_literal(literal), } Ok(()) } @@ -57,10 +87,12 @@ impl Interpret for NextItem { impl HasSpanRange for NextItem { fn span_range(&self) -> SpanRange { match self { - NextItem::CommandInvocation(command_invocation) => command_invocation.span_range(), + NextItem::Command(command_invocation) => command_invocation.span_range(), NextItem::Variable(variable_substitution) => variable_substitution.span_range(), NextItem::Group(group) => group.span_range(), - NextItem::Leaf(token_tree) => token_tree.span_range(), + NextItem::Punct(punct) => punct.span_range(), + NextItem::Ident(ident) => ident.span_range(), + NextItem::Literal(literal) => literal.span_range(), } } } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index d61aac97..92a755dd 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -7,10 +7,23 @@ pub(crate) struct Variable { } impl Variable { - pub(crate) fn new(marker: Punct, variable_name: Ident) -> Self { - Self { - marker, - variable_name, + pub(super) fn parse_consuming_only_if_match( + punct: &Punct, + parse_stream: &mut InterpreterParseStream, + ) -> Option { + if punct.as_char() != '#' { + return None; + } + match parse_stream.peek_token_tree() { + Some(TokenTree::Ident(_)) => {} + _ => return None, + } + match parse_stream.next_token_tree_or_end() { + Some(TokenTree::Ident(variable_name)) => Some(Self { + marker: punct.clone(), + variable_name, + }), + _ => unreachable!("We just peeked a token of this type"), } } diff --git a/src/lib.rs b/src/lib.rs index ff7ce23a..03c1ddc8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -538,14 +538,18 @@ use internal_prelude::*; /// See the [crate-level documentation](crate) for full details. #[proc_macro] pub fn preinterpret(token_stream: proc_macro::TokenStream) -> proc_macro::TokenStream { - let mut interpreter = Interpreter::new(); - proc_macro2::TokenStream::from(token_stream) - .interpret_as_tokens(&mut interpreter) - .map(InterpretedStream::into_token_stream) + preinterpret_internal(proc_macro2::TokenStream::from(token_stream)) .unwrap_or_else(|err| err.to_compile_error()) .into() } +fn preinterpret_internal(input: TokenStream) -> Result { + let mut interpreter = Interpreter::new(); + let interpretation_stream = InterpretationStream::parse_from_token_stream(input, Span::call_site().span_range())?; + let interpreted_stream = interpretation_stream.interpret_as_tokens(&mut interpreter)?; + Ok(interpreted_stream.into_token_stream()) +} + // This is the recommended way to run the doc tests in the readme #[doc = include_str!("../README.md")] #[cfg(doctest)] // Don't actually export this! diff --git a/src/traits.rs b/src/traits.rs index 7a8f53de..f105f81b 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -6,6 +6,7 @@ pub(crate) trait Interpret: Sized { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()>; + fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(); self.interpret_as_tokens_into(interpreter, &mut output)?; @@ -17,6 +18,7 @@ pub(crate) trait Interpret: Sized { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()>; + fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { let mut output = ExpressionStream::new(); self.interpret_as_expression_into(interpreter, &mut output)?; @@ -39,71 +41,22 @@ impl TokenTreeExt for TokenTree { } } -impl Interpret for TokenStream { - fn interpret_as_tokens_into( - self, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()> { - let span_range = self.span_range(); - InterpreterParseStream::new(self, span_range).interpret_as_tokens_into(interpreter, output) - } - - fn interpret_as_expression_into( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, - ) -> Result<()> { - let span_range = self.span_range(); - InterpreterParseStream::new(self, span_range) - .interpret_as_expression_into(interpreter, expression_stream) - } -} - -impl Interpret for Group { - fn interpret_as_tokens_into( - self, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()> { - output.push_new_group( - self.stream().interpret_as_tokens(interpreter)?, - self.delimiter(), - self.span_range(), - ); - Ok(()) - } - - fn interpret_as_expression_into( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, - ) -> Result<()> { - expression_stream.push_expression_group( - self.stream().interpret_as_expression(interpreter)?, - self.delimiter(), - self.span_range(), - ); - Ok(()) - } -} - pub(crate) trait SpanErrorExt: Sized { - fn err(self, message: impl std::fmt::Display) -> syn::Result { + fn err(&self, message: impl std::fmt::Display) -> syn::Result { Err(self.error(message)) } - fn error(self, message: impl std::fmt::Display) -> syn::Error; + fn error(&self, message: impl std::fmt::Display) -> syn::Error; } -impl SpanErrorExt for Span { - fn error(self, message: impl std::fmt::Display) -> syn::Error { - syn::Error::new(self, message) +impl SpanErrorExt for T { + fn error(&self, message: impl std::fmt::Display) -> syn::Error { + self.span_range().error(message) } } impl SpanErrorExt for SpanRange { - fn error(self, message: impl std::fmt::Display) -> syn::Error { + fn error(&self, message: impl std::fmt::Display) -> syn::Error { syn::Error::new_spanned(self, message) } } @@ -135,6 +88,10 @@ impl WithSpanExt for Group { pub(crate) trait HasSpanRange { fn span_range(&self) -> SpanRange; + + fn span(&self) -> Span { + self.span_range().span() + } } /// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] @@ -237,6 +194,9 @@ macro_rules! impl_auto_span_range { impl_auto_span_range! { TokenStream, + Ident, + Punct, + Literal, syn::Expr, syn::ExprBinary, syn::ExprUnary, From 0eb014f0876a46fb8017897ec35d2d84e92cd996 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 9 Jan 2025 02:43:47 +0000 Subject: [PATCH 015/126] fix: Styling fix --- src/commands/concat_commands.rs | 12 ++++--- src/commands/control_flow_commands.rs | 21 ++++++++++--- src/expressions/expression_stream.rs | 2 +- src/interpretation/command.rs | 10 +++--- src/interpretation/interpretation_stream.rs | 28 ++++++++++------- src/interpretation/interpreter.rs | 6 +++- .../interpreter_parse_stream.rs | 31 +++++++++++++------ src/interpretation/mod.rs | 2 +- src/interpretation/next_item.rs | 3 +- src/lib.rs | 3 +- 10 files changed, 78 insertions(+), 40 deletions(-) diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index fabe95cb..7829a200 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -57,7 +57,6 @@ fn concat_into_literal( Ok(InterpretedStream::of_literal(literal)) } - fn concat_recursive(arguments: InterpretedStream) -> String { fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { for token_tree in arguments { @@ -118,9 +117,9 @@ macro_rules! define_concat_command { impl CommandDefinition for $command { const COMMAND_NAME: &'static str = $command_name; - + const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; - + fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, @@ -129,11 +128,14 @@ macro_rules! define_concat_command { } impl CommandInvocation for $command { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute( + self: Box, + interpreter: &mut Interpreter, + ) -> Result { $output_fn(self.arguments, interpreter, $conversion_fn) } } - } + }; } //======================================= diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index a13c8232..5f10101c 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -16,12 +16,18 @@ impl CommandDefinition for IfCommand { static ERROR: &str = "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"; let condition = arguments.next_item(ERROR)?; - let true_code = arguments.next_as_kinded_group(Delimiter::Brace, ERROR)?.into_inner_stream(); + let true_code = arguments + .next_as_kinded_group(Delimiter::Brace, ERROR)? + .into_inner_stream(); let false_code = if !arguments.is_empty() { arguments.next_as_punct_matching('!', ERROR)?; arguments.next_as_ident_matching("else", ERROR)?; arguments.next_as_punct_matching('!', ERROR)?; - Some(arguments.next_as_kinded_group(Delimiter::Brace, ERROR)?.into_inner_stream()) + Some( + arguments + .next_as_kinded_group(Delimiter::Brace, ERROR)? + .into_inner_stream(), + ) } else { None }; @@ -69,7 +75,9 @@ impl CommandDefinition for WhileCommand { static ERROR: &str = "Expected [!while! (condition) { code }]"; let condition = arguments.next_item(ERROR)?; - let loop_code = arguments.next_as_kinded_group(Delimiter::Brace, ERROR)?.into_inner_stream(); + let loop_code = arguments + .next_as_kinded_group(Delimiter::Brace, ERROR)? + .into_inner_stream(); arguments.assert_end(ERROR)?; Ok(Self { @@ -84,7 +92,8 @@ impl CommandInvocation for WhileCommand { let mut output = InterpretedStream::new(); let mut iteration_count = 0; loop { - let evaluated_condition = self.condition + let evaluated_condition = self + .condition .clone() .interpret_as_expression(interpreter)? .evaluate()? @@ -99,7 +108,9 @@ impl CommandInvocation for WhileCommand { interpreter .config() .check_iteration_count(&self.condition, iteration_count)?; - self.loop_code.clone().interpret_as_tokens_into(interpreter, &mut output)?; + self.loop_code + .clone() + .interpret_as_tokens_into(interpreter, &mut output)?; } Ok(output) diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index cb9edf0f..7d84cb32 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -4,7 +4,7 @@ use super::*; /// /// Ideally we'd parse expressions at parse time, but that requires writing a custom parser for /// a subset of the rust expression tree... -/// +/// /// Instead, to be lazy for now, we interpret the stream at intepretation time to substitute /// in variables and commands, and then parse the resulting expression with syn. #[derive(Clone)] diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index ff53688f..cd9f56f6 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -106,7 +106,7 @@ impl Command { let ident = tokens.next_as_ident("").ok()?; Some((ident, tokens)) } - + fn extract_command_data( command_ident: &Ident, parse_stream: &mut InterpreterParseStream, @@ -115,14 +115,14 @@ impl Command { parse_stream.next_as_punct_matching('!', "").ok()?; Some(command_kind) } - + // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, // so return `Ok(None)` let (command_ident, mut parse_stream) = match matches_command_start(group) { Some(command_start) => command_start, None => return Ok(None), }; - + // We have now checked enough that we're confident the user is pretty intentionally using // the call convention. Any issues we hit from this point will be a helpful compiler error. match extract_command_data(&command_ident, &mut parse_stream) { @@ -154,7 +154,9 @@ impl Command { CommandOutputBehaviour::GroupedStream => { output.push_new_group(substitution, Delimiter::None, self.source_group_span_range); } - CommandOutputBehaviour::EmptyStream | CommandOutputBehaviour::SingleToken | CommandOutputBehaviour::AppendStream => { + CommandOutputBehaviour::EmptyStream + | CommandOutputBehaviour::SingleToken + | CommandOutputBehaviour::AppendStream => { output.extend(substitution); } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 03ea9a9d..970677cd 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -8,23 +8,26 @@ pub(crate) struct InterpretationStream { } impl InterpretationStream { - pub(crate) fn parse_from_token_stream(token_stream: TokenStream, span_range: SpanRange) -> Result { + pub(crate) fn parse_from_token_stream( + token_stream: TokenStream, + span_range: SpanRange, + ) -> Result { InterpreterParseStream::new(token_stream, span_range).parse_all_for_interpretation() } - pub(crate) fn parse(parse_stream: &mut InterpreterParseStream, span_range: SpanRange) -> Result { + pub(crate) fn parse( + parse_stream: &mut InterpreterParseStream, + span_range: SpanRange, + ) -> Result { let mut items = Vec::new(); while let Some(next_item) = NextItem::parse(parse_stream)? { items.push(next_item); } - Ok(Self { - items, - span_range, - }) + Ok(Self { items, span_range }) } } -impl<'a> Interpret for InterpretationStream { +impl Interpret for InterpretationStream { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, @@ -69,8 +72,9 @@ pub(crate) struct InterpretationGroup { impl InterpretationGroup { pub(super) fn parse(source_group: Group) -> Result { - let interpretation_stream = InterpreterParseStream::new(source_group.stream(), source_group.span_range()) - .parse_all_for_interpretation()?; + let interpretation_stream = + InterpreterParseStream::new(source_group.stream(), source_group.span_range()) + .parse_all_for_interpretation()?; Ok(Self { source_group, interpretation_stream, @@ -93,7 +97,8 @@ impl Interpret for InterpretationGroup { output: &mut InterpretedStream, ) -> Result<()> { output.push_new_group( - self.interpretation_stream.interpret_as_tokens(interpreter)?, + self.interpretation_stream + .interpret_as_tokens(interpreter)?, self.source_group.delimiter(), self.source_group.span_range(), ); @@ -106,7 +111,8 @@ impl Interpret for InterpretationGroup { expression_stream: &mut ExpressionStream, ) -> Result<()> { expression_stream.push_expression_group( - self.interpretation_stream.interpret_as_expression(interpreter)?, + self.interpretation_stream + .interpret_as_expression(interpreter)?, self.source_group.delimiter(), self.source_group.span_range(), ); diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index ce9b6f50..85793cfc 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -39,7 +39,11 @@ impl Default for InterpreterConfig { } impl InterpreterConfig { - pub(crate) fn check_iteration_count(&self, span_source: &impl HasSpanRange, count: usize) -> Result<()> { + pub(crate) fn check_iteration_count( + &self, + span_source: &impl HasSpanRange, + count: usize, + ) -> Result<()> { if let Some(limit) = self.iteration_limit { if count > limit { return span_source.err(format!("Iteration limit of {} exceeded", limit)); diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 45e9c9e2..391af287 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -67,7 +67,7 @@ impl InterpreterParseStream { Some(next_item) => { self.latest_item_span_range = next_item.span_range(); Some(next_item) - }, + } None => None, }) } @@ -75,7 +75,9 @@ impl InterpreterParseStream { pub(crate) fn next_item(&mut self, error_message: &'static str) -> Result { match self.next_item_or_end()? { Some(item) => Ok(item), - None => self.latest_item_span_range.err(format!("Unexpected end: {error_message}")), + None => self + .latest_item_span_range + .err(format!("Unexpected end: {error_message}")), } } @@ -86,9 +88,13 @@ impl InterpreterParseStream { } } - pub(crate) fn next_as_ident_matching(&mut self, ident_name: &str, error_message: &'static str) -> Result { + pub(crate) fn next_as_ident_matching( + &mut self, + ident_name: &str, + error_message: &'static str, + ) -> Result { match self.next_item(error_message)? { - NextItem::Ident(ident) if &ident.to_string() == ident_name => Ok(ident), + NextItem::Ident(ident) if ident.to_string() == ident_name => Ok(ident), other => other.err(error_message), } } @@ -100,24 +106,29 @@ impl InterpreterParseStream { } } - pub(crate) fn next_as_punct_matching(&mut self, char: char, error_message: &'static str) -> Result { + pub(crate) fn next_as_punct_matching( + &mut self, + char: char, + error_message: &'static str, + ) -> Result { match self.next_item(error_message)? { NextItem::Punct(punct) if punct.as_char() == char => Ok(punct), other => other.err(error_message), } } - pub(crate) fn next_as_kinded_group(&mut self, delimiter: Delimiter, error_message: &'static str) -> Result { + pub(crate) fn next_as_kinded_group( + &mut self, + delimiter: Delimiter, + error_message: &'static str, + ) -> Result { match self.next_item(error_message)? { NextItem::Group(group) if group.delimiter() == delimiter => Ok(group), other => other.err(error_message), } } - pub(crate) fn next_as_variable( - &mut self, - error_message: &'static str, - ) -> Result { + pub(crate) fn next_as_variable(&mut self, error_message: &'static str) -> Result { match self.next_item(error_message)? { NextItem::Variable(variable_substitution) => Ok(variable_substitution), other => other.err(error_message), diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 9e855686..5b7a685f 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -7,9 +7,9 @@ mod next_item; mod variable; pub(crate) use command::*; +pub(crate) use interpretation_stream::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; -pub(crate) use interpretation_stream::*; pub(crate) use interpreter_parse_stream::*; pub(crate) use next_item::*; pub(crate) use variable::*; diff --git a/src/interpretation/next_item.rs b/src/interpretation/next_item.rs index 240f1289..5c73f161 100644 --- a/src/interpretation/next_item.rs +++ b/src/interpretation/next_item.rs @@ -25,7 +25,8 @@ impl NextItem { } } TokenTree::Punct(punct) => { - if let Some(variable) = Variable::parse_consuming_only_if_match(&punct, parse_stream) + if let Some(variable) = + Variable::parse_consuming_only_if_match(&punct, parse_stream) { NextItem::Variable(variable) } else { diff --git a/src/lib.rs b/src/lib.rs index 03c1ddc8..58b3100d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -545,7 +545,8 @@ pub fn preinterpret(token_stream: proc_macro::TokenStream) -> proc_macro::TokenS fn preinterpret_internal(input: TokenStream) -> Result { let mut interpreter = Interpreter::new(); - let interpretation_stream = InterpretationStream::parse_from_token_stream(input, Span::call_site().span_range())?; + let interpretation_stream = + InterpretationStream::parse_from_token_stream(input, Span::call_site().span_range())?; let interpreted_stream = interpretation_stream.interpret_as_tokens(&mut interpreter)?; Ok(interpreted_stream.into_token_stream()) } From ecaaa06f61b62e45472e6e206dfcbf9f0019d01e Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 9 Jan 2025 02:47:45 +0000 Subject: [PATCH 016/126] fix: Fix ident equality check --- src/interpretation/interpreter_parse_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 391af287..95c1cbe0 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -94,7 +94,7 @@ impl InterpreterParseStream { error_message: &'static str, ) -> Result { match self.next_item(error_message)? { - NextItem::Ident(ident) if ident.to_string() == ident_name => Ok(ident), + NextItem::Ident(ident) if ident == ident_name => Ok(ident), other => other.err(error_message), } } From ed69555c391845dc28c3a85e9bd97d8474449e51 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 9 Jan 2025 02:55:18 +0000 Subject: [PATCH 017/126] fix: Fix style --- CHANGELOG.md | 3 ++- src/interpretation/variable.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29687e5b..6e759649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Grouping... Proposal: * #x is a group, #..x is flattened * Whether a command is flattened or not is dependent on the command + * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!..! ...]` may also be used instead. * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces @@ -32,7 +33,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * e.g. for various expressions * e.g. for long sums * Add compile failure tests -* `[!range! 0..5]` outputs `[0 1 2 3 4]` +* `[!range! 0..5]` outputs `0 1 2 3 4` * `[!error! "message" token stream for span]` * Reconfiguring iteration limit * Support `!else if!` in `!if!` diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 92a755dd..9fc68917 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -63,7 +63,7 @@ impl Variable { } } -impl<'a> Interpret for &'a Variable { +impl Interpret for &Variable { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, From 1fcb5d490b08839f78fc9fe0631718831499f60b Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 11 Jan 2025 02:08:32 +0000 Subject: [PATCH 018/126] Add compile error tests --- CHANGELOG.md | 5 +- Cargo.toml | 3 + regenerate-compilation-failures.sh | 5 ++ src/commands/concat_commands.rs | 28 +++--- src/commands/control_flow_commands.rs | 28 +++--- src/commands/core_commands.rs | 59 ++++++++++--- src/commands/expression_commands.rs | 14 ++- src/commands/mod.rs | 1 + src/commands/token_commands.rs | 32 ++----- src/expressions/evaluation_tree.rs | 3 +- src/expressions/expression_stream.rs | 6 +- src/expressions/value.rs | 6 +- src/interpretation/command.rs | 85 +++++++++---------- src/interpretation/interpretation_stream.rs | 2 +- src/interpretation/interpreted_stream.rs | 38 ++++++--- .../interpreter_parse_stream.rs | 4 + src/interpretation/variable.rs | 4 +- src/traits.rs | 52 ++++++++++-- .../compilation_failures/core/error_spans.rs | 15 ++++ .../core/error_spans.stderr | 10 +++ .../expressions/add_float_and_int.rs | 7 ++ .../expressions/add_float_and_int.stderr | 5 ++ .../expressions/braces.rs | 7 ++ .../expressions/braces.stderr | 5 ++ .../expressions/cast_int_to_bool.rs | 7 ++ .../expressions/cast_int_to_bool.stderr | 5 ++ .../expressions/compare_int_and_float.rs | 7 ++ .../expressions/compare_int_and_float.stderr | 5 ++ ...ix_me_comparison_operators_are_not_lazy.rs | 11 +++ ...e_comparison_operators_are_not_lazy.stderr | 13 +++ .../fix_me_negative_max_int_fails.rs | 12 +++ .../fix_me_negative_max_int_fails.stderr | 5 ++ tests/core.rs | 45 ++++++++++ tests/expressions.rs | 11 ++- tests/simple_test.rs | 1 + 35 files changed, 390 insertions(+), 156 deletions(-) create mode 100755 regenerate-compilation-failures.sh create mode 100644 tests/compilation_failures/core/error_spans.rs create mode 100644 tests/compilation_failures/core/error_spans.stderr create mode 100644 tests/compilation_failures/expressions/add_float_and_int.rs create mode 100644 tests/compilation_failures/expressions/add_float_and_int.stderr create mode 100644 tests/compilation_failures/expressions/braces.rs create mode 100644 tests/compilation_failures/expressions/braces.stderr create mode 100644 tests/compilation_failures/expressions/cast_int_to_bool.rs create mode 100644 tests/compilation_failures/expressions/cast_int_to_bool.stderr create mode 100644 tests/compilation_failures/expressions/compare_int_and_float.rs create mode 100644 tests/compilation_failures/expressions/compare_int_and_float.stderr create mode 100644 tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs create mode 100644 tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr create mode 100644 tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs create mode 100644 tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr create mode 100644 tests/core.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e759649..e150d65e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. +* Other commands: + * `[!error! ..]` I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. @@ -34,7 +36,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * e.g. for long sums * Add compile failure tests * `[!range! 0..5]` outputs `0 1 2 3 4` -* `[!error! "message" token stream for span]` +* `[!error! "message" [token stream for span]]` +* Parse fields `{ ... }` and change `!error! to use them as per https://github.com/rust-lang/rust/issues/54140#issuecomment-2585002922. * Reconfiguring iteration limit * Support `!else if!` in `!if!` * `[!extend! #x += ...]` to make such actions more performant diff --git a/Cargo.toml b/Cargo.toml index 7213cd89..f1b023e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,6 @@ proc-macro = true proc-macro2 = { version = "1.0" } syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing"] } quote = { version = "1.0", default-features = false } + +[dev-dependencies] +trybuild = { version = "1.0.66", features = ["diff"] } diff --git a/regenerate-compilation-failures.sh b/regenerate-compilation-failures.sh new file mode 100755 index 00000000..ceefa46e --- /dev/null +++ b/regenerate-compilation-failures.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +TRYBUILD=overwrite cargo test \ No newline at end of file diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 7829a200..01134ab4 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -28,33 +28,33 @@ fn concat_into_string( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> Result { let output_span = input.span(); let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); let string_literal = string_literal(&conversion_fn(&concatenated), output_span); - Ok(InterpretedStream::of_literal(string_literal)) + Ok(CommandOutput::Literal(string_literal)) } fn concat_into_ident( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> Result { let output_span = input.span(); let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); let ident = parse_ident(&conversion_fn(&concatenated), output_span)?; - Ok(InterpretedStream::of_ident(ident)) + Ok(CommandOutput::Ident(ident)) } fn concat_into_literal( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> Result { let output_span = input.span(); let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); let literal = parse_literal(&conversion_fn(&concatenated), output_span)?; - Ok(InterpretedStream::of_literal(literal)) + Ok(CommandOutput::Literal(literal)) } fn concat_recursive(arguments: InterpretedStream) -> String { @@ -62,15 +62,9 @@ fn concat_recursive(arguments: InterpretedStream) -> String { for token_tree in arguments { match token_tree { TokenTree::Literal(literal) => { - let lit: Lit = parse_str(&literal.to_string()).expect( - "All proc_macro2::Literal values should be decodable as a syn::Lit", - ); - match lit { - Lit::Str(lit_str) => output.push_str(&lit_str.value()), - Lit::Char(lit_char) => output.push(lit_char.value()), - _ => { - output.push_str(&literal.to_string()); - } + match literal.content_if_string_or_char() { + Some(content) => output.push_str(&content), + None => output.push_str(&literal.to_string()), } } TokenTree::Group(group) => match group.delimiter() { @@ -118,8 +112,6 @@ macro_rules! define_concat_command { impl CommandDefinition for $command { const COMMAND_NAME: &'static str = $command_name; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; - fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, @@ -131,7 +123,7 @@ macro_rules! define_concat_command { fn execute( self: Box, interpreter: &mut Interpreter, - ) -> Result { + ) -> Result { $output_fn(self.arguments, interpreter, $conversion_fn) } } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 5f10101c..8f58b0ce 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -5,13 +5,12 @@ pub(crate) struct IfCommand { condition: NextItem, true_code: InterpretationStream, false_code: Option, + nothing_span_range: SpanRange, } impl CommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { static ERROR: &str = "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"; @@ -37,12 +36,13 @@ impl CommandDefinition for IfCommand { condition, true_code, false_code, + nothing_span_range: arguments.full_span_range(), }) } } impl CommandInvocation for IfCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let evaluated_condition = self .condition .interpret_as_expression(interpreter)? @@ -50,13 +50,15 @@ impl CommandInvocation for IfCommand { .expect_bool("An if condition must evaluate to a boolean")? .value(); - if evaluated_condition { - self.true_code.interpret_as_tokens(interpreter) + let output = if evaluated_condition { + self.true_code.interpret_as_tokens(interpreter)? } else if let Some(false_code) = self.false_code { - false_code.interpret_as_tokens(interpreter) + false_code.interpret_as_tokens(interpreter)? } else { - Ok(InterpretedStream::new()) - } + InterpretedStream::new(self.nothing_span_range) + }; + + Ok(CommandOutput::AppendStream(output)) } } @@ -64,13 +66,12 @@ impl CommandInvocation for IfCommand { pub(crate) struct WhileCommand { condition: NextItem, loop_code: InterpretationStream, + nothing_span_range: SpanRange, } impl CommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { static ERROR: &str = "Expected [!while! (condition) { code }]"; @@ -83,13 +84,14 @@ impl CommandDefinition for WhileCommand { Ok(Self { condition, loop_code, + nothing_span_range: arguments.full_span_range(), }) } } impl CommandInvocation for WhileCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(); + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let mut output = InterpretedStream::new(self.nothing_span_range); let mut iteration_count = 0; loop { let evaluated_condition = self @@ -113,6 +115,6 @@ impl CommandInvocation for WhileCommand { .interpret_as_tokens_into(interpreter, &mut output)?; } - Ok(output) + Ok(CommandOutput::AppendStream(output)) } } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 82759db3..aa3f45d9 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -11,8 +11,6 @@ pub(crate) struct SetCommand { impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { static ERROR: &str = "Expected [!set! #variable = ... ]"; Ok(Self { @@ -24,34 +22,34 @@ impl CommandDefinition for SetCommand { } impl CommandInvocation for SetCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let result_tokens = self.arguments.interpret_as_tokens(interpreter)?; self.variable.set(interpreter, result_tokens); - Ok(InterpretedStream::new()) + Ok(CommandOutput::Empty) } } #[derive(Clone)] pub(crate) struct RawCommand { + arguments_span_range: SpanRange, token_stream: TokenStream, } impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { + arguments_span_range: arguments.full_span_range(), token_stream: arguments.read_all_as_raw_token_stream(), }) } } impl CommandInvocation for RawCommand { - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(InterpretedStream::raw(self.token_stream)) + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { + Ok(CommandOutput::AppendStream(InterpretedStream::raw(self.arguments_span_range, self.token_stream))) } } @@ -61,15 +59,52 @@ pub(crate) struct IgnoreCommand; impl CommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; - fn parse(_arguments: InterpreterParseStream) -> Result { Ok(Self) } } impl CommandInvocation for IgnoreCommand { - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(InterpretedStream::new()) + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { + Ok(CommandOutput::Empty) } } + +#[derive(Clone)] +pub(crate) struct ErrorCommand { + message: NextItem, + error_span_stream: InterpretationStream, +} + +impl CommandDefinition for ErrorCommand { + const COMMAND_NAME: &'static str = "error"; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + static ERROR: &str = "Expected [!error! \"Error message\" [tokens spanning error]], for example:\n* [!error! \"Compiler error message\" [$tokens_covering_error]]\n * [!error! [!string! \"My Error\" \"in bits\"] []]"; + let parsed = Self { + message: arguments.next_item(ERROR)?, + error_span_stream: arguments.next_as_kinded_group(Delimiter::Bracket, ERROR)?.into_inner_stream(), + }; + arguments.assert_end(ERROR)?; + Ok(parsed) + } +} + +impl CommandInvocation for ErrorCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + static ERROR: &str = "Expected a single string literal as the error message"; + let message_span_range = self.message.span_range(); + let message = self.message.interpret_as_tokens(interpreter)? + .as_singleton(ERROR)? + .to_literal(ERROR)? + .content_if_string() + .ok_or_else(|| message_span_range.error(ERROR))?; + let error_span_stream = self.error_span_stream.interpret_as_tokens(interpreter)?; + + if error_span_stream.is_empty() { + return Span::call_site().err(message); + } else { + error_span_stream.into_token_stream().span_range().err(message) + } + } +} \ No newline at end of file diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index bd53d953..1446c8c3 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -8,8 +8,6 @@ pub(crate) struct EvaluateCommand { impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; - fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { expression: arguments.parse_all_for_interpretation()?, @@ -18,9 +16,9 @@ impl CommandDefinition for EvaluateCommand { } impl CommandInvocation for EvaluateCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let expression = self.expression.interpret_as_expression(interpreter)?; - Ok(expression.evaluate()?.into_interpreted_stream()) + Ok(CommandOutput::GroupedStream(expression.evaluate()?.into_interpreted_stream())) } } @@ -36,8 +34,6 @@ pub(crate) struct AssignCommand { impl CommandDefinition for AssignCommand { const COMMAND_NAME: &'static str = "assign"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { static ERROR: &str = "Expected [!assign! #variable += ...] for + or some other operator supported in an expression"; Ok(Self { @@ -57,7 +53,7 @@ impl CommandDefinition for AssignCommand { } impl CommandInvocation for AssignCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let Self { variable, operator, @@ -65,7 +61,7 @@ impl CommandInvocation for AssignCommand { expression, } = *self; - let mut expression_stream = ExpressionStream::new(); + let mut expression_stream = ExpressionStream::new(expression.span_range()); variable.interpret_as_expression_into(interpreter, &mut expression_stream)?; expression_stream.push_punct(operator); expression.interpret_as_expression_into(interpreter, &mut expression_stream)?; @@ -73,6 +69,6 @@ impl CommandInvocation for AssignCommand { let output = expression_stream.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output); - Ok(InterpretedStream::new()) + Ok(CommandOutput::Empty) } } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 6635c646..ab39f40a 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -16,6 +16,7 @@ define_command_kind! { SetCommand, RawCommand, IgnoreCommand, + ErrorCommand, // Concat & Type Convert Commands StringCommand, diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 527e0af8..f5a9b5d3 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -6,8 +6,6 @@ pub(crate) struct EmptyCommand; impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::EmptyStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { arguments.assert_end("The !empty! command does not take any arguments")?; Ok(Self) @@ -15,8 +13,8 @@ impl CommandDefinition for EmptyCommand { } impl CommandInvocation for EmptyCommand { - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(InterpretedStream::new()) + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { + Ok(CommandOutput::Empty) } } @@ -28,8 +26,6 @@ pub(crate) struct IsEmptyCommand { impl CommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; - fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, @@ -38,10 +34,10 @@ impl CommandDefinition for IsEmptyCommand { } impl CommandInvocation for IsEmptyCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_as_tokens(interpreter)?; - Ok(TokenTree::bool(interpreted.is_empty(), output_span).into()) + Ok(CommandOutput::Ident(Ident::new_bool(interpreted.is_empty(), output_span))) } } @@ -53,8 +49,6 @@ pub(crate) struct LengthCommand { impl CommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::SingleToken; - fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, @@ -63,12 +57,12 @@ impl CommandDefinition for LengthCommand { } impl CommandInvocation for LengthCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_as_tokens(interpreter)?; let stream_length = interpreted.into_token_stream().into_iter().count(); let length_literal = Literal::usize_unsuffixed(stream_length).with_span(output_span); - Ok(InterpretedStream::of_literal(length_literal)) + Ok(CommandOutput::Literal(length_literal)) } } @@ -80,8 +74,6 @@ pub(crate) struct GroupCommand { impl CommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour = CommandOutputBehaviour::AppendStream; - fn parse(mut arguments: InterpreterParseStream) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, @@ -90,14 +82,8 @@ impl CommandDefinition for GroupCommand { } impl CommandInvocation for GroupCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(); - let span_range = self.arguments.span_range(); - output.push_new_group( - self.arguments.interpret_as_tokens(interpreter)?, - Delimiter::None, - span_range, - ); - Ok(output) + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let output = self.arguments.interpret_as_tokens(interpreter)?; + Ok(CommandOutput::GroupedStream(output)) } } diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index 088f41bb..a84a821f 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -434,7 +434,8 @@ impl EvaluationOutput { } pub(crate) fn into_interpreted_stream(self) -> InterpretedStream { - InterpretedStream::raw(self.into_token_stream()) + let value = self.into_value(); + InterpretedStream::raw(value.source_span(), value.into_token_stream()) } } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 7d84cb32..d15f001c 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -13,9 +13,9 @@ pub(crate) struct ExpressionStream { } impl ExpressionStream { - pub(crate) fn new() -> Self { + pub(crate) fn new(source_span_range: SpanRange) -> Self { Self { - interpreted_stream: InterpretedStream::new(), + interpreted_stream: InterpretedStream::new(source_span_range), } } @@ -32,7 +32,7 @@ impl ExpressionStream { self.interpreted_stream.push_punct(punct); } - pub(crate) fn push_interpreted_group( + pub(crate) fn push_grouped_interpreted_stream( &mut self, contents: InterpretedStream, span_range: SpanRange, diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 114277a7..fcf2fe63 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -69,11 +69,7 @@ impl EvaluationValue { impl HasSpanRange for EvaluationValue { fn span_range(&self) -> SpanRange { - match self { - Self::Integer(int) => int.source_span, - Self::Float(float) => float.source_span, - Self::Boolean(bool) => bool.source_span, - } + self.source_span() } } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index cd9f56f6..2a016885 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,22 +1,21 @@ use crate::internal_prelude::*; -pub(crate) enum CommandOutputBehaviour { - EmptyStream, - #[allow(unused)] // Likely useful in future - GroupedStream, - AppendStream, - SingleToken, -} - pub(crate) trait CommandDefinition: CommandInvocation + Clone { const COMMAND_NAME: &'static str; - const OUTPUT_BEHAVIOUR: CommandOutputBehaviour; fn parse(arguments: InterpreterParseStream) -> Result; } pub(crate) trait CommandInvocation { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; +} + +pub(crate) enum CommandOutput { + Empty, + Literal(Literal), + Ident(Ident), + AppendStream(InterpretedStream), + GroupedStream(InterpretedStream), } pub(crate) trait ClonableCommandInvocation: CommandInvocation { @@ -60,14 +59,6 @@ macro_rules! define_command_kind { }) } - pub(crate) fn output_behaviour(&self) -> CommandOutputBehaviour { - match self { - $( - Self::$command => <$command as CommandDefinition>::OUTPUT_BEHAVIOUR, - )* - } - } - pub(crate) fn for_ident(ident: &Ident) -> Option { Some(match ident.to_string().as_ref() { $( @@ -90,9 +81,8 @@ pub(crate) use define_command_kind; #[derive(Clone)] pub(crate) struct Command { - command_kind: CommandKind, - source_group_span_range: SpanRange, invocation: Box, + source_group_span_range: SpanRange, } impl Command { @@ -129,9 +119,8 @@ impl Command { Some(command_kind) => { let invocation = command_kind.parse_invocation( parse_stream)?; Ok(Some(Self { - command_kind, - source_group_span_range: group.span_range(), invocation, + source_group_span_range: group.span_range(), })) }, None => Err(command_ident.span().error( @@ -143,25 +132,6 @@ impl Command { )), } } - - fn execute_into( - self, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()> { - let substitution = self.invocation.execute(interpreter)?; - match self.command_kind.output_behaviour() { - CommandOutputBehaviour::GroupedStream => { - output.push_new_group(substitution, Delimiter::None, self.source_group_span_range); - } - CommandOutputBehaviour::EmptyStream - | CommandOutputBehaviour::SingleToken - | CommandOutputBehaviour::AppendStream => { - output.extend(substitution); - } - } - Ok(()) - } } impl HasSpanRange for Command { @@ -176,7 +146,22 @@ impl Interpret for Command { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.execute_into(interpreter, output) + match self.invocation.execute(interpreter)? { + CommandOutput::Empty => {}, + CommandOutput::Literal(literal) => { + output.push_literal(literal); + }, + CommandOutput::Ident(ident) => { + output.push_ident(ident); + }, + CommandOutput::AppendStream(stream) => { + output.extend(stream); + }, + CommandOutput::GroupedStream(stream) => { + output.push_new_group(stream, Delimiter::None, self.source_group_span_range); + }, + }; + Ok(()) } fn interpret_as_expression_into( @@ -184,9 +169,19 @@ impl Interpret for Command { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - let span_range = self.span_range(); - expression_stream - .push_interpreted_group(self.interpret_as_tokens(interpreter)?, span_range); + match self.invocation.execute(interpreter)? { + CommandOutput::Empty => {}, + CommandOutput::Literal(literal) => { + expression_stream.push_literal(literal); + }, + CommandOutput::Ident(ident) => { + expression_stream.push_ident(ident); + }, + CommandOutput::AppendStream(stream) + | CommandOutput::GroupedStream(stream) => { + expression_stream.push_grouped_interpreted_stream(stream, self.source_group_span_range); + }, + }; Ok(()) } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 970677cd..77a16a7c 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -44,7 +44,7 @@ impl Interpret for InterpretationStream { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - let mut inner_expression_stream = ExpressionStream::new(); + let mut inner_expression_stream = ExpressionStream::new(self.span_range); for item in self.items { item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)?; } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index f95552d1..5aa20cb1 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -2,26 +2,23 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct InterpretedStream { + source_span_range: SpanRange, token_stream: TokenStream, } impl InterpretedStream { - pub(crate) fn new() -> Self { + pub(crate) fn new(source_span_range: SpanRange) -> Self { Self { + source_span_range, token_stream: TokenStream::new(), } } - pub(crate) fn raw(token_stream: TokenStream) -> Self { - Self { token_stream } - } - - pub(crate) fn of_ident(ident: Ident) -> Self { - TokenTree::Ident(ident).into() - } - - pub(crate) fn of_literal(literal: Literal) -> Self { - TokenTree::Literal(literal).into() + pub(crate) fn raw(empty_span_range: SpanRange, token_stream: TokenStream) -> Self { + Self { + source_span_range: empty_span_range, + token_stream, + } } pub(crate) fn extend(&mut self, interpreted_stream: InterpretedStream) { @@ -61,6 +58,18 @@ impl InterpretedStream { self.token_stream.is_empty() } + pub(crate) fn as_singleton(self, error_message: &str) -> Result { + if self.is_empty() { + return self.source_span_range.err(error_message); + } + let mut iter = self.token_stream.into_iter(); + let first = iter.next().unwrap(); // No panic because we're not empty + match iter.next() { + Some(_) => self.source_span_range.err(error_message), + None => Ok(first), + } + } + pub(crate) fn into_token_stream(self) -> TokenStream { self.token_stream } @@ -69,6 +78,7 @@ impl InterpretedStream { impl From for InterpretedStream { fn from(value: TokenTree) -> Self { InterpretedStream { + source_span_range: value.span_range(), token_stream: value.into(), } } @@ -76,6 +86,10 @@ impl From for InterpretedStream { impl HasSpanRange for InterpretedStream { fn span_range(&self) -> SpanRange { - self.token_stream.span_range() + if self.token_stream.is_empty() { + self.source_span_range + } else { + self.token_stream.span_range() + } } } diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 95c1cbe0..58df1136 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -53,6 +53,10 @@ impl InterpreterParseStream { } } + pub(crate) fn full_span_range(&self) -> SpanRange { + self.full_span_range + } + pub(super) fn peek_token_tree(&mut self) -> Option<&TokenTree> { self.tokens.peek() } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 9fc68917..4401a623 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -78,12 +78,12 @@ impl Interpret for &Variable { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - expression_stream.push_interpreted_group(self.substitute(interpreter)?, self.span_range()); + expression_stream.push_grouped_interpreted_stream(self.substitute(interpreter)?, self.span_range()); Ok(()) } } -impl HasSpanRange for Variable { +impl HasSpanRange for &Variable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span(), self.variable_name.span()) } diff --git a/src/traits.rs b/src/traits.rs index f105f81b..cfe94d21 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,6 +1,6 @@ use crate::internal_prelude::*; -pub(crate) trait Interpret: Sized { +pub(crate) trait Interpret: Sized + HasSpanRange { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, @@ -8,7 +8,7 @@ pub(crate) trait Interpret: Sized { ) -> Result<()>; fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(); + let mut output = InterpretedStream::new(self.span_range()); self.interpret_as_tokens_into(interpreter, &mut output)?; Ok(output) } @@ -20,25 +20,61 @@ pub(crate) trait Interpret: Sized { ) -> Result<()>; fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { - let mut output = ExpressionStream::new(); + let mut output = ExpressionStream::new(self.span_range()); self.interpret_as_expression_into(interpreter, &mut output)?; Ok(output) } } +pub(crate) trait IdentExt: Sized { + fn new_bool(value: bool, span: Span) -> Self; +} + +impl IdentExt for Ident { + fn new_bool(value: bool, span: Span) -> Self { + Ident::new(&value.to_string(), span) + } +} + + +pub(crate) trait LiteralExt: Sized { + fn content_if_string(&self) -> Option; + fn content_if_string_or_char(&self) -> Option; +} + +impl LiteralExt for Literal { + fn content_if_string(&self) -> Option { + match parse_str::(&self.to_string()).unwrap() { + Lit::Str(lit_str) => Some(lit_str.value()), + _ => None, + } + } + + fn content_if_string_or_char(&self) -> Option { + match parse_str::(&self.to_string()).unwrap() { + Lit::Str(lit_str) => Some(lit_str.value()), + Lit::Char(lit_char) => Some(lit_char.value().to_string()), + _ => None, + } + } +} + pub(crate) trait TokenTreeExt: Sized { - fn bool(value: bool, span: Span) -> Self; fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; + fn to_literal(self, error_message: &str) -> Result; } impl TokenTreeExt for TokenTree { - fn bool(value: bool, span: Span) -> Self { - TokenTree::Ident(Ident::new(&value.to_string(), span)) - } - fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) } + + fn to_literal(self, error_message: &str) -> Result { + match self { + TokenTree::Literal(literal) => Ok(literal), + other => other.err(error_message), + } + } } pub(crate) trait SpanErrorExt: Sized { diff --git a/tests/compilation_failures/core/error_spans.rs b/tests/compilation_failures/core/error_spans.rs new file mode 100644 index 00000000..2032661b --- /dev/null +++ b/tests/compilation_failures/core/error_spans.rs @@ -0,0 +1,15 @@ +use preinterpret::*; + +macro_rules! assert_is_100 { + ($input:literal) => {preinterpret!{ + [!if! ($input != 100) { + [!error! [!string! "Expected 100, got " $input] [$input]] + }] + }}; +} + +fn main() { + // In rust analyzer, the span is in the correct location + // But in rustc it's not... I'm not really sure why. + assert_is_100!(5); +} \ No newline at end of file diff --git a/tests/compilation_failures/core/error_spans.stderr b/tests/compilation_failures/core/error_spans.stderr new file mode 100644 index 00000000..a1fc2fb5 --- /dev/null +++ b/tests/compilation_failures/core/error_spans.stderr @@ -0,0 +1,10 @@ +error: Expected 100, got 5 + --> tests/compilation_failures/core/error_spans.rs:6:62 + | +6 | [!error! [!string! "Expected 100, got " $input] [$input]] + | ^^^^^^ +... +14 | assert_is_100!(5); + | ----------------- in this macro invocation + | + = note: this error originates in the macro `assert_is_100` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/compilation_failures/expressions/add_float_and_int.rs b/tests/compilation_failures/expressions/add_float_and_int.rs new file mode 100644 index 00000000..9daca0b4 --- /dev/null +++ b/tests/compilation_failures/expressions/add_float_and_int.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!evaluate! 1.2 + 1] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/add_float_and_int.stderr b/tests/compilation_failures/expressions/add_float_and_int.stderr new file mode 100644 index 00000000..7cd3f383 --- /dev/null +++ b/tests/compilation_failures/expressions/add_float_and_int.stderr @@ -0,0 +1,5 @@ +error: The + operator cannot infer a common operand type from untyped float and untyped integer. Consider using `as` to cast to matching types. + --> tests/compilation_failures/expressions/add_float_and_int.rs:5:25 + | +5 | [!evaluate! 1.2 + 1] + | ^ diff --git a/tests/compilation_failures/expressions/braces.rs b/tests/compilation_failures/expressions/braces.rs new file mode 100644 index 00000000..21f141fd --- /dev/null +++ b/tests/compilation_failures/expressions/braces.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!evaluate! {true}] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr new file mode 100644 index 00000000..b07fc912 --- /dev/null +++ b/tests/compilation_failures/expressions/braces.stderr @@ -0,0 +1,5 @@ +error: This expression is not supported in preinterpret expressions + --> tests/compilation_failures/expressions/braces.rs:5:21 + | +5 | [!evaluate! {true}] + | ^^^^^^ diff --git a/tests/compilation_failures/expressions/cast_int_to_bool.rs b/tests/compilation_failures/expressions/cast_int_to_bool.rs new file mode 100644 index 00000000..891ef418 --- /dev/null +++ b/tests/compilation_failures/expressions/cast_int_to_bool.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!evaluate! 1 as bool] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/cast_int_to_bool.stderr b/tests/compilation_failures/expressions/cast_int_to_bool.stderr new file mode 100644 index 00000000..a8c67dc7 --- /dev/null +++ b/tests/compilation_failures/expressions/cast_int_to_bool.stderr @@ -0,0 +1,5 @@ +error: This cast is not supported + --> tests/compilation_failures/expressions/cast_int_to_bool.rs:5:23 + | +5 | [!evaluate! 1 as bool] + | ^^ diff --git a/tests/compilation_failures/expressions/compare_int_and_float.rs b/tests/compilation_failures/expressions/compare_int_and_float.rs new file mode 100644 index 00000000..e2f7b29b --- /dev/null +++ b/tests/compilation_failures/expressions/compare_int_and_float.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!evaluate! 5 < 6.4] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/compare_int_and_float.stderr b/tests/compilation_failures/expressions/compare_int_and_float.stderr new file mode 100644 index 00000000..4208a21a --- /dev/null +++ b/tests/compilation_failures/expressions/compare_int_and_float.stderr @@ -0,0 +1,5 @@ +error: The < operator cannot infer a common operand type from untyped integer and untyped float. Consider using `as` to cast to matching types. + --> tests/compilation_failures/expressions/compare_int_and_float.rs:5:23 + | +5 | [!evaluate! 5 < 6.4] + | ^ diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs new file mode 100644 index 00000000..d267595a --- /dev/null +++ b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + preinterpret!{ + [!set! #is_eager = false] + let _ = [!evaluate! false && ([!set! #is_eager = true] true)]; + [!if! #is_eager { + [!error! "The && expression is not evaluated lazily" []] + }] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr new file mode 100644 index 00000000..6362c4d5 --- /dev/null +++ b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr @@ -0,0 +1,13 @@ +error: The && expression is not evaluated lazily + --> tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs:4:5 + | +4 | / preinterpret!{ +5 | | [!set! #is_eager = false] +6 | | let _ = [!evaluate! false && ([!set! #is_eager = true] true)]; +7 | | [!if! #is_eager { +8 | | [!error! "The && expression is not evaluated lazily" []] +9 | | }] +10 | | } + | |_____^ + | + = note: this error originates in the macro `preinterpret` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs new file mode 100644 index 00000000..a051e68c --- /dev/null +++ b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs @@ -0,0 +1,12 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret! { + // This should not fail, and should be fixed. + // This test just records the fact it doesn't work as a known issue. + // A fix of this should remove this test and move it to a working test. + [!evaluate! -128i8] + // This should also not fail according to the rules of rustc. + [!evaluate! -(--128i8)] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr new file mode 100644 index 00000000..03455a9d --- /dev/null +++ b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr @@ -0,0 +1,5 @@ +error: number too large to fit in target type + --> tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs:8:22 + | +8 | [!evaluate! -128i8] + | ^^^^^ diff --git a/tests/core.rs b/tests/core.rs new file mode 100644 index 00000000..9e5471e3 --- /dev/null +++ b/tests/core.rs @@ -0,0 +1,45 @@ +use preinterpret::preinterpret; + +macro_rules! my_assert_eq { + ($input:tt, $($output:tt)*) => { + assert_eq!(preinterpret!($input), $($output)*); + }; +} + + +#[test] +fn test_core_compilation_failures() { + let t = trybuild::TestCases::new(); + // In particular, the "error" command is tested here. + t.compile_fail("tests/compilation_failures/core/*.rs"); +} + +#[test] +fn test_set() { + my_assert_eq!({ + [!set! #output = "Hello World!"] + #output + }, "Hello World!"); + my_assert_eq!({ + [!set! #hello = "Hello"] + [!set! #world = "World"] + [!set! #output = #hello " " #world "!"] + [!set! #output = [!string! #output]] + #output + }, "Hello World!"); +} + +#[test] +fn test_raw() { + my_assert_eq!({ + [!string! [!raw! #variable and [!command!] are not interpreted or error]] + }, "#variableand[!command!]arenotinterpretedorerror"); +} + +#[test] +fn test_ignore() { + my_assert_eq!({ + true + [!ignore! this is all just ignored!] + }, true); +} diff --git a/tests/expressions.rs b/tests/expressions.rs index 20b253bb..5c869e0f 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -6,6 +6,12 @@ macro_rules! assert_preinterpret_eq { }; } +#[test] +fn test_expression_compilation_failures() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/compilation_failures/expressions/*.rs"); +} + #[test] fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! !!(!!(true))], true); @@ -17,6 +23,8 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! 3.6 + 3999999999999999992.0], 3.6 + 3999999999999999992.0); assert_preinterpret_eq!([!evaluate! -3.2], -3.2); assert_preinterpret_eq!([!evaluate! true && true || false], true); + assert_preinterpret_eq!([!evaluate! true || false && false], true); // The && has priority + assert_preinterpret_eq!([!evaluate! true | false & false], true); // The & has priority assert_preinterpret_eq!([!evaluate! true as u32 + 2], 3); assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u32); assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u64); @@ -52,6 +60,3 @@ fn assign_works() { 12 ); } - -// TODO - Add failing tests for these: -// assert_preinterpret_eq!([!evaluate! !!(!!({true}))], true); diff --git a/tests/simple_test.rs b/tests/simple_test.rs index 2775098b..cb67e500 100644 --- a/tests/simple_test.rs +++ b/tests/simple_test.rs @@ -3,6 +3,7 @@ use preinterpret::preinterpret; preinterpret! { [!set! #bytes = 32] [!set! #postfix = Hello World #bytes] + [!set! #some_symbols = and some symbols such as [!raw! #] and #123] [!set! #MyRawVar = [!raw! Test no #str [!ident! replacement]]] [!ignore! non - sensical !code :D - ignored (!)] struct MyStruct; From dbf207f6be49e3cd7bb31a6ffc4f4845923d48d4 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 11 Jan 2025 02:09:46 +0000 Subject: [PATCH 019/126] fix: Fix style --- src/commands/concat_commands.rs | 15 +++++---------- src/commands/core_commands.rs | 24 +++++++++++++++++------- src/commands/expression_commands.rs | 4 +++- src/commands/token_commands.rs | 5 ++++- src/interpretation/command.rs | 24 ++++++++++++------------ src/interpretation/interpreted_stream.rs | 2 +- src/interpretation/variable.rs | 3 ++- src/traits.rs | 5 ++--- tests/core.rs | 8 ++++---- tests/expressions.rs | 2 +- 10 files changed, 51 insertions(+), 41 deletions(-) diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 01134ab4..3132c646 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -61,12 +61,10 @@ fn concat_recursive(arguments: InterpretedStream) -> String { fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { for token_tree in arguments { match token_tree { - TokenTree::Literal(literal) => { - match literal.content_if_string_or_char() { - Some(content) => output.push_str(&content), - None => output.push_str(&literal.to_string()), - } - } + TokenTree::Literal(literal) => match literal.content_if_string_or_char() { + Some(content) => output.push_str(&content), + None => output.push_str(&literal.to_string()), + }, TokenTree::Group(group) => match group.delimiter() { Delimiter::Parenthesis => { output.push('('); @@ -120,10 +118,7 @@ macro_rules! define_concat_command { } impl CommandInvocation for $command { - fn execute( - self: Box, - interpreter: &mut Interpreter, - ) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { $output_fn(self.arguments, interpreter, $conversion_fn) } } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index aa3f45d9..b6ee431e 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -49,7 +49,10 @@ impl CommandDefinition for RawCommand { impl CommandInvocation for RawCommand { fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::AppendStream(InterpretedStream::raw(self.arguments_span_range, self.token_stream))) + Ok(CommandOutput::AppendStream(InterpretedStream::raw( + self.arguments_span_range, + self.token_stream, + ))) } } @@ -83,7 +86,9 @@ impl CommandDefinition for ErrorCommand { static ERROR: &str = "Expected [!error! \"Error message\" [tokens spanning error]], for example:\n* [!error! \"Compiler error message\" [$tokens_covering_error]]\n * [!error! [!string! \"My Error\" \"in bits\"] []]"; let parsed = Self { message: arguments.next_item(ERROR)?, - error_span_stream: arguments.next_as_kinded_group(Delimiter::Bracket, ERROR)?.into_inner_stream(), + error_span_stream: arguments + .next_as_kinded_group(Delimiter::Bracket, ERROR)? + .into_inner_stream(), }; arguments.assert_end(ERROR)?; Ok(parsed) @@ -94,17 +99,22 @@ impl CommandInvocation for ErrorCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { static ERROR: &str = "Expected a single string literal as the error message"; let message_span_range = self.message.span_range(); - let message = self.message.interpret_as_tokens(interpreter)? - .as_singleton(ERROR)? + let message = self + .message + .interpret_as_tokens(interpreter)? + .into_singleton(ERROR)? .to_literal(ERROR)? .content_if_string() .ok_or_else(|| message_span_range.error(ERROR))?; let error_span_stream = self.error_span_stream.interpret_as_tokens(interpreter)?; if error_span_stream.is_empty() { - return Span::call_site().err(message); + Span::call_site().err(message) } else { - error_span_stream.into_token_stream().span_range().err(message) + error_span_stream + .into_token_stream() + .span_range() + .err(message) } } -} \ No newline at end of file +} diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 1446c8c3..505a6f41 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -18,7 +18,9 @@ impl CommandDefinition for EvaluateCommand { impl CommandInvocation for EvaluateCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let expression = self.expression.interpret_as_expression(interpreter)?; - Ok(CommandOutput::GroupedStream(expression.evaluate()?.into_interpreted_stream())) + Ok(CommandOutput::GroupedStream( + expression.evaluate()?.into_interpreted_stream(), + )) } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index f5a9b5d3..adb249bd 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -37,7 +37,10 @@ impl CommandInvocation for IsEmptyCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_as_tokens(interpreter)?; - Ok(CommandOutput::Ident(Ident::new_bool(interpreted.is_empty(), output_span))) + Ok(CommandOutput::Ident(Ident::new_bool( + interpreted.is_empty(), + output_span, + ))) } } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 2a016885..51af8038 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -147,19 +147,19 @@ impl Interpret for Command { output: &mut InterpretedStream, ) -> Result<()> { match self.invocation.execute(interpreter)? { - CommandOutput::Empty => {}, + CommandOutput::Empty => {} CommandOutput::Literal(literal) => { output.push_literal(literal); - }, + } CommandOutput::Ident(ident) => { output.push_ident(ident); - }, + } CommandOutput::AppendStream(stream) => { output.extend(stream); - }, + } CommandOutput::GroupedStream(stream) => { output.push_new_group(stream, Delimiter::None, self.source_group_span_range); - }, + } }; Ok(()) } @@ -170,17 +170,17 @@ impl Interpret for Command { expression_stream: &mut ExpressionStream, ) -> Result<()> { match self.invocation.execute(interpreter)? { - CommandOutput::Empty => {}, + CommandOutput::Empty => {} CommandOutput::Literal(literal) => { expression_stream.push_literal(literal); - }, + } CommandOutput::Ident(ident) => { expression_stream.push_ident(ident); - }, - CommandOutput::AppendStream(stream) - | CommandOutput::GroupedStream(stream) => { - expression_stream.push_grouped_interpreted_stream(stream, self.source_group_span_range); - }, + } + CommandOutput::AppendStream(stream) | CommandOutput::GroupedStream(stream) => { + expression_stream + .push_grouped_interpreted_stream(stream, self.source_group_span_range); + } }; Ok(()) } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 5aa20cb1..747bc4a6 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -58,7 +58,7 @@ impl InterpretedStream { self.token_stream.is_empty() } - pub(crate) fn as_singleton(self, error_message: &str) -> Result { + pub(crate) fn into_singleton(self, error_message: &str) -> Result { if self.is_empty() { return self.source_span_range.err(error_message); } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 4401a623..23907a1c 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -78,7 +78,8 @@ impl Interpret for &Variable { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - expression_stream.push_grouped_interpreted_stream(self.substitute(interpreter)?, self.span_range()); + expression_stream + .push_grouped_interpreted_stream(self.substitute(interpreter)?, self.span_range()); Ok(()) } } diff --git a/src/traits.rs b/src/traits.rs index cfe94d21..6d428120 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -36,7 +36,6 @@ impl IdentExt for Ident { } } - pub(crate) trait LiteralExt: Sized { fn content_if_string(&self) -> Option; fn content_if_string_or_char(&self) -> Option; @@ -68,8 +67,8 @@ impl TokenTreeExt for TokenTree { fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) } - - fn to_literal(self, error_message: &str) -> Result { + + fn to_literal(self, error_message: &str) -> Result { match self { TokenTree::Literal(literal) => Ok(literal), other => other.err(error_message), diff --git a/tests/core.rs b/tests/core.rs index 9e5471e3..1e4199d5 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -6,7 +6,6 @@ macro_rules! my_assert_eq { }; } - #[test] fn test_core_compilation_failures() { let t = trybuild::TestCases::new(); @@ -31,9 +30,10 @@ fn test_set() { #[test] fn test_raw() { - my_assert_eq!({ - [!string! [!raw! #variable and [!command!] are not interpreted or error]] - }, "#variableand[!command!]arenotinterpretedorerror"); + my_assert_eq!( + { [!string! [!raw! #variable and [!command!] are not interpreted or error]] }, + "#variableand[!command!]arenotinterpretedorerror" + ); } #[test] diff --git a/tests/expressions.rs b/tests/expressions.rs index 5c869e0f..0af48f26 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -24,7 +24,7 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! -3.2], -3.2); assert_preinterpret_eq!([!evaluate! true && true || false], true); assert_preinterpret_eq!([!evaluate! true || false && false], true); // The && has priority - assert_preinterpret_eq!([!evaluate! true | false & false], true); // The & has priority + assert_preinterpret_eq!([!evaluate! true | false & false], true); // The & has priority assert_preinterpret_eq!([!evaluate! true as u32 + 2], 3); assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u32); assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u64); From faa96cf1777787152d6bd69c0f89970cf228e651 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 11 Jan 2025 14:36:38 +0000 Subject: [PATCH 020/126] fix: Error span unwraps transparent groups --- CHANGELOG.md | 4 +- src/commands/concat_commands.rs | 2 +- src/commands/core_commands.rs | 33 +++++++- src/interpretation/interpreted_stream.rs | 7 ++ .../interpreter_parse_stream.rs | 84 ++++++++++++------- src/traits.rs | 24 +++++- .../compilation_failures/core/error_spans.rs | 2 - .../core/error_spans.stderr | 11 +-- 8 files changed, 122 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e150d65e..370cff72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,11 +22,11 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Update big comment in parse stream file * Grouping... Proposal: * #x is a group, #..x is flattened * Whether a command is flattened or not is dependent on the command - * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!..! ...]` may also be used instead. + * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!command! ...]` may also be used instead. + * For more complicated argument-lists, we can consider using `{ .. }` field-based arguments * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 3132c646..1b4b8d8d 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -61,7 +61,7 @@ fn concat_recursive(arguments: InterpretedStream) -> String { fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { for token_tree in arguments { match token_tree { - TokenTree::Literal(literal) => match literal.content_if_string_or_char() { + TokenTree::Literal(literal) => match literal.content_if_string_like() { Some(content) => output.push_str(&content), None => output.push_str(&literal.to_string()), }, diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index b6ee431e..62891e37 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -98,15 +98,46 @@ impl CommandDefinition for ErrorCommand { impl CommandInvocation for ErrorCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { static ERROR: &str = "Expected a single string literal as the error message"; + let message_span_range = self.message.span_range(); let message = self .message .interpret_as_tokens(interpreter)? + .flatten_transparent_groups() .into_singleton(ERROR)? .to_literal(ERROR)? .content_if_string() .ok_or_else(|| message_span_range.error(ERROR))?; - let error_span_stream = self.error_span_stream.interpret_as_tokens(interpreter)?; + + // Consider the case where preinterpret embeds in a declarative macro, and we have + // an error like this: + // [!error! [!string! "Expected 100, got " $input] [$input]] + // + // In cases like this, rustc wraps $input in a transparent group, which means that + // the span of that group is the span of the tokens "$input" in the definition of the + // declarative macro. This is not what we want. We want the span of the tokens which + // were fed into $input in the declarative macro. + // + // The simplest solution here is to get rid of all transparent groups, to get back to the + // source spans. + // + // Once this workstream with macro diagnostics is stabilised: + // https://github.com/rust-lang/rust/issues/54140#issuecomment-802701867 + // + // Then we can revisit this and do something better, and include all spans as separate spans + // in the error message, which will allow a user to trace an error through N different layers + // of macros. + // + // (Possibly we can try to join spans together, and if they don't join, they become separate + // spans which get printed to the error message). + // + // Coincidentally, rust analyzer currently does not properly support + // transparent groups (as of Jan 2025), so gets it right without this flattening: + // https://github.com/rust-lang/rust-analyzer/issues/18211 + let error_span_stream = self + .error_span_stream + .interpret_as_tokens(interpreter)? + .flatten_transparent_groups(); if error_span_stream.is_empty() { Span::call_site().err(message) diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 747bc4a6..3cf506b0 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -58,6 +58,13 @@ impl InterpretedStream { self.token_stream.is_empty() } + pub(crate) fn flatten_transparent_groups(self) -> InterpretedStream { + InterpretedStream { + source_span_range: self.source_span_range, + token_stream: self.token_stream.flatten_transparent_groups(), + } + } + pub(crate) fn into_singleton(self, error_message: &str) -> Result { if self.is_empty() { return self.source_span_range.err(error_message); diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 58df1136..24f0964b 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -4,36 +4,62 @@ use crate::internal_prelude::*; // How syn features fits with preinterpret parsing // =============================================== // +// There are a few places where we parse in preinterpret: +// * Parse the initial input from the macro, into an interpretable structure +// * Parsing as part of interpretation +// * e.g. of raw tokens, either from the preinterpret declaration, or from a variable +// * e.g. of a variable, as part of incremental parsing (while_parse style loops) +// // I spent quite a while considering whether this could be wrapping a -// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`, instead -// of a custom Peekable -// -// I came to the conclusion it doesn't make much sense for now, but -// could be explored in future: -// -// * ParseBuffer / ParseStream is powerful but restrictive -// > Due to how it works, we'd need to use it to parse the initial input -// in a single initial pass. -// > We currently have an Interpreter with us as we parse, which the `Parse` -// trait doesn't allow. -// > We could consider splitting into a two-pass approach, where we start -// with a Parse step, and then we interpret after, but it would be quite -// a big change internally -// * Cursor needs to reference into some TokenBuffer -// > We would convert the input TokenStream into a TokenBuffer and -// Cursor into that -// > But this can't be converted into a ParseBuffer outside of the syn crate, -// and so it doesn't get much benefit -// -// Either of these approaches appear disjoint from the parse/destructuring -// operations... -// -// * For parse/destructuring operations, we may need to temporarily -// create parse streams in scope of a command execution -// * For #VARIABLES which can be incrementally consumed / parsed, -// it would be nice to be able to store a Cursor into a TokenBuffer, -// but annoyingly it isn't possible to convert this to a ParseStream -// outside of syn. +// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... +// +// Parsing the initial input +// ------------------------- +// +// This is where InterpreterParseStream comes in. +// +// Now that I've changed how preinterpret works to be a two-pass approach +// (first parsing, then interpreting), we could consider swapping out the +// InterpreterParseStream to wrap a syn::ParseStream instead. +// +// This probably could work, but has a little more overhead to swap to a +// TokenBuffer (and so ParseStream) and back. +// +// Parsing in the context of a command execution +// --------------------------------------------- +// +// For parse/destructuring operations, we could temporarily create parse +// streams in scope of a command execution. +// +// Parsing a variable or other token stream +// ---------------------------------------- +// +// Some commands want to performantly parse a variable or other token stream. +// +// Here we want variables to support: +// * Easy appending of tokens +// * Incremental parsing +// +// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to +// append to it, and freely convert it to a syn::ParseStream, possibly even storing +// a cursor position into it. +// +// Unfortunately this isn't at all possible: +// * TokenBuffer appending isn't a thing, you can only create one (recursively) from +// a TokenStream +// * TokenBuffer can't be converted to a ParseStream outside of the syn crate +// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be +// used against a fixed buffer. +// +// We could probably work around these limitations by sacrificing performance and transforming +// to TokenStream and back, but probably there's a better way. +// +// What we probably want is our own abstraction, likely a fork from `syn`, which supports +// converting a Cursor into an indexed based cursor, which can safely be stored separately +// from the TokenBuffer. +// +// We could use this abstraction for InterpretedStream; and our variables could store a +// tuple of (IndexCursor, PreinterpretTokenBuffer) #[derive(Clone)] pub(crate) struct InterpreterParseStream { diff --git a/src/traits.rs b/src/traits.rs index 6d428120..91f99aec 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -38,7 +38,7 @@ impl IdentExt for Ident { pub(crate) trait LiteralExt: Sized { fn content_if_string(&self) -> Option; - fn content_if_string_or_char(&self) -> Option; + fn content_if_string_like(&self) -> Option; } impl LiteralExt for Literal { @@ -49,15 +49,35 @@ impl LiteralExt for Literal { } } - fn content_if_string_or_char(&self) -> Option { + fn content_if_string_like(&self) -> Option { match parse_str::(&self.to_string()).unwrap() { Lit::Str(lit_str) => Some(lit_str.value()), Lit::Char(lit_char) => Some(lit_char.value().to_string()), + Lit::CStr(lit_cstr) => Some(lit_cstr.value().to_string_lossy().to_string()), _ => None, } } } +pub(crate) trait TokenStreamExt: Sized { + fn flatten_transparent_groups(self) -> Self; +} + +impl TokenStreamExt for TokenStream { + fn flatten_transparent_groups(self) -> Self { + let mut output = TokenStream::new(); + for token in self { + match token { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + output.extend(group.stream().flatten_transparent_groups()); + } + other => output.extend(iter::once(other)), + } + } + output + } +} + pub(crate) trait TokenTreeExt: Sized { fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; fn to_literal(self, error_message: &str) -> Result; diff --git a/tests/compilation_failures/core/error_spans.rs b/tests/compilation_failures/core/error_spans.rs index 2032661b..62eb37ff 100644 --- a/tests/compilation_failures/core/error_spans.rs +++ b/tests/compilation_failures/core/error_spans.rs @@ -9,7 +9,5 @@ macro_rules! assert_is_100 { } fn main() { - // In rust analyzer, the span is in the correct location - // But in rustc it's not... I'm not really sure why. assert_is_100!(5); } \ No newline at end of file diff --git a/tests/compilation_failures/core/error_spans.stderr b/tests/compilation_failures/core/error_spans.stderr index a1fc2fb5..9f732bce 100644 --- a/tests/compilation_failures/core/error_spans.stderr +++ b/tests/compilation_failures/core/error_spans.stderr @@ -1,10 +1,5 @@ error: Expected 100, got 5 - --> tests/compilation_failures/core/error_spans.rs:6:62 + --> tests/compilation_failures/core/error_spans.rs:12:20 | -6 | [!error! [!string! "Expected 100, got " $input] [$input]] - | ^^^^^^ -... -14 | assert_is_100!(5); - | ----------------- in this macro invocation - | - = note: this error originates in the macro `assert_is_100` (in Nightly builds, run with -Z macro-backtrace for more info) +12 | assert_is_100!(5); + | ^ From b0f455a3dd10484e89f4840f3f13964183543e02 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 12 Jan 2025 00:38:10 +0000 Subject: [PATCH 021/126] feature: Add field passing --- CHANGELOG.md | 11 +- src/commands/core_commands.rs | 60 +++--- src/interpretation/interpreted_stream.rs | 195 +++++++++++++++++- .../interpreter_parse_stream.rs | 12 +- src/traits.rs | 17 +- .../compilation_failures/core/error_spans.rs | 5 +- .../core/error_spans.stderr | 4 +- ...ix_me_comparison_operators_are_not_lazy.rs | 2 +- ...e_comparison_operators_are_not_lazy.stderr | 2 +- 9 files changed, 262 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 370cff72..e742c048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,10 +23,9 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come * Grouping... Proposal: - * #x is a group, #..x is flattened - * Whether a command is flattened or not is dependent on the command + * In output land: #x is a group, #..x is flattened + * In parse land: #x matches a single token, #..x consumes the rest of a stream * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!command! ...]` may also be used instead. - * For more complicated argument-lists, we can consider using `{ .. }` field-based arguments * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces @@ -36,8 +35,6 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * e.g. for long sums * Add compile failure tests * `[!range! 0..5]` outputs `0 1 2 3 4` -* `[!error! "message" [token stream for span]]` -* Parse fields `{ ... }` and change `!error! to use them as per https://github.com/rust-lang/rust/issues/54140#issuecomment-2585002922. * Reconfiguring iteration limit * Support `!else if!` in `!if!` * `[!extend! #x += ...]` to make such actions more performant @@ -57,6 +54,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * How does this extend to parsing scenarios, such as a punctuated `,`? * It doesn't explicitly... * But `[!for! #x in [!split! [Hello, World,] on ,]]` could work, if `[!split!]` outputs transparent groups, and discards empty final items + * `[!comma_split! Hello, World,]` + * `[!split! { input: [Hello, World,], separator: [,], ignore_empty_at_end?: true, require_trailing_separator?: false, }]` * How does this handle zipping / unzipping and un-grouping, and/or parsing a more complicated group? * Proposal: The parsing operates over the content of a single token tree, possibly via explicitly ungrouping. * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` @@ -82,6 +81,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings * `[!OPTIONAL! ...]` * Groups or `[!GROUP! ...]` + * `#x` binding reads a token tree + * `#..x` binding reads the rest of the stream * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` * `[!match!]` (with `#..x` as a catch-all) * Work on book diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 62891e37..3feb008f 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -75,39 +75,54 @@ impl CommandInvocation for IgnoreCommand { #[derive(Clone)] pub(crate) struct ErrorCommand { - message: NextItem, - error_span_stream: InterpretationStream, + arguments: InterpretationStream, } impl CommandDefinition for ErrorCommand { const COMMAND_NAME: &'static str = "error"; fn parse(mut arguments: InterpreterParseStream) -> Result { - static ERROR: &str = "Expected [!error! \"Error message\" [tokens spanning error]], for example:\n* [!error! \"Compiler error message\" [$tokens_covering_error]]\n * [!error! [!string! \"My Error\" \"in bits\"] []]"; - let parsed = Self { - message: arguments.next_item(ERROR)?, - error_span_stream: arguments - .next_as_kinded_group(Delimiter::Bracket, ERROR)? - .into_inner_stream(), - }; - arguments.assert_end(ERROR)?; - Ok(parsed) + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) } } +#[derive(Default)] +struct ErrorCommandArguments { + message: Option, + error_spans: Option, +} + impl CommandInvocation for ErrorCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - static ERROR: &str = "Expected a single string literal as the error message"; + let fields_parser = FieldsParseDefinition::new(ErrorCommandArguments::default()) + .add_required_field( + "message", + "\"Error message to display\"", + None, + |params, val| params.message = Some(val), + ) + .add_optional_field( + "spans", + "[$abc]", + Some("An optional [token stream], to determine where to show the error message"), + |params, val| params.error_spans = Some(val), + ); + + let arguments = self.arguments + .interpret_as_tokens(interpreter)? + .parse_into_fields(fields_parser)?; - let message_span_range = self.message.span_range(); - let message = self + let message = arguments .message - .interpret_as_tokens(interpreter)? - .flatten_transparent_groups() - .into_singleton(ERROR)? - .to_literal(ERROR)? - .content_if_string() - .ok_or_else(|| message_span_range.error(ERROR))?; + .unwrap() // Field was required + .value(); + + let error_span_stream = arguments + .error_spans + .map(|b| b.token_stream) + .unwrap_or_default(); // Consider the case where preinterpret embeds in a declarative macro, and we have // an error like this: @@ -134,10 +149,7 @@ impl CommandInvocation for ErrorCommand { // Coincidentally, rust analyzer currently does not properly support // transparent groups (as of Jan 2025), so gets it right without this flattening: // https://github.com/rust-lang/rust-analyzer/issues/18211 - let error_span_stream = self - .error_span_stream - .interpret_as_tokens(interpreter)? - .flatten_transparent_groups(); + let error_span_stream = error_span_stream.flatten_transparent_groups(); if error_span_stream.is_empty() { Span::call_site().err(message) diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 3cf506b0..54e76d5e 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -1,3 +1,5 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; + use crate::internal_prelude::*; #[derive(Clone)] @@ -58,13 +60,18 @@ impl InterpretedStream { self.token_stream.is_empty() } - pub(crate) fn flatten_transparent_groups(self) -> InterpretedStream { - InterpretedStream { - source_span_range: self.source_span_range, - token_stream: self.token_stream.flatten_transparent_groups(), - } + pub(crate) fn parse_into_fields(self, parser: FieldsParseDefinition) -> Result { + let source_span_range = self.source_span_range; + self.syn_parse(parser.create_syn_parser(source_span_range)) } + /// For a type `T` which implements `syn::Parse`, you can call this as `syn_parse(self, T::parse)`. + /// For more complicated parsers, just pass the parsing function to this function. + pub(crate) fn syn_parse(self, parser: P) -> Result { + parser.parse2(self.token_stream) + } + + #[allow(unused)] pub(crate) fn into_singleton(self, error_message: &str) -> Result { if self.is_empty() { return self.source_span_range.err(error_message); @@ -100,3 +107,181 @@ impl HasSpanRange for InterpretedStream { } } } + +/// Parses a [..] block. +pub(crate) struct BracketedTokenStream { + #[allow(unused)] + pub(crate) brackets: syn::token::Bracket, + pub(crate) token_stream: TokenStream, +} + +impl syn::parse::Parse for BracketedTokenStream { + fn parse(input: syn::parse::ParseStream) -> Result { + let content; + Ok(Self { + brackets: syn::bracketed!(content in input), + token_stream: content.parse()?, + }) + } +} + + +pub(crate) struct FieldsParseDefinition { + new_builder: T, + field_definitions: FieldDefinitions, +} + +impl FieldsParseDefinition { + pub(crate) fn new(new_builder: T) -> Self { + Self { + new_builder, + field_definitions: FieldDefinitions(BTreeMap::new()), + } + } + + pub(crate) fn add_required_field( + self, + field_name: &str, + example: &str, + explanation: Option<&str>, + set: impl Fn(&mut T, F) + 'static, + ) -> Self { + self.add_field(field_name, example, explanation, true, F::parse, set) + } + + pub(crate) fn add_optional_field( + self, + field_name: &str, + example: &str, + explanation: Option<&str>, + set: impl Fn(&mut T, F) + 'static, + ) -> Self { + self.add_field(field_name, example, explanation, false, F::parse, set) + } + + pub(crate) fn add_field( + mut self, + field_name: &str, + example: &str, + explanation: Option<&str>, + is_required: bool, + parse: impl Fn(syn::parse::ParseStream) -> Result + 'static, + set: impl Fn(&mut T, F) + 'static, + ) -> Self { + if self.field_definitions.0.insert( + field_name.to_string(), + FieldParseDefinition { + is_required, + example: example.into(), + explanation: explanation.map(|s| s.to_string()), + parse_and_set: Box::new(move |builder, content| { + let value = parse(content)?; + set(builder, value); + Ok(()) + }), + }, + ).is_some() { + panic!("Duplicate field name: {field_name:?}"); + } + self + } + + pub(crate) fn create_syn_parser(self, error_span_range: SpanRange) -> impl FnOnce(syn::parse::ParseStream) -> Result { + fn inner( + input: syn::parse::ParseStream, + new_builder: T, + field_definitions: &FieldDefinitions, + error_span_range: SpanRange, + ) -> Result { + let mut builder = new_builder; + let content; + let _ = syn::braced!(content in input); + + let mut required_field_names: BTreeSet<_> = field_definitions + .0 + .iter() + .filter_map(|(field_name, field_definition)| { + match field_definition.is_required { + true => Some(field_name.clone()), + false => None, + } + }) + .collect(); + let mut seen_field_names = HashSet::new(); + + while !content.is_empty() { + let field_name = content.parse::()?; + let field_name_value = field_name.to_string(); + if !seen_field_names.insert(field_name_value.clone()) { + return field_name.err("Duplicate field name"); + } + required_field_names.remove(field_name_value.as_str()); + let _ = content.parse::()?; + let field_definition = field_definitions + .0 + .get(field_name_value.as_str()) + .ok_or_else(|| field_name.error(format!("Unsupported field name")))?; + (field_definition.parse_and_set)(&mut builder, &content)?; + if !content.is_empty() { + content.parse::()?; + } + } + + if !required_field_names.is_empty() { + return error_span_range.err(format!( + "Missing required fields: {missing_fields:?}", + missing_fields = required_field_names, + )); + } + + Ok(builder) + } + move |input: syn::parse::ParseStream| { + inner(input, self.new_builder, &self.field_definitions, error_span_range) + .map_err(|error| { + // Sadly error combination is just buggy - the two outputted + // compile_error! invocations are back to back which causes a rustc + // parse error. Instead, let's do this. + error + .concat("\n") + .concat(&self.field_definitions.error_message()) + }) + } + } +} + + +struct FieldDefinitions(BTreeMap>); + +impl FieldDefinitions { + fn error_message(&self) -> String{ + use std::fmt::Write; + let mut message = "Expected: {\n".to_string(); + for (field_name, field_definition) in &self.0 { + write!( + &mut message, + " {}{}: {}", + field_name, + if field_definition.is_required { "" } else { "?" }, + field_definition.example, + ).unwrap(); + match field_definition.explanation { + Some(ref explanation) => write!( + &mut message, + ", // {explanation}\n", + explanation = explanation, + ).unwrap(), + None => write!(&mut message, ",\n").unwrap(), + } + } + message.push('}'); + message + } +} + +struct FieldParseDefinition { + is_required: bool, + example: String, + explanation: Option, + parse_and_set: Box Result<()>>, +} diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 24f0964b..37dcd6d3 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -91,6 +91,16 @@ impl InterpreterParseStream { self.tokens.next() } + #[allow(unused)] + pub(super) fn next_token_tree(&mut self, error_message: &str) -> Result { + match self.next_token_tree_or_end() { + Some(token_tree) => Ok(token_tree), + None => self + .latest_item_span_range + .err(format!("Unexpected end: {error_message}")), + } + } + fn next_item_or_end(&mut self) -> Result> { let next_item = NextItem::parse(self)?; Ok(match next_item { @@ -102,7 +112,7 @@ impl InterpreterParseStream { }) } - pub(crate) fn next_item(&mut self, error_message: &'static str) -> Result { + pub(crate) fn next_item(&mut self, error_message: &str) -> Result { match self.next_item_or_end()? { Some(item) => Ok(item), None => self diff --git a/src/traits.rs b/src/traits.rs index 91f99aec..2c632fb4 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -37,6 +37,7 @@ impl IdentExt for Ident { } pub(crate) trait LiteralExt: Sized { + #[allow(unused)] fn content_if_string(&self) -> Option; fn content_if_string_like(&self) -> Option; } @@ -80,19 +81,23 @@ impl TokenStreamExt for TokenStream { pub(crate) trait TokenTreeExt: Sized { fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; - fn to_literal(self, error_message: &str) -> Result; } impl TokenTreeExt for TokenTree { fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) } +} - fn to_literal(self, error_message: &str) -> Result { - match self { - TokenTree::Literal(literal) => Ok(literal), - other => other.err(error_message), - } +pub(crate) trait SynErrorExt: Sized { + fn concat(self, extra: &str) -> Self; +} + +impl SynErrorExt for syn::Error { + fn concat(self, extra: &str) -> Self { + let mut message = self.to_string(); + message.push_str(extra); + Self::new(self.span(), message) } } diff --git a/tests/compilation_failures/core/error_spans.rs b/tests/compilation_failures/core/error_spans.rs index 62eb37ff..7c2d1dd2 100644 --- a/tests/compilation_failures/core/error_spans.rs +++ b/tests/compilation_failures/core/error_spans.rs @@ -3,7 +3,10 @@ use preinterpret::*; macro_rules! assert_is_100 { ($input:literal) => {preinterpret!{ [!if! ($input != 100) { - [!error! [!string! "Expected 100, got " $input] [$input]] + [!error! { + message: [!string! "Expected 100, got " $input], + spans: [$input], + }] }] }}; } diff --git a/tests/compilation_failures/core/error_spans.stderr b/tests/compilation_failures/core/error_spans.stderr index 9f732bce..8f5b6365 100644 --- a/tests/compilation_failures/core/error_spans.stderr +++ b/tests/compilation_failures/core/error_spans.stderr @@ -1,5 +1,5 @@ error: Expected 100, got 5 - --> tests/compilation_failures/core/error_spans.rs:12:20 + --> tests/compilation_failures/core/error_spans.rs:15:20 | -12 | assert_is_100!(5); +15 | assert_is_100!(5); | ^ diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs index d267595a..7d0cba29 100644 --- a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs +++ b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs @@ -5,7 +5,7 @@ fn main() { [!set! #is_eager = false] let _ = [!evaluate! false && ([!set! #is_eager = true] true)]; [!if! #is_eager { - [!error! "The && expression is not evaluated lazily" []] + [!error! { message: "The && expression is not evaluated lazily" }] }] } } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr index 6362c4d5..92c1ef0e 100644 --- a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr +++ b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr @@ -5,7 +5,7 @@ error: The && expression is not evaluated lazily 5 | | [!set! #is_eager = false] 6 | | let _ = [!evaluate! false && ([!set! #is_eager = true] true)]; 7 | | [!if! #is_eager { -8 | | [!error! "The && expression is not evaluated lazily" []] +8 | | [!error! { message: "The && expression is not evaluated lazily" }] 9 | | }] 10 | | } | |_____^ From 5f1cee0110c5ecda0d614f1c077085b482ea0dff Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 12 Jan 2025 00:39:05 +0000 Subject: [PATCH 022/126] fix: Style fix --- src/commands/core_commands.rs | 3 +- src/interpretation/interpreted_stream.rs | 97 ++++++++++++++---------- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 3feb008f..bb1a76ad 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -110,7 +110,8 @@ impl CommandInvocation for ErrorCommand { |params, val| params.error_spans = Some(val), ); - let arguments = self.arguments + let arguments = self + .arguments .interpret_as_tokens(interpreter)? .parse_into_fields(fields_parser)?; diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 54e76d5e..4e19a6e9 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -60,7 +60,10 @@ impl InterpretedStream { self.token_stream.is_empty() } - pub(crate) fn parse_into_fields(self, parser: FieldsParseDefinition) -> Result { + pub(crate) fn parse_into_fields( + self, + parser: FieldsParseDefinition, + ) -> Result { let source_span_range = self.source_span_range; self.syn_parse(parser.create_syn_parser(source_span_range)) } @@ -125,7 +128,6 @@ impl syn::parse::Parse for BracketedTokenStream { } } - pub(crate) struct FieldsParseDefinition { new_builder: T, field_definitions: FieldDefinitions, @@ -168,25 +170,33 @@ impl FieldsParseDefinition { parse: impl Fn(syn::parse::ParseStream) -> Result + 'static, set: impl Fn(&mut T, F) + 'static, ) -> Self { - if self.field_definitions.0.insert( - field_name.to_string(), - FieldParseDefinition { - is_required, - example: example.into(), - explanation: explanation.map(|s| s.to_string()), - parse_and_set: Box::new(move |builder, content| { - let value = parse(content)?; - set(builder, value); - Ok(()) - }), - }, - ).is_some() { + if self + .field_definitions + .0 + .insert( + field_name.to_string(), + FieldParseDefinition { + is_required, + example: example.into(), + explanation: explanation.map(|s| s.to_string()), + parse_and_set: Box::new(move |builder, content| { + let value = parse(content)?; + set(builder, value); + Ok(()) + }), + }, + ) + .is_some() + { panic!("Duplicate field name: {field_name:?}"); } self } - pub(crate) fn create_syn_parser(self, error_span_range: SpanRange) -> impl FnOnce(syn::parse::ParseStream) -> Result { + pub(crate) fn create_syn_parser( + self, + error_span_range: SpanRange, + ) -> impl FnOnce(syn::parse::ParseStream) -> Result { fn inner( input: syn::parse::ParseStream, new_builder: T, @@ -200,12 +210,12 @@ impl FieldsParseDefinition { let mut required_field_names: BTreeSet<_> = field_definitions .0 .iter() - .filter_map(|(field_name, field_definition)| { - match field_definition.is_required { + .filter_map( + |(field_name, field_definition)| match field_definition.is_required { true => Some(field_name.clone()), false => None, - } - }) + }, + ) .collect(); let mut seen_field_names = HashSet::new(); @@ -220,7 +230,7 @@ impl FieldsParseDefinition { let field_definition = field_definitions .0 .get(field_name_value.as_str()) - .ok_or_else(|| field_name.error(format!("Unsupported field name")))?; + .ok_or_else(|| field_name.error("Unsupported field name".to_string()))?; (field_definition.parse_and_set)(&mut builder, &content)?; if !content.is_empty() { content.parse::()?; @@ -237,24 +247,28 @@ impl FieldsParseDefinition { Ok(builder) } move |input: syn::parse::ParseStream| { - inner(input, self.new_builder, &self.field_definitions, error_span_range) - .map_err(|error| { - // Sadly error combination is just buggy - the two outputted - // compile_error! invocations are back to back which causes a rustc - // parse error. Instead, let's do this. - error - .concat("\n") - .concat(&self.field_definitions.error_message()) - }) + inner( + input, + self.new_builder, + &self.field_definitions, + error_span_range, + ) + .map_err(|error| { + // Sadly error combination is just buggy - the two outputted + // compile_error! invocations are back to back which causes a rustc + // parse error. Instead, let's do this. + error + .concat("\n") + .concat(&self.field_definitions.error_message()) + }) } } } - struct FieldDefinitions(BTreeMap>); impl FieldDefinitions { - fn error_message(&self) -> String{ + fn error_message(&self) -> String { use std::fmt::Write; let mut message = "Expected: {\n".to_string(); for (field_name, field_definition) in &self.0 { @@ -262,16 +276,22 @@ impl FieldDefinitions { &mut message, " {}{}: {}", field_name, - if field_definition.is_required { "" } else { "?" }, + if field_definition.is_required { + "" + } else { + "?" + }, field_definition.example, - ).unwrap(); + ) + .unwrap(); match field_definition.explanation { - Some(ref explanation) => write!( + Some(ref explanation) => writeln!( &mut message, - ", // {explanation}\n", + ", // {explanation}", explanation = explanation, - ).unwrap(), - None => write!(&mut message, ",\n").unwrap(), + ) + .unwrap(), + None => writeln!(&mut message, ",").unwrap(), } } message.push('}'); @@ -283,5 +303,6 @@ struct FieldParseDefinition { is_required: bool, example: String, explanation: Option, + #[allow(clippy::type_complexity)] parse_and_set: Box Result<()>>, } From 88ffe18d83812daeea82de6d8a395c65c727cad8 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 14 Jan 2025 22:39:31 +0000 Subject: [PATCH 023/126] tweak: Minor refactors --- CHANGELOG.md | 10 +- src/commands/control_flow_commands.rs | 4 +- src/commands/core_commands.rs | 24 +++ src/commands/mod.rs | 1 + src/internal_prelude.rs | 1 + .../{next_item.rs => interpretation_item.rs} | 58 ++--- src/interpretation/interpretation_stream.rs | 4 +- src/interpretation/interpreted_stream.rs | 198 ------------------ .../interpreter_parse_stream.rs | 18 +- src/interpretation/mod.rs | 4 +- src/lib.rs | 1 + src/parsing/building_blocks.rs | 18 ++ src/parsing/fields.rs | 181 ++++++++++++++++ src/parsing/mod.rs | 5 + 14 files changed, 284 insertions(+), 243 deletions(-) rename src/interpretation/{next_item.rs => interpretation_item.rs} (51%) create mode 100644 src/parsing/building_blocks.rs create mode 100644 src/parsing/fields.rs create mode 100644 src/parsing/mod.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e742c048..65289e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,16 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. ### To come - + +* Support field parsing both before and at command execution. + * Trial moving to `ParseStream`-based parsing + * Move `[!error!]` field parsing to be at parse time + => Input fields defined via macro, including static parse step e.g. `message: InterpretationItem, spans: Option` + => Then each input can be interpreted as a specific type during execution. * Grouping... Proposal: * In output land: #x is a group, #..x is flattened * In parse land: #x matches a single token, #..x consumes the rest of a stream - * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!command! ...]` may also be used instead. + * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!command! ...]` or `#x` may also be used instead. * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces @@ -78,6 +83,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Basic place parsing * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! * Explicit Punct, Idents, Literals + * `[!STREAM! ]` method to take a stream * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings * `[!OPTIONAL! ...]` * Groups or `[!GROUP! ...]` diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 8f58b0ce..ce812966 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IfCommand { - condition: NextItem, + condition: InterpretationItem, true_code: InterpretationStream, false_code: Option, nothing_span_range: SpanRange, @@ -64,7 +64,7 @@ impl CommandInvocation for IfCommand { #[derive(Clone)] pub(crate) struct WhileCommand { - condition: NextItem, + condition: InterpretationItem, loop_code: InterpretationStream, nothing_span_range: SpanRange, } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index bb1a76ad..70024ffc 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -73,6 +73,30 @@ impl CommandInvocation for IgnoreCommand { } } +/// This serves as a no-op command to take a stream when the grammar only allows a single item +#[derive(Clone)] +pub(crate) struct StreamCommand { + arguments: InterpretationStream, +} + +impl CommandDefinition for StreamCommand { + const COMMAND_NAME: &'static str = "stream"; + + fn parse(mut arguments: InterpreterParseStream) -> Result { + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) + } +} + +impl CommandInvocation for StreamCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + Ok(CommandOutput::AppendStream( + self.arguments.interpret_as_tokens(interpreter)?, + )) + } +} + #[derive(Clone)] pub(crate) struct ErrorCommand { arguments: InterpretationStream, diff --git a/src/commands/mod.rs b/src/commands/mod.rs index ab39f40a..1899b100 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -16,6 +16,7 @@ define_command_kind! { SetCommand, RawCommand, IgnoreCommand, + StreamCommand, ErrorCommand, // Concat & Type Convert Commands diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 8078b557..d5d46bd5 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -7,5 +7,6 @@ pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, R pub(crate) use crate::commands::*; pub(crate) use crate::expressions::*; pub(crate) use crate::interpretation::*; +pub(crate) use crate::parsing::*; pub(crate) use crate::string_conversion::*; pub(crate) use crate::traits::*; diff --git a/src/interpretation/next_item.rs b/src/interpretation/interpretation_item.rs similarity index 51% rename from src/interpretation/next_item.rs rename to src/interpretation/interpretation_item.rs index 5c73f161..0d5d8571 100644 --- a/src/interpretation/next_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -1,7 +1,7 @@ use crate::internal_prelude::*; #[derive(Clone)] -pub(crate) enum NextItem { +pub(crate) enum InterpretationItem { Command(Command), Variable(Variable), Group(InterpretationGroup), @@ -10,7 +10,7 @@ pub(crate) enum NextItem { Literal(Literal), } -impl NextItem { +impl InterpretationItem { pub(super) fn parse(parse_stream: &mut InterpreterParseStream) -> Result> { let next = match parse_stream.next_token_tree_or_end() { Some(next) => next, @@ -19,45 +19,45 @@ impl NextItem { Ok(Some(match next { TokenTree::Group(group) => { if let Some(command) = Command::attempt_parse_from_group(&group)? { - NextItem::Command(command) + InterpretationItem::Command(command) } else { - NextItem::Group(InterpretationGroup::parse(group)?) + InterpretationItem::Group(InterpretationGroup::parse(group)?) } } TokenTree::Punct(punct) => { if let Some(variable) = Variable::parse_consuming_only_if_match(&punct, parse_stream) { - NextItem::Variable(variable) + InterpretationItem::Variable(variable) } else { - NextItem::Punct(punct) + InterpretationItem::Punct(punct) } } - TokenTree::Ident(ident) => NextItem::Ident(ident), - TokenTree::Literal(literal) => NextItem::Literal(literal), + TokenTree::Ident(ident) => InterpretationItem::Ident(ident), + TokenTree::Literal(literal) => InterpretationItem::Literal(literal), })) } } -impl Interpret for NextItem { +impl Interpret for InterpretationItem { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { match self { - NextItem::Command(command_invocation) => { + InterpretationItem::Command(command_invocation) => { command_invocation.interpret_as_tokens_into(interpreter, output)?; } - NextItem::Variable(variable) => { + InterpretationItem::Variable(variable) => { variable.interpret_as_tokens_into(interpreter, output)?; } - NextItem::Group(group) => { + InterpretationItem::Group(group) => { group.interpret_as_tokens_into(interpreter, output)?; } - NextItem::Punct(punct) => output.push_punct(punct), - NextItem::Ident(ident) => output.push_ident(ident), - NextItem::Literal(literal) => output.push_literal(literal), + InterpretationItem::Punct(punct) => output.push_punct(punct), + InterpretationItem::Ident(ident) => output.push_ident(ident), + InterpretationItem::Literal(literal) => output.push_literal(literal), } Ok(()) } @@ -68,32 +68,34 @@ impl Interpret for NextItem { expression_stream: &mut ExpressionStream, ) -> Result<()> { match self { - NextItem::Command(command_invocation) => { + InterpretationItem::Command(command_invocation) => { command_invocation.interpret_as_expression_into(interpreter, expression_stream)?; } - NextItem::Variable(variable) => { + InterpretationItem::Variable(variable) => { variable.interpret_as_expression_into(interpreter, expression_stream)?; } - NextItem::Group(group) => { + InterpretationItem::Group(group) => { group.interpret_as_expression_into(interpreter, expression_stream)?; } - NextItem::Punct(punct) => expression_stream.push_punct(punct), - NextItem::Ident(ident) => expression_stream.push_ident(ident), - NextItem::Literal(literal) => expression_stream.push_literal(literal), + InterpretationItem::Punct(punct) => expression_stream.push_punct(punct), + InterpretationItem::Ident(ident) => expression_stream.push_ident(ident), + InterpretationItem::Literal(literal) => expression_stream.push_literal(literal), } Ok(()) } } -impl HasSpanRange for NextItem { +impl HasSpanRange for InterpretationItem { fn span_range(&self) -> SpanRange { match self { - NextItem::Command(command_invocation) => command_invocation.span_range(), - NextItem::Variable(variable_substitution) => variable_substitution.span_range(), - NextItem::Group(group) => group.span_range(), - NextItem::Punct(punct) => punct.span_range(), - NextItem::Ident(ident) => ident.span_range(), - NextItem::Literal(literal) => literal.span_range(), + InterpretationItem::Command(command_invocation) => command_invocation.span_range(), + InterpretationItem::Variable(variable_substitution) => { + variable_substitution.span_range() + } + InterpretationItem::Group(group) => group.span_range(), + InterpretationItem::Punct(punct) => punct.span_range(), + InterpretationItem::Ident(ident) => ident.span_range(), + InterpretationItem::Literal(literal) => literal.span_range(), } } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 77a16a7c..1b36191f 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -3,7 +3,7 @@ use crate::internal_prelude::*; /// A parsed stream ready for interpretation #[derive(Clone)] pub(crate) struct InterpretationStream { - items: Vec, + items: Vec, span_range: SpanRange, } @@ -20,7 +20,7 @@ impl InterpretationStream { span_range: SpanRange, ) -> Result { let mut items = Vec::new(); - while let Some(next_item) = NextItem::parse(parse_stream)? { + while let Some(next_item) = InterpretationItem::parse(parse_stream)? { items.push(next_item); } Ok(Self { items, span_range }) diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 4e19a6e9..50521a6d 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -1,5 +1,3 @@ -use std::collections::{BTreeMap, BTreeSet, HashSet}; - use crate::internal_prelude::*; #[derive(Clone)] @@ -110,199 +108,3 @@ impl HasSpanRange for InterpretedStream { } } } - -/// Parses a [..] block. -pub(crate) struct BracketedTokenStream { - #[allow(unused)] - pub(crate) brackets: syn::token::Bracket, - pub(crate) token_stream: TokenStream, -} - -impl syn::parse::Parse for BracketedTokenStream { - fn parse(input: syn::parse::ParseStream) -> Result { - let content; - Ok(Self { - brackets: syn::bracketed!(content in input), - token_stream: content.parse()?, - }) - } -} - -pub(crate) struct FieldsParseDefinition { - new_builder: T, - field_definitions: FieldDefinitions, -} - -impl FieldsParseDefinition { - pub(crate) fn new(new_builder: T) -> Self { - Self { - new_builder, - field_definitions: FieldDefinitions(BTreeMap::new()), - } - } - - pub(crate) fn add_required_field( - self, - field_name: &str, - example: &str, - explanation: Option<&str>, - set: impl Fn(&mut T, F) + 'static, - ) -> Self { - self.add_field(field_name, example, explanation, true, F::parse, set) - } - - pub(crate) fn add_optional_field( - self, - field_name: &str, - example: &str, - explanation: Option<&str>, - set: impl Fn(&mut T, F) + 'static, - ) -> Self { - self.add_field(field_name, example, explanation, false, F::parse, set) - } - - pub(crate) fn add_field( - mut self, - field_name: &str, - example: &str, - explanation: Option<&str>, - is_required: bool, - parse: impl Fn(syn::parse::ParseStream) -> Result + 'static, - set: impl Fn(&mut T, F) + 'static, - ) -> Self { - if self - .field_definitions - .0 - .insert( - field_name.to_string(), - FieldParseDefinition { - is_required, - example: example.into(), - explanation: explanation.map(|s| s.to_string()), - parse_and_set: Box::new(move |builder, content| { - let value = parse(content)?; - set(builder, value); - Ok(()) - }), - }, - ) - .is_some() - { - panic!("Duplicate field name: {field_name:?}"); - } - self - } - - pub(crate) fn create_syn_parser( - self, - error_span_range: SpanRange, - ) -> impl FnOnce(syn::parse::ParseStream) -> Result { - fn inner( - input: syn::parse::ParseStream, - new_builder: T, - field_definitions: &FieldDefinitions, - error_span_range: SpanRange, - ) -> Result { - let mut builder = new_builder; - let content; - let _ = syn::braced!(content in input); - - let mut required_field_names: BTreeSet<_> = field_definitions - .0 - .iter() - .filter_map( - |(field_name, field_definition)| match field_definition.is_required { - true => Some(field_name.clone()), - false => None, - }, - ) - .collect(); - let mut seen_field_names = HashSet::new(); - - while !content.is_empty() { - let field_name = content.parse::()?; - let field_name_value = field_name.to_string(); - if !seen_field_names.insert(field_name_value.clone()) { - return field_name.err("Duplicate field name"); - } - required_field_names.remove(field_name_value.as_str()); - let _ = content.parse::()?; - let field_definition = field_definitions - .0 - .get(field_name_value.as_str()) - .ok_or_else(|| field_name.error("Unsupported field name".to_string()))?; - (field_definition.parse_and_set)(&mut builder, &content)?; - if !content.is_empty() { - content.parse::()?; - } - } - - if !required_field_names.is_empty() { - return error_span_range.err(format!( - "Missing required fields: {missing_fields:?}", - missing_fields = required_field_names, - )); - } - - Ok(builder) - } - move |input: syn::parse::ParseStream| { - inner( - input, - self.new_builder, - &self.field_definitions, - error_span_range, - ) - .map_err(|error| { - // Sadly error combination is just buggy - the two outputted - // compile_error! invocations are back to back which causes a rustc - // parse error. Instead, let's do this. - error - .concat("\n") - .concat(&self.field_definitions.error_message()) - }) - } - } -} - -struct FieldDefinitions(BTreeMap>); - -impl FieldDefinitions { - fn error_message(&self) -> String { - use std::fmt::Write; - let mut message = "Expected: {\n".to_string(); - for (field_name, field_definition) in &self.0 { - write!( - &mut message, - " {}{}: {}", - field_name, - if field_definition.is_required { - "" - } else { - "?" - }, - field_definition.example, - ) - .unwrap(); - match field_definition.explanation { - Some(ref explanation) => writeln!( - &mut message, - ", // {explanation}", - explanation = explanation, - ) - .unwrap(), - None => writeln!(&mut message, ",").unwrap(), - } - } - message.push('}'); - message - } -} - -struct FieldParseDefinition { - is_required: bool, - example: String, - explanation: Option, - #[allow(clippy::type_complexity)] - parse_and_set: Box Result<()>>, -} diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs index 37dcd6d3..4f07776d 100644 --- a/src/interpretation/interpreter_parse_stream.rs +++ b/src/interpretation/interpreter_parse_stream.rs @@ -101,8 +101,8 @@ impl InterpreterParseStream { } } - fn next_item_or_end(&mut self) -> Result> { - let next_item = NextItem::parse(self)?; + fn next_item_or_end(&mut self) -> Result> { + let next_item = InterpretationItem::parse(self)?; Ok(match next_item { Some(next_item) => { self.latest_item_span_range = next_item.span_range(); @@ -112,7 +112,7 @@ impl InterpreterParseStream { }) } - pub(crate) fn next_item(&mut self, error_message: &str) -> Result { + pub(crate) fn next_item(&mut self, error_message: &str) -> Result { match self.next_item_or_end()? { Some(item) => Ok(item), None => self @@ -123,7 +123,7 @@ impl InterpreterParseStream { pub(crate) fn next_as_ident(&mut self, error_message: &'static str) -> Result { match self.next_item(error_message)? { - NextItem::Ident(ident) => Ok(ident), + InterpretationItem::Ident(ident) => Ok(ident), other => other.err(error_message), } } @@ -134,14 +134,14 @@ impl InterpreterParseStream { error_message: &'static str, ) -> Result { match self.next_item(error_message)? { - NextItem::Ident(ident) if ident == ident_name => Ok(ident), + InterpretationItem::Ident(ident) if ident == ident_name => Ok(ident), other => other.err(error_message), } } pub(crate) fn next_as_punct(&mut self, error_message: &'static str) -> Result { match self.next_item(error_message)? { - NextItem::Punct(punct) => Ok(punct), + InterpretationItem::Punct(punct) => Ok(punct), other => other.err(error_message), } } @@ -152,7 +152,7 @@ impl InterpreterParseStream { error_message: &'static str, ) -> Result { match self.next_item(error_message)? { - NextItem::Punct(punct) if punct.as_char() == char => Ok(punct), + InterpretationItem::Punct(punct) if punct.as_char() == char => Ok(punct), other => other.err(error_message), } } @@ -163,14 +163,14 @@ impl InterpreterParseStream { error_message: &'static str, ) -> Result { match self.next_item(error_message)? { - NextItem::Group(group) if group.delimiter() == delimiter => Ok(group), + InterpretationItem::Group(group) if group.delimiter() == delimiter => Ok(group), other => other.err(error_message), } } pub(crate) fn next_as_variable(&mut self, error_message: &'static str) -> Result { match self.next_item(error_message)? { - NextItem::Variable(variable_substitution) => Ok(variable_substitution), + InterpretationItem::Variable(variable_substitution) => Ok(variable_substitution), other => other.err(error_message), } } diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 5b7a685f..da4dedb5 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,15 +1,15 @@ mod command; +mod interpretation_item; mod interpretation_stream; mod interpreted_stream; mod interpreter; mod interpreter_parse_stream; -mod next_item; mod variable; pub(crate) use command::*; +pub(crate) use interpretation_item::*; pub(crate) use interpretation_stream::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; pub(crate) use interpreter_parse_stream::*; -pub(crate) use next_item::*; pub(crate) use variable::*; diff --git a/src/lib.rs b/src/lib.rs index 58b3100d..f3e7a2ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -514,6 +514,7 @@ mod commands; mod expressions; mod internal_prelude; mod interpretation; +mod parsing; mod string_conversion; mod traits; diff --git a/src/parsing/building_blocks.rs b/src/parsing/building_blocks.rs new file mode 100644 index 00000000..4f28b04a --- /dev/null +++ b/src/parsing/building_blocks.rs @@ -0,0 +1,18 @@ +use crate::internal_prelude::*; + +/// Parses a [..] block. +pub(crate) struct BracketedTokenStream { + #[allow(unused)] + pub(crate) brackets: syn::token::Bracket, + pub(crate) token_stream: TokenStream, +} + +impl syn::parse::Parse for BracketedTokenStream { + fn parse(input: syn::parse::ParseStream) -> Result { + let content; + Ok(Self { + brackets: syn::bracketed!(content in input), + token_stream: content.parse()?, + }) + } +} diff --git a/src/parsing/fields.rs b/src/parsing/fields.rs new file mode 100644 index 00000000..c3cdd6b3 --- /dev/null +++ b/src/parsing/fields.rs @@ -0,0 +1,181 @@ +use crate::internal_prelude::*; +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +pub(crate) struct FieldsParseDefinition { + new_builder: T, + field_definitions: FieldDefinitions, +} + +impl FieldsParseDefinition { + pub(crate) fn new(new_builder: T) -> Self { + Self { + new_builder, + field_definitions: FieldDefinitions(BTreeMap::new()), + } + } + + pub(crate) fn add_required_field( + self, + field_name: &str, + example: &str, + explanation: Option<&str>, + set: impl Fn(&mut T, F) + 'static, + ) -> Self { + self.add_field(field_name, example, explanation, true, F::parse, set) + } + + pub(crate) fn add_optional_field( + self, + field_name: &str, + example: &str, + explanation: Option<&str>, + set: impl Fn(&mut T, F) + 'static, + ) -> Self { + self.add_field(field_name, example, explanation, false, F::parse, set) + } + + pub(crate) fn add_field( + mut self, + field_name: &str, + example: &str, + explanation: Option<&str>, + is_required: bool, + parse: impl Fn(syn::parse::ParseStream) -> Result + 'static, + set: impl Fn(&mut T, F) + 'static, + ) -> Self { + if self + .field_definitions + .0 + .insert( + field_name.to_string(), + FieldParseDefinition { + is_required, + example: example.into(), + explanation: explanation.map(|s| s.to_string()), + parse_and_set: Box::new(move |builder, content| { + let value = parse(content)?; + set(builder, value); + Ok(()) + }), + }, + ) + .is_some() + { + panic!("Duplicate field name: {field_name:?}"); + } + self + } + + pub(crate) fn create_syn_parser( + self, + error_span_range: SpanRange, + ) -> impl FnOnce(syn::parse::ParseStream) -> Result { + fn inner( + input: syn::parse::ParseStream, + new_builder: T, + field_definitions: &FieldDefinitions, + error_span_range: SpanRange, + ) -> Result { + let mut builder = new_builder; + let content; + let _ = syn::braced!(content in input); + + let mut required_field_names: BTreeSet<_> = field_definitions + .0 + .iter() + .filter_map( + |(field_name, field_definition)| match field_definition.is_required { + true => Some(field_name.clone()), + false => None, + }, + ) + .collect(); + let mut seen_field_names = HashSet::new(); + + while !content.is_empty() { + let field_name = content.parse::()?; + let field_name_value = field_name.to_string(); + if !seen_field_names.insert(field_name_value.clone()) { + return field_name.err("Duplicate field name"); + } + required_field_names.remove(field_name_value.as_str()); + let _ = content.parse::()?; + let field_definition = field_definitions + .0 + .get(field_name_value.as_str()) + .ok_or_else(|| field_name.error("Unsupported field name".to_string()))?; + (field_definition.parse_and_set)(&mut builder, &content)?; + if !content.is_empty() { + content.parse::()?; + } + } + + if !required_field_names.is_empty() { + return error_span_range.err(format!( + "Missing required fields: {missing_fields:?}", + missing_fields = required_field_names, + )); + } + + Ok(builder) + } + move |input: syn::parse::ParseStream| { + inner( + input, + self.new_builder, + &self.field_definitions, + error_span_range, + ) + .map_err(|error| { + // Sadly error combination is just buggy - the two outputted + // compile_error! invocations are back to back which causes a rustc + // parse error. Instead, let's do this. + error + .concat("\n") + .concat(&self.field_definitions.error_message()) + }) + } + } +} + +struct FieldDefinitions(BTreeMap>); + +impl FieldDefinitions { + fn error_message(&self) -> String { + use std::fmt::Write; + let mut message = "Expected: {\n".to_string(); + for (field_name, field_definition) in &self.0 { + write!( + &mut message, + " {}{}: {}", + field_name, + if field_definition.is_required { + "" + } else { + "?" + }, + field_definition.example, + ) + .unwrap(); + match field_definition.explanation { + Some(ref explanation) => writeln!( + &mut message, + ", // {explanation}", + explanation = explanation, + ) + .unwrap(), + None => writeln!(&mut message, ",").unwrap(), + } + } + message.push('}'); + message + } +} + +struct FieldParseDefinition { + is_required: bool, + example: String, + explanation: Option, + #[allow(clippy::type_complexity)] + parse_and_set: Box Result<()>>, +} diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs new file mode 100644 index 00000000..20e8c1f7 --- /dev/null +++ b/src/parsing/mod.rs @@ -0,0 +1,5 @@ +mod building_blocks; +mod fields; + +pub(crate) use building_blocks::*; +pub(crate) use fields::*; From 19348c120fef4fe94ce69ecd251b6b02d6868552 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 15 Jan 2025 11:49:05 +0000 Subject: [PATCH 024/126] refactor: Move to `ParseStream`-based parsing --- CHANGELOG.md | 1 - Cargo.toml | 2 +- src/commands/concat_commands.rs | 2 +- src/commands/control_flow_commands.rs | 73 +++---- src/commands/core_commands.rs | 31 +-- src/commands/expression_commands.rs | 36 ++-- src/commands/token_commands.rs | 10 +- src/expressions/expression_stream.rs | 8 +- src/internal_prelude.rs | 5 +- src/interpretation/command.rs | 87 ++++---- src/interpretation/command_arguments.rs | 115 ++++++++++ src/interpretation/interpretation_item.rs | 64 +++--- src/interpretation/interpretation_stream.rs | 48 +++-- src/interpretation/interpreted_stream.rs | 4 +- .../interpreter_parse_stream.rs | 196 ------------------ src/interpretation/mod.rs | 4 +- src/interpretation/variable.rs | 35 ++-- src/parsing/building_blocks.rs | 2 +- src/parsing/fields.rs | 4 +- src/traits.rs | 38 +++- 20 files changed, 358 insertions(+), 407 deletions(-) create mode 100644 src/interpretation/command_arguments.rs delete mode 100644 src/interpretation/interpreter_parse_stream.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 65289e8a..ab986a62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,6 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come * Support field parsing both before and at command execution. - * Trial moving to `ParseStream`-based parsing * Move `[!error!]` field parsing to be at parse time => Input fields defined via macro, including static parse step e.g. `message: InterpretationItem, spans: Option` => Then each input can be interpreted as a specific type during execution. diff --git a/Cargo.toml b/Cargo.toml index f1b023e0..000deacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0" } -syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing"] } +syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing", "clone-impls"] } quote = { version = "1.0", default-features = false } [dev-dependencies] diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 1b4b8d8d..3562cd72 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -110,7 +110,7 @@ macro_rules! define_concat_command { impl CommandDefinition for $command { const COMMAND_NAME: &'static str = $command_name; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index ce812966..08f110a1 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -11,33 +11,27 @@ pub(crate) struct IfCommand { impl CommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - fn parse(mut arguments: InterpreterParseStream) -> Result { - static ERROR: &str = "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code}]"; - - let condition = arguments.next_item(ERROR)?; - let true_code = arguments - .next_as_kinded_group(Delimiter::Brace, ERROR)? - .into_inner_stream(); - let false_code = if !arguments.is_empty() { - arguments.next_as_punct_matching('!', ERROR)?; - arguments.next_as_ident_matching("else", ERROR)?; - arguments.next_as_punct_matching('!', ERROR)?; - Some( - arguments - .next_as_kinded_group(Delimiter::Brace, ERROR)? - .into_inner_stream(), - ) - } else { - None - }; - arguments.assert_end(ERROR)?; - - Ok(Self { - condition, - true_code, - false_code, - nothing_span_range: arguments.full_span_range(), - }) + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + condition: input.parse()?, + true_code: input.parse_code_group_for_interpretation()?, + false_code: { + if !input.is_empty() { + input.parse::()?; + input.parse::()?; + input.parse::()?; + Some(input.parse_code_group_for_interpretation()?) + } else { + None + } + }, + nothing_span_range: arguments.full_span_range(), + }) + }, + "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code }]", + ) } } @@ -72,20 +66,17 @@ pub(crate) struct WhileCommand { impl CommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; - fn parse(mut arguments: InterpreterParseStream) -> Result { - static ERROR: &str = "Expected [!while! (condition) { code }]"; - - let condition = arguments.next_item(ERROR)?; - let loop_code = arguments - .next_as_kinded_group(Delimiter::Brace, ERROR)? - .into_inner_stream(); - arguments.assert_end(ERROR)?; - - Ok(Self { - condition, - loop_code, - nothing_span_range: arguments.full_span_range(), - }) + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + condition: input.parse()?, + loop_code: input.parse_code_group_for_interpretation()?, + nothing_span_range: arguments.full_span_range(), + }) + }, + "Expected [!while! (condition) { code }]", + ) } } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 70024ffc..a1947ea3 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -4,20 +4,25 @@ use crate::internal_prelude::*; pub(crate) struct SetCommand { variable: Variable, #[allow(unused)] - equals: Punct, + equals: Token![=], arguments: InterpretationStream, } impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; - fn parse(mut arguments: InterpreterParseStream) -> Result { - static ERROR: &str = "Expected [!set! #variable = ... ]"; - Ok(Self { - variable: arguments.next_as_variable(ERROR)?, - equals: arguments.next_as_punct_matching('=', ERROR)?, - arguments: arguments.parse_all_for_interpretation()?, - }) + fn parse(arguments: CommandArguments) -> Result { + arguments + .fully_parse_or_error( + |input| { + Ok(Self { + variable: input.parse()?, + equals: input.parse()?, + arguments: input.parse_with(arguments.full_span_range())?, + }) + }, + "Expected [!set! #variable = ... ]", + ) } } @@ -39,7 +44,7 @@ pub(crate) struct RawCommand { impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments_span_range: arguments.full_span_range(), token_stream: arguments.read_all_as_raw_token_stream(), @@ -62,7 +67,9 @@ pub(crate) struct IgnoreCommand; impl CommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; - fn parse(_arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { + // Avoid a syn parse error by reading all the tokens + let _ = arguments.read_all_as_raw_token_stream(); Ok(Self) } } @@ -82,7 +89,7 @@ pub(crate) struct StreamCommand { impl CommandDefinition for StreamCommand { const COMMAND_NAME: &'static str = "stream"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) @@ -105,7 +112,7 @@ pub(crate) struct ErrorCommand { impl CommandDefinition for ErrorCommand { const COMMAND_NAME: &'static str = "error"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 505a6f41..e5a97a0b 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -8,7 +8,7 @@ pub(crate) struct EvaluateCommand { impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { expression: arguments.parse_all_for_interpretation()?, }) @@ -29,28 +29,32 @@ pub(crate) struct AssignCommand { variable: Variable, operator: Punct, #[allow(unused)] - equals: Punct, + equals: Token![=], expression: InterpretationStream, } impl CommandDefinition for AssignCommand { const COMMAND_NAME: &'static str = "assign"; - fn parse(mut arguments: InterpreterParseStream) -> Result { - static ERROR: &str = "Expected [!assign! #variable += ...] for + or some other operator supported in an expression"; - Ok(Self { - variable: arguments.next_as_variable(ERROR)?, - operator: { - let operator = arguments.next_as_punct(ERROR)?; - match operator.as_char() { - '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => return operator.err("Expected one of + - * / % & | or ^"), - } - operator + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + variable: input.parse()?, + operator: { + let operator: Punct = input.parse()?; + match operator.as_char() { + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} + _ => return operator.err("Expected one of + - * / % & | or ^"), + } + operator + }, + equals: input.parse()?, + expression: input.parse_with(arguments.full_span_range())?, + }) }, - equals: arguments.next_as_punct_matching('=', ERROR)?, - expression: arguments.parse_all_for_interpretation()?, - }) + "Expected [!assign! #variable += ...] for + or some other operator supported in an expression", + ) } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index adb249bd..e4a6b559 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -6,8 +6,8 @@ pub(crate) struct EmptyCommand; impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; - fn parse(mut arguments: InterpreterParseStream) -> Result { - arguments.assert_end("The !empty! command does not take any arguments")?; + fn parse(arguments: CommandArguments) -> Result { + arguments.assert_empty("The !empty! command does not take any arguments")?; Ok(Self) } } @@ -26,7 +26,7 @@ pub(crate) struct IsEmptyCommand { impl CommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) @@ -52,7 +52,7 @@ pub(crate) struct LengthCommand { impl CommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) @@ -77,7 +77,7 @@ pub(crate) struct GroupCommand { impl CommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - fn parse(mut arguments: InterpreterParseStream) -> Result { + fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index d15f001c..94fbeea6 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -35,20 +35,20 @@ impl ExpressionStream { pub(crate) fn push_grouped_interpreted_stream( &mut self, contents: InterpretedStream, - span_range: SpanRange, + span: Span, ) { self.interpreted_stream - .push_new_group(contents, Delimiter::None, span_range); + .push_new_group(contents, Delimiter::None, span); } pub(crate) fn push_expression_group( &mut self, contents: Self, delimiter: Delimiter, - span_range: SpanRange, + span: Span, ) { self.interpreted_stream - .push_new_group(contents.interpreted_stream, delimiter, span_range); + .push_new_group(contents.interpreted_stream, delimiter, span); } pub(crate) fn evaluate(self) -> Result { diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index d5d46bd5..2c0355ec 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -1,8 +1,11 @@ pub(crate) use core::iter; pub(crate) use proc_macro2::*; +pub(crate) use proc_macro2::extra::*; pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; -pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, UnOp}; +pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, UnOp, token, Token}; +pub(crate) use syn::parse::{Parse, ParseBuffer, ParseStream, Parser, discouraged::AnyDelimiter}; +pub(crate) use syn::ext::IdentExt as SynIdentExt; pub(crate) use crate::commands::*; pub(crate) use crate::expressions::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 51af8038..d807fc34 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -3,7 +3,7 @@ use crate::internal_prelude::*; pub(crate) trait CommandDefinition: CommandInvocation + Clone { const COMMAND_NAME: &'static str; - fn parse(arguments: InterpreterParseStream) -> Result; + fn parse(arguments: CommandArguments) -> Result; } pub(crate) trait CommandInvocation { @@ -49,7 +49,7 @@ macro_rules! define_command_kind { } impl CommandKind { - pub(crate) fn parse_invocation(&self, arguments: InterpreterParseStream) -> Result> { + pub(crate) fn parse_invocation(&self, arguments: CommandArguments) -> Result> { Ok(match self { $( Self::$command => Box::new( @@ -79,64 +79,47 @@ macro_rules! define_command_kind { } pub(crate) use define_command_kind; +impl Parse for CommandKind { + fn parse(input: ParseStream) -> Result { + // Support parsing any ident + let ident = input.call(Ident::parse_any)?; + match Self::for_ident(&ident) { + Some(command_kind) => Ok(command_kind), + None => ident.span().err( + format!( + "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", + Self::list_all(), + ident, + ), + ), + } + } +} + #[derive(Clone)] pub(crate) struct Command { invocation: Box, - source_group_span_range: SpanRange, + source_group_span: DelimSpan, } -impl Command { - pub(super) fn attempt_parse_from_group(group: &Group) -> Result> { - fn matches_command_start(group: &Group) -> Option<(Ident, InterpreterParseStream)> { - if group.delimiter() != Delimiter::Bracket { - return None; - } - let mut tokens = InterpreterParseStream::new(group.stream(), group.span_range()); - tokens.next_as_punct_matching('!', "").ok()?; - let ident = tokens.next_as_ident("").ok()?; - Some((ident, tokens)) - } - - fn extract_command_data( - command_ident: &Ident, - parse_stream: &mut InterpreterParseStream, - ) -> Option { - let command_kind = CommandKind::for_ident(command_ident)?; - parse_stream.next_as_punct_matching('!', "").ok()?; - Some(command_kind) - } - - // Attempt to match `[!ident`, if that doesn't match, we assume it's not a command invocation, - // so return `Ok(None)` - let (command_ident, mut parse_stream) = match matches_command_start(group) { - Some(command_start) => command_start, - None => return Ok(None), - }; - - // We have now checked enough that we're confident the user is pretty intentionally using - // the call convention. Any issues we hit from this point will be a helpful compiler error. - match extract_command_data(&command_ident, &mut parse_stream) { - Some(command_kind) => { - let invocation = command_kind.parse_invocation( parse_stream)?; - Ok(Some(Self { - invocation, - source_group_span_range: group.span_range(), - })) - }, - None => Err(command_ident.span().error( - format!( - "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", - CommandKind::list_all(), - command_ident, - ), - )), - } +impl Parse for Command { + fn parse(input: ParseStream) -> Result { + let content; + let open_bracket = syn::bracketed!(content in input); + content.parse::()?; + let command_kind = content.parse::()?; + content.parse::()?; + let invocation = command_kind.parse_invocation( CommandArguments::new(&content, open_bracket.span.span_range()))?; + Ok(Self { + invocation, + source_group_span: open_bracket.span, + }) } } impl HasSpanRange for Command { fn span_range(&self) -> SpanRange { - self.source_group_span_range + self.source_group_span.span_range() } } @@ -158,7 +141,7 @@ impl Interpret for Command { output.extend(stream); } CommandOutput::GroupedStream(stream) => { - output.push_new_group(stream, Delimiter::None, self.source_group_span_range); + output.push_new_group(stream, Delimiter::None, self.source_group_span.join()); } }; Ok(()) @@ -179,7 +162,7 @@ impl Interpret for Command { } CommandOutput::AppendStream(stream) | CommandOutput::GroupedStream(stream) => { expression_stream - .push_grouped_interpreted_stream(stream, self.source_group_span_range); + .push_grouped_interpreted_stream(stream, self.source_group_span.join()); } }; Ok(()) diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs new file mode 100644 index 00000000..cd17f021 --- /dev/null +++ b/src/interpretation/command_arguments.rs @@ -0,0 +1,115 @@ +use crate::internal_prelude::*; + +// =============================================== +// How syn features fits with preinterpret parsing +// =============================================== +// +// There are a few places where we parse in preinterpret: +// * Parse the initial input from the macro, into an interpretable structure +// * Parsing as part of interpretation +// * e.g. of raw tokens, either from the preinterpret declaration, or from a variable +// * e.g. of a variable, as part of incremental parsing (while_parse style loops) +// +// I spent quite a while considering whether this could be wrapping a +// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... +// +// Parsing the initial input +// ------------------------- +// +// This is where CommandArguments comes in. +// +// Now that I've changed how preinterpret works to be a two-pass approach +// (first parsing, then interpreting), we could consider swapping out the +// CommandArguments to wrap a syn::ParseStream instead. +// +// This probably could work, but has a little more overhead to swap to a +// TokenBuffer (and so ParseStream) and back. +// +// Parsing in the context of a command execution +// --------------------------------------------- +// +// For parse/destructuring operations, we could temporarily create parse +// streams in scope of a command execution. +// +// Parsing a variable or other token stream +// ---------------------------------------- +// +// Some commands want to performantly parse a variable or other token stream. +// +// Here we want variables to support: +// * Easy appending of tokens +// * Incremental parsing +// +// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to +// append to it, and freely convert it to a syn::ParseStream, possibly even storing +// a cursor position into it. +// +// Unfortunately this isn't at all possible: +// * TokenBuffer appending isn't a thing, you can only create one (recursively) from +// a TokenStream +// * TokenBuffer can't be converted to a ParseStream outside of the syn crate +// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be +// used against a fixed buffer. +// +// We could probably work around these limitations by sacrificing performance and transforming +// to TokenStream and back, but probably there's a better way. +// +// What we probably want is our own abstraction, likely a fork from `syn`, which supports +// converting a Cursor into an indexed based cursor, which can safely be stored separately +// from the TokenBuffer. +// +// We could use this abstraction for InterpretedStream; and our variables could store a +// tuple of (IndexCursor, PreinterpretTokenBuffer) + +#[derive(Clone)] +pub(crate) struct CommandArguments<'a> { + parse_stream: ParseStream<'a>, + /// The span range of the original stream, before tokens were consumed + full_span_range: SpanRange, + /// The span of the last item consumed (or the full span range if no items have been consumed yet) + latest_item_span_range: SpanRange, +} + +impl<'a> CommandArguments<'a> { + pub(crate) fn new(parse_stream: ParseStream<'a>, span_range: SpanRange) -> Self { + Self { + parse_stream, + full_span_range: span_range, + latest_item_span_range: span_range, + } + } + + pub(crate) fn full_span_range(&self) -> SpanRange { + self.full_span_range + } + + /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message + pub(crate) fn assert_empty(&self, error_message: &'static str) -> Result<()> { + if self.parse_stream.is_empty() { + Ok(()) + } else { + self.latest_item_span_range.err(error_message) + } + } + + pub(crate) fn fully_parse_or_error( + &self, + parse_function: impl FnOnce(ParseStream) -> Result, + error_message: &'static str + ) -> Result { + let parsed = parse_function(self.parse_stream) + .or_else(|_| self.full_span_range.err(error_message))?; + + self.assert_empty(error_message)?; + + Ok(parsed) + } + + pub(crate) fn parse_all_for_interpretation(&self) -> Result { + self.parse_stream.parse_all_for_interpretation(self.full_span_range) + } + + pub(crate) fn read_all_as_raw_token_stream(&self) -> TokenStream { + self.parse_stream.parse::().unwrap() + } +} diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 0d5d8571..507cc6cf 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -10,32 +10,48 @@ pub(crate) enum InterpretationItem { Literal(Literal), } -impl InterpretationItem { - pub(super) fn parse(parse_stream: &mut InterpreterParseStream) -> Result> { - let next = match parse_stream.next_token_tree_or_end() { - Some(next) => next, - None => return Ok(None), - }; - Ok(Some(match next { - TokenTree::Group(group) => { - if let Some(command) = Command::attempt_parse_from_group(&group)? { - InterpretationItem::Command(command) - } else { - InterpretationItem::Group(InterpretationGroup::parse(group)?) - } - } - TokenTree::Punct(punct) => { - if let Some(variable) = - Variable::parse_consuming_only_if_match(&punct, parse_stream) - { - InterpretationItem::Variable(variable) - } else { - InterpretationItem::Punct(punct) - } - } +enum GroupMatch { + Command, + OtherGroup, + None, +} + +fn attempt_match_group(cursor: syn::buffer::Cursor) -> GroupMatch { + let next = match cursor.any_group() { + Some((next, delimiter, _, _)) if delimiter == Delimiter::Bracket => next, + Some(_) => return GroupMatch::OtherGroup, + None => return GroupMatch::None, + }; + let next = match next.punct() { + Some((punct, next)) if punct.as_char() == '!' => next, + _ => return GroupMatch::OtherGroup, + }; + let next = match next.ident() { + Some((_, next)) => next, + _ => return GroupMatch::OtherGroup, + }; + match next.punct() { + Some((punct, _)) if punct.as_char() == '!' => GroupMatch::Command, + _ => GroupMatch::OtherGroup, + } +} + +impl Parse for InterpretationItem { + fn parse(input: ParseStream) -> Result { + match attempt_match_group(input.cursor()) { + GroupMatch::Command => return Ok(InterpretationItem::Command(input.parse()?)), + GroupMatch::OtherGroup => return Ok(InterpretationItem::Group(input.parse()?)), + GroupMatch::None => {}, + } + if input.peek(token::Pound) && input.peek2(syn::Ident) { + return Ok(InterpretationItem::Variable(input.parse()?)) + } + Ok(match input.parse::()? { + TokenTree::Group(_) => unreachable!("Should have been already handled by the first branch above"), + TokenTree::Punct(punct) => InterpretationItem::Punct(punct), TokenTree::Ident(ident) => InterpretationItem::Ident(ident), TokenTree::Literal(literal) => InterpretationItem::Literal(literal), - })) + }) } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 1b36191f..0ee090ee 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -12,16 +12,17 @@ impl InterpretationStream { token_stream: TokenStream, span_range: SpanRange, ) -> Result { - InterpreterParseStream::new(token_stream, span_range).parse_all_for_interpretation() + Self::create_parser(span_range).parse2(token_stream) } +} - pub(crate) fn parse( - parse_stream: &mut InterpreterParseStream, - span_range: SpanRange, - ) -> Result { +impl ContextualParse for InterpretationStream { + type Context = SpanRange; + + fn parse_with_context(input: ParseStream, span_range: Self::Context) -> Result { let mut items = Vec::new(); - while let Some(next_item) = InterpretationItem::parse(parse_stream)? { - items.push(next_item); + while !input.is_empty() { + items.push(input.parse()?); } Ok(Self { items, span_range }) } @@ -51,7 +52,7 @@ impl Interpret for InterpretationStream { expression_stream.push_expression_group( inner_expression_stream, Delimiter::None, - self.span_range, + self.span_range.span(), ); Ok(()) } @@ -66,23 +67,26 @@ impl HasSpanRange for InterpretationStream { /// A parsed group ready for interpretation #[derive(Clone)] pub(crate) struct InterpretationGroup { - source_group: Group, + source_delimeter: Delimiter, + source_delim_span: DelimSpan, interpretation_stream: InterpretationStream, } -impl InterpretationGroup { - pub(super) fn parse(source_group: Group) -> Result { - let interpretation_stream = - InterpreterParseStream::new(source_group.stream(), source_group.span_range()) - .parse_all_for_interpretation()?; +impl Parse for InterpretationGroup { + fn parse(input: ParseStream) -> Result { + let (delimeter, delim_span, content) = input.parse_any_delimiter()?; + let span_range = delim_span.span_range(); Ok(Self { - source_group, - interpretation_stream, + source_delimeter: delimeter, + source_delim_span: delim_span, + interpretation_stream: content.parse_with(span_range)?, }) } +} +impl InterpretationGroup { pub(crate) fn delimiter(&self) -> Delimiter { - self.source_group.delimiter() + self.source_delimeter } pub(crate) fn into_inner_stream(self) -> InterpretationStream { @@ -99,8 +103,8 @@ impl Interpret for InterpretationGroup { output.push_new_group( self.interpretation_stream .interpret_as_tokens(interpreter)?, - self.source_group.delimiter(), - self.source_group.span_range(), + self.source_delimeter, + self.source_delim_span.join(), ); Ok(()) } @@ -113,8 +117,8 @@ impl Interpret for InterpretationGroup { expression_stream.push_expression_group( self.interpretation_stream .interpret_as_expression(interpreter)?, - self.source_group.delimiter(), - self.source_group.span_range(), + self.source_delimeter, + self.source_delim_span.join(), ); Ok(()) } @@ -122,6 +126,6 @@ impl Interpret for InterpretationGroup { impl HasSpanRange for InterpretationGroup { fn span_range(&self) -> SpanRange { - self.source_group.span_range() + self.source_delim_span.span_range() } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 50521a6d..c0d71ac3 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -41,12 +41,12 @@ impl InterpretedStream { &mut self, inner_tokens: InterpretedStream, delimiter: Delimiter, - span_range: SpanRange, + span: Span, ) { self.push_raw_token_tree(TokenTree::group( inner_tokens.token_stream, delimiter, - span_range.span(), + span, )); } diff --git a/src/interpretation/interpreter_parse_stream.rs b/src/interpretation/interpreter_parse_stream.rs deleted file mode 100644 index 4f07776d..00000000 --- a/src/interpretation/interpreter_parse_stream.rs +++ /dev/null @@ -1,196 +0,0 @@ -use crate::internal_prelude::*; - -// =============================================== -// How syn features fits with preinterpret parsing -// =============================================== -// -// There are a few places where we parse in preinterpret: -// * Parse the initial input from the macro, into an interpretable structure -// * Parsing as part of interpretation -// * e.g. of raw tokens, either from the preinterpret declaration, or from a variable -// * e.g. of a variable, as part of incremental parsing (while_parse style loops) -// -// I spent quite a while considering whether this could be wrapping a -// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... -// -// Parsing the initial input -// ------------------------- -// -// This is where InterpreterParseStream comes in. -// -// Now that I've changed how preinterpret works to be a two-pass approach -// (first parsing, then interpreting), we could consider swapping out the -// InterpreterParseStream to wrap a syn::ParseStream instead. -// -// This probably could work, but has a little more overhead to swap to a -// TokenBuffer (and so ParseStream) and back. -// -// Parsing in the context of a command execution -// --------------------------------------------- -// -// For parse/destructuring operations, we could temporarily create parse -// streams in scope of a command execution. -// -// Parsing a variable or other token stream -// ---------------------------------------- -// -// Some commands want to performantly parse a variable or other token stream. -// -// Here we want variables to support: -// * Easy appending of tokens -// * Incremental parsing -// -// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to -// append to it, and freely convert it to a syn::ParseStream, possibly even storing -// a cursor position into it. -// -// Unfortunately this isn't at all possible: -// * TokenBuffer appending isn't a thing, you can only create one (recursively) from -// a TokenStream -// * TokenBuffer can't be converted to a ParseStream outside of the syn crate -// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be -// used against a fixed buffer. -// -// We could probably work around these limitations by sacrificing performance and transforming -// to TokenStream and back, but probably there's a better way. -// -// What we probably want is our own abstraction, likely a fork from `syn`, which supports -// converting a Cursor into an indexed based cursor, which can safely be stored separately -// from the TokenBuffer. -// -// We could use this abstraction for InterpretedStream; and our variables could store a -// tuple of (IndexCursor, PreinterpretTokenBuffer) - -#[derive(Clone)] -pub(crate) struct InterpreterParseStream { - tokens: iter::Peekable<::IntoIter>, - /// The span range of the original stream, before tokens were consumed - full_span_range: SpanRange, - /// The span of the last item consumed (or the full span range if no items have been consumed yet) - latest_item_span_range: SpanRange, -} - -impl InterpreterParseStream { - pub(crate) fn new(token_stream: TokenStream, span_range: SpanRange) -> Self { - Self { - tokens: token_stream.into_iter().peekable(), - full_span_range: span_range, - latest_item_span_range: span_range, - } - } - - pub(crate) fn full_span_range(&self) -> SpanRange { - self.full_span_range - } - - pub(super) fn peek_token_tree(&mut self) -> Option<&TokenTree> { - self.tokens.peek() - } - - pub(super) fn next_token_tree_or_end(&mut self) -> Option { - self.tokens.next() - } - - #[allow(unused)] - pub(super) fn next_token_tree(&mut self, error_message: &str) -> Result { - match self.next_token_tree_or_end() { - Some(token_tree) => Ok(token_tree), - None => self - .latest_item_span_range - .err(format!("Unexpected end: {error_message}")), - } - } - - fn next_item_or_end(&mut self) -> Result> { - let next_item = InterpretationItem::parse(self)?; - Ok(match next_item { - Some(next_item) => { - self.latest_item_span_range = next_item.span_range(); - Some(next_item) - } - None => None, - }) - } - - pub(crate) fn next_item(&mut self, error_message: &str) -> Result { - match self.next_item_or_end()? { - Some(item) => Ok(item), - None => self - .latest_item_span_range - .err(format!("Unexpected end: {error_message}")), - } - } - - pub(crate) fn next_as_ident(&mut self, error_message: &'static str) -> Result { - match self.next_item(error_message)? { - InterpretationItem::Ident(ident) => Ok(ident), - other => other.err(error_message), - } - } - - pub(crate) fn next_as_ident_matching( - &mut self, - ident_name: &str, - error_message: &'static str, - ) -> Result { - match self.next_item(error_message)? { - InterpretationItem::Ident(ident) if ident == ident_name => Ok(ident), - other => other.err(error_message), - } - } - - pub(crate) fn next_as_punct(&mut self, error_message: &'static str) -> Result { - match self.next_item(error_message)? { - InterpretationItem::Punct(punct) => Ok(punct), - other => other.err(error_message), - } - } - - pub(crate) fn next_as_punct_matching( - &mut self, - char: char, - error_message: &'static str, - ) -> Result { - match self.next_item(error_message)? { - InterpretationItem::Punct(punct) if punct.as_char() == char => Ok(punct), - other => other.err(error_message), - } - } - - pub(crate) fn next_as_kinded_group( - &mut self, - delimiter: Delimiter, - error_message: &'static str, - ) -> Result { - match self.next_item(error_message)? { - InterpretationItem::Group(group) if group.delimiter() == delimiter => Ok(group), - other => other.err(error_message), - } - } - - pub(crate) fn next_as_variable(&mut self, error_message: &'static str) -> Result { - match self.next_item(error_message)? { - InterpretationItem::Variable(variable_substitution) => Ok(variable_substitution), - other => other.err(error_message), - } - } - - pub(crate) fn is_empty(&mut self) -> bool { - self.peek_token_tree().is_none() - } - - pub(crate) fn assert_end(&mut self, error_message: &'static str) -> Result<()> { - match self.next_token_tree_or_end() { - Some(token) => token.span_range().err(error_message), - None => Ok(()), - } - } - - pub(crate) fn parse_all_for_interpretation(&mut self) -> Result { - InterpretationStream::parse(self, self.full_span_range) - } - - pub(crate) fn read_all_as_raw_token_stream(&mut self) -> TokenStream { - core::mem::replace(&mut self.tokens, TokenStream::new().into_iter().peekable()).collect() - } -} diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index da4dedb5..61afb09d 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -3,7 +3,7 @@ mod interpretation_item; mod interpretation_stream; mod interpreted_stream; mod interpreter; -mod interpreter_parse_stream; +mod command_arguments; mod variable; pub(crate) use command::*; @@ -11,5 +11,5 @@ pub(crate) use interpretation_item::*; pub(crate) use interpretation_stream::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; -pub(crate) use interpreter_parse_stream::*; +pub(crate) use command_arguments::*; pub(crate) use variable::*; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 23907a1c..53618167 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -2,31 +2,20 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct Variable { - marker: Punct, // # + marker: Token![#], variable_name: Ident, } -impl Variable { - pub(super) fn parse_consuming_only_if_match( - punct: &Punct, - parse_stream: &mut InterpreterParseStream, - ) -> Option { - if punct.as_char() != '#' { - return None; - } - match parse_stream.peek_token_tree() { - Some(TokenTree::Ident(_)) => {} - _ => return None, - } - match parse_stream.next_token_tree_or_end() { - Some(TokenTree::Ident(variable_name)) => Some(Self { - marker: punct.clone(), - variable_name, - }), - _ => unreachable!("We just peeked a token of this type"), - } +impl Parse for Variable { + fn parse(input: ParseStream) -> Result { + Ok(Self { + marker: input.parse()?, + variable_name: input.parse()?, + }) } +} +impl Variable { pub(crate) fn variable_name(&self) -> String { self.variable_name.to_string() } @@ -79,19 +68,19 @@ impl Interpret for &Variable { expression_stream: &mut ExpressionStream, ) -> Result<()> { expression_stream - .push_grouped_interpreted_stream(self.substitute(interpreter)?, self.span_range()); + .push_grouped_interpreted_stream(self.substitute(interpreter)?, self.span_range().span()); Ok(()) } } impl HasSpanRange for &Variable { fn span_range(&self) -> SpanRange { - SpanRange::new_between(self.marker.span(), self.variable_name.span()) + SpanRange::new_between(self.marker.span, self.variable_name.span()) } } impl core::fmt::Display for Variable { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "{}{}", self.marker.as_char(), self.variable_name) + write!(f, "#{}", self.variable_name) } } diff --git a/src/parsing/building_blocks.rs b/src/parsing/building_blocks.rs index 4f28b04a..1d5245e7 100644 --- a/src/parsing/building_blocks.rs +++ b/src/parsing/building_blocks.rs @@ -3,7 +3,7 @@ use crate::internal_prelude::*; /// Parses a [..] block. pub(crate) struct BracketedTokenStream { #[allow(unused)] - pub(crate) brackets: syn::token::Bracket, + pub(crate) brackets: token::Bracket, pub(crate) token_stream: TokenStream, } diff --git a/src/parsing/fields.rs b/src/parsing/fields.rs index c3cdd6b3..f47eb694 100644 --- a/src/parsing/fields.rs +++ b/src/parsing/fields.rs @@ -99,14 +99,14 @@ impl FieldsParseDefinition { return field_name.err("Duplicate field name"); } required_field_names.remove(field_name_value.as_str()); - let _ = content.parse::()?; + let _ = content.parse::()?; let field_definition = field_definitions .0 .get(field_name_value.as_str()) .ok_or_else(|| field_name.error("Unsupported field name".to_string()))?; (field_definition.parse_and_set)(&mut builder, &content)?; if !content.is_empty() { - content.parse::()?; + content.parse::()?; } } diff --git a/src/traits.rs b/src/traits.rs index 2c632fb4..2fecba35 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -89,6 +89,42 @@ impl TokenTreeExt for TokenTree { } } +pub(crate) trait ParserExt { + fn parse_with(&self, context: T::Context) -> Result; + fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result; + fn parse_code_group_for_interpretation(&self) -> Result; +} + +impl<'a> ParserExt for ParseBuffer<'a> { + fn parse_with(&self, context: T::Context) -> Result { + T::parse_with_context(self, context) + } + + fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result { + self.parse_with(span_range) + } + + fn parse_code_group_for_interpretation(&self) -> Result { + let group = self.parse::()?; + if group.delimiter() != Delimiter::Brace { + return group.err("expected {"); + } + Ok(group.into_inner_stream()) + } +} + +pub(crate) trait ContextualParse: Sized { + type Context; + + fn parse_with_context(input: ParseStream, context: Self::Context) -> Result; + + fn create_parser(context: Self::Context) -> impl FnOnce(ParseStream) -> Result { + move |input: ParseStream| { + Self::parse_with_context(input, context) + } + } +} + pub(crate) trait SynErrorExt: Sized { fn concat(self, extra: &str) -> Self; } @@ -225,7 +261,7 @@ impl HasSpanRange for Group { } } -impl HasSpanRange for proc_macro2::extra::DelimSpan { +impl HasSpanRange for DelimSpan { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.open(), self.close()) } From 944fb086236d44b5a4e10a219ebf52866ac96d7f Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 15 Jan 2025 11:50:39 +0000 Subject: [PATCH 025/126] fix: Style fix --- src/commands/core_commands.rs | 21 ++++++++++----------- src/internal_prelude.rs | 8 +++++--- src/interpretation/command.rs | 5 ++++- src/interpretation/command_arguments.rs | 5 +++-- src/interpretation/interpretation_item.rs | 10 ++++++---- src/interpretation/interpreted_stream.rs | 6 +----- src/interpretation/mod.rs | 4 ++-- src/interpretation/variable.rs | 6 ++++-- src/traits.rs | 4 +--- 9 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index a1947ea3..5a324c53 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -12,17 +12,16 @@ impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; fn parse(arguments: CommandArguments) -> Result { - arguments - .fully_parse_or_error( - |input| { - Ok(Self { - variable: input.parse()?, - equals: input.parse()?, - arguments: input.parse_with(arguments.full_span_range())?, - }) - }, - "Expected [!set! #variable = ... ]", - ) + arguments.fully_parse_or_error( + |input| { + Ok(Self { + variable: input.parse()?, + equals: input.parse()?, + arguments: input.parse_with(arguments.full_span_range())?, + }) + }, + "Expected [!set! #variable = ... ]", + ) } } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 2c0355ec..87704b6b 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -1,11 +1,13 @@ pub(crate) use core::iter; -pub(crate) use proc_macro2::*; pub(crate) use proc_macro2::extra::*; +pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; -pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, UnOp, token, Token}; -pub(crate) use syn::parse::{Parse, ParseBuffer, ParseStream, Parser, discouraged::AnyDelimiter}; pub(crate) use syn::ext::IdentExt as SynIdentExt; +pub(crate) use syn::parse::{discouraged::AnyDelimiter, Parse, ParseBuffer, ParseStream, Parser}; +pub(crate) use syn::{ + parse_str, token, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, +}; pub(crate) use crate::commands::*; pub(crate) use crate::expressions::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index d807fc34..69df8edc 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -109,7 +109,10 @@ impl Parse for Command { content.parse::()?; let command_kind = content.parse::()?; content.parse::()?; - let invocation = command_kind.parse_invocation( CommandArguments::new(&content, open_bracket.span.span_range()))?; + let invocation = command_kind.parse_invocation(CommandArguments::new( + &content, + open_bracket.span.span_range(), + ))?; Ok(Self { invocation, source_group_span: open_bracket.span, diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index cd17f021..74018241 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -95,7 +95,7 @@ impl<'a> CommandArguments<'a> { pub(crate) fn fully_parse_or_error( &self, parse_function: impl FnOnce(ParseStream) -> Result, - error_message: &'static str + error_message: &'static str, ) -> Result { let parsed = parse_function(self.parse_stream) .or_else(|_| self.full_span_range.err(error_message))?; @@ -106,7 +106,8 @@ impl<'a> CommandArguments<'a> { } pub(crate) fn parse_all_for_interpretation(&self) -> Result { - self.parse_stream.parse_all_for_interpretation(self.full_span_range) + self.parse_stream + .parse_all_for_interpretation(self.full_span_range) } pub(crate) fn read_all_as_raw_token_stream(&self) -> TokenStream { diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 507cc6cf..c895e7e5 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -18,7 +18,7 @@ enum GroupMatch { fn attempt_match_group(cursor: syn::buffer::Cursor) -> GroupMatch { let next = match cursor.any_group() { - Some((next, delimiter, _, _)) if delimiter == Delimiter::Bracket => next, + Some((next, Delimiter::Bracket, _, _)) => next, Some(_) => return GroupMatch::OtherGroup, None => return GroupMatch::None, }; @@ -41,13 +41,15 @@ impl Parse for InterpretationItem { match attempt_match_group(input.cursor()) { GroupMatch::Command => return Ok(InterpretationItem::Command(input.parse()?)), GroupMatch::OtherGroup => return Ok(InterpretationItem::Group(input.parse()?)), - GroupMatch::None => {}, + GroupMatch::None => {} } if input.peek(token::Pound) && input.peek2(syn::Ident) { - return Ok(InterpretationItem::Variable(input.parse()?)) + return Ok(InterpretationItem::Variable(input.parse()?)); } Ok(match input.parse::()? { - TokenTree::Group(_) => unreachable!("Should have been already handled by the first branch above"), + TokenTree::Group(_) => { + unreachable!("Should have been already handled by the first branch above") + } TokenTree::Punct(punct) => InterpretationItem::Punct(punct), TokenTree::Ident(ident) => InterpretationItem::Ident(ident), TokenTree::Literal(literal) => InterpretationItem::Literal(literal), diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index c0d71ac3..9e719c0d 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -43,11 +43,7 @@ impl InterpretedStream { delimiter: Delimiter, span: Span, ) { - self.push_raw_token_tree(TokenTree::group( - inner_tokens.token_stream, - delimiter, - span, - )); + self.push_raw_token_tree(TokenTree::group(inner_tokens.token_stream, delimiter, span)); } fn push_raw_token_tree(&mut self, token_tree: TokenTree) { diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 61afb09d..c7e623d3 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,15 +1,15 @@ mod command; +mod command_arguments; mod interpretation_item; mod interpretation_stream; mod interpreted_stream; mod interpreter; -mod command_arguments; mod variable; pub(crate) use command::*; +pub(crate) use command_arguments::*; pub(crate) use interpretation_item::*; pub(crate) use interpretation_stream::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; -pub(crate) use command_arguments::*; pub(crate) use variable::*; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 53618167..ab17fcc2 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -67,8 +67,10 @@ impl Interpret for &Variable { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - expression_stream - .push_grouped_interpreted_stream(self.substitute(interpreter)?, self.span_range().span()); + expression_stream.push_grouped_interpreted_stream( + self.substitute(interpreter)?, + self.span_range().span(), + ); Ok(()) } } diff --git a/src/traits.rs b/src/traits.rs index 2fecba35..c636c68b 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -119,9 +119,7 @@ pub(crate) trait ContextualParse: Sized { fn parse_with_context(input: ParseStream, context: Self::Context) -> Result; fn create_parser(context: Self::Context) -> impl FnOnce(ParseStream) -> Result { - move |input: ParseStream| { - Self::parse_with_context(input, context) - } + move |input: ParseStream| Self::parse_with_context(input, context) } } From 020dc7655b54f2b7ed51c96cc41e466bbcf892f9 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 15 Jan 2025 12:06:32 +0000 Subject: [PATCH 026/126] fix: Fix for msrv --- src/expressions/expression_stream.rs | 2 -- src/interpretation/interpretation_stream.rs | 4 ++++ src/traits.rs | 4 ---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 94fbeea6..6a5bc2b2 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -52,8 +52,6 @@ impl ExpressionStream { } pub(crate) fn evaluate(self) -> Result { - use syn::parse::{Parse, Parser}; - // Parsing into a rust expression is overkill here. // // In future we could choose to implement a subset of the grammar which we actually can use/need. diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 0ee090ee..5aa68576 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -14,6 +14,10 @@ impl InterpretationStream { ) -> Result { Self::create_parser(span_range).parse2(token_stream) } + + fn create_parser(context: SpanRange) -> impl FnOnce(ParseStream) -> Result { + move |input: ParseStream| Self::parse_with_context(input, context) + } } impl ContextualParse for InterpretationStream { diff --git a/src/traits.rs b/src/traits.rs index c636c68b..f8e28fb7 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -117,10 +117,6 @@ pub(crate) trait ContextualParse: Sized { type Context; fn parse_with_context(input: ParseStream, context: Self::Context) -> Result; - - fn create_parser(context: Self::Context) -> impl FnOnce(ParseStream) -> Result { - move |input: ParseStream| Self::parse_with_context(input, context) - } } pub(crate) trait SynErrorExt: Sized { From 3393bc0b13f1b8ff9f6e4586b56c8e4eedc2f7fb Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 16 Jan 2025 11:07:24 +0000 Subject: [PATCH 027/126] feature: Rework error command parsing to be on the input --- CHANGELOG.md | 12 +- src/commands/core_commands.rs | 183 +++++++++++------- src/expressions/expression_stream.rs | 14 ++ src/internal_prelude.rs | 2 +- src/interpretation/command.rs | 32 ++- src/interpretation/command_arguments.rs | 38 +++- src/interpretation/interpret_traits.rs | 40 ++++ src/interpretation/interpretation_item.rs | 50 ++--- src/interpretation/interpretation_stream.rs | 4 + src/interpretation/interpretation_value.rs | 54 ++++++ src/interpretation/interpreted_stream.rs | 5 + src/interpretation/mod.rs | 4 + src/interpretation/variable.rs | 31 ++- src/parsing/building_blocks.rs | 51 ++++- src/parsing/fields.rs | 3 + src/traits.rs | 32 +-- tests/compilation_failures/complex/nested.rs | 13 ++ .../complex/nested.stderr | 14 ++ .../control_flow/while_infinite_loop.rs | 5 + .../control_flow/while_infinite_loop.stderr | 5 + .../core/error_no_span.rs | 15 ++ .../core/error_no_span.stderr | 17 ++ .../core/error_span_multiple.rs | 16 ++ .../core/error_span_multiple.stderr | 5 + .../core/error_span_repeat.rs | 17 ++ .../core/error_span_repeat.stderr | 5 + .../{error_spans.rs => error_span_single.rs} | 0 ..._spans.stderr => error_span_single.stderr} | 2 +- tests/{simple_test.rs => complex.rs} | 8 +- tests/control_flow.rs | 12 +- tests/tokens.rs | 6 +- 31 files changed, 526 insertions(+), 169 deletions(-) create mode 100644 src/interpretation/interpret_traits.rs create mode 100644 src/interpretation/interpretation_value.rs create mode 100644 tests/compilation_failures/complex/nested.rs create mode 100644 tests/compilation_failures/complex/nested.stderr create mode 100644 tests/compilation_failures/control_flow/while_infinite_loop.rs create mode 100644 tests/compilation_failures/control_flow/while_infinite_loop.stderr create mode 100644 tests/compilation_failures/core/error_no_span.rs create mode 100644 tests/compilation_failures/core/error_no_span.stderr create mode 100644 tests/compilation_failures/core/error_span_multiple.rs create mode 100644 tests/compilation_failures/core/error_span_multiple.stderr create mode 100644 tests/compilation_failures/core/error_span_repeat.rs create mode 100644 tests/compilation_failures/core/error_span_repeat.stderr rename tests/compilation_failures/core/{error_spans.rs => error_span_single.rs} (100%) rename tests/compilation_failures/core/{error_spans.stderr => error_span_single.stderr} (56%) rename tests/{simple_test.rs => complex.rs} (80%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab986a62..56dc5a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,14 +22,11 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Support field parsing both before and at command execution. - * Move `[!error!]` field parsing to be at parse time - => Input fields defined via macro, including static parse step e.g. `message: InterpretationItem, spans: Option` - => Then each input can be interpreted as a specific type during execution. +* Create fields parsing macro +* Add compile tests for incorrectly formatted nested commands, e.g. an error inside an if * Grouping... Proposal: - * In output land: #x is a group, #..x is flattened - * In parse land: #x matches a single token, #..x consumes the rest of a stream * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!command! ...]` or `#x` may also be used instead. + * Add test that I can load !error! spans from `#x = Hello World` or `#..x = [Hello World]` * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces @@ -80,6 +77,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!split!]` and `[!split_no_trailing!]` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` * Basic place parsing + * In parse land: #x matches a single token, #..x consumes the rest of a stream * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! * Explicit Punct, Idents, Literals * `[!STREAM! ]` method to take a stream @@ -90,6 +88,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `#..x` binding reads the rest of the stream * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` * `[!match!]` (with `#..x` as a catch-all) +* Check all #[allow(unused)] and remove any which aren't needed +* Rework expression parsing * Work on book * Including documenting expressions diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 5a324c53..b82ee1dd 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -105,7 +105,7 @@ impl CommandInvocation for StreamCommand { #[derive(Clone)] pub(crate) struct ErrorCommand { - arguments: InterpretationStream, + arguments: ErrorArguments, } impl CommandDefinition for ErrorCommand { @@ -113,82 +113,125 @@ impl CommandDefinition for ErrorCommand { fn parse(arguments: CommandArguments) -> Result { Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, + arguments: arguments.fully_parse_as()?, }) } } -#[derive(Default)] -struct ErrorCommandArguments { - message: Option, - error_spans: Option, +#[derive(Clone)] +struct ErrorArguments { + message: InterpretationValue, + spans: Option>, +} + +impl ArgumentsContent for ErrorArguments { + fn error_message() -> String { + r#"Expected: { + // The error message to display + message: "...", + // An optional [token stream], to determine where to show the error message + spans?: [$abc], +}"# + .to_string() + } +} + +impl Parse for ErrorArguments { + fn parse(input: ParseStream) -> Result { + let mut message = None; + let mut spans = None; + + let content; + let brace = syn::braced!(content in input); + while !content.is_empty() { + let ident: Ident = content.parse()?; + content.parse::()?; + match ident.to_string().as_str() { + "message" => { + if message.is_some() { + return ident.err("duplicate field"); + } + message = Some(content.parse()?); + } + "spans" => { + if spans.is_some() { + return ident.err("duplicate field"); + } + spans = Some(content.parse()?); + } + _ => return ident.err("unexpected field"), + } + if !content.is_empty() { + content.parse::()?; + } + } + let mut missing_fields: Vec = vec![]; + + if message.is_none() { + missing_fields.push("message".to_string()); + } + + if !missing_fields.is_empty() { + return brace.span.err(format!( + "required fields are missing: {}", + missing_fields.join(", ") + )); + } + + Ok(Self { + message: message.unwrap(), + spans, + }) + } } impl CommandInvocation for ErrorCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let fields_parser = FieldsParseDefinition::new(ErrorCommandArguments::default()) - .add_required_field( - "message", - "\"Error message to display\"", - None, - |params, val| params.message = Some(val), - ) - .add_optional_field( - "spans", - "[$abc]", - Some("An optional [token stream], to determine where to show the error message"), - |params, val| params.error_spans = Some(val), - ); - - let arguments = self - .arguments - .interpret_as_tokens(interpreter)? - .parse_into_fields(fields_parser)?; - - let message = arguments - .message - .unwrap() // Field was required - .value(); - - let error_span_stream = arguments - .error_spans - .map(|b| b.token_stream) - .unwrap_or_default(); - - // Consider the case where preinterpret embeds in a declarative macro, and we have - // an error like this: - // [!error! [!string! "Expected 100, got " $input] [$input]] - // - // In cases like this, rustc wraps $input in a transparent group, which means that - // the span of that group is the span of the tokens "$input" in the definition of the - // declarative macro. This is not what we want. We want the span of the tokens which - // were fed into $input in the declarative macro. - // - // The simplest solution here is to get rid of all transparent groups, to get back to the - // source spans. - // - // Once this workstream with macro diagnostics is stabilised: - // https://github.com/rust-lang/rust/issues/54140#issuecomment-802701867 - // - // Then we can revisit this and do something better, and include all spans as separate spans - // in the error message, which will allow a user to trace an error through N different layers - // of macros. - // - // (Possibly we can try to join spans together, and if they don't join, they become separate - // spans which get printed to the error message). - // - // Coincidentally, rust analyzer currently does not properly support - // transparent groups (as of Jan 2025), so gets it right without this flattening: - // https://github.com/rust-lang/rust-analyzer/issues/18211 - let error_span_stream = error_span_stream.flatten_transparent_groups(); - - if error_span_stream.is_empty() { - Span::call_site().err(message) - } else { - error_span_stream - .into_token_stream() - .span_range() - .err(message) - } + let message = self.arguments.message.interpret(interpreter)?.value(); + + let error_span = match self.arguments.spans { + Some(spans) => { + let error_span_stream = spans.interpret(interpreter)?; + + // Consider the case where preinterpret embeds in a declarative macro, and we have + // an error like this: + // [!error! [!string! "Expected 100, got " $input] [$input]] + // + // In cases like this, rustc wraps $input in a transparent group, which means that + // the span of that group is the span of the tokens "$input" in the definition of the + // declarative macro. This is not what we want. We want the span of the tokens which + // were fed into $input in the declarative macro. + // + // The simplest solution here is to get rid of all transparent groups, to get back to the + // source spans. + // + // Once this workstream with macro diagnostics is stabilised: + // https://github.com/rust-lang/rust/issues/54140#issuecomment-802701867 + // + // Then we can revisit this and do something better, and include all spans as separate spans + // in the error message, which will allow a user to trace an error through N different layers + // of macros. + // + // (Possibly we can try to join spans together, and if they don't join, they become separate + // spans which get printed to the error message). + // + // Coincidentally, rust analyzer currently does not properly support + // transparent groups (as of Jan 2025), so gets it right without this flattening: + // https://github.com/rust-lang/rust-analyzer/issues/18211 + + let error_span_stream = error_span_stream + .interpreted_stream + .into_token_stream() + .flatten_transparent_groups(); + if error_span_stream.is_empty() { + Span::call_site().span_range() + } else { + error_span_stream.span_range() + } + } + None => Span::call_site().span_range(), + }; + + error_span.err(message) } } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 6a5bc2b2..210c559a 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -1,5 +1,19 @@ use super::*; +pub(crate) trait Express: Sized + HasSpanRange { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()>; + + fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { + let mut output = ExpressionStream::new(self.span_range()); + self.interpret_as_expression_into(interpreter, &mut output)?; + Ok(output) + } +} + /// This abstraction is a bit ropey... /// /// Ideally we'd parse expressions at parse time, but that requires writing a custom parser for diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 87704b6b..e16bda27 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -4,7 +4,7 @@ pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; pub(crate) use syn::ext::IdentExt as SynIdentExt; -pub(crate) use syn::parse::{discouraged::AnyDelimiter, Parse, ParseBuffer, ParseStream, Parser}; +pub(crate) use syn::parse::{discouraged::*, Parse, ParseBuffer, ParseStream, Parser}; pub(crate) use syn::{ parse_str, token, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, }; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 69df8edc..dad9c4a0 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -79,23 +79,6 @@ macro_rules! define_command_kind { } pub(crate) use define_command_kind; -impl Parse for CommandKind { - fn parse(input: ParseStream) -> Result { - // Support parsing any ident - let ident = input.call(Ident::parse_any)?; - match Self::for_ident(&ident) { - Some(command_kind) => Ok(command_kind), - None => ident.span().err( - format!( - "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", - Self::list_all(), - ident, - ), - ), - } - } -} - #[derive(Clone)] pub(crate) struct Command { invocation: Box, @@ -107,10 +90,21 @@ impl Parse for Command { let content; let open_bracket = syn::bracketed!(content in input); content.parse::()?; - let command_kind = content.parse::()?; + let command_name = content.call(Ident::parse_any)?; + let command_kind = match CommandKind::for_ident(&command_name) { + Some(command_kind) => command_kind, + None => command_name.span().err( + format!( + "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", + CommandKind::list_all(), + command_name, + ), + )?, + }; content.parse::()?; let invocation = command_kind.parse_invocation(CommandArguments::new( &content, + command_name, open_bracket.span.span_range(), ))?; Ok(Self { @@ -149,7 +143,9 @@ impl Interpret for Command { }; Ok(()) } +} +impl Express for Command { fn interpret_as_expression_into( self, interpreter: &mut Interpreter, diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index 74018241..2624fabc 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -64,6 +64,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct CommandArguments<'a> { parse_stream: ParseStream<'a>, + command_name: Ident, /// The span range of the original stream, before tokens were consumed full_span_range: SpanRange, /// The span of the last item consumed (or the full span range if no items have been consumed yet) @@ -71,9 +72,14 @@ pub(crate) struct CommandArguments<'a> { } impl<'a> CommandArguments<'a> { - pub(crate) fn new(parse_stream: ParseStream<'a>, span_range: SpanRange) -> Self { + pub(crate) fn new( + parse_stream: ParseStream<'a>, + command_name: Ident, + span_range: SpanRange, + ) -> Self { Self { parse_stream, + command_name, full_span_range: span_range, latest_item_span_range: span_range, } @@ -84,7 +90,7 @@ impl<'a> CommandArguments<'a> { } /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message - pub(crate) fn assert_empty(&self, error_message: &'static str) -> Result<()> { + pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> Result<()> { if self.parse_stream.is_empty() { Ok(()) } else { @@ -92,13 +98,31 @@ impl<'a> CommandArguments<'a> { } } + pub(crate) fn fully_parse_as(&self) -> Result { + self.fully_parse_or_error(T::parse, T::error_message()) + } + pub(crate) fn fully_parse_or_error( &self, parse_function: impl FnOnce(ParseStream) -> Result, - error_message: &'static str, + error_message: impl std::fmt::Display, ) -> Result { - let parsed = parse_function(self.parse_stream) - .or_else(|_| self.full_span_range.err(error_message))?; + let parsed = parse_function(self.parse_stream).or_else(|error| { + // In future, when the diagnostic API is stable, + // we can add this context directly onto the command ident... + // Rather than just selectively adding it to the inner-most error. + let error_string = error.to_string(); + + // We avoid adding this additional context if it's already been added in an + // inner error, because that's likely the correct error to show. + if error_string.contains("\nOccurred whilst parsing") { + return Err(error); + } + error.span().err(format!( + "{}\nOccurred whilst parsing [!{}! ..] - {}", + error_string, self.command_name, error_message, + )) + })?; self.assert_empty(error_message)?; @@ -114,3 +138,7 @@ impl<'a> CommandArguments<'a> { self.parse_stream.parse::().unwrap() } } + +pub(crate) trait ArgumentsContent: Parse { + fn error_message() -> String; +} diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs new file mode 100644 index 00000000..c646d5d3 --- /dev/null +++ b/src/interpretation/interpret_traits.rs @@ -0,0 +1,40 @@ +use crate::internal_prelude::*; + +pub(crate) trait Interpret: Sized + HasSpanRange { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()>; + + fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { + let mut output = InterpretedStream::new(self.span_range()); + self.interpret_as_tokens_into(interpreter, &mut output)?; + Ok(output) + } +} + +pub(crate) trait InterpretValue: Sized { + type InterpretedValue; + + fn interpret(self, interpreter: &mut Interpreter) -> Result; +} + +impl + HasSpanRange, I: ToTokens> Interpret for T { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + output.extend_raw(self.interpret(interpreter)?); + Ok(()) + } +} + +impl InterpretValue for T { + type InterpretedValue = Self; + + fn interpret(self, _interpreter: &mut Interpreter) -> Result { + Ok(self) + } +} diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index c895e7e5..48b3bc8c 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -10,13 +10,38 @@ pub(crate) enum InterpretationItem { Literal(Literal), } +impl Parse for InterpretationItem { + fn parse(input: ParseStream) -> Result { + match detect_group(input.cursor()) { + GroupMatch::Command => return Ok(InterpretationItem::Command(input.parse()?)), + GroupMatch::OtherGroup => return Ok(InterpretationItem::Group(input.parse()?)), + GroupMatch::None => {} + } + if input.peek(token::Pound) { + let fork = input.fork(); + if let Ok(variable) = fork.parse() { + input.advance_to(&fork); + return Ok(InterpretationItem::Variable(variable)); + } + } + Ok(match input.parse::()? { + TokenTree::Group(_) => { + unreachable!("Should have been already handled by the first branch above") + } + TokenTree::Punct(punct) => InterpretationItem::Punct(punct), + TokenTree::Ident(ident) => InterpretationItem::Ident(ident), + TokenTree::Literal(literal) => InterpretationItem::Literal(literal), + }) + } +} + enum GroupMatch { Command, OtherGroup, None, } -fn attempt_match_group(cursor: syn::buffer::Cursor) -> GroupMatch { +fn detect_group(cursor: syn::buffer::Cursor) -> GroupMatch { let next = match cursor.any_group() { Some((next, Delimiter::Bracket, _, _)) => next, Some(_) => return GroupMatch::OtherGroup, @@ -36,27 +61,6 @@ fn attempt_match_group(cursor: syn::buffer::Cursor) -> GroupMatch { } } -impl Parse for InterpretationItem { - fn parse(input: ParseStream) -> Result { - match attempt_match_group(input.cursor()) { - GroupMatch::Command => return Ok(InterpretationItem::Command(input.parse()?)), - GroupMatch::OtherGroup => return Ok(InterpretationItem::Group(input.parse()?)), - GroupMatch::None => {} - } - if input.peek(token::Pound) && input.peek2(syn::Ident) { - return Ok(InterpretationItem::Variable(input.parse()?)); - } - Ok(match input.parse::()? { - TokenTree::Group(_) => { - unreachable!("Should have been already handled by the first branch above") - } - TokenTree::Punct(punct) => InterpretationItem::Punct(punct), - TokenTree::Ident(ident) => InterpretationItem::Ident(ident), - TokenTree::Literal(literal) => InterpretationItem::Literal(literal), - }) - } -} - impl Interpret for InterpretationItem { fn interpret_as_tokens_into( self, @@ -79,7 +83,9 @@ impl Interpret for InterpretationItem { } Ok(()) } +} +impl Express for InterpretationItem { fn interpret_as_expression_into( self, interpreter: &mut Interpreter, diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 5aa68576..b6eb3d2d 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -43,7 +43,9 @@ impl Interpret for InterpretationStream { } Ok(()) } +} +impl Express for InterpretationStream { fn interpret_as_expression_into( self, interpreter: &mut Interpreter, @@ -112,7 +114,9 @@ impl Interpret for InterpretationGroup { ); Ok(()) } +} +impl Express for InterpretationGroup { fn interpret_as_expression_into( self, interpreter: &mut Interpreter, diff --git a/src/interpretation/interpretation_value.rs b/src/interpretation/interpretation_value.rs new file mode 100644 index 00000000..81d37bb1 --- /dev/null +++ b/src/interpretation/interpretation_value.rs @@ -0,0 +1,54 @@ +use crate::internal_prelude::*; + +/// This can be use to represent a value, or a source of that value at parse time. +/// +/// For example, `InterpretationValue` could be used to represent a literal, +/// or a variable/command which could convert to a literal after interpretation. +#[derive(Clone)] +pub(crate) enum InterpretationValue { + Command(Command), + Variable(Variable), + Value(T), +} + +impl Parse for InterpretationValue { + fn parse(input: ParseStream) -> Result { + let fork = input.fork(); + if let Ok(command) = fork.parse() { + input.advance_to(&fork); + return Ok(InterpretationValue::Command(command)); + } + let fork = input.fork(); + if let Ok(command) = fork.parse() { + input.advance_to(&fork); + return Ok(InterpretationValue::Variable(command)); + } + Ok(InterpretationValue::Value(input.parse()?)) + } +} + +impl HasSpanRange for InterpretationValue { + fn span_range(&self) -> SpanRange { + match self { + InterpretationValue::Command(command) => command.span_range(), + InterpretationValue::Variable(variable) => variable.span_range(), + InterpretationValue::Value(value) => value.span_range(), + } + } +} + +impl, I: Parse> InterpretValue for InterpretationValue { + type InterpretedValue = I; + + fn interpret(self, interpreter: &mut Interpreter) -> Result { + match self { + InterpretationValue::Command(command) => command + .interpret_as_tokens(interpreter)? + .syn_parse(I::parse), + InterpretationValue::Variable(variable) => variable + .interpret_as_tokens(interpreter)? + .syn_parse(I::parse), + InterpretationValue::Value(value) => value.interpret(interpreter), + } + } +} diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 9e719c0d..dcf38b85 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -46,6 +46,10 @@ impl InterpretedStream { self.push_raw_token_tree(TokenTree::group(inner_tokens.token_stream, delimiter, span)); } + pub(crate) fn extend_raw(&mut self, tokens: impl ToTokens) { + tokens.to_tokens(&mut self.token_stream); + } + fn push_raw_token_tree(&mut self, token_tree: TokenTree) { self.token_stream.extend(iter::once(token_tree)); } @@ -54,6 +58,7 @@ impl InterpretedStream { self.token_stream.is_empty() } + #[allow(unused)] pub(crate) fn parse_into_fields( self, parser: FieldsParseDefinition, diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index c7e623d3..e4b888a9 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,15 +1,19 @@ mod command; mod command_arguments; +mod interpret_traits; mod interpretation_item; mod interpretation_stream; +mod interpretation_value; mod interpreted_stream; mod interpreter; mod variable; pub(crate) use command::*; pub(crate) use command_arguments::*; +pub(crate) use interpret_traits::*; pub(crate) use interpretation_item::*; pub(crate) use interpretation_stream::*; +pub(crate) use interpretation_value::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; pub(crate) use variable::*; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index ab17fcc2..c8861022 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -3,15 +3,30 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct Variable { marker: Token![#], + is_flattened: bool, variable_name: Ident, } impl Parse for Variable { fn parse(input: ParseStream) -> Result { - Ok(Self { - marker: input.parse()?, - variable_name: input.parse()?, - }) + let marker = input.parse()?; + let lookahead = input.lookahead1(); + if lookahead.peek(Ident::peek_any) { + return Ok(Self { + marker, + is_flattened: false, + variable_name: input.parse()?, + }); + } + if lookahead.peek(Token![..]) { + let _ = input.parse::(); + return Ok(Self { + marker, + is_flattened: true, + variable_name: input.parse()?, + }); + } + Err(lookahead.error()) } } @@ -58,10 +73,16 @@ impl Interpret for &Variable { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.extend(self.substitute(interpreter)?); + if self.is_flattened { + output.extend(self.substitute(interpreter)?); + } else { + output.push_new_group(self.substitute(interpreter)?, Delimiter::None, self.span()); + } Ok(()) } +} +impl Express for &Variable { fn interpret_as_expression_into( self, interpreter: &mut Interpreter, diff --git a/src/parsing/building_blocks.rs b/src/parsing/building_blocks.rs index 1d5245e7..234cf3db 100644 --- a/src/parsing/building_blocks.rs +++ b/src/parsing/building_blocks.rs @@ -1,18 +1,59 @@ use crate::internal_prelude::*; +#[derive(Clone)] +pub(crate) struct InterpretationBracketedGroup { + #[allow(unused)] + pub(crate) brackets: token::Bracket, + pub(crate) interpretation_stream: InterpretationStream, +} + +impl HasSpanRange for InterpretationBracketedGroup { + fn span_range(&self) -> SpanRange { + self.brackets.span.span_range() + } +} + +impl Parse for InterpretationBracketedGroup { + fn parse(input: ParseStream) -> Result { + let content; + let brackets = syn::bracketed!(content in input); + let interpretation_stream = content.parse_with(brackets.span.span_range())?; + Ok(Self { + brackets, + interpretation_stream, + }) + } +} + +impl InterpretValue for InterpretationBracketedGroup { + type InterpretedValue = InterpretedBracketedGroup; + + fn interpret(self, interpreter: &mut Interpreter) -> Result { + Ok(InterpretedBracketedGroup { + brackets: self.brackets, + interpreted_stream: self + .interpretation_stream + .interpret_as_tokens(interpreter)?, + }) + } +} + /// Parses a [..] block. -pub(crate) struct BracketedTokenStream { +pub(crate) struct InterpretedBracketedGroup { #[allow(unused)] pub(crate) brackets: token::Bracket, - pub(crate) token_stream: TokenStream, + pub(crate) interpreted_stream: InterpretedStream, } -impl syn::parse::Parse for BracketedTokenStream { +impl syn::parse::Parse for InterpretedBracketedGroup { fn parse(input: syn::parse::ParseStream) -> Result { let content; + let brackets = syn::bracketed!(content in input); + let interpreted_stream = + InterpretedStream::raw(brackets.span.span_range(), content.parse()?); Ok(Self { - brackets: syn::bracketed!(content in input), - token_stream: content.parse()?, + brackets, + interpreted_stream, }) } } diff --git a/src/parsing/fields.rs b/src/parsing/fields.rs index f47eb694..ae3a5b58 100644 --- a/src/parsing/fields.rs +++ b/src/parsing/fields.rs @@ -1,11 +1,13 @@ use crate::internal_prelude::*; use std::collections::{BTreeMap, BTreeSet, HashSet}; +#[allow(unused)] pub(crate) struct FieldsParseDefinition { new_builder: T, field_definitions: FieldDefinitions, } +#[allow(unused)] impl FieldsParseDefinition { pub(crate) fn new(new_builder: T) -> Self { Self { @@ -172,6 +174,7 @@ impl FieldDefinitions { } } +#[allow(unused)] struct FieldParseDefinition { is_required: bool, example: String, diff --git a/src/traits.rs b/src/traits.rs index f8e28fb7..5f5d3340 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,31 +1,5 @@ use crate::internal_prelude::*; -pub(crate) trait Interpret: Sized + HasSpanRange { - fn interpret_as_tokens_into( - self, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()>; - - fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(self.span_range()); - self.interpret_as_tokens_into(interpreter, &mut output)?; - Ok(output) - } - - fn interpret_as_expression_into( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, - ) -> Result<()>; - - fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { - let mut output = ExpressionStream::new(self.span_range()); - self.interpret_as_expression_into(interpreter, &mut output)?; - Ok(output) - } -} - pub(crate) trait IdentExt: Sized { fn new_bool(value: bool, span: Span) -> Self; } @@ -119,6 +93,7 @@ pub(crate) trait ContextualParse: Sized { fn parse_with_context(input: ParseStream, context: Self::Context) -> Result; } +#[allow(unused)] pub(crate) trait SynErrorExt: Sized { fn concat(self, extra: &str) -> Self; } @@ -257,7 +232,10 @@ impl HasSpanRange for Group { impl HasSpanRange for DelimSpan { fn span_range(&self) -> SpanRange { - SpanRange::new_between(self.open(), self.close()) + // We could use self.open() => self.close() here, but using + // self.join() is better as it can be round-tripped to a span + // as the whole span, rather than just the start or end. + SpanRange::new_between(self.join(), self.join()) } } diff --git a/tests/compilation_failures/complex/nested.rs b/tests/compilation_failures/complex/nested.rs new file mode 100644 index 00000000..273fb4ae --- /dev/null +++ b/tests/compilation_failures/complex/nested.rs @@ -0,0 +1,13 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!if! true { + [!if! true { + [!if! true { + [!error! { + // Missing message + }] + }] + }] + }]); +} \ No newline at end of file diff --git a/tests/compilation_failures/complex/nested.stderr b/tests/compilation_failures/complex/nested.stderr new file mode 100644 index 00000000..31fce6ce --- /dev/null +++ b/tests/compilation_failures/complex/nested.stderr @@ -0,0 +1,14 @@ +error: required fields are missing: message + Occurred whilst parsing [!error! ..] - Expected: { + // The error message to display + message: "...", + // An optional [token stream], to determine where to show the error message + spans?: [$abc], + } + --> tests/compilation_failures/complex/nested.rs:7:26 + | +7 | [!error! { + | __________________________^ +8 | | // Missing message +9 | | }] + | |_________________^ diff --git a/tests/compilation_failures/control_flow/while_infinite_loop.rs b/tests/compilation_failures/control_flow/while_infinite_loop.rs new file mode 100644 index 00000000..ad2ad9de --- /dev/null +++ b/tests/compilation_failures/control_flow/while_infinite_loop.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!while! true {}]); +} \ No newline at end of file diff --git a/tests/compilation_failures/control_flow/while_infinite_loop.stderr b/tests/compilation_failures/control_flow/while_infinite_loop.stderr new file mode 100644 index 00000000..9e6b5e43 --- /dev/null +++ b/tests/compilation_failures/control_flow/while_infinite_loop.stderr @@ -0,0 +1,5 @@ +error: Iteration limit of 10000 exceeded + --> tests/compilation_failures/control_flow/while_infinite_loop.rs:4:28 + | +4 | preinterpret!([!while! true {}]); + | ^^^^ diff --git a/tests/compilation_failures/core/error_no_span.rs b/tests/compilation_failures/core/error_no_span.rs new file mode 100644 index 00000000..2471f8e9 --- /dev/null +++ b/tests/compilation_failures/core/error_no_span.rs @@ -0,0 +1,15 @@ +use preinterpret::*; + +macro_rules! assert_literals_eq_no_spans { + ($input1:literal and $input2:literal) => {preinterpret!{ + [!if! ($input1 != $input2) { + [!error! { + message: [!string! "Expected " $input1 " to equal " $input2], + }] + }] + }}; +} + +fn main() { + assert_literals_eq_no_spans!(102 and 64); +} \ No newline at end of file diff --git a/tests/compilation_failures/core/error_no_span.stderr b/tests/compilation_failures/core/error_no_span.stderr new file mode 100644 index 00000000..f29628b2 --- /dev/null +++ b/tests/compilation_failures/core/error_no_span.stderr @@ -0,0 +1,17 @@ +error: Expected 102 to equal 64 + --> tests/compilation_failures/core/error_no_span.rs:4:47 + | +4 | ($input1:literal and $input2:literal) => {preinterpret!{ + | _______________________________________________^ +5 | | [!if! ($input1 != $input2) { +6 | | [!error! { +7 | | message: [!string! "Expected " $input1 " to equal " $input2], +8 | | }] +9 | | }] +10 | | }}; + | |_____^ +... +14 | assert_literals_eq_no_spans!(102 and 64); + | ---------------------------------------- in this macro invocation + | + = note: this error originates in the macro `preinterpret` which comes from the expansion of the macro `assert_literals_eq_no_spans` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/compilation_failures/core/error_span_multiple.rs b/tests/compilation_failures/core/error_span_multiple.rs new file mode 100644 index 00000000..f890d643 --- /dev/null +++ b/tests/compilation_failures/core/error_span_multiple.rs @@ -0,0 +1,16 @@ +use preinterpret::*; + +macro_rules! assert_literals_eq { + ($input1:literal and $input2:literal) => {preinterpret!{ + [!if! ($input1 != $input2) { + [!error! { + message: [!string! "Expected " $input1 " to equal " $input2], + spans: [$input1, $input2], + }] + }] + }}; +} + +fn main() { + assert_literals_eq!(102 and 64); +} \ No newline at end of file diff --git a/tests/compilation_failures/core/error_span_multiple.stderr b/tests/compilation_failures/core/error_span_multiple.stderr new file mode 100644 index 00000000..33d69e86 --- /dev/null +++ b/tests/compilation_failures/core/error_span_multiple.stderr @@ -0,0 +1,5 @@ +error: Expected 102 to equal 64 + --> tests/compilation_failures/core/error_span_multiple.rs:15:25 + | +15 | assert_literals_eq!(102 and 64); + | ^^^^^^^^^^ diff --git a/tests/compilation_failures/core/error_span_repeat.rs b/tests/compilation_failures/core/error_span_repeat.rs new file mode 100644 index 00000000..48bc84ba --- /dev/null +++ b/tests/compilation_failures/core/error_span_repeat.rs @@ -0,0 +1,17 @@ +use preinterpret::*; + +macro_rules! assert_input_length_of_3 { + ($($input:literal)+) => {preinterpret!{ + [!set! #input_length = [!length! $($input)+]]; + [!if! (#input_length != 3) { + [!error! { + message: [!string! "Expected 3 inputs, got " #input_length], + spans: [$($input)+], + }] + }] + }}; +} + +fn main() { + assert_input_length_of_3!(42 101 666 1024); +} \ No newline at end of file diff --git a/tests/compilation_failures/core/error_span_repeat.stderr b/tests/compilation_failures/core/error_span_repeat.stderr new file mode 100644 index 00000000..3f27bea3 --- /dev/null +++ b/tests/compilation_failures/core/error_span_repeat.stderr @@ -0,0 +1,5 @@ +error: Expected 3 inputs, got 4 + --> tests/compilation_failures/core/error_span_repeat.rs:16:31 + | +16 | assert_input_length_of_3!(42 101 666 1024); + | ^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/core/error_spans.rs b/tests/compilation_failures/core/error_span_single.rs similarity index 100% rename from tests/compilation_failures/core/error_spans.rs rename to tests/compilation_failures/core/error_span_single.rs diff --git a/tests/compilation_failures/core/error_spans.stderr b/tests/compilation_failures/core/error_span_single.stderr similarity index 56% rename from tests/compilation_failures/core/error_spans.stderr rename to tests/compilation_failures/core/error_span_single.stderr index 8f5b6365..476323e0 100644 --- a/tests/compilation_failures/core/error_spans.stderr +++ b/tests/compilation_failures/core/error_span_single.stderr @@ -1,5 +1,5 @@ error: Expected 100, got 5 - --> tests/compilation_failures/core/error_spans.rs:15:20 + --> tests/compilation_failures/core/error_span_single.rs:15:20 | 15 | assert_is_100!(5); | ^ diff --git a/tests/simple_test.rs b/tests/complex.rs similarity index 80% rename from tests/simple_test.rs rename to tests/complex.rs index cb67e500..e08f81cb 100644 --- a/tests/simple_test.rs +++ b/tests/complex.rs @@ -1,4 +1,4 @@ -use preinterpret::preinterpret; +use preinterpret::*; preinterpret! { [!set! #bytes = 32] @@ -13,6 +13,12 @@ preinterpret! { const SNAKE_CASE: &str = [!snake! MyVar]; } +#[test] +fn test_complex_compilation_failures() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/compilation_failures/complex/*.rs"); +} + #[test] fn complex_example_evaluates_correctly() { let _x: XBooHello1HelloWorld32 = MyStruct; diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 582e2572..e9a70425 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -8,6 +8,13 @@ macro_rules! assert_preinterpret_eq { }; } +#[test] +fn test_control_flow_compilation_failures() { + let t = trybuild::TestCases::new(); + // In particular, the "error" command is tested here. + t.compile_fail("tests/compilation_failures/control_flow/*.rs"); +} + #[test] fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); @@ -38,8 +45,3 @@ fn test_while() { #x }, 5); } - -// TODO: Check compilation error for: -// assert_preinterpret_eq!({ -// [!while! true {}] -// }, 5); diff --git a/tests/tokens.rs b/tests/tokens.rs index f3f0737f..2ab4ff8f 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -16,7 +16,7 @@ fn test_empty_and_is_empty() { assert_preinterpret_eq!([!is_empty! Not Empty], false); assert_preinterpret_eq!({ [!set! #x = [!empty!]] - [!is_empty! #x] + [!is_empty! #..x] }, true); assert_preinterpret_eq!({ [!set! #x = [!empty!]] @@ -34,10 +34,10 @@ fn test_length_and_group() { assert_preinterpret_eq!({ [!length! [!group! "hello" World]] }, 1); assert_preinterpret_eq!({ [!set! #x = Hello "World" (1 2 3 4 5)] - [!length! #x] + [!length! #..x] }, 3); assert_preinterpret_eq!({ [!set! #x = Hello "World" (1 2 3 4 5)] - [!length! [!group! #x]] + [!length! [!group! #..x]] }, 1); } From 8e8bb6a640fc8b425e9eac4377e131bb730e56ed Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 16 Jan 2025 16:56:42 +0000 Subject: [PATCH 028/126] tweak: Add explicit CodeInput and StreamInput --- CHANGELOG.md | 9 +- src/commands/control_flow_commands.rs | 12 +- src/commands/core_commands.rs | 5 +- src/commands/token_commands.rs | 4 +- src/interpretation/command_code_input.rs | 36 ++++++ src/interpretation/command_stream_input.rs | 111 ++++++++++++++++++ src/interpretation/interpretation_stream.rs | 64 ++++++---- src/interpretation/interpreted_stream.rs | 8 ++ src/interpretation/mod.rs | 4 + src/interpretation/variable.rs | 79 +++++++++---- src/parsing/building_blocks.rs | 58 --------- src/parsing/mod.rs | 2 + src/traits.rs | 9 -- .../tokens/empty_with_input.rs | 5 + .../tokens/empty_with_input.stderr | 5 + tests/tokens.rs | 6 + 16 files changed, 286 insertions(+), 131 deletions(-) create mode 100644 src/interpretation/command_code_input.rs create mode 100644 src/interpretation/command_stream_input.rs create mode 100644 tests/compilation_failures/tokens/empty_with_input.rs create mode 100644 tests/compilation_failures/tokens/empty_with_input.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 56dc5a6e..ec6e71bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,10 +23,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come * Create fields parsing macro -* Add compile tests for incorrectly formatted nested commands, e.g. an error inside an if -* Grouping... Proposal: - * Command arguments which are streams should be surrounded by `[ ... ]`... a `[!command! ...]` or `#x` may also be used instead. - * Add test that I can load !error! spans from `#x = Hello World` or `#..x = [Hello World]` +* Add tests for `$x` being raw. +* Add tests for CommandStreamInput (via `[!split! ]` or `[!intersperse! ]`?) from `#x = Hello World` or `#..x = [Hello World]` or `$x` * ? Use `[!let! #x = 12]` instead of `[!set! ...]` * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * Fix `if` and `while` to read expression until braces @@ -75,6 +73,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Even though this might be more performant, I'm not too much of a fan of this, as it's hard to understand * Perhaps we can leave it to the `[!while_parse! #x[!OPTIONAL! ,] from #X]` style commands? * `[!split!]` and `[!split_no_trailing!]` +* `[!intersperse! { items: X, with: X, add_trailing?: false, override_final?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` * Basic place parsing * In parse land: #x matches a single token, #..x consumes the rest of a stream @@ -88,7 +87,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `#..x` binding reads the rest of the stream * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` * `[!match!]` (with `#..x` as a catch-all) -* Check all #[allow(unused)] and remove any which aren't needed +* Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing * Work on book * Including documenting expressions diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 08f110a1..06ce8b40 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -3,8 +3,8 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IfCommand { condition: InterpretationItem, - true_code: InterpretationStream, - false_code: Option, + true_code: CommandCodeInput, + false_code: Option, nothing_span_range: SpanRange, } @@ -16,13 +16,13 @@ impl CommandDefinition for IfCommand { |input| { Ok(Self { condition: input.parse()?, - true_code: input.parse_code_group_for_interpretation()?, + true_code: input.parse()?, false_code: { if !input.is_empty() { input.parse::()?; input.parse::()?; input.parse::()?; - Some(input.parse_code_group_for_interpretation()?) + Some(input.parse()?) } else { None } @@ -59,7 +59,7 @@ impl CommandInvocation for IfCommand { #[derive(Clone)] pub(crate) struct WhileCommand { condition: InterpretationItem, - loop_code: InterpretationStream, + loop_code: CommandCodeInput, nothing_span_range: SpanRange, } @@ -71,7 +71,7 @@ impl CommandDefinition for WhileCommand { |input| { Ok(Self { condition: input.parse()?, - loop_code: input.parse_code_group_for_interpretation()?, + loop_code: input.parse()?, nothing_span_range: arguments.full_span_range(), }) }, diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index b82ee1dd..77877bf7 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -121,7 +121,7 @@ impl CommandDefinition for ErrorCommand { #[derive(Clone)] struct ErrorArguments { message: InterpretationValue, - spans: Option>, + spans: Option, } impl ArgumentsContent for ErrorArguments { @@ -191,7 +191,7 @@ impl CommandInvocation for ErrorCommand { let error_span = match self.arguments.spans { Some(spans) => { - let error_span_stream = spans.interpret(interpreter)?; + let error_span_stream = spans.interpret_as_tokens(interpreter)?; // Consider the case where preinterpret embeds in a declarative macro, and we have // an error like this: @@ -220,7 +220,6 @@ impl CommandInvocation for ErrorCommand { // https://github.com/rust-lang/rust-analyzer/issues/18211 let error_span_stream = error_span_stream - .interpreted_stream .into_token_stream() .flatten_transparent_groups(); if error_span_stream.is_empty() { diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index e4a6b559..4c1d2659 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -7,7 +7,9 @@ impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; fn parse(arguments: CommandArguments) -> Result { - arguments.assert_empty("The !empty! command does not take any arguments")?; + arguments.assert_empty( + "The !empty! command does not take any arguments. Perhaps you want !is_empty! instead?", + )?; Ok(Self) } } diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs new file mode 100644 index 00000000..667774ed --- /dev/null +++ b/src/interpretation/command_code_input.rs @@ -0,0 +1,36 @@ +use crate::internal_prelude::*; + +/// Parses a group { .. } for interpretation +#[derive(Clone)] +pub(crate) struct CommandCodeInput { + delim_span: DelimSpan, + inner: InterpretationStream, +} + +impl Parse for CommandCodeInput { + fn parse(input: ParseStream) -> Result { + let content; + let bracket = syn::braced!(content in input); + let inner = content.parse_with(bracket.span.span_range())?; + Ok(Self { + delim_span: bracket.span, + inner, + }) + } +} + +impl HasSpanRange for CommandCodeInput { + fn span_range(&self) -> SpanRange { + self.delim_span.span_range() + } +} + +impl Interpret for CommandCodeInput { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + self.inner.interpret_as_tokens_into(interpreter, output) + } +} diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs new file mode 100644 index 00000000..d34949ae --- /dev/null +++ b/src/interpretation/command_stream_input.rs @@ -0,0 +1,111 @@ +use crate::internal_prelude::*; + +/// For use as a stream input to a command, when the whole command isn't the stream. +/// +/// It accepts any of the following: +/// * A `[..]` group - the input stream is the interpreted contents of the brackets +/// * A [!command! ...] - the input stream is the command's output +/// * A $macro_variable or other transparent group - the input stream is the *raw* contents of the group +/// * A `#variable` - the input stream is the content of the variable +/// * A flattened `#..variable` - the variable must contain a single `[..]` or transparent group, the input stream are the group contents +#[derive(Clone)] +pub(crate) enum CommandStreamInput { + Command(Command), + Variable(Variable), + Bracketed { + delim_span: DelimSpan, + inner: InterpretationStream, + }, + Raw { + delim_span: DelimSpan, + inner: TokenStream, + }, +} + +impl Parse for CommandStreamInput { + fn parse(input: ParseStream) -> Result { + let fork = input.fork(); + if let Ok(command) = fork.parse() { + input.advance_to(&fork); + return Ok(CommandStreamInput::Command(command)); + } + let fork = input.fork(); + if let Ok(command) = fork.parse() { + input.advance_to(&fork); + return Ok(CommandStreamInput::Variable(command)); + } + let error_span = input.span(); + match input.parse_any_delimiter() { + Ok((Delimiter::Bracket, delim_span, content)) => Ok(CommandStreamInput::Bracketed { + delim_span, + inner: content.parse_with(delim_span.span_range())?, + }), + Ok((Delimiter::None, delim_span, content)) => Ok(CommandStreamInput::Raw { + delim_span, + inner: content.parse()?, + }), + _ => error_span + .err("expected [ .... ] or a [!command! ..], #variable, or #macro_variable"), + } + } +} + +impl HasSpanRange for CommandStreamInput { + fn span_range(&self) -> SpanRange { + match self { + CommandStreamInput::Command(command) => command.span_range(), + CommandStreamInput::Variable(variable) => variable.span_range(), + CommandStreamInput::Bracketed { + delim_span: span, .. + } => span.span_range(), + CommandStreamInput::Raw { + delim_span: span, .. + } => span.span_range(), + } + } +} + +impl Interpret for CommandStreamInput { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + match self { + CommandStreamInput::Command(command) => { + command.interpret_as_tokens_into(interpreter, output) + } + CommandStreamInput::Variable(variable) => { + if variable.is_flattened() { + let tokens = variable.interpret_as_new_stream(interpreter)? + .syn_parse(|input: ParseStream| -> Result { + let (delimiter, _, content) = input.parse_any_delimiter()?; + match delimiter { + Delimiter::Bracket | Delimiter::None if input.is_empty() => { + content.parse() + }, + _ => { + variable.err(format!( + "expected variable to contain a single [ .. ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", + variable.display_unflattened_variable_token(), + )) + }, + } + })?; + + output.extend_raw(tokens); + Ok(()) + } else { + variable.interpret_into_stream(interpreter, output) + } + } + CommandStreamInput::Bracketed { inner, .. } => { + inner.interpret_as_tokens_into(interpreter, output) + } + CommandStreamInput::Raw { inner, .. } => { + output.extend_raw(inner); + Ok(()) + } + } + } +} diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index b6eb3d2d..462fedda 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -75,43 +75,48 @@ impl HasSpanRange for InterpretationStream { pub(crate) struct InterpretationGroup { source_delimeter: Delimiter, source_delim_span: DelimSpan, - interpretation_stream: InterpretationStream, + content: InterpretationGroupContent, +} + +#[derive(Clone)] +enum InterpretationGroupContent { + Interpeted(InterpretationStream), + Raw(TokenStream), } impl Parse for InterpretationGroup { fn parse(input: ParseStream) -> Result { - let (delimeter, delim_span, content) = input.parse_any_delimiter()?; + let (delimiter, delim_span, content) = input.parse_any_delimiter()?; let span_range = delim_span.span_range(); + let content = match delimiter { + // This is likely from a macro variable or macro expansion. + // Either way, we shouldn't be interpreting it. + Delimiter::None => InterpretationGroupContent::Raw(content.parse()?), + _ => InterpretationGroupContent::Interpeted(content.parse_with(span_range)?), + }; Ok(Self { - source_delimeter: delimeter, + source_delimeter: delimiter, source_delim_span: delim_span, - interpretation_stream: content.parse_with(span_range)?, + content, }) } } -impl InterpretationGroup { - pub(crate) fn delimiter(&self) -> Delimiter { - self.source_delimeter - } - - pub(crate) fn into_inner_stream(self) -> InterpretationStream { - self.interpretation_stream - } -} - impl Interpret for InterpretationGroup { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.push_new_group( - self.interpretation_stream - .interpret_as_tokens(interpreter)?, - self.source_delimeter, - self.source_delim_span.join(), - ); + let inner = match self.content { + InterpretationGroupContent::Interpeted(stream) => { + stream.interpret_as_tokens(interpreter)? + } + InterpretationGroupContent::Raw(token_stream) => { + InterpretedStream::raw(self.source_delim_span.span_range(), token_stream) + } + }; + output.push_new_group(inner, self.source_delimeter, self.source_delim_span.join()); Ok(()) } } @@ -122,12 +127,19 @@ impl Express for InterpretationGroup { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - expression_stream.push_expression_group( - self.interpretation_stream - .interpret_as_expression(interpreter)?, - self.source_delimeter, - self.source_delim_span.join(), - ); + match self.content { + InterpretationGroupContent::Interpeted(stream) => expression_stream + .push_expression_group( + stream.interpret_as_expression(interpreter)?, + self.source_delimeter, + self.source_delim_span.join(), + ), + InterpretationGroupContent::Raw(token_stream) => expression_stream + .push_grouped_interpreted_stream( + InterpretedStream::raw(self.source_delim_span.span_range(), token_stream), + self.source_delim_span.join(), + ), + } Ok(()) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index dcf38b85..f168a69f 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -86,9 +86,17 @@ impl InterpretedStream { } } + pub(crate) fn set_span_range(&mut self, span_range: SpanRange) { + self.source_span_range = span_range; + } + pub(crate) fn into_token_stream(self) -> TokenStream { self.token_stream } + + pub(crate) fn append_cloned_into(&self, output: &mut InterpretedStream) { + output.token_stream.extend(self.token_stream.clone()) + } } impl From for InterpretedStream { diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index e4b888a9..0edfdc41 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,5 +1,7 @@ mod command; mod command_arguments; +mod command_code_input; +mod command_stream_input; mod interpret_traits; mod interpretation_item; mod interpretation_stream; @@ -10,6 +12,8 @@ mod variable; pub(crate) use command::*; pub(crate) use command_arguments::*; +pub(crate) use command_code_input::*; +pub(crate) use command_stream_input::*; pub(crate) use interpret_traits::*; pub(crate) use interpretation_item::*; pub(crate) use interpretation_stream::*; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index c8861022..d749e17b 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -35,29 +35,57 @@ impl Variable { self.variable_name.to_string() } + pub(crate) fn is_flattened(&self) -> bool { + self.is_flattened + } + pub(crate) fn set(&self, interpreter: &mut Interpreter, value: InterpretedStream) { interpreter.set_variable(self.variable_name(), value); } - fn substitute(&self, interpreter: &Interpreter) -> Result { - Ok(self.read_or_else( - interpreter, - || format!( - "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self, - self, - ) - )?.clone()) + pub(super) fn interpret_into_stream( + &self, + interpreter: &Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + self.read_existing(interpreter)?.append_cloned_into(output); + Ok(()) } - fn read_or_else<'i>( + pub(super) fn interpret_as_new_stream( &self, - interpreter: &'i Interpreter, - create_error: impl FnOnce() -> String, - ) -> Result<&'i InterpretedStream> { + interpreter: &Interpreter, + ) -> Result { + let mut cloned = self.read_existing(interpreter)?.clone(); + cloned.set_span_range(self.span_range()); + Ok(cloned) + } + + pub(crate) fn substitute_into( + &self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + if self.is_flattened { + self.interpret_into_stream(interpreter, output) + } else { + output.push_new_group( + self.interpret_as_new_stream(interpreter)?, + Delimiter::None, + self.span(), + ); + Ok(()) + } + } + + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i InterpretedStream> { match self.read_option(interpreter) { Some(token_stream) => Ok(token_stream), - None => self.span_range().err(create_error()), + None => self.span_range().err(format!( + "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", + self, + self, + )), } } @@ -65,6 +93,10 @@ impl Variable { let Variable { variable_name, .. } = self; interpreter.get_variable(&variable_name.to_string()) } + + pub(crate) fn display_unflattened_variable_token(&self) -> String { + format!("#{}", self.variable_name) + } } impl Interpret for &Variable { @@ -73,12 +105,7 @@ impl Interpret for &Variable { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - if self.is_flattened { - output.extend(self.substitute(interpreter)?); - } else { - output.push_new_group(self.substitute(interpreter)?, Delimiter::None, self.span()); - } - Ok(()) + self.substitute_into(interpreter, output) } } @@ -89,19 +116,25 @@ impl Express for &Variable { expression_stream: &mut ExpressionStream, ) -> Result<()> { expression_stream.push_grouped_interpreted_stream( - self.substitute(interpreter)?, - self.span_range().span(), + self.interpret_as_new_stream(interpreter)?, + self.span(), ); Ok(()) } } -impl HasSpanRange for &Variable { +impl HasSpanRange for Variable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) } } +impl HasSpanRange for &Variable { + fn span_range(&self) -> SpanRange { + Variable::span_range(self) + } +} + impl core::fmt::Display for Variable { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "#{}", self.variable_name) diff --git a/src/parsing/building_blocks.rs b/src/parsing/building_blocks.rs index 234cf3db..db17e26e 100644 --- a/src/parsing/building_blocks.rs +++ b/src/parsing/building_blocks.rs @@ -1,59 +1 @@ use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) struct InterpretationBracketedGroup { - #[allow(unused)] - pub(crate) brackets: token::Bracket, - pub(crate) interpretation_stream: InterpretationStream, -} - -impl HasSpanRange for InterpretationBracketedGroup { - fn span_range(&self) -> SpanRange { - self.brackets.span.span_range() - } -} - -impl Parse for InterpretationBracketedGroup { - fn parse(input: ParseStream) -> Result { - let content; - let brackets = syn::bracketed!(content in input); - let interpretation_stream = content.parse_with(brackets.span.span_range())?; - Ok(Self { - brackets, - interpretation_stream, - }) - } -} - -impl InterpretValue for InterpretationBracketedGroup { - type InterpretedValue = InterpretedBracketedGroup; - - fn interpret(self, interpreter: &mut Interpreter) -> Result { - Ok(InterpretedBracketedGroup { - brackets: self.brackets, - interpreted_stream: self - .interpretation_stream - .interpret_as_tokens(interpreter)?, - }) - } -} - -/// Parses a [..] block. -pub(crate) struct InterpretedBracketedGroup { - #[allow(unused)] - pub(crate) brackets: token::Bracket, - pub(crate) interpreted_stream: InterpretedStream, -} - -impl syn::parse::Parse for InterpretedBracketedGroup { - fn parse(input: syn::parse::ParseStream) -> Result { - let content; - let brackets = syn::bracketed!(content in input); - let interpreted_stream = - InterpretedStream::raw(brackets.span.span_range(), content.parse()?); - Ok(Self { - brackets, - interpreted_stream, - }) - } -} diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs index 20e8c1f7..41689ae0 100644 --- a/src/parsing/mod.rs +++ b/src/parsing/mod.rs @@ -1,5 +1,7 @@ +#[allow(unused)] mod building_blocks; mod fields; +#[allow(unused)] pub(crate) use building_blocks::*; pub(crate) use fields::*; diff --git a/src/traits.rs b/src/traits.rs index 5f5d3340..e2abaff6 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -66,7 +66,6 @@ impl TokenTreeExt for TokenTree { pub(crate) trait ParserExt { fn parse_with(&self, context: T::Context) -> Result; fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result; - fn parse_code_group_for_interpretation(&self) -> Result; } impl<'a> ParserExt for ParseBuffer<'a> { @@ -77,14 +76,6 @@ impl<'a> ParserExt for ParseBuffer<'a> { fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result { self.parse_with(span_range) } - - fn parse_code_group_for_interpretation(&self) -> Result { - let group = self.parse::()?; - if group.delimiter() != Delimiter::Brace { - return group.err("expected {"); - } - Ok(group.into_inner_stream()) - } } pub(crate) trait ContextualParse: Sized { diff --git a/tests/compilation_failures/tokens/empty_with_input.rs b/tests/compilation_failures/tokens/empty_with_input.rs new file mode 100644 index 00000000..932daedd --- /dev/null +++ b/tests/compilation_failures/tokens/empty_with_input.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!empty! should not have arguments]); +} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/empty_with_input.stderr b/tests/compilation_failures/tokens/empty_with_input.stderr new file mode 100644 index 00000000..7fb6d15e --- /dev/null +++ b/tests/compilation_failures/tokens/empty_with_input.stderr @@ -0,0 +1,5 @@ +error: The !empty! command does not take any arguments. Perhaps you want !is_empty! instead? + --> tests/compilation_failures/tokens/empty_with_input.rs:4:19 + | +4 | preinterpret!([!empty! should not have arguments]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/tokens.rs b/tests/tokens.rs index 2ab4ff8f..55a98b3e 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -6,6 +6,12 @@ macro_rules! assert_preinterpret_eq { }; } +#[test] +fn test_tokens_compilation_failures() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/compilation_failures/tokens/*.rs"); +} + #[test] fn test_empty_and_is_empty() { assert_preinterpret_eq!({ From 4332be6e868537832d3df28ebbe1dc801bf5c176 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 16 Jan 2025 22:58:45 +0000 Subject: [PATCH 029/126] feature: Add extend command --- CHANGELOG.md | 13 +- src/commands/core_commands.rs | 38 ++++- src/commands/expression_commands.rs | 4 +- src/commands/mod.rs | 1 + src/expressions/expression_stream.rs | 7 + src/interpretation/command_stream_input.rs | 56 ++++--- src/interpretation/interpretation_item.rs | 27 +++- src/interpretation/interpretation_value.rs | 18 ++- src/interpretation/interpreter.rs | 4 + src/interpretation/variable.rs | 168 +++++++++++++++------ tests/complex.rs | 1 + tests/control_flow.rs | 1 + tests/core.rs | 1 + tests/expressions.rs | 1 + tests/tokens.rs | 1 + 15 files changed, 248 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec6e71bc..7140375b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,9 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. ### To come - + +* Tests for `[!extend!]` +* Compile error test for `[!set #..x = f]` and `[!extend! #..x += f]` * Create fields parsing macro * Add tests for `$x` being raw. * Add tests for CommandStreamInput (via `[!split! ]` or `[!intersperse! ]`?) from `#x = Hello World` or `#..x = [Hello World]` or `$x` @@ -36,7 +38,6 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!range! 0..5]` outputs `0 1 2 3 4` * Reconfiguring iteration limit * Support `!else if!` in `!if!` -* `[!extend! #x += ...]` to make such actions more performant * Support `!for!` so we can use it for simple generation scenarios without needing macros at all: * Complexities: * Parsing `,` @@ -72,8 +73,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!for! [!SPLIT! #x,] in [Hello, World,]]` * Even though this might be more performant, I'm not too much of a fan of this, as it's hard to understand * Perhaps we can leave it to the `[!while_parse! #x[!OPTIONAL! ,] from #X]` style commands? -* `[!split!]` and `[!split_no_trailing!]` -* `[!intersperse! { items: X, with: X, add_trailing?: false, override_final?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` +* `[!split! { items: X, with: X, drop_trailing_empty?: true }]` +* `[!intersperse! { items: X, with: X, add_trailing?: false, override_final_with?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` * Basic place parsing * In parse land: #x matches a single token, #..x consumes the rest of a stream @@ -91,6 +92,10 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Rework expression parsing * Work on book * Including documenting expressions + * There are three main kinds of commands: + * Those taking a stream as-is + * Those taking some { fields } + * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` # Major Version 0.2 diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 77877bf7..9e1ca18a 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct SetCommand { - variable: Variable, + variable: GroupedVariable, #[allow(unused)] equals: Token![=], arguments: InterpretationStream, @@ -28,12 +28,46 @@ impl CommandDefinition for SetCommand { impl CommandInvocation for SetCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let result_tokens = self.arguments.interpret_as_tokens(interpreter)?; - self.variable.set(interpreter, result_tokens); + self.variable.set(interpreter, result_tokens)?; Ok(CommandOutput::Empty) } } +#[derive(Clone)] +pub(crate) struct ExtendCommand { + variable: GroupedVariable, + #[allow(unused)] + plus_equals: Token![+=], + arguments: InterpretationStream, +} + +impl CommandDefinition for ExtendCommand { + const COMMAND_NAME: &'static str = "extend"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + variable: input.parse()?, + plus_equals: input.parse()?, + arguments: input.parse_all_for_interpretation(arguments.full_span_range())?, + }) + }, + "Expected [!extend! #variable += .. tokens ..]" + ) + } +} + +impl CommandInvocation for ExtendCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let output = self.arguments.interpret_as_tokens(interpreter)?; + self.variable.get_mut(interpreter)? + .extend(output); + Ok(CommandOutput::Empty) + } +} + #[derive(Clone)] pub(crate) struct RawCommand { arguments_span_range: SpanRange, diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index e5a97a0b..9b7aa9a5 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -26,7 +26,7 @@ impl CommandInvocation for EvaluateCommand { #[derive(Clone)] pub(crate) struct AssignCommand { - variable: Variable, + variable: GroupedVariable, operator: Punct, #[allow(unused)] equals: Token![=], @@ -73,7 +73,7 @@ impl CommandInvocation for AssignCommand { expression.interpret_as_expression_into(interpreter, &mut expression_stream)?; let output = expression_stream.evaluate()?.into_interpreted_stream(); - variable.set(interpreter, output); + variable.set(interpreter, output)?; Ok(CommandOutput::Empty) } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 1899b100..68237891 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -14,6 +14,7 @@ use token_commands::*; define_command_kind! { // Core Commands SetCommand, + ExtendCommand, RawCommand, IgnoreCommand, StreamCommand, diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 210c559a..b0a4f5c1 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -55,6 +55,13 @@ impl ExpressionStream { .push_new_group(contents, Delimiter::None, span); } + pub(crate) fn push_interpreted_stream( + &mut self, + contents: InterpretedStream, + ) { + self.interpreted_stream.extend(contents); + } + pub(crate) fn push_expression_group( &mut self, contents: Self, diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index d34949ae..d37c347e 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -11,7 +11,8 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum CommandStreamInput { Command(Command), - Variable(Variable), + GroupedVariable(GroupedVariable), + FlattenedVariable(FlattenedVariable), Bracketed { delim_span: DelimSpan, inner: InterpretationStream, @@ -32,7 +33,12 @@ impl Parse for CommandStreamInput { let fork = input.fork(); if let Ok(command) = fork.parse() { input.advance_to(&fork); - return Ok(CommandStreamInput::Variable(command)); + return Ok(CommandStreamInput::GroupedVariable(command)); + } + let fork = input.fork(); + if let Ok(command) = fork.parse() { + input.advance_to(&fork); + return Ok(CommandStreamInput::FlattenedVariable(command)); } let error_span = input.span(); match input.parse_any_delimiter() { @@ -54,7 +60,8 @@ impl HasSpanRange for CommandStreamInput { fn span_range(&self) -> SpanRange { match self { CommandStreamInput::Command(command) => command.span_range(), - CommandStreamInput::Variable(variable) => variable.span_range(), + CommandStreamInput::GroupedVariable(variable) => variable.span_range(), + CommandStreamInput::FlattenedVariable(variable) => variable.span_range(), CommandStreamInput::Bracketed { delim_span: span, .. } => span.span_range(), @@ -75,29 +82,28 @@ impl Interpret for CommandStreamInput { CommandStreamInput::Command(command) => { command.interpret_as_tokens_into(interpreter, output) } - CommandStreamInput::Variable(variable) => { - if variable.is_flattened() { - let tokens = variable.interpret_as_new_stream(interpreter)? - .syn_parse(|input: ParseStream| -> Result { - let (delimiter, _, content) = input.parse_any_delimiter()?; - match delimiter { - Delimiter::Bracket | Delimiter::None if input.is_empty() => { - content.parse() - }, - _ => { - variable.err(format!( - "expected variable to contain a single [ .. ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", - variable.display_unflattened_variable_token(), - )) - }, - } - })?; + CommandStreamInput::FlattenedVariable(variable) => { + let tokens = variable.interpret_as_new_stream(interpreter)? + .syn_parse(|input: ParseStream| -> Result { + let (delimiter, _, content) = input.parse_any_delimiter()?; + match delimiter { + Delimiter::Bracket | Delimiter::None if input.is_empty() => { + content.parse() + }, + _ => { + variable.err(format!( + "expected variable to contain a single [ .. ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", + variable.display_grouped_variable_token(), + )) + }, + } + })?; - output.extend_raw(tokens); - Ok(()) - } else { - variable.interpret_into_stream(interpreter, output) - } + output.extend_raw(tokens); + Ok(()) + } + CommandStreamInput::GroupedVariable(variable) => { + variable.interpret_as_tokens_into(interpreter, output) } CommandStreamInput::Bracketed { inner, .. } => { inner.interpret_as_tokens_into(interpreter, output) diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 48b3bc8c..d6e6b738 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -3,7 +3,8 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum InterpretationItem { Command(Command), - Variable(Variable), + GroupedVariable(GroupedVariable), + FlattenedVariable(FlattenedVariable), Group(InterpretationGroup), Punct(Punct), Ident(Ident), @@ -21,7 +22,12 @@ impl Parse for InterpretationItem { let fork = input.fork(); if let Ok(variable) = fork.parse() { input.advance_to(&fork); - return Ok(InterpretationItem::Variable(variable)); + return Ok(InterpretationItem::GroupedVariable(variable)); + } + let fork = input.fork(); + if let Ok(variable) = fork.parse() { + input.advance_to(&fork); + return Ok(InterpretationItem::FlattenedVariable(variable)); } } Ok(match input.parse::()? { @@ -71,7 +77,10 @@ impl Interpret for InterpretationItem { InterpretationItem::Command(command_invocation) => { command_invocation.interpret_as_tokens_into(interpreter, output)?; } - InterpretationItem::Variable(variable) => { + InterpretationItem::GroupedVariable(variable) => { + variable.interpret_as_tokens_into(interpreter, output)?; + } + InterpretationItem::FlattenedVariable(variable) => { variable.interpret_as_tokens_into(interpreter, output)?; } InterpretationItem::Group(group) => { @@ -95,7 +104,10 @@ impl Express for InterpretationItem { InterpretationItem::Command(command_invocation) => { command_invocation.interpret_as_expression_into(interpreter, expression_stream)?; } - InterpretationItem::Variable(variable) => { + InterpretationItem::FlattenedVariable(variable) => { + variable.interpret_as_expression_into(interpreter, expression_stream)?; + } + InterpretationItem::GroupedVariable(variable) => { variable.interpret_as_expression_into(interpreter, expression_stream)?; } InterpretationItem::Group(group) => { @@ -113,8 +125,11 @@ impl HasSpanRange for InterpretationItem { fn span_range(&self) -> SpanRange { match self { InterpretationItem::Command(command_invocation) => command_invocation.span_range(), - InterpretationItem::Variable(variable_substitution) => { - variable_substitution.span_range() + InterpretationItem::FlattenedVariable(variable) => { + variable.span_range() + } + InterpretationItem::GroupedVariable(variable) => { + variable.span_range() } InterpretationItem::Group(group) => group.span_range(), InterpretationItem::Punct(punct) => punct.span_range(), diff --git a/src/interpretation/interpretation_value.rs b/src/interpretation/interpretation_value.rs index 81d37bb1..91e06a0c 100644 --- a/src/interpretation/interpretation_value.rs +++ b/src/interpretation/interpretation_value.rs @@ -7,7 +7,8 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum InterpretationValue { Command(Command), - Variable(Variable), + GroupedVariable(GroupedVariable), + FlattenedVariable(FlattenedVariable), Value(T), } @@ -21,7 +22,12 @@ impl Parse for InterpretationValue { let fork = input.fork(); if let Ok(command) = fork.parse() { input.advance_to(&fork); - return Ok(InterpretationValue::Variable(command)); + return Ok(InterpretationValue::GroupedVariable(command)); + } + let fork = input.fork(); + if let Ok(command) = fork.parse() { + input.advance_to(&fork); + return Ok(InterpretationValue::FlattenedVariable(command)); } Ok(InterpretationValue::Value(input.parse()?)) } @@ -31,7 +37,8 @@ impl HasSpanRange for InterpretationValue { fn span_range(&self) -> SpanRange { match self { InterpretationValue::Command(command) => command.span_range(), - InterpretationValue::Variable(variable) => variable.span_range(), + InterpretationValue::GroupedVariable(variable) => variable.span_range(), + InterpretationValue::FlattenedVariable(variable) => variable.span_range(), InterpretationValue::Value(value) => value.span_range(), } } @@ -45,7 +52,10 @@ impl, I: Parse> InterpretValue for Inter InterpretationValue::Command(command) => command .interpret_as_tokens(interpreter)? .syn_parse(I::parse), - InterpretationValue::Variable(variable) => variable + InterpretationValue::GroupedVariable(variable) => variable + .interpret_as_tokens(interpreter)? + .syn_parse(I::parse), + InterpretationValue::FlattenedVariable(variable) => variable .interpret_as_tokens(interpreter)? .syn_parse(I::parse), InterpretationValue::Value(value) => value.interpret(interpreter), diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 85793cfc..9da61b31 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -21,6 +21,10 @@ impl Interpreter { self.variables.get(name) } + pub(crate) fn get_variable_mut<'i>(&'i mut self, name: &str) -> Option<&'i mut InterpretedStream> { + self.variables.get_mut(name) + } + pub(crate) fn config(&self) -> &InterpreterConfig { &self.config } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index d749e17b..055d3bb4 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -1,57 +1,134 @@ use crate::internal_prelude::*; #[derive(Clone)] -pub(crate) struct Variable { +pub(crate) struct GroupedVariable { marker: Token![#], - is_flattened: bool, variable_name: Ident, } -impl Parse for Variable { +impl Parse for GroupedVariable { fn parse(input: ParseStream) -> Result { - let marker = input.parse()?; - let lookahead = input.lookahead1(); - if lookahead.peek(Ident::peek_any) { - return Ok(Self { - marker, - is_flattened: false, - variable_name: input.parse()?, - }); - } - if lookahead.peek(Token![..]) { - let _ = input.parse::(); - return Ok(Self { - marker, - is_flattened: true, - variable_name: input.parse()?, - }); - } - Err(lookahead.error()) + Ok(Self { + marker: input.parse()?, + variable_name: input.call(Ident::parse_any)?, + }) } } -impl Variable { +impl GroupedVariable { pub(crate) fn variable_name(&self) -> String { self.variable_name.to_string() } - pub(crate) fn is_flattened(&self) -> bool { - self.is_flattened + pub(crate) fn set(&self, interpreter: &mut Interpreter, value: InterpretedStream) -> Result<()> { + interpreter.set_variable(self.variable_name(), value); + Ok(()) } - pub(crate) fn set(&self, interpreter: &mut Interpreter, value: InterpretedStream) { - interpreter.set_variable(self.variable_name(), value); + pub(crate) fn get_mut<'i>(&self, interpreter: &'i mut Interpreter) -> Result<&'i mut InterpretedStream> { + interpreter.get_variable_mut(&self.variable_name()) + .ok_or_else(|| self.error(format!("The variable {} wasn't already set", self))) } - pub(super) fn interpret_into_stream( + pub(super) fn interpret_as_new_stream( &self, interpreter: &Interpreter, + ) -> Result { + let mut cloned = self.read_existing(interpreter)?.clone(); + cloned.set_span_range(self.span_range()); + Ok(cloned) + } + + pub(crate) fn substitute_into( + &self, + interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.read_existing(interpreter)?.append_cloned_into(output); + output.push_new_group( + self.interpret_as_new_stream(interpreter)?, + Delimiter::None, + self.span(), + ); + Ok(()) + } + + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i InterpretedStream> { + match self.read_option(interpreter) { + Some(token_stream) => Ok(token_stream), + None => self.span_range().err(format!( + "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", + self, + self, + )), + } + } + + fn read_option<'i>(&self, interpreter: &'i Interpreter) -> Option<&'i InterpretedStream> { + interpreter.get_variable(&self.variable_name.to_string()) + } +} + +impl Interpret for &GroupedVariable { + fn interpret_as_tokens_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + self.substitute_into(interpreter, output) + } +} + +impl Express for &GroupedVariable { + fn interpret_as_expression_into( + self, + interpreter: &mut Interpreter, + expression_stream: &mut ExpressionStream, + ) -> Result<()> { + expression_stream.push_grouped_interpreted_stream( + self.interpret_as_new_stream(interpreter)?, + self.span(), + ); Ok(()) } +} + +impl HasSpanRange for GroupedVariable { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.marker.span, self.variable_name.span()) + } +} + +impl HasSpanRange for &GroupedVariable { + fn span_range(&self) -> SpanRange { + GroupedVariable::span_range(self) + } +} + +impl core::fmt::Display for GroupedVariable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "#..{}", self.variable_name) + } +} + +#[derive(Clone)] +pub(crate) struct FlattenedVariable { + marker: Token![#], + #[allow(unused)] + flatten: Token![..], + variable_name: Ident, +} +impl Parse for FlattenedVariable { + fn parse(input: ParseStream) -> Result { + Ok(Self { + marker: input.parse()?, + flatten: input.parse()?, + variable_name: input.call(Ident::parse_any)?, + }) + } +} + +impl FlattenedVariable { pub(super) fn interpret_as_new_stream( &self, interpreter: &Interpreter, @@ -66,16 +143,8 @@ impl Variable { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - if self.is_flattened { - self.interpret_into_stream(interpreter, output) - } else { - output.push_new_group( - self.interpret_as_new_stream(interpreter)?, - Delimiter::None, - self.span(), - ); - Ok(()) - } + self.read_existing(interpreter)?.append_cloned_into(output); + Ok(()) } fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i InterpretedStream> { @@ -90,16 +159,16 @@ impl Variable { } fn read_option<'i>(&self, interpreter: &'i Interpreter) -> Option<&'i InterpretedStream> { - let Variable { variable_name, .. } = self; + let FlattenedVariable { variable_name, .. } = self; interpreter.get_variable(&variable_name.to_string()) } - pub(crate) fn display_unflattened_variable_token(&self) -> String { + pub(crate) fn display_grouped_variable_token(&self) -> String { format!("#{}", self.variable_name) } } -impl Interpret for &Variable { +impl Interpret for &FlattenedVariable { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, @@ -109,34 +178,33 @@ impl Interpret for &Variable { } } -impl Express for &Variable { +impl Express for &FlattenedVariable { fn interpret_as_expression_into( self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - expression_stream.push_grouped_interpreted_stream( - self.interpret_as_new_stream(interpreter)?, - self.span(), + expression_stream.push_interpreted_stream( + self.interpret_as_tokens(interpreter)?, ); Ok(()) } } -impl HasSpanRange for Variable { +impl HasSpanRange for FlattenedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) } } -impl HasSpanRange for &Variable { +impl HasSpanRange for &FlattenedVariable { fn span_range(&self) -> SpanRange { - Variable::span_range(self) + FlattenedVariable::span_range(self) } } -impl core::fmt::Display for Variable { +impl core::fmt::Display for FlattenedVariable { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "#{}", self.variable_name) + write!(f, "#..{}", self.variable_name) } } diff --git a/tests/complex.rs b/tests/complex.rs index e08f81cb..1331a6fa 100644 --- a/tests/complex.rs +++ b/tests/complex.rs @@ -14,6 +14,7 @@ preinterpret! { } #[test] +#[cfg_attr(miri, ignore = "incompatible with miri")] fn test_complex_compilation_failures() { let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/complex/*.rs"); diff --git a/tests/control_flow.rs b/tests/control_flow.rs index e9a70425..bb8912c5 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -9,6 +9,7 @@ macro_rules! assert_preinterpret_eq { } #[test] +#[cfg_attr(miri, ignore = "incompatible with miri")] fn test_control_flow_compilation_failures() { let t = trybuild::TestCases::new(); // In particular, the "error" command is tested here. diff --git a/tests/core.rs b/tests/core.rs index 1e4199d5..760e4008 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -7,6 +7,7 @@ macro_rules! my_assert_eq { } #[test] +#[cfg_attr(miri, ignore = "incompatible with miri")] fn test_core_compilation_failures() { let t = trybuild::TestCases::new(); // In particular, the "error" command is tested here. diff --git a/tests/expressions.rs b/tests/expressions.rs index 0af48f26..f07f124b 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -7,6 +7,7 @@ macro_rules! assert_preinterpret_eq { } #[test] +#[cfg_attr(miri, ignore = "incompatible with miri")] fn test_expression_compilation_failures() { let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/expressions/*.rs"); diff --git a/tests/tokens.rs b/tests/tokens.rs index 55a98b3e..6bf92b40 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -7,6 +7,7 @@ macro_rules! assert_preinterpret_eq { } #[test] +#[cfg_attr(miri, ignore = "incompatible with miri")] fn test_tokens_compilation_failures() { let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/tokens/*.rs"); From da1b94bb721c1da91f3ac838d6041624c2223546 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 16 Jan 2025 22:59:04 +0000 Subject: [PATCH 030/126] fix: Fix styling --- src/commands/core_commands.rs | 5 ++--- src/expressions/expression_stream.rs | 5 +---- src/interpretation/interpretation_item.rs | 8 ++------ src/interpretation/interpreter.rs | 5 ++++- src/interpretation/variable.rs | 18 ++++++++++++------ 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 9e1ca18a..e1540d4b 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -54,7 +54,7 @@ impl CommandDefinition for ExtendCommand { arguments: input.parse_all_for_interpretation(arguments.full_span_range())?, }) }, - "Expected [!extend! #variable += .. tokens ..]" + "Expected [!extend! #variable += .. tokens ..]", ) } } @@ -62,8 +62,7 @@ impl CommandDefinition for ExtendCommand { impl CommandInvocation for ExtendCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output = self.arguments.interpret_as_tokens(interpreter)?; - self.variable.get_mut(interpreter)? - .extend(output); + self.variable.get_mut(interpreter)?.extend(output); Ok(CommandOutput::Empty) } } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index b0a4f5c1..3a906afc 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -55,10 +55,7 @@ impl ExpressionStream { .push_new_group(contents, Delimiter::None, span); } - pub(crate) fn push_interpreted_stream( - &mut self, - contents: InterpretedStream, - ) { + pub(crate) fn push_interpreted_stream(&mut self, contents: InterpretedStream) { self.interpreted_stream.extend(contents); } diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index d6e6b738..5fbaec6b 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -125,12 +125,8 @@ impl HasSpanRange for InterpretationItem { fn span_range(&self) -> SpanRange { match self { InterpretationItem::Command(command_invocation) => command_invocation.span_range(), - InterpretationItem::FlattenedVariable(variable) => { - variable.span_range() - } - InterpretationItem::GroupedVariable(variable) => { - variable.span_range() - } + InterpretationItem::FlattenedVariable(variable) => variable.span_range(), + InterpretationItem::GroupedVariable(variable) => variable.span_range(), InterpretationItem::Group(group) => group.span_range(), InterpretationItem::Punct(punct) => punct.span_range(), InterpretationItem::Ident(ident) => ident.span_range(), diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 9da61b31..50db11e2 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -21,7 +21,10 @@ impl Interpreter { self.variables.get(name) } - pub(crate) fn get_variable_mut<'i>(&'i mut self, name: &str) -> Option<&'i mut InterpretedStream> { + pub(crate) fn get_variable_mut<'i>( + &'i mut self, + name: &str, + ) -> Option<&'i mut InterpretedStream> { self.variables.get_mut(name) } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 055d3bb4..24db3f79 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -20,13 +20,21 @@ impl GroupedVariable { self.variable_name.to_string() } - pub(crate) fn set(&self, interpreter: &mut Interpreter, value: InterpretedStream) -> Result<()> { + pub(crate) fn set( + &self, + interpreter: &mut Interpreter, + value: InterpretedStream, + ) -> Result<()> { interpreter.set_variable(self.variable_name(), value); Ok(()) } - pub(crate) fn get_mut<'i>(&self, interpreter: &'i mut Interpreter) -> Result<&'i mut InterpretedStream> { - interpreter.get_variable_mut(&self.variable_name()) + pub(crate) fn get_mut<'i>( + &self, + interpreter: &'i mut Interpreter, + ) -> Result<&'i mut InterpretedStream> { + interpreter + .get_variable_mut(&self.variable_name()) .ok_or_else(|| self.error(format!("The variable {} wasn't already set", self))) } @@ -184,9 +192,7 @@ impl Express for &FlattenedVariable { interpreter: &mut Interpreter, expression_stream: &mut ExpressionStream, ) -> Result<()> { - expression_stream.push_interpreted_stream( - self.interpret_as_tokens(interpreter)?, - ); + expression_stream.push_interpreted_stream(self.interpret_as_tokens(interpreter)?); Ok(()) } } From 0da56d9baf9e2a9d689570357c884e82418d2802 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 16 Jan 2025 23:04:06 +0000 Subject: [PATCH 031/126] fix: Fix style --- src/traits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/traits.rs b/src/traits.rs index e2abaff6..1d97b049 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -68,7 +68,7 @@ pub(crate) trait ParserExt { fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result; } -impl<'a> ParserExt for ParseBuffer<'a> { +impl ParserExt for ParseBuffer<'_> { fn parse_with(&self, context: T::Context) -> Result { T::parse_with_context(self, context) } From 039fe690e157d2c0b38c809b4c4576d8172e28d9 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 17 Jan 2025 22:22:38 +0000 Subject: [PATCH 032/126] feat: Made expressions more flexible and added !range! --- CHANGELOG.md | 38 ++-- Cargo.toml | 2 +- src/commands/control_flow_commands.rs | 10 +- src/commands/core_commands.rs | 90 ++------ src/commands/expression_commands.rs | 139 ++++++++++-- src/commands/mod.rs | 1 + src/expressions/expression_stream.rs | 200 ++++++++++++++++-- src/expressions/integer.rs | 24 +++ src/internal_prelude.rs | 3 +- src/interpretation/command.rs | 9 +- src/interpretation/command_fields_input.rs | 115 ++++++++++ src/interpretation/command_stream_input.rs | 70 ++---- src/interpretation/interpret_traits.rs | 2 +- src/interpretation/interpretation_item.rs | 138 ++++++------ src/interpretation/interpretation_stream.rs | 118 ++++++----- src/interpretation/interpretation_value.rs | 22 +- src/interpretation/interpreted_stream.rs | 6 +- src/interpretation/mod.rs | 2 + src/interpretation/variable.rs | 38 ++-- src/traits.rs | 47 +++- .../core/extend_flattened_variable.rs | 8 + .../core/extend_flattened_variable.stderr | 6 + .../core/set_flattened_variable.rs | 7 + .../core/set_flattened_variable.stderr | 6 + .../expressions/braces.stderr | 5 +- ...ped_variable_with_incomplete_expression.rs | 8 + ...variable_with_incomplete_expression.stderr | 5 + .../expressions/inner_braces.rs | 7 + .../expressions/inner_braces.stderr | 6 + tests/control_flow.rs | 4 +- tests/core.rs | 27 +++ tests/expressions.rs | 20 ++ 32 files changed, 831 insertions(+), 352 deletions(-) create mode 100644 src/interpretation/command_fields_input.rs create mode 100644 tests/compilation_failures/core/extend_flattened_variable.rs create mode 100644 tests/compilation_failures/core/extend_flattened_variable.stderr create mode 100644 tests/compilation_failures/core/set_flattened_variable.rs create mode 100644 tests/compilation_failures/core/set_flattened_variable.stderr create mode 100644 tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs create mode 100644 tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr create mode 100644 tests/compilation_failures/expressions/inner_braces.rs create mode 100644 tests/compilation_failures/expressions/inner_braces.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 7140375b..81434f4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,40 +4,40 @@ ### New Commands +* Core commands: + * `[!error! ...]` + * `[!stream! ...]` command which just returns its contents, but can be used in places where a single item is expected when parsing. * Expression commands: * `[!evaluate! ...]` * `[!assign! #x += ...]` for `+` and other supported operators + * `[!range! 0..5]` outputs `0 1 2 3 4` * Control flow commands: * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` - * `[!while! cond {}]` + * `[!while! COND {}]` * Token-stream utility commands: * `[!empty!]` * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. -* Other commands: - * `[!error! ..]` I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. ### To come -* Tests for `[!extend!]` -* Compile error test for `[!set #..x = f]` and `[!extend! #..x += f]` -* Create fields parsing macro -* Add tests for `$x` being raw. -* Add tests for CommandStreamInput (via `[!split! ]` or `[!intersperse! ]`?) from `#x = Hello World` or `#..x = [Hello World]` or `$x` -* ? Use `[!let! #x = 12]` instead of `[!set! ...]` - * ...Or maybe not. Maybe `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` -* Fix `if` and `while` to read expression until braces +* Explore getting rid of lots of the span range stuff +* Support `!else if!` in `!if!` * Support string & char literals (for comparisons & casts) in expressions +* Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` +* Other token stream commands +* Add tests for CommandStreamInput (via `[!split! ]` or `[!intersperse! ]`?): + => From `#x = Hello World` + => Or `#..x = [Hello World]` + => But not `$x` - it has to be wrapped in `[]` + => Improve tests for `[!range!]` * Add more tests * e.g. for various expressions * e.g. for long sums * Add compile failure tests -* `[!range! 0..5]` outputs `0 1 2 3 4` -* Reconfiguring iteration limit -* Support `!else if!` in `!if!` * Support `!for!` so we can use it for simple generation scenarios without needing macros at all: * Complexities: * Parsing `,` @@ -77,17 +77,21 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!intersperse! { items: X, with: X, add_trailing?: false, override_final_with?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` * Basic place parsing + * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * In parse land: #x matches a single token, #..x consumes the rest of a stream * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! * Explicit Punct, Idents, Literals - * `[!STREAM! ]` method to take a stream + * `[!PARSER! ]` method to take a stream * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings - * `[!OPTIONAL! ...]` + * `[!OPTIONAL! ...]` and/or possibly `#(..)?` and `[!is_set! #x]`? * Groups or `[!GROUP! ...]` + * Regarding `#(..)+` and `#(..)*`... + * Any binding could be set to an array, but it gets complicated fast with nested bindings. + * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. * `#x` binding reads a token tree * `#..x` binding reads the rest of the stream * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` -* `[!match!]` (with `#..x` as a catch-all) + * `[!match!]` (with `#..x` as a catch-all) * Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing * Work on book diff --git a/Cargo.toml b/Cargo.toml index 000deacf..339a560f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0" } -syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing", "clone-impls"] } +syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing", "clone-impls", "full"] } quote = { version = "1.0", default-features = false } [dev-dependencies] diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 06ce8b40..520590bd 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IfCommand { - condition: InterpretationItem, + condition: ExpressionInput, true_code: CommandCodeInput, false_code: Option, nothing_span_range: SpanRange, @@ -39,8 +39,7 @@ impl CommandInvocation for IfCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let evaluated_condition = self .condition - .interpret_as_expression(interpreter)? - .evaluate()? + .evaluate(interpreter)? .expect_bool("An if condition must evaluate to a boolean")? .value(); @@ -58,7 +57,7 @@ impl CommandInvocation for IfCommand { #[derive(Clone)] pub(crate) struct WhileCommand { - condition: InterpretationItem, + condition: ExpressionInput, loop_code: CommandCodeInput, nothing_span_range: SpanRange, } @@ -88,8 +87,7 @@ impl CommandInvocation for WhileCommand { let evaluated_condition = self .condition .clone() - .interpret_as_expression(interpreter)? - .evaluate()? + .evaluate(interpreter)? .expect_bool("An if condition must evaluate to a boolean")? .value(); diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index e1540d4b..64269c29 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -20,7 +20,7 @@ impl CommandDefinition for SetCommand { arguments: input.parse_with(arguments.full_span_range())?, }) }, - "Expected [!set! #variable = ... ]", + "Expected [!set! #variable = ..]", ) } } @@ -54,7 +54,7 @@ impl CommandDefinition for ExtendCommand { arguments: input.parse_all_for_interpretation(arguments.full_span_range())?, }) }, - "Expected [!extend! #variable += .. tokens ..]", + "Expected [!extend! #variable += ..]", ) } } @@ -138,91 +138,35 @@ impl CommandInvocation for StreamCommand { #[derive(Clone)] pub(crate) struct ErrorCommand { - arguments: ErrorArguments, + inputs: ErrorInputs, } -impl CommandDefinition for ErrorCommand { - const COMMAND_NAME: &'static str = "error"; - - fn parse(arguments: CommandArguments) -> Result { - Ok(Self { - arguments: arguments.fully_parse_as()?, - }) - } -} - -#[derive(Clone)] -struct ErrorArguments { - message: InterpretationValue, - spans: Option, -} - -impl ArgumentsContent for ErrorArguments { - fn error_message() -> String { - r#"Expected: { - // The error message to display - message: "...", - // An optional [token stream], to determine where to show the error message - spans?: [$abc], -}"# - .to_string() +define_field_inputs! { + ErrorInputs { + required: { + message: InterpretationValue = r#""...""# ("The error message to display"), + }, + optional: { + spans: CommandStreamInput = "[$abc]" ("An optional [token stream], to determine where to show the error message"), + } } } -impl Parse for ErrorArguments { - fn parse(input: ParseStream) -> Result { - let mut message = None; - let mut spans = None; - - let content; - let brace = syn::braced!(content in input); - while !content.is_empty() { - let ident: Ident = content.parse()?; - content.parse::()?; - match ident.to_string().as_str() { - "message" => { - if message.is_some() { - return ident.err("duplicate field"); - } - message = Some(content.parse()?); - } - "spans" => { - if spans.is_some() { - return ident.err("duplicate field"); - } - spans = Some(content.parse()?); - } - _ => return ident.err("unexpected field"), - } - if !content.is_empty() { - content.parse::()?; - } - } - let mut missing_fields: Vec = vec![]; - - if message.is_none() { - missing_fields.push("message".to_string()); - } - - if !missing_fields.is_empty() { - return brace.span.err(format!( - "required fields are missing: {}", - missing_fields.join(", ") - )); - } +impl CommandDefinition for ErrorCommand { + const COMMAND_NAME: &'static str = "error"; + fn parse(arguments: CommandArguments) -> Result { Ok(Self { - message: message.unwrap(), - spans, + inputs: arguments.fully_parse_as()?, }) } } impl CommandInvocation for ErrorCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let message = self.arguments.message.interpret(interpreter)?.value(); + let message = self.inputs.message.interpret(interpreter)?.value(); - let error_span = match self.arguments.spans { + let error_span = match self.inputs.spans { Some(spans) => { let error_span_stream = spans.interpret_as_tokens(interpreter)?; diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 9b7aa9a5..4d9b0e36 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -2,22 +2,27 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct EvaluateCommand { - expression: InterpretationStream, + expression: ExpressionInput, } impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; fn parse(arguments: CommandArguments) -> Result { - Ok(Self { - expression: arguments.parse_all_for_interpretation()?, - }) + arguments.fully_parse_or_error( + |input| { + Ok(Self { + expression: input.parse()?, + }) + }, + "Expected [!evaluate! ...] containing a valid preinterpret expression", + ) } } impl CommandInvocation for EvaluateCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let expression = self.expression.interpret_as_expression(interpreter)?; + let expression = self.expression.start_expression_builder(interpreter)?; Ok(CommandOutput::GroupedStream( expression.evaluate()?.into_interpreted_stream(), )) @@ -30,7 +35,7 @@ pub(crate) struct AssignCommand { operator: Punct, #[allow(unused)] equals: Token![=], - expression: InterpretationStream, + expression: ExpressionInput, } impl CommandDefinition for AssignCommand { @@ -50,7 +55,7 @@ impl CommandDefinition for AssignCommand { operator }, equals: input.parse()?, - expression: input.parse_with(arguments.full_span_range())?, + expression: input.parse()?, }) }, "Expected [!assign! #variable += ...] for + or some other operator supported in an expression", @@ -67,14 +72,124 @@ impl CommandInvocation for AssignCommand { expression, } = *self; - let mut expression_stream = ExpressionStream::new(expression.span_range()); - variable.interpret_as_expression_into(interpreter, &mut expression_stream)?; - expression_stream.push_punct(operator); - expression.interpret_as_expression_into(interpreter, &mut expression_stream)?; + let mut builder = ExpressionBuilder::new(); + variable.add_to_expression(interpreter, &mut builder)?; + builder.push_punct(operator); + builder.extend_with_interpreted_stream( + expression.evaluate(interpreter)?.into_interpreted_stream(), + ); - let output = expression_stream.evaluate()?.into_interpreted_stream(); + let output = builder.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output)?; Ok(CommandOutput::Empty) } } + +#[derive(Clone)] +pub(crate) struct RangeCommand { + left: ExpressionInput, + range_limits: RangeLimits, + right: ExpressionInput, +} + +impl CommandDefinition for RangeCommand { + const COMMAND_NAME: &'static str = "range"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + left: input.parse()?, + range_limits: input.parse()?, + right: input.parse()?, + }) + }, + "Expected a rust range expression such as [!range! 1..4]", + ) + } +} + +impl CommandInvocation for RangeCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let range_span_range = self.range_limits.span_range(); + + let left = self + .left + .evaluate(interpreter)? + .expect_integer("The left side of the range must be an integer")? + .try_into_i128()?; + let right = self + .right + .evaluate(interpreter)? + .expect_integer("The right side of the range must be an integer")? + .try_into_i128()?; + if left > right { + return Ok(CommandOutput::Empty); + } + + let length = self + .range_limits + .length_of_range(left, right) + .ok_or_else(|| { + range_span_range.error("The range is too large to be represented as a usize") + })?; + interpreter + .config() + .check_iteration_count(&range_span_range, length)?; + + let mut output = InterpretedStream::new(range_span_range); + match self.range_limits { + RangeLimits::HalfOpen(_) => { + let iter = + (left..right).map(|value| TokenTree::Literal(Literal::i128_unsuffixed(value))); + output.extend_raw_token_iter(iter) + } + RangeLimits::Closed(_) => { + let iter = + (left..=right).map(|value| TokenTree::Literal(Literal::i128_unsuffixed(value))); + output.extend_raw_token_iter(iter) + } + }; + Ok(CommandOutput::GroupedStream(output)) + } +} + +// A copy of syn::RangeLimits to avoid needing a `full` dependency on syn +#[derive(Clone)] +enum RangeLimits { + HalfOpen(Token![..]), + Closed(Token![..=]), +} + +impl Parse for RangeLimits { + fn parse(input: ParseStream) -> Result { + if input.peek(Token![..=]) { + Ok(RangeLimits::Closed(input.parse()?)) + } else { + Ok(RangeLimits::HalfOpen(input.parse()?)) + } + } +} + +impl ToTokens for RangeLimits { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + RangeLimits::HalfOpen(token) => token.to_tokens(tokens), + RangeLimits::Closed(token) => token.to_tokens(tokens), + } + } +} + +impl AutoSpanRange for RangeLimits {} + +impl RangeLimits { + fn length_of_range(&self, left: i128, right: i128) -> Option { + match self { + RangeLimits::HalfOpen(_) => usize::try_from(right.checked_sub(left)?).ok(), + RangeLimits::Closed(_) => { + usize::try_from(right.checked_sub(left)?.checked_add(1)?).ok() + } + } + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 68237891..5547029c 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -46,6 +46,7 @@ define_command_kind! { // Expression Commands EvaluateCommand, AssignCommand, + RangeCommand, // Control flow commands IfCommand, diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 3a906afc..4f9aad91 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -1,35 +1,203 @@ use super::*; -pub(crate) trait Express: Sized + HasSpanRange { - fn interpret_as_expression_into( +pub(crate) trait Express: Sized { + fn add_to_expression( self, interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, + builder: &mut ExpressionBuilder, ) -> Result<()>; - fn interpret_as_expression(self, interpreter: &mut Interpreter) -> Result { - let mut output = ExpressionStream::new(self.span_range()); - self.interpret_as_expression_into(interpreter, &mut output)?; + fn start_expression_builder(self, interpreter: &mut Interpreter) -> Result { + let mut output = ExpressionBuilder::new(); + self.add_to_expression(interpreter, &mut output)?; Ok(output) } } /// This abstraction is a bit ropey... /// -/// Ideally we'd parse expressions at parse time, but that requires writing a custom parser for -/// a subset of the rust expression tree... +/// Ideally we'd properly handle building expressions into an expression tree, +/// but that requires duplicating some portion of the syn parser for the rust expression tree... /// /// Instead, to be lazy for now, we interpret the stream at intepretation time to substitute /// in variables and commands, and then parse the resulting expression with syn. #[derive(Clone)] -pub(crate) struct ExpressionStream { +pub(crate) struct ExpressionInput { + items: Vec, +} + +impl Parse for ExpressionInput { + fn parse(input: ParseStream) -> Result { + let mut items = Vec::new(); + while !input.is_empty() { + // Until we create a proper ExpressionInput parser which builds up a syntax tree + // then we need to have a way to stop parsing... currently ExpressionInput comes + // before code blocks or .. in [!range!] so we can break on those. + // These aren't valid inside expressions we support anyway, so it's good enough for now. + let item = match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::Command => ExpressionItem::Command(input.parse()?), + PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), + PeekMatch::InterpretationGroup(Delimiter::Brace | Delimiter::Bracket) => break, + PeekMatch::InterpretationGroup(_) => { + ExpressionItem::ExpressionGroup(input.parse()?) + } + PeekMatch::Other => { + if input.cursor().punct_matching('.').is_some() { + break; + } + match input.parse::()? { + TokenTree::Group(_) => { + unreachable!( + "Should have been already handled by InterpretationGroup above" + ) + } + TokenTree::Punct(punct) => ExpressionItem::Punct(punct), + TokenTree::Ident(ident) => ExpressionItem::Ident(ident), + TokenTree::Literal(literal) => ExpressionItem::Literal(literal), + } + } + }; + items.push(item); + } + if items.is_empty() { + return input.span().err("Expected an expression"); + } + Ok(Self { items }) + } +} + +impl HasSpanRange for ExpressionInput { + fn span_range(&self) -> SpanRange { + SpanRange::new_between( + self.items.first().unwrap().span(), + self.items.last().unwrap().span(), + ) + } +} + +impl Express for ExpressionInput { + fn add_to_expression( + self, + interpreter: &mut Interpreter, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + for item in self.items { + item.add_to_expression(interpreter, builder)?; + } + Ok(()) + } +} + +impl ExpressionInput { + pub(crate) fn evaluate(self, interpreter: &mut Interpreter) -> Result { + self.start_expression_builder(interpreter)?.evaluate() + } +} + +#[derive(Clone)] +pub(crate) enum ExpressionItem { + Command(Command), + GroupedVariable(GroupedVariable), + FlattenedVariable(FlattenedVariable), + ExpressionGroup(ExpressionGroup), + Punct(Punct), + Ident(Ident), + Literal(Literal), +} + +impl HasSpanRange for ExpressionItem { + fn span_range(&self) -> SpanRange { + match self { + ExpressionItem::Command(command) => command.span_range(), + ExpressionItem::GroupedVariable(grouped_variable) => grouped_variable.span_range(), + ExpressionItem::FlattenedVariable(flattened_variable) => { + flattened_variable.span_range() + } + ExpressionItem::ExpressionGroup(expression_group) => expression_group.span_range(), + ExpressionItem::Punct(punct) => punct.span_range(), + ExpressionItem::Ident(ident) => ident.span_range(), + ExpressionItem::Literal(literal) => literal.span_range(), + } + } +} + +impl Express for ExpressionItem { + fn add_to_expression( + self, + interpreter: &mut Interpreter, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + match self { + ExpressionItem::Command(command_invocation) => { + command_invocation.add_to_expression(interpreter, builder)?; + } + ExpressionItem::FlattenedVariable(variable) => { + variable.add_to_expression(interpreter, builder)?; + } + ExpressionItem::GroupedVariable(variable) => { + variable.add_to_expression(interpreter, builder)?; + } + ExpressionItem::ExpressionGroup(group) => { + group.add_to_expression(interpreter, builder)?; + } + ExpressionItem::Punct(punct) => builder.push_punct(punct), + ExpressionItem::Ident(ident) => builder.push_ident(ident), + ExpressionItem::Literal(literal) => builder.push_literal(literal), + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct ExpressionGroup { + source_delimiter: Delimiter, + source_delim_span: DelimSpan, + content: ExpressionInput, +} + +impl Parse for ExpressionGroup { + fn parse(input: ParseStream) -> Result { + let (delimiter, delim_span, content) = input.parse_any_delimiter()?; + Ok(Self { + source_delimiter: delimiter, + source_delim_span: delim_span, + content: content.parse()?, + }) + } +} + +impl Express for ExpressionGroup { + fn add_to_expression( + self, + interpreter: &mut Interpreter, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + builder.push_expression_group( + self.content.start_expression_builder(interpreter)?, + self.source_delimiter, + self.source_delim_span.join(), + ); + Ok(()) + } +} + +impl HasSpanRange for ExpressionGroup { + fn span_range(&self) -> SpanRange { + self.source_delim_span.span_range() + } +} + +#[derive(Clone)] +pub(crate) struct ExpressionBuilder { interpreted_stream: InterpretedStream, } -impl ExpressionStream { - pub(crate) fn new(source_span_range: SpanRange) -> Self { +impl ExpressionBuilder { + pub(crate) fn new() -> Self { + let unused_span_range = Span::call_site().span_range(); Self { - interpreted_stream: InterpretedStream::new(source_span_range), + interpreted_stream: InterpretedStream::new(unused_span_range), } } @@ -51,11 +219,15 @@ impl ExpressionStream { contents: InterpretedStream, span: Span, ) { + // Currently using Expr::Parse, it ignores transparent groups, which is + // a little too permissive. + // Instead, we use parentheses to ensure that the group has to be a valid + // expression itself, without being flattened self.interpreted_stream - .push_new_group(contents, Delimiter::None, span); + .push_new_group(contents, Delimiter::Parenthesis, span); } - pub(crate) fn push_interpreted_stream(&mut self, contents: InterpretedStream) { + pub(crate) fn extend_with_interpreted_stream(&mut self, contents: InterpretedStream) { self.interpreted_stream.extend(contents); } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 17ee5a9e..8032efda 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -17,6 +17,30 @@ impl EvaluationInteger { }) } + pub(crate) fn try_into_i128(self) -> Result { + let option_of_fallback = match self.value { + EvaluationIntegerValue::Untyped(x) => x.parse_fallback().ok(), + EvaluationIntegerValue::U8(x) => Some(x.into()), + EvaluationIntegerValue::U16(x) => Some(x.into()), + EvaluationIntegerValue::U32(x) => Some(x.into()), + EvaluationIntegerValue::U64(x) => Some(x.into()), + EvaluationIntegerValue::U128(x) => x.try_into().ok(), + EvaluationIntegerValue::Usize(x) => x.try_into().ok(), + EvaluationIntegerValue::I8(x) => Some(x.into()), + EvaluationIntegerValue::I16(x) => Some(x.into()), + EvaluationIntegerValue::I32(x) => Some(x.into()), + EvaluationIntegerValue::I64(x) => Some(x.into()), + EvaluationIntegerValue::I128(x) => Some(x), + EvaluationIntegerValue::Isize(x) => x.try_into().ok(), + }; + match option_of_fallback { + Some(value) => Ok(value), + None => self + .source_span + .err("The integer does not fit in a i128".to_string()), + } + } + pub(super) fn handle_unary_operation( self, operation: UnaryOperation, diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index e16bda27..e0566c5b 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -3,10 +3,11 @@ pub(crate) use proc_macro2::extra::*; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; +pub(crate) use syn::buffer::Cursor; pub(crate) use syn::ext::IdentExt as SynIdentExt; pub(crate) use syn::parse::{discouraged::*, Parse, ParseBuffer, ParseStream, Parser}; pub(crate) use syn::{ - parse_str, token, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, + parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, }; pub(crate) use crate::commands::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index dad9c4a0..49d8f6a4 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -146,10 +146,10 @@ impl Interpret for Command { } impl Express for Command { - fn interpret_as_expression_into( + fn add_to_expression( self, interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, + expression_stream: &mut ExpressionBuilder, ) -> Result<()> { match self.invocation.execute(interpreter)? { CommandOutput::Empty => {} @@ -159,7 +159,10 @@ impl Express for Command { CommandOutput::Ident(ident) => { expression_stream.push_ident(ident); } - CommandOutput::AppendStream(stream) | CommandOutput::GroupedStream(stream) => { + CommandOutput::AppendStream(stream) => { + expression_stream.extend_with_interpreted_stream(stream); + } + CommandOutput::GroupedStream(stream) => { expression_stream .push_grouped_interpreted_stream(stream, self.source_group_span.join()); } diff --git a/src/interpretation/command_fields_input.rs b/src/interpretation/command_fields_input.rs new file mode 100644 index 00000000..e77b6c20 --- /dev/null +++ b/src/interpretation/command_fields_input.rs @@ -0,0 +1,115 @@ +macro_rules! define_field_inputs { + ( + $inputs_type:ident { + required: { + $( + $required_field:ident: $required_type:ty = $required_example:literal $(($required_description:literal))? + ),* $(,)? + }$(,)? + optional: { + $( + $optional_field:ident: $optional_type:ty = $optional_example:literal $(($optional_description:literal))? + ),* $(,)? + }$(,)? + } + ) => { + #[derive(Clone)] + struct $inputs_type { + $( + $required_field: $required_type, + )* + + $( + $optional_field: Option<$optional_type>, + )* + } + + impl Parse for $inputs_type { + fn parse(input: ParseStream) -> Result { + $( + let mut $required_field: Option<$required_type> = None; + )* + $( + let mut $optional_field: Option<$optional_type> = None; + )* + + let content; + let brace = syn::braced!(content in input); + + while !content.is_empty() { + let ident = content.call(Ident::parse_any)?; + content.parse::()?; + match ident.to_string().as_str() { + $( + stringify!($required_field) => { + if $required_field.is_some() { + return ident.err("duplicate field"); + } + $required_field = Some(content.parse()?); + } + )* + $( + stringify!($optional_field) => { + if $optional_field.is_some() { + return ident.err("duplicate field"); + } + $optional_field = Some(content.parse()?); + } + )* + _ => return ident.err("unexpected field"), + } + if !content.is_empty() { + content.parse::()?; + } + } + let mut missing_fields: Vec = vec![]; + + $( + if $required_field.is_none() { + missing_fields.push(stringify!($required_field).to_string()); + } + )* + + if !missing_fields.is_empty() { + return brace.span.err(format!( + "required fields are missing: {}", + missing_fields.join(", ") + )); + } + + $( + let $required_field = $required_field.unwrap(); + )* + + Ok(Self { + $( + $required_field, + )* + $( + $optional_field, + )* + }) + } + } + + impl ArgumentsContent for $inputs_type { + fn error_message() -> String { + use std::fmt::Write; + let mut message = "Expected: {\n".to_string(); + let buffer = &mut message; + $( + $(writeln!(buffer, " // {}", $required_description).unwrap();)? + writeln!(buffer, " {}: {},", stringify!($required_field), $required_example).unwrap(); + )* + $( + $(writeln!(buffer, " // {}", $optional_description).unwrap();)? + writeln!(buffer, " {}?: {},", stringify!($optional_field), $optional_example).unwrap(); + )* + buffer.push('}'); + message + } + } + }; +} + +pub(crate) use define_field_inputs; diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index d37c347e..9ea4b186 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -5,54 +5,31 @@ use crate::internal_prelude::*; /// It accepts any of the following: /// * A `[..]` group - the input stream is the interpreted contents of the brackets /// * A [!command! ...] - the input stream is the command's output -/// * A $macro_variable or other transparent group - the input stream is the *raw* contents of the group /// * A `#variable` - the input stream is the content of the variable /// * A flattened `#..variable` - the variable must contain a single `[..]` or transparent group, the input stream are the group contents +/// +/// We don't support transparent groups, because they are not used consistently and it would expose this +/// inconsistency to the user, and be a potentially breaking change for other tooling to add/remove them. +/// For example, as of Jan 2025, in declarative macro substitutions a $literal gets a wrapping group, +/// but a $tt or $($tt)* does not. #[derive(Clone)] pub(crate) enum CommandStreamInput { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), - Bracketed { - delim_span: DelimSpan, - inner: InterpretationStream, - }, - Raw { - delim_span: DelimSpan, - inner: TokenStream, - }, + ExplicitStream(InterpretationGroup), } impl Parse for CommandStreamInput { fn parse(input: ParseStream) -> Result { - let fork = input.fork(); - if let Ok(command) = fork.parse() { - input.advance_to(&fork); - return Ok(CommandStreamInput::Command(command)); - } - let fork = input.fork(); - if let Ok(command) = fork.parse() { - input.advance_to(&fork); - return Ok(CommandStreamInput::GroupedVariable(command)); - } - let fork = input.fork(); - if let Ok(command) = fork.parse() { - input.advance_to(&fork); - return Ok(CommandStreamInput::FlattenedVariable(command)); - } - let error_span = input.span(); - match input.parse_any_delimiter() { - Ok((Delimiter::Bracket, delim_span, content)) => Ok(CommandStreamInput::Bracketed { - delim_span, - inner: content.parse_with(delim_span.span_range())?, - }), - Ok((Delimiter::None, delim_span, content)) => Ok(CommandStreamInput::Raw { - delim_span, - inner: content.parse()?, - }), - _ => error_span - .err("expected [ .... ] or a [!command! ..], #variable, or #macro_variable"), - } + Ok(match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::Command => Self::Command(input.parse()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + PeekMatch::InterpretationGroup(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), + PeekMatch::InterpretationGroup(_) | PeekMatch::Other => input.span() + .err("Expected [ ..input stream.. ] or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, + }) } } @@ -62,12 +39,7 @@ impl HasSpanRange for CommandStreamInput { CommandStreamInput::Command(command) => command.span_range(), CommandStreamInput::GroupedVariable(variable) => variable.span_range(), CommandStreamInput::FlattenedVariable(variable) => variable.span_range(), - CommandStreamInput::Bracketed { - delim_span: span, .. - } => span.span_range(), - CommandStreamInput::Raw { - delim_span: span, .. - } => span.span_range(), + CommandStreamInput::ExplicitStream(group) => group.span_range(), } } } @@ -99,19 +71,15 @@ impl Interpret for CommandStreamInput { } })?; - output.extend_raw(tokens); + output.extend_raw_tokens(tokens); Ok(()) } CommandStreamInput::GroupedVariable(variable) => { variable.interpret_as_tokens_into(interpreter, output) } - CommandStreamInput::Bracketed { inner, .. } => { - inner.interpret_as_tokens_into(interpreter, output) - } - CommandStreamInput::Raw { inner, .. } => { - output.extend_raw(inner); - Ok(()) - } + CommandStreamInput::ExplicitStream(group) => group + .into_content() + .interpret_as_tokens_into(interpreter, output), } } } diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index c646d5d3..f6fd729b 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -26,7 +26,7 @@ impl + HasSpanRange, I: ToTokens> Interp interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.extend_raw(self.interpret(interpreter)?); + output.extend_raw_tokens(self.interpret(interpreter)?); Ok(()) } } diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 5fbaec6b..8c910ac9 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -5,7 +5,7 @@ pub(crate) enum InterpretationItem { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), - Group(InterpretationGroup), + InterpretationGroup(InterpretationGroup), Punct(Punct), Ident(Ident), Literal(Literal), @@ -13,58 +13,77 @@ pub(crate) enum InterpretationItem { impl Parse for InterpretationItem { fn parse(input: ParseStream) -> Result { - match detect_group(input.cursor()) { - GroupMatch::Command => return Ok(InterpretationItem::Command(input.parse()?)), - GroupMatch::OtherGroup => return Ok(InterpretationItem::Group(input.parse()?)), - GroupMatch::None => {} - } - if input.peek(token::Pound) { - let fork = input.fork(); - if let Ok(variable) = fork.parse() { - input.advance_to(&fork); - return Ok(InterpretationItem::GroupedVariable(variable)); - } - let fork = input.fork(); - if let Ok(variable) = fork.parse() { - input.advance_to(&fork); - return Ok(InterpretationItem::FlattenedVariable(variable)); - } - } - Ok(match input.parse::()? { - TokenTree::Group(_) => { - unreachable!("Should have been already handled by the first branch above") + Ok(match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::Command => InterpretationItem::Command(input.parse()?), + PeekMatch::InterpretationGroup(_) => { + InterpretationItem::InterpretationGroup(input.parse()?) } - TokenTree::Punct(punct) => InterpretationItem::Punct(punct), - TokenTree::Ident(ident) => InterpretationItem::Ident(ident), - TokenTree::Literal(literal) => InterpretationItem::Literal(literal), + PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), + PeekMatch::Other => match input.parse::()? { + TokenTree::Group(_) => { + unreachable!("Should have been already handled by InterpretationGroup above") + } + TokenTree::Punct(punct) => InterpretationItem::Punct(punct), + TokenTree::Ident(ident) => InterpretationItem::Ident(ident), + TokenTree::Literal(literal) => InterpretationItem::Literal(literal), + }, }) } } -enum GroupMatch { +pub(crate) enum PeekMatch { Command, - OtherGroup, - None, + GroupedVariable, + FlattenedVariable, + InterpretationGroup(Delimiter), + Other, } -fn detect_group(cursor: syn::buffer::Cursor) -> GroupMatch { - let next = match cursor.any_group() { - Some((next, Delimiter::Bracket, _, _)) => next, - Some(_) => return GroupMatch::OtherGroup, - None => return GroupMatch::None, - }; - let next = match next.punct() { - Some((punct, next)) if punct.as_char() == '!' => next, - _ => return GroupMatch::OtherGroup, - }; - let next = match next.ident() { - Some((_, next)) => next, - _ => return GroupMatch::OtherGroup, - }; - match next.punct() { - Some((punct, _)) if punct.as_char() == '!' => GroupMatch::Command, - _ => GroupMatch::OtherGroup, +pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMatch { + // We have to check groups first, so that we handle transparent groups + // and avoid the self.ignore_none() calls inside cursor + if let Some((next, delimiter, _, _)) = cursor.any_group() { + if delimiter == Delimiter::Bracket { + if let Some((_, next)) = next.punct_matching('!') { + if let Some((_, next)) = next.ident() { + if next.punct_matching('!').is_some() { + return PeekMatch::Command; + } + } + } + } + + // Ideally we'd like to detect $($tt)* substitutions from macros and interpret them as + // a Raw (uninterpreted) group, because typically that's what a user would typically intend. + // + // You'd think mapping a Delimiter::None to a PeekMatch::RawGroup would be a good way + // of doing this, but unfortunately this behaviour is very arbitrary and not in a helpful way: + // => A $tt or $($tt)* is not grouped... + // => A $literal or $($literal)* _is_ outputted in a group... + // + // So this isn't possible. It's unlikely to matter much, and a user can always do: + // [!raw! $($tt)*] anyway. + + return PeekMatch::InterpretationGroup(delimiter); } + if let Some((_, next)) = cursor.punct_matching('#') { + if next.ident().is_some() { + return PeekMatch::GroupedVariable; + } + if let Some((_, next)) = next.punct_matching('.') { + if let Some((_, next)) = next.punct_matching('.') { + if next.ident().is_some() { + return PeekMatch::FlattenedVariable; + } + } + } + } + // This is rather annoying for our purposes, but the Cursor (and even `impl Parse on Punct`) + // treats `'` specially, and there's no way to peek a ' token which isn't part of a lifetime. + // Instead of dividing up specific cases here, we let the caller handle it by parsing as a + // TokenTree if they need to, which doesn't have these limitations. + PeekMatch::Other } impl Interpret for InterpretationItem { @@ -83,7 +102,7 @@ impl Interpret for InterpretationItem { InterpretationItem::FlattenedVariable(variable) => { variable.interpret_as_tokens_into(interpreter, output)?; } - InterpretationItem::Group(group) => { + InterpretationItem::InterpretationGroup(group) => { group.interpret_as_tokens_into(interpreter, output)?; } InterpretationItem::Punct(punct) => output.push_punct(punct), @@ -94,40 +113,13 @@ impl Interpret for InterpretationItem { } } -impl Express for InterpretationItem { - fn interpret_as_expression_into( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, - ) -> Result<()> { - match self { - InterpretationItem::Command(command_invocation) => { - command_invocation.interpret_as_expression_into(interpreter, expression_stream)?; - } - InterpretationItem::FlattenedVariable(variable) => { - variable.interpret_as_expression_into(interpreter, expression_stream)?; - } - InterpretationItem::GroupedVariable(variable) => { - variable.interpret_as_expression_into(interpreter, expression_stream)?; - } - InterpretationItem::Group(group) => { - group.interpret_as_expression_into(interpreter, expression_stream)?; - } - InterpretationItem::Punct(punct) => expression_stream.push_punct(punct), - InterpretationItem::Ident(ident) => expression_stream.push_ident(ident), - InterpretationItem::Literal(literal) => expression_stream.push_literal(literal), - } - Ok(()) - } -} - impl HasSpanRange for InterpretationItem { fn span_range(&self) -> SpanRange { match self { InterpretationItem::Command(command_invocation) => command_invocation.span_range(), InterpretationItem::FlattenedVariable(variable) => variable.span_range(), InterpretationItem::GroupedVariable(variable) => variable.span_range(), - InterpretationItem::Group(group) => group.span_range(), + InterpretationItem::InterpretationGroup(group) => group.span_range(), InterpretationItem::Punct(punct) => punct.span_range(), InterpretationItem::Ident(ident) => ident.span_range(), InterpretationItem::Literal(literal) => literal.span_range(), diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 462fedda..dabf517a 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -45,55 +45,75 @@ impl Interpret for InterpretationStream { } } -impl Express for InterpretationStream { - fn interpret_as_expression_into( +impl HasSpanRange for InterpretationStream { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +/// A parsed group ready for interpretation +#[derive(Clone)] +pub(crate) struct InterpretationGroup { + source_delimiter: Delimiter, + source_delim_span: DelimSpan, + content: InterpretationStream, +} + +impl InterpretationGroup { + pub(crate) fn into_content(self) -> InterpretationStream { + self.content + } +} + +impl Parse for InterpretationGroup { + fn parse(input: ParseStream) -> Result { + let (delimiter, delim_span, content) = input.parse_any_delimiter()?; + let content = content.parse_with(delim_span.span_range())?; + Ok(Self { + source_delimiter: delimiter, + source_delim_span: delim_span, + content, + }) + } +} + +impl Interpret for InterpretationGroup { + fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, + output: &mut InterpretedStream, ) -> Result<()> { - let mut inner_expression_stream = ExpressionStream::new(self.span_range); - for item in self.items { - item.interpret_as_expression_into(interpreter, &mut inner_expression_stream)?; - } - expression_stream.push_expression_group( - inner_expression_stream, - Delimiter::None, - self.span_range.span(), - ); + let inner = self.content.interpret_as_tokens(interpreter)?; + output.push_new_group(inner, self.source_delimiter, self.source_delim_span.join()); Ok(()) } } -impl HasSpanRange for InterpretationStream { +impl HasSpanRange for InterpretationGroup { fn span_range(&self) -> SpanRange { - self.span_range + self.source_delim_span.span_range() } } -/// A parsed group ready for interpretation +/// A parsed group intended to be raw tokens #[derive(Clone)] -pub(crate) struct InterpretationGroup { +pub(crate) struct RawGroup { source_delimeter: Delimiter, source_delim_span: DelimSpan, - content: InterpretationGroupContent, + content: TokenStream, } -#[derive(Clone)] -enum InterpretationGroupContent { - Interpeted(InterpretationStream), - Raw(TokenStream), +#[allow(unused)] +impl RawGroup { + pub(crate) fn into_content(self) -> TokenStream { + self.content + } } -impl Parse for InterpretationGroup { +impl Parse for RawGroup { fn parse(input: ParseStream) -> Result { let (delimiter, delim_span, content) = input.parse_any_delimiter()?; - let span_range = delim_span.span_range(); - let content = match delimiter { - // This is likely from a macro variable or macro expansion. - // Either way, we shouldn't be interpreting it. - Delimiter::None => InterpretationGroupContent::Raw(content.parse()?), - _ => InterpretationGroupContent::Interpeted(content.parse_with(span_range)?), - }; + let content = content.parse()?; Ok(Self { source_delimeter: delimiter, source_delim_span: delim_span, @@ -102,49 +122,33 @@ impl Parse for InterpretationGroup { } } -impl Interpret for InterpretationGroup { +impl Interpret for RawGroup { fn interpret_as_tokens_into( self, - interpreter: &mut Interpreter, + _: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - let inner = match self.content { - InterpretationGroupContent::Interpeted(stream) => { - stream.interpret_as_tokens(interpreter)? - } - InterpretationGroupContent::Raw(token_stream) => { - InterpretedStream::raw(self.source_delim_span.span_range(), token_stream) - } - }; + let inner = InterpretedStream::raw(self.source_delim_span.span_range(), self.content); output.push_new_group(inner, self.source_delimeter, self.source_delim_span.join()); Ok(()) } } -impl Express for InterpretationGroup { - fn interpret_as_expression_into( +impl Express for RawGroup { + fn add_to_expression( self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, + _: &mut Interpreter, + expression_stream: &mut ExpressionBuilder, ) -> Result<()> { - match self.content { - InterpretationGroupContent::Interpeted(stream) => expression_stream - .push_expression_group( - stream.interpret_as_expression(interpreter)?, - self.source_delimeter, - self.source_delim_span.join(), - ), - InterpretationGroupContent::Raw(token_stream) => expression_stream - .push_grouped_interpreted_stream( - InterpretedStream::raw(self.source_delim_span.span_range(), token_stream), - self.source_delim_span.join(), - ), - } + expression_stream.push_grouped_interpreted_stream( + InterpretedStream::raw(self.source_delim_span.span_range(), self.content), + self.source_delim_span.join(), + ); Ok(()) } } -impl HasSpanRange for InterpretationGroup { +impl HasSpanRange for RawGroup { fn span_range(&self) -> SpanRange { self.source_delim_span.span_range() } diff --git a/src/interpretation/interpretation_value.rs b/src/interpretation/interpretation_value.rs index 91e06a0c..0815aa6a 100644 --- a/src/interpretation/interpretation_value.rs +++ b/src/interpretation/interpretation_value.rs @@ -14,22 +14,12 @@ pub(crate) enum InterpretationValue { impl Parse for InterpretationValue { fn parse(input: ParseStream) -> Result { - let fork = input.fork(); - if let Ok(command) = fork.parse() { - input.advance_to(&fork); - return Ok(InterpretationValue::Command(command)); - } - let fork = input.fork(); - if let Ok(command) = fork.parse() { - input.advance_to(&fork); - return Ok(InterpretationValue::GroupedVariable(command)); - } - let fork = input.fork(); - if let Ok(command) = fork.parse() { - input.advance_to(&fork); - return Ok(InterpretationValue::FlattenedVariable(command)); - } - Ok(InterpretationValue::Value(input.parse()?)) + Ok(match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::Command => Self::Command(input.parse()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + PeekMatch::InterpretationGroup(_) | PeekMatch::Other => Self::Value(input.parse()?), + }) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index f168a69f..7c398880 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -46,10 +46,14 @@ impl InterpretedStream { self.push_raw_token_tree(TokenTree::group(inner_tokens.token_stream, delimiter, span)); } - pub(crate) fn extend_raw(&mut self, tokens: impl ToTokens) { + pub(crate) fn extend_raw_tokens(&mut self, tokens: impl ToTokens) { tokens.to_tokens(&mut self.token_stream); } + pub(crate) fn extend_raw_token_iter(&mut self, tokens: impl IntoIterator) { + self.token_stream.extend(tokens); + } + fn push_raw_token_tree(&mut self, token_tree: TokenTree) { self.token_stream.extend(iter::once(token_tree)); } diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 0edfdc41..33f5125d 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,6 +1,7 @@ mod command; mod command_arguments; mod command_code_input; +mod command_fields_input; mod command_stream_input; mod interpret_traits; mod interpretation_item; @@ -13,6 +14,7 @@ mod variable; pub(crate) use command::*; pub(crate) use command_arguments::*; pub(crate) use command_code_input::*; +pub(crate) use command_fields_input::*; pub(crate) use command_stream_input::*; pub(crate) use interpret_traits::*; pub(crate) use interpretation_item::*; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 24db3f79..4a47a700 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -8,10 +8,15 @@ pub(crate) struct GroupedVariable { impl Parse for GroupedVariable { fn parse(input: ParseStream) -> Result { - Ok(Self { - marker: input.parse()?, - variable_name: input.call(Ident::parse_any)?, - }) + input.try_parse_or_message( + |input| { + Ok(Self { + marker: input.parse()?, + variable_name: input.call(Ident::parse_any)?, + }) + }, + "Expected #variable", + ) } } @@ -87,10 +92,10 @@ impl Interpret for &GroupedVariable { } impl Express for &GroupedVariable { - fn interpret_as_expression_into( + fn add_to_expression( self, interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, + expression_stream: &mut ExpressionBuilder, ) -> Result<()> { expression_stream.push_grouped_interpreted_stream( self.interpret_as_new_stream(interpreter)?, @@ -128,11 +133,16 @@ pub(crate) struct FlattenedVariable { impl Parse for FlattenedVariable { fn parse(input: ParseStream) -> Result { - Ok(Self { - marker: input.parse()?, - flatten: input.parse()?, - variable_name: input.call(Ident::parse_any)?, - }) + input.try_parse_or_message( + |input| { + Ok(Self { + marker: input.parse()?, + flatten: input.parse()?, + variable_name: input.call(Ident::parse_any)?, + }) + }, + "Expected #..variable", + ) } } @@ -187,12 +197,12 @@ impl Interpret for &FlattenedVariable { } impl Express for &FlattenedVariable { - fn interpret_as_expression_into( + fn add_to_expression( self, interpreter: &mut Interpreter, - expression_stream: &mut ExpressionStream, + expression_stream: &mut ExpressionBuilder, ) -> Result<()> { - expression_stream.push_interpreted_stream(self.interpret_as_tokens(interpreter)?); + expression_stream.extend_with_interpreted_stream(self.interpret_as_tokens(interpreter)?); Ok(()) } } diff --git a/src/traits.rs b/src/traits.rs index 1d97b049..40a25727 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -10,6 +10,19 @@ impl IdentExt for Ident { } } +pub(crate) trait CursorExt: Sized { + fn punct_matching(self, char: char) -> Option<(Punct, Self)>; +} + +impl CursorExt for Cursor<'_> { + fn punct_matching(self, char: char) -> Option<(Punct, Self)> { + match self.punct() { + Some((punct, next)) if punct.as_char() == char => Some((punct, next)), + _ => None, + } + } +} + pub(crate) trait LiteralExt: Sized { #[allow(unused)] fn content_if_string(&self) -> Option; @@ -66,6 +79,11 @@ impl TokenTreeExt for TokenTree { pub(crate) trait ParserExt { fn parse_with(&self, context: T::Context) -> Result; fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result; + fn try_parse_or_message Result, M: std::fmt::Display>( + &self, + func: F, + message: M, + ) -> Result; } impl ParserExt for ParseBuffer<'_> { @@ -76,6 +94,15 @@ impl ParserExt for ParseBuffer<'_> { fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result { self.parse_with(span_range) } + + fn try_parse_or_message Result, M: std::fmt::Display>( + &self, + parse: F, + message: M, + ) -> Result { + let error_span = self.span(); + parse(self).map_err(|_| error_span.error(message)) + } } pub(crate) trait ContextualParse: Sized { @@ -107,13 +134,7 @@ pub(crate) trait SpanErrorExt: Sized { impl SpanErrorExt for T { fn error(&self, message: impl std::fmt::Display) -> syn::Error { - self.span_range().error(message) - } -} - -impl SpanErrorExt for SpanRange { - fn error(&self, message: impl std::fmt::Display) -> syn::Error { - syn::Error::new_spanned(self, message) + self.span_range().create_error(message) } } @@ -170,6 +191,10 @@ impl SpanRange { Self { start, end } } + fn create_error(&self, message: impl std::fmt::Display) -> syn::Error { + syn::Error::new_spanned(self, message) + } + /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) pub(crate) fn span(&self) -> Span { @@ -203,6 +228,12 @@ impl ToTokens for SpanRange { } } +impl HasSpanRange for SpanRange { + fn span_range(&self) -> SpanRange { + *self + } +} + impl HasSpanRange for Span { fn span_range(&self) -> SpanRange { SpanRange::new_between(*self, *self) @@ -241,7 +272,7 @@ impl HasSpanRange for T { /// This should only be used for syn built-ins or when there isn't a better /// span range available -trait AutoSpanRange {} +pub(crate) trait AutoSpanRange {} macro_rules! impl_auto_span_range { ($($ty:ty),* $(,)?) => { diff --git a/tests/compilation_failures/core/extend_flattened_variable.rs b/tests/compilation_failures/core/extend_flattened_variable.rs new file mode 100644 index 00000000..fff05516 --- /dev/null +++ b/tests/compilation_failures/core/extend_flattened_variable.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!set! #variable = 1] + [!extend! #..variable += 2] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_flattened_variable.stderr b/tests/compilation_failures/core/extend_flattened_variable.stderr new file mode 100644 index 00000000..11789f0c --- /dev/null +++ b/tests/compilation_failures/core/extend_flattened_variable.stderr @@ -0,0 +1,6 @@ +error: Expected #variable + Occurred whilst parsing [!extend! ..] - Expected [!extend! #variable += ..] + --> tests/compilation_failures/core/extend_flattened_variable.rs:6:19 + | +6 | [!extend! #..variable += 2] + | ^ diff --git a/tests/compilation_failures/core/set_flattened_variable.rs b/tests/compilation_failures/core/set_flattened_variable.rs new file mode 100644 index 00000000..f1264dcf --- /dev/null +++ b/tests/compilation_failures/core/set_flattened_variable.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!set! #..variable = 1] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/core/set_flattened_variable.stderr b/tests/compilation_failures/core/set_flattened_variable.stderr new file mode 100644 index 00000000..3629546d --- /dev/null +++ b/tests/compilation_failures/core/set_flattened_variable.stderr @@ -0,0 +1,6 @@ +error: Expected #variable + Occurred whilst parsing [!set! ..] - Expected [!set! #variable = ..] + --> tests/compilation_failures/core/set_flattened_variable.rs:5:16 + | +5 | [!set! #..variable = 1] + | ^ diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr index b07fc912..7ecf4b90 100644 --- a/tests/compilation_failures/expressions/braces.stderr +++ b/tests/compilation_failures/expressions/braces.stderr @@ -1,5 +1,6 @@ -error: This expression is not supported in preinterpret expressions +error: Expected an expression + Occurred whilst parsing [!evaluate! ..] - Expected [!evaluate! ...] containing a valid preinterpret expression --> tests/compilation_failures/expressions/braces.rs:5:21 | 5 | [!evaluate! {true}] - | ^^^^^^ + | ^ diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs new file mode 100644 index 00000000..df4b2cc3 --- /dev/null +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!set! #x = + 1] + [!evaluate! 1 #x] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr new file mode 100644 index 00000000..68aa6002 --- /dev/null +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr @@ -0,0 +1,5 @@ +error: expected an expression + --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:21 + | +5 | [!set! #x = + 1] + | ^ diff --git a/tests/compilation_failures/expressions/inner_braces.rs b/tests/compilation_failures/expressions/inner_braces.rs new file mode 100644 index 00000000..464c4653 --- /dev/null +++ b/tests/compilation_failures/expressions/inner_braces.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!evaluate! ({true})] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/inner_braces.stderr b/tests/compilation_failures/expressions/inner_braces.stderr new file mode 100644 index 00000000..98ba4703 --- /dev/null +++ b/tests/compilation_failures/expressions/inner_braces.stderr @@ -0,0 +1,6 @@ +error: Expected an expression + Occurred whilst parsing [!evaluate! ..] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/inner_braces.rs:5:22 + | +5 | [!evaluate! ({true})] + | ^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index bb8912c5..ad18c96a 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -26,7 +26,7 @@ fn test_if() { assert_preinterpret_eq!({ [!set! #x = 1] [!set! #y = 2] - [!if! (#x == #y) { "YES" } !else! { "NO" }] + [!if! #x == #y { "YES" } !else! { "NO" }] }, "NO"); assert_preinterpret_eq!({ 0 @@ -42,7 +42,7 @@ fn test_if() { fn test_while() { assert_preinterpret_eq!({ [!set! #x = 0] - [!while! (#x < 5) { [!assign! #x += 1] }] + [!while! #x < 5 { [!assign! #x += 1] }] #x }, 5); } diff --git a/tests/core.rs b/tests/core.rs index 760e4008..34b05b2a 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -37,6 +37,33 @@ fn test_raw() { ); } +#[test] +fn test_extend() { + my_assert_eq!( + { + [!set! #variable = "Hello"] + [!extend! #variable += " World!"] + [!string! #variable] + }, + "Hello World!" + ); + my_assert_eq!( + { + [!set! #i = 1] + [!set! #output = [!empty!]] + [!while! (#i <= 4) { + [!extend! #output += #i] + [!if! (#i <= 3) { + [!extend! #output += ", "] + }] + [!assign! #i += 1] + }] + [!string! #output] + }, + "1, 2, 3, 4" + ); +} + #[test] fn test_ignore() { my_assert_eq!({ diff --git a/tests/expressions.rs b/tests/expressions.rs index f07f124b..a46a661d 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -47,6 +47,13 @@ fn test_basic_evaluate_works() { }, 36 ); + assert_preinterpret_eq!( + { + [!set! #partial_sum = + 2] + [!evaluate! 5 #..partial_sum] + }, + 7 + ); } #[test] @@ -61,3 +68,16 @@ fn assign_works() { 12 ); } + +#[test] +fn range_works() { + assert_preinterpret_eq!([!string! [!range! -2..5]], "-2-101234"); + assert_preinterpret_eq!( + { + [!set! #x = 2] + [!string! [!range! (#x + #x)..=5]] + }, + "45" + ); + assert_preinterpret_eq!({ [!string! [!range! 8..=5]] }, ""); +} From f99d4f306b76e6bbc1cabf0c08c1b23a44baf944 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 18 Jan 2025 14:27:47 +0000 Subject: [PATCH 033/126] feature: Add !intersperse! command --- CHANGELOG.md | 5 + src/commands/control_flow_commands.rs | 4 +- src/commands/core_commands.rs | 6 +- src/commands/expression_commands.rs | 4 +- src/commands/mod.rs | 1 + src/commands/token_commands.rs | 145 +++++++++- src/internal_prelude.rs | 2 +- src/interpretation/command.rs | 13 +- ...ields_input.rs => command_field_inputs.rs} | 0 src/interpretation/command_stream_input.rs | 65 +++-- src/interpretation/command_value_input.rs | 72 +++++ src/interpretation/interpretation_value.rs | 54 ---- src/interpretation/interpreted_stream.rs | 2 +- src/interpretation/mod.rs | 8 +- src/traits.rs | 11 + .../intersperse_bool_input_braces_issue.rs | 13 + ...intersperse_bool_input_braces_issue.stderr | 6 + .../intersperse_stream_input_braces_issue.rs | 10 + ...tersperse_stream_input_braces_issue.stderr | 5 + ...intersperse_stream_input_variable_issue.rs | 11 + ...rsperse_stream_input_variable_issue.stderr | 5 + tests/tokens.rs | 254 ++++++++++++++++++ 22 files changed, 599 insertions(+), 97 deletions(-) rename src/interpretation/{command_fields_input.rs => command_field_inputs.rs} (100%) create mode 100644 src/interpretation/command_value_input.rs delete mode 100644 src/interpretation/interpretation_value.rs create mode 100644 tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs create mode 100644 tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr create mode 100644 tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs create mode 100644 tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr create mode 100644 tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs create mode 100644 tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 81434f4e..ae05986d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,11 +19,13 @@ * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. + * `[!intersperse! { .. }]` which inserts separator tokens between each token tree in a stream. I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. ### To come +* Consider [!..flattened! ] and getting rid of !group! * Explore getting rid of lots of the span range stuff * Support `!else if!` in `!if!` * Support string & char literals (for comparisons & casts) in expressions @@ -95,6 +97,9 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing * Work on book + * Input paradigms: + * Streams + * StreamInput / ValueInput / CodeInput * Including documenting expressions * There are three main kinds of commands: * Those taking a stream as-is diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 520590bd..699bc567 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -51,7 +51,7 @@ impl CommandInvocation for IfCommand { InterpretedStream::new(self.nothing_span_range) }; - Ok(CommandOutput::AppendStream(output)) + Ok(CommandOutput::Stream(output)) } } @@ -104,6 +104,6 @@ impl CommandInvocation for WhileCommand { .interpret_as_tokens_into(interpreter, &mut output)?; } - Ok(CommandOutput::AppendStream(output)) + Ok(CommandOutput::Stream(output)) } } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 64269c29..1a2a139b 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -86,7 +86,7 @@ impl CommandDefinition for RawCommand { impl CommandInvocation for RawCommand { fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::AppendStream(InterpretedStream::raw( + Ok(CommandOutput::Stream(InterpretedStream::raw( self.arguments_span_range, self.token_stream, ))) @@ -130,7 +130,7 @@ impl CommandDefinition for StreamCommand { impl CommandInvocation for StreamCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::AppendStream( + Ok(CommandOutput::Stream( self.arguments.interpret_as_tokens(interpreter)?, )) } @@ -144,7 +144,7 @@ pub(crate) struct ErrorCommand { define_field_inputs! { ErrorInputs { required: { - message: InterpretationValue = r#""...""# ("The error message to display"), + message: CommandValueInput = r#""...""# ("The error message to display"), }, optional: { spans: CommandStreamInput = "[$abc]" ("An optional [token stream], to determine where to show the error message"), diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 4d9b0e36..ca1abed0 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -23,7 +23,7 @@ impl CommandDefinition for EvaluateCommand { impl CommandInvocation for EvaluateCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let expression = self.expression.start_expression_builder(interpreter)?; - Ok(CommandOutput::GroupedStream( + Ok(CommandOutput::Stream( expression.evaluate()?.into_interpreted_stream(), )) } @@ -151,7 +151,7 @@ impl CommandInvocation for RangeCommand { output.extend_raw_token_iter(iter) } }; - Ok(CommandOutput::GroupedStream(output)) + Ok(CommandOutput::Stream(output)) } } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 5547029c..2c3abcee 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -57,4 +57,5 @@ define_command_kind! { IsEmptyCommand, LengthCommand, GroupCommand, + IntersperseCommand, } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 4c1d2659..cfe815af 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -88,7 +88,148 @@ impl CommandDefinition for GroupCommand { impl CommandInvocation for GroupCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let output = self.arguments.interpret_as_tokens(interpreter)?; - Ok(CommandOutput::GroupedStream(output)) + let mut output = InterpretedStream::new(self.arguments.span_range()); + let group_span = self.arguments.span(); + let inner = self.arguments.interpret_as_tokens(interpreter)?; + output.push_new_group(inner, Delimiter::None, group_span); + Ok(CommandOutput::Stream(output)) } } + +#[derive(Clone)] +pub(crate) struct IntersperseCommand { + span_range: SpanRange, + inputs: IntersperseInputs, +} + +define_field_inputs! { + IntersperseInputs { + required: { + items: CommandStreamInput = "[Hello World] or #var or [!cmd! ...]", + separator: CommandStreamInput = "[,]" ("The token/s to add between each item"), + }, + optional: { + add_trailing: CommandValueInput = "false" ("Whether to add the separator after the last item (default: false)"), + final_separator: CommandStreamInput = "[or]" ("Define a different final separator (default: same as normal separator)"), + } + } +} + +impl CommandDefinition for IntersperseCommand { + const COMMAND_NAME: &'static str = "intersperse"; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + span_range: arguments.full_span_range(), + inputs: arguments.fully_parse_as()?, + }) + } +} + +impl CommandInvocation for IntersperseCommand { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let items = self + .inputs + .items + .interpret_as_tokens(interpreter)? + .into_token_stream(); + let add_trailing = match self.inputs.add_trailing { + Some(add_trailing) => add_trailing.interpret(interpreter)?.value(), + None => false, + }; + + let mut output = InterpretedStream::new(self.span_range); + + if items.is_empty() { + return Ok(CommandOutput::Stream(output)); + } + + let mut appender = SeparatorAppender { + separator: self.inputs.separator, + final_separator: self.inputs.final_separator, + add_trailing, + }; + + let mut items = items.into_iter().peekable(); + let mut this_item = items.next().unwrap(); // Safe to unwrap as non-empty + loop { + output.push_raw_token_tree(this_item); + let next_item = items.next(); + match next_item { + Some(next_item) => { + let remaining = if items.peek().is_some() { + RemainingItemCount::MoreThanOne + } else { + RemainingItemCount::ExactlyOne + }; + appender.add_separator(interpreter, remaining, &mut output)?; + this_item = next_item; + } + None => { + appender.add_separator(interpreter, RemainingItemCount::None, &mut output)?; + break; + } + } + } + + Ok(CommandOutput::Stream(output)) + } +} + +struct SeparatorAppender { + separator: CommandStreamInput, + final_separator: Option, + add_trailing: bool, +} + +impl SeparatorAppender { + fn add_separator( + &mut self, + interpreter: &mut Interpreter, + remaining: RemainingItemCount, + output: &mut InterpretedStream, + ) -> Result<()> { + let extender = match self.separator(remaining) { + TrailingSeparator::Normal => self.separator.clone().interpret_as_tokens(interpreter)?, + TrailingSeparator::Final => match self.final_separator.take() { + Some(final_separator) => final_separator.interpret_as_tokens(interpreter)?, + None => self.separator.clone().interpret_as_tokens(interpreter)?, + }, + TrailingSeparator::None => InterpretedStream::new(SpanRange::ignored()), + }; + output.extend(extender); + Ok(()) + } + + fn separator(&self, remaining_item_count: RemainingItemCount) -> TrailingSeparator { + match remaining_item_count { + RemainingItemCount::None => { + if self.add_trailing { + TrailingSeparator::Final + } else { + TrailingSeparator::None + } + } + RemainingItemCount::ExactlyOne => { + if self.add_trailing { + TrailingSeparator::Normal + } else { + TrailingSeparator::Final + } + } + RemainingItemCount::MoreThanOne => TrailingSeparator::Normal, + } + } +} + +enum RemainingItemCount { + None, + ExactlyOne, + MoreThanOne, +} + +enum TrailingSeparator { + Normal, + Final, + None, +} diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index e0566c5b..1ef1f529 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -7,7 +7,7 @@ pub(crate) use syn::buffer::Cursor; pub(crate) use syn::ext::IdentExt as SynIdentExt; pub(crate) use syn::parse::{discouraged::*, Parse, ParseBuffer, ParseStream, Parser}; pub(crate) use syn::{ - parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, + parse_str, Error, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, }; pub(crate) use crate::commands::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 49d8f6a4..803fca37 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -14,8 +14,7 @@ pub(crate) enum CommandOutput { Empty, Literal(Literal), Ident(Ident), - AppendStream(InterpretedStream), - GroupedStream(InterpretedStream), + Stream(InterpretedStream), } pub(crate) trait ClonableCommandInvocation: CommandInvocation { @@ -134,12 +133,9 @@ impl Interpret for Command { CommandOutput::Ident(ident) => { output.push_ident(ident); } - CommandOutput::AppendStream(stream) => { + CommandOutput::Stream(stream) => { output.extend(stream); } - CommandOutput::GroupedStream(stream) => { - output.push_new_group(stream, Delimiter::None, self.source_group_span.join()); - } }; Ok(()) } @@ -159,10 +155,7 @@ impl Express for Command { CommandOutput::Ident(ident) => { expression_stream.push_ident(ident); } - CommandOutput::AppendStream(stream) => { - expression_stream.extend_with_interpreted_stream(stream); - } - CommandOutput::GroupedStream(stream) => { + CommandOutput::Stream(stream) => { expression_stream .push_grouped_interpreted_stream(stream, self.source_group_span.join()); } diff --git a/src/interpretation/command_fields_input.rs b/src/interpretation/command_field_inputs.rs similarity index 100% rename from src/interpretation/command_fields_input.rs rename to src/interpretation/command_field_inputs.rs diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 9ea4b186..7d0135c6 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -17,6 +17,7 @@ pub(crate) enum CommandStreamInput { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), + Code(CommandCodeInput), ExplicitStream(InterpretationGroup), } @@ -27,8 +28,9 @@ impl Parse for CommandStreamInput { PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::InterpretationGroup(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), + PeekMatch::InterpretationGroup(Delimiter::Brace) => Self::Code(input.parse()?), PeekMatch::InterpretationGroup(_) | PeekMatch::Other => input.span() - .err("Expected [ ..input stream.. ] or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, + .err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) } } @@ -39,6 +41,7 @@ impl HasSpanRange for CommandStreamInput { CommandStreamInput::Command(command) => command.span_range(), CommandStreamInput::GroupedVariable(variable) => variable.span_range(), CommandStreamInput::FlattenedVariable(variable) => variable.span_range(), + CommandStreamInput::Code(code) => code.span_range(), CommandStreamInput::ExplicitStream(group) => group.span_range(), } } @@ -55,27 +58,30 @@ impl Interpret for CommandStreamInput { command.interpret_as_tokens_into(interpreter, output) } CommandStreamInput::FlattenedVariable(variable) => { - let tokens = variable.interpret_as_new_stream(interpreter)? - .syn_parse(|input: ParseStream| -> Result { - let (delimiter, _, content) = input.parse_any_delimiter()?; - match delimiter { - Delimiter::Bracket | Delimiter::None if input.is_empty() => { - content.parse() - }, - _ => { - variable.err(format!( - "expected variable to contain a single [ .. ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", - variable.display_grouped_variable_token(), - )) - }, - } - })?; - + let tokens = parse_as_stream_input( + variable.interpret_as_new_stream(interpreter)?, + || { + variable.error(format!( + "Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", + variable.display_grouped_variable_token(), + )) + }, + )?; output.extend_raw_tokens(tokens); Ok(()) } CommandStreamInput::GroupedVariable(variable) => { - variable.interpret_as_tokens_into(interpreter, output) + let ungrouped_variable_contents = variable.interpret_as_new_stream(interpreter)?; + output.extend(ungrouped_variable_contents); + Ok(()) + } + CommandStreamInput::Code(code) => { + let span = code.span(); + let tokens = parse_as_stream_input(code.interpret_as_tokens(interpreter)?, || { + span.error("Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string()) + })?; + output.extend_raw_tokens(tokens); + Ok(()) } CommandStreamInput::ExplicitStream(group) => group .into_content() @@ -83,3 +89,26 @@ impl Interpret for CommandStreamInput { } } } + +fn parse_as_stream_input( + interpreted: InterpretedStream, + on_error: impl FnOnce() -> Error, +) -> Result { + fn get_group(interpreted: InterpretedStream) -> Option { + let mut token_iter = interpreted.into_token_stream().into_iter(); + let group = match token_iter.next()? { + TokenTree::Group(group) + if matches!(group.delimiter(), Delimiter::Bracket | Delimiter::None) => + { + Some(group) + } + _ => return None, + }; + if token_iter.next().is_some() { + return None; + } + group + } + let group = get_group(interpreted).ok_or_else(on_error)?; + Ok(group.stream()) +} diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs new file mode 100644 index 00000000..bdc89ac0 --- /dev/null +++ b/src/interpretation/command_value_input.rs @@ -0,0 +1,72 @@ +use crate::internal_prelude::*; + +/// This can be use to represent a value, or a source of that value at parse time. +/// +/// For example, `CommandValueInput` could be used to represent a literal, +/// or a variable/command which could convert to a literal after interpretation. +#[derive(Clone)] +pub(crate) enum CommandValueInput { + Command(Command), + GroupedVariable(GroupedVariable), + FlattenedVariable(FlattenedVariable), + Code(CommandCodeInput), + Value(T), +} + +impl Parse for CommandValueInput { + fn parse(input: ParseStream) -> Result { + Ok(match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::Command => Self::Command(input.parse()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + PeekMatch::InterpretationGroup(Delimiter::Brace) => Self::Code(input.parse()?), + PeekMatch::InterpretationGroup(_) | PeekMatch::Other => Self::Value(input.parse()?), + }) + } +} + +impl HasSpanRange for CommandValueInput { + fn span_range(&self) -> SpanRange { + match self { + CommandValueInput::Command(command) => command.span_range(), + CommandValueInput::GroupedVariable(variable) => variable.span_range(), + CommandValueInput::FlattenedVariable(variable) => variable.span_range(), + CommandValueInput::Code(code) => code.span_range(), + CommandValueInput::Value(value) => value.span_range(), + } + } +} + +impl, I: Parse> InterpretValue for CommandValueInput { + type InterpretedValue = I; + + fn interpret(self, interpreter: &mut Interpreter) -> Result { + let descriptor = match self { + CommandValueInput::Command(_) => "command output", + CommandValueInput::GroupedVariable(_) => "grouped variable output", + CommandValueInput::FlattenedVariable(_) => "flattened variable output", + CommandValueInput::Code(_) => "output from the { ... } block", + CommandValueInput::Value(_) => "value", + }; + let interpreted_stream = match self { + CommandValueInput::Command(command) => command + .interpret_as_tokens(interpreter)?, + CommandValueInput::GroupedVariable(variable) => variable + .interpret_as_tokens(interpreter)?, + CommandValueInput::FlattenedVariable(variable) => variable + .interpret_as_tokens(interpreter)?, + CommandValueInput::Code(code) => { + code.interpret_as_tokens(interpreter)? + } + CommandValueInput::Value(value) => return value.interpret(interpreter), + }; + match interpreted_stream.syn_parse(I::parse) { + Ok(value) => Ok(value), + Err(err) => Err(err.concat(&format!( + "\nOccurred whilst parsing the {} to a {}.", + descriptor, + std::any::type_name::() + ))), + } + } +} diff --git a/src/interpretation/interpretation_value.rs b/src/interpretation/interpretation_value.rs deleted file mode 100644 index 0815aa6a..00000000 --- a/src/interpretation/interpretation_value.rs +++ /dev/null @@ -1,54 +0,0 @@ -use crate::internal_prelude::*; - -/// This can be use to represent a value, or a source of that value at parse time. -/// -/// For example, `InterpretationValue` could be used to represent a literal, -/// or a variable/command which could convert to a literal after interpretation. -#[derive(Clone)] -pub(crate) enum InterpretationValue { - Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), - Value(T), -} - -impl Parse for InterpretationValue { - fn parse(input: ParseStream) -> Result { - Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command => Self::Command(input.parse()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::InterpretationGroup(_) | PeekMatch::Other => Self::Value(input.parse()?), - }) - } -} - -impl HasSpanRange for InterpretationValue { - fn span_range(&self) -> SpanRange { - match self { - InterpretationValue::Command(command) => command.span_range(), - InterpretationValue::GroupedVariable(variable) => variable.span_range(), - InterpretationValue::FlattenedVariable(variable) => variable.span_range(), - InterpretationValue::Value(value) => value.span_range(), - } - } -} - -impl, I: Parse> InterpretValue for InterpretationValue { - type InterpretedValue = I; - - fn interpret(self, interpreter: &mut Interpreter) -> Result { - match self { - InterpretationValue::Command(command) => command - .interpret_as_tokens(interpreter)? - .syn_parse(I::parse), - InterpretationValue::GroupedVariable(variable) => variable - .interpret_as_tokens(interpreter)? - .syn_parse(I::parse), - InterpretationValue::FlattenedVariable(variable) => variable - .interpret_as_tokens(interpreter)? - .syn_parse(I::parse), - InterpretationValue::Value(value) => value.interpret(interpreter), - } - } -} diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 7c398880..4cab3178 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -54,7 +54,7 @@ impl InterpretedStream { self.token_stream.extend(tokens); } - fn push_raw_token_tree(&mut self, token_tree: TokenTree) { + pub(crate) fn push_raw_token_tree(&mut self, token_tree: TokenTree) { self.token_stream.extend(iter::once(token_tree)); } diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 33f5125d..e6af51f9 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,12 +1,12 @@ mod command; mod command_arguments; mod command_code_input; -mod command_fields_input; +mod command_field_inputs; mod command_stream_input; +mod command_value_input; mod interpret_traits; mod interpretation_item; mod interpretation_stream; -mod interpretation_value; mod interpreted_stream; mod interpreter; mod variable; @@ -14,12 +14,12 @@ mod variable; pub(crate) use command::*; pub(crate) use command_arguments::*; pub(crate) use command_code_input::*; -pub(crate) use command_fields_input::*; +pub(crate) use command_field_inputs::*; pub(crate) use command_stream_input::*; +pub(crate) use command_value_input::*; pub(crate) use interpret_traits::*; pub(crate) use interpretation_item::*; pub(crate) use interpretation_stream::*; -pub(crate) use interpretation_value::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; pub(crate) use variable::*; diff --git a/src/traits.rs b/src/traits.rs index 40a25727..69a3662f 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -48,10 +48,16 @@ impl LiteralExt for Literal { } pub(crate) trait TokenStreamExt: Sized { + #[allow(unused)] + fn push(&mut self, token: TokenTree); fn flatten_transparent_groups(self) -> Self; } impl TokenStreamExt for TokenStream { + fn push(&mut self, token: TokenTree) { + self.extend(iter::once(token)); + } + fn flatten_transparent_groups(self) -> Self { let mut output = TokenStream::new(); for token in self { @@ -187,6 +193,11 @@ pub(crate) struct SpanRange { #[allow(unused)] impl SpanRange { + /// For use where the span range is unused, but needs to be provided + pub(crate) fn ignored() -> Self { + Span::call_site().span_range() + } + pub(crate) fn new_between(start: Span, end: Span) -> Self { Self { start, end } } diff --git a/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs b/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs new file mode 100644 index 00000000..87ed1bb9 --- /dev/null +++ b/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs @@ -0,0 +1,13 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!intersperse! { + items: [1 2], + separator: [], + add_trailing: { + true false + } + }] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr b/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr new file mode 100644 index 00000000..afea4d6b --- /dev/null +++ b/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr @@ -0,0 +1,6 @@ +error: unexpected token + Occurred whilst parsing the output from the { ... } block to a syn::lit::LitBool. + --> tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs:9:22 + | +9 | true false + | ^^^^^ diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs new file mode 100644 index 00000000..3a81831a --- /dev/null +++ b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs @@ -0,0 +1,10 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!intersperse! { + items: { 1 2 3 }, + separator: [] + }] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr new file mode 100644 index 00000000..10070cbb --- /dev/null +++ b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr @@ -0,0 +1,5 @@ +error: Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream. + --> tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs:6:20 + | +6 | items: { 1 2 3 }, + | ^^^^^^^^^ diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs new file mode 100644 index 00000000..e8d5ffa7 --- /dev/null +++ b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!set! #x = 1 2] + [!intersperse! { + items: #..x, + separator: [] + }] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr new file mode 100644 index 00000000..95fb83a0 --- /dev/null +++ b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr @@ -0,0 +1,5 @@ +error: Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use #x instead, to use the content of the variable as the stream. + --> tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs:7:20 + | +7 | items: #..x, + | ^^^^ diff --git a/tests/tokens.rs b/tests/tokens.rs index 6bf92b40..e2dd798b 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -48,3 +48,257 @@ fn test_length_and_group() { [!length! [!group! #..x]] }, 1); } + +#[test] +fn test_intersperse() { + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [Hello World], + separator: [", "], + }]] + }, + "Hello, World" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [Hello World], + separator: [_ "and" _], + }]] + }, + "Hello_and_World" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [Hello World], + separator: [_ "and" _], + add_trailing: true, + }]] + }, + "Hello_and_World_and_" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [The Quick Brown Fox], + separator: [], + }]] + }, + "TheQuickBrownFox" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [The Quick Brown Fox], + separator: [,], + add_trailing: true, + }]] + }, + "The,Quick,Brown,Fox," + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [Red Green Blue], + separator: [", "], + final_separator: [" and "], + }]] + }, + "Red, Green and Blue" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [Red Green Blue], + separator: [", "], + add_trailing: true, + final_separator: [" and "], + }]] + }, + "Red, Green, Blue and " + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [], + separator: [", "], + add_trailing: true, + final_separator: [" and "], + }]] + }, + "" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [SingleItem], + separator: [","], + final_separator: ["!"], + }]] + }, + "SingleItem" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [SingleItem], + separator: [","], + final_separator: ["!"], + add_trailing: true, + }]] + }, + "SingleItem!" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [SingleItem], + separator: [","], + add_trailing: true, + }]] + }, + "SingleItem," + ); +} + +#[test] +fn complex_cases_for_intersperse_and_input_types() { + // Normal separator is not interpreted if it is unneeded + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [], + separator: [[!error! { message: "FAIL" }]], + add_trailing: true, + }]] + }, + "" + ); + // Final separator is not interpreted if it is unneeded + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [], + separator: [], + final_separator: [[!error! { message: "FAIL" }]], + add_trailing: true, + }]] + }, + "" + ); + // The separator is interpreted each time it is included + assert_preinterpret_eq!({ + [!set! #i = 0] + [!string! [!intersperse! { + items: [A B C D E F G], + separator: [ + (#i) + [!assign! #i += 1] + ], + add_trailing: true, + }]] + }, "A(0)B(1)C(2)D(3)E(4)F(5)G(6)"); + // Command can be used for items + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [!range! 0..4], + separator: [_], + }]] + }, + "0_1_2_3" + ); + // Grouped Variable can be used for items + assert_preinterpret_eq!({ + [!set! #items = 0 1 2 3] + [!string! [!intersperse! { + items: #items, + separator: [_], + }]] + }, "0_1_2_3"); + // Grouped variable containing flattened command can be used for items + assert_preinterpret_eq!({ + [!set! #items = [!range! 0..4]] + [!string! [!intersperse! { + items: #items, + separator: [_], + }]] + }, "0_1_2_3"); + // Flattened variable containing [ ... ] group + assert_preinterpret_eq!({ + [!set! #items = [0 1 2 3]] + [!string! [!intersperse! { + items: #..items, + separator: [_], + }]] + }, "0_1_2_3"); + // Flattened variable containing transparent group + assert_preinterpret_eq!({ + [!set! #items = 0 1 2 3] + [!set! #wrapped_items = #items] // [!GROUP! 0 1 2 3] + [!string! [!intersperse! { + items: #..wrapped_items, // [!GROUP! 0 1 2 3] + separator: [_], + }]] + }, "0_1_2_3"); + // { ... } block returning transparent group (from variable) + assert_preinterpret_eq!({ + [!set! #items = 0 1 2 3] + [!string! [!intersperse! { + items: { + #items + }, + separator: [_], + }]] + }, "0_1_2_3"); + // { ... } block returning [ ... ] group + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: { + [0 1 2 3] + }, + separator: [_], + }]] + }, + "0_1_2_3" + ); + // Grouped variable containing two groups + assert_preinterpret_eq!({ + [!set! #items = 0 1] + [!set! #item_groups = #items #items] // [!GROUP! 0 1] [!GROUP! 0 1] + [!string! [!intersperse! { + items: #item_groups, // [!GROUP! [!GROUP! 0 1] [!GROUP! 0 1]] + separator: [_], + }]] + }, "01_01"); + // All inputs can be variables + // Inputs can be in any order + assert_preinterpret_eq!({ + [!set! #people = Anna Barbara Charlie] + [!set! #separator = ", "] + [!set! #final_separator = " and "] + [!set! #add_trailing = false] + [!string! [!intersperse! { + separator: #separator, + final_separator: #final_separator, + add_trailing: #add_trailing, + items: #people, + }]] + }, "Anna, Barbara and Charlie"); + // Add trailing is executed even if it's irrelevant because there are no items + assert_preinterpret_eq!({ + [!set! #x = "NOT_EXECUTED"] + [!intersperse! { + items: [], + separator: [], + add_trailing: { + [!set! #x = "EXECUTED"] + false + }, + }] + #x + }, "EXECUTED"); +} From 214b807ab5daff11cb0a3672f677f7db34275f62 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 18 Jan 2025 14:35:18 +0000 Subject: [PATCH 034/126] fix: Move the syn comment and fix styling --- src/interpretation/command_arguments.rs | 61 ----------------------- src/interpretation/command_value_input.rs | 15 +++--- src/interpretation/interpreted_stream.rs | 44 ++++++++++++++++ 3 files changed, 51 insertions(+), 69 deletions(-) diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index 2624fabc..de07e9c3 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -1,66 +1,5 @@ use crate::internal_prelude::*; -// =============================================== -// How syn features fits with preinterpret parsing -// =============================================== -// -// There are a few places where we parse in preinterpret: -// * Parse the initial input from the macro, into an interpretable structure -// * Parsing as part of interpretation -// * e.g. of raw tokens, either from the preinterpret declaration, or from a variable -// * e.g. of a variable, as part of incremental parsing (while_parse style loops) -// -// I spent quite a while considering whether this could be wrapping a -// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... -// -// Parsing the initial input -// ------------------------- -// -// This is where CommandArguments comes in. -// -// Now that I've changed how preinterpret works to be a two-pass approach -// (first parsing, then interpreting), we could consider swapping out the -// CommandArguments to wrap a syn::ParseStream instead. -// -// This probably could work, but has a little more overhead to swap to a -// TokenBuffer (and so ParseStream) and back. -// -// Parsing in the context of a command execution -// --------------------------------------------- -// -// For parse/destructuring operations, we could temporarily create parse -// streams in scope of a command execution. -// -// Parsing a variable or other token stream -// ---------------------------------------- -// -// Some commands want to performantly parse a variable or other token stream. -// -// Here we want variables to support: -// * Easy appending of tokens -// * Incremental parsing -// -// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to -// append to it, and freely convert it to a syn::ParseStream, possibly even storing -// a cursor position into it. -// -// Unfortunately this isn't at all possible: -// * TokenBuffer appending isn't a thing, you can only create one (recursively) from -// a TokenStream -// * TokenBuffer can't be converted to a ParseStream outside of the syn crate -// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be -// used against a fixed buffer. -// -// We could probably work around these limitations by sacrificing performance and transforming -// to TokenStream and back, but probably there's a better way. -// -// What we probably want is our own abstraction, likely a fork from `syn`, which supports -// converting a Cursor into an indexed based cursor, which can safely be stored separately -// from the TokenBuffer. -// -// We could use this abstraction for InterpretedStream; and our variables could store a -// tuple of (IndexCursor, PreinterpretTokenBuffer) - #[derive(Clone)] pub(crate) struct CommandArguments<'a> { parse_stream: ParseStream<'a>, diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index bdc89ac0..4d65c166 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -49,15 +49,14 @@ impl, I: Parse> InterpretValue for Comma CommandValueInput::Value(_) => "value", }; let interpreted_stream = match self { - CommandValueInput::Command(command) => command - .interpret_as_tokens(interpreter)?, - CommandValueInput::GroupedVariable(variable) => variable - .interpret_as_tokens(interpreter)?, - CommandValueInput::FlattenedVariable(variable) => variable - .interpret_as_tokens(interpreter)?, - CommandValueInput::Code(code) => { - code.interpret_as_tokens(interpreter)? + CommandValueInput::Command(command) => command.interpret_as_tokens(interpreter)?, + CommandValueInput::GroupedVariable(variable) => { + variable.interpret_as_tokens(interpreter)? } + CommandValueInput::FlattenedVariable(variable) => { + variable.interpret_as_tokens(interpreter)? + } + CommandValueInput::Code(code) => code.interpret_as_tokens(interpreter)?, CommandValueInput::Value(value) => return value.interpret(interpreter), }; match interpreted_stream.syn_parse(I::parse) { diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 4cab3178..be1bacfa 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -1,5 +1,49 @@ use crate::internal_prelude::*; +// ====================================== +// How syn fits with preinterpret parsing +// ====================================== +// +// TLDR: This is discussed on this syn issue, where David Tolnay suggested +// forking syn to get what we want (i.e. a more versatile TokenBuffer): +// ==> https://github.com/dtolnay/syn/issues/1842 +// +// There are a few places where we support (or might wish to support) parsing +// as part of interpretation: +// * e.g. of a token stream in `CommandValueInput` +// * e.g. as part of a PARSER, from an InterpretedStream +// * e.g. of a variable, as part of incremental parsing (while_parse style loops) +// +// I spent quite a while considering whether this could be wrapping a +// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... +// +// Some commands want to performantly parse a variable or other token stream. +// +// Here we want variables to support: +// * Easy appending of tokens +// * Incremental parsing +// +// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to +// append to it, and freely convert it to a syn::ParseStream, possibly even storing +// a cursor position into it. +// +// Unfortunately this isn't at all possible: +// * TokenBuffer appending isn't a thing, you can only create one (recursively) from +// a TokenStream +// * TokenBuffer can't be converted to a ParseStream outside of the syn crate +// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be +// used against a fixed buffer. +// +// We could probably work around these limitations by sacrificing performance and transforming +// to TokenStream and back, but probably there's a better way. +// +// What we probably want is our own abstraction, likely a fork from `syn`, which supports +// converting a Cursor into an indexed based cursor, which can safely be stored separately +// from the TokenBuffer. +// +// We could use this abstraction for InterpretedStream; and our variables could store a +// tuple of (IndexCursor, PreinterpretTokenBuffer) + #[derive(Clone)] pub(crate) struct InterpretedStream { source_span_range: SpanRange, From 5020633de5097fc561b94974a0dedb9156ba9e40 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 18 Jan 2025 17:51:10 +0000 Subject: [PATCH 035/126] refactor: Command output types are static --- CHANGELOG.md | 3 + src/commands/concat_commands.rs | 83 ++++++++++------ src/commands/control_flow_commands.rs | 14 ++- src/commands/core_commands.rs | 44 ++++----- src/commands/expression_commands.rs | 27 +++--- src/commands/token_commands.rs | 40 ++++---- src/expressions/boolean.rs | 11 ++- src/expressions/evaluation_tree.rs | 6 ++ src/expressions/expression_stream.rs | 3 +- src/expressions/float.rs | 11 ++- src/expressions/integer.rs | 11 ++- src/expressions/value.rs | 7 ++ src/interpretation/command.rs | 105 +++++++++++++++++++-- src/interpretation/command_stream_input.rs | 3 +- src/interpretation/command_value_input.rs | 3 +- src/interpretation/interpretation_item.rs | 17 +++- 16 files changed, 262 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae05986d..83116cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,9 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come * Consider [!..flattened! ] and getting rid of !group! + => Make the type of commands static + => Disallow most command types in expressions + => Get rid of [!empty!] and [!stream!] and replace with [!..group!] or [!group!] * Explore getting rid of lots of the span range stuff * Support `!else if!` in `!if!` * Support string & char literals (for comparisons & casts) in expressions diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 3562cd72..d353b11f 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -28,33 +28,33 @@ fn concat_into_string( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> Result { let output_span = input.span(); let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); let string_literal = string_literal(&conversion_fn(&concatenated), output_span); - Ok(CommandOutput::Literal(string_literal)) + Ok(string_literal) } fn concat_into_ident( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> Result { let output_span = input.span(); let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); let ident = parse_ident(&conversion_fn(&concatenated), output_span)?; - Ok(CommandOutput::Ident(ident)) + Ok(ident) } fn concat_into_literal( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> Result { let output_span = input.span(); let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); let literal = parse_literal(&conversion_fn(&concatenated), output_span)?; - Ok(CommandOutput::Literal(literal)) + Ok(literal) } fn concat_recursive(arguments: InterpretedStream) -> String { @@ -98,7 +98,7 @@ fn concat_recursive(arguments: InterpretedStream) -> String { output } -macro_rules! define_concat_command { +macro_rules! define_literal_concat_command { ( $command_name:literal => $command:ident: $output_fn:ident($conversion_fn:expr) ) => { @@ -109,17 +109,42 @@ macro_rules! define_concat_command { impl CommandDefinition for $command { const COMMAND_NAME: &'static str = $command_name; + type OutputKind = OutputKindValue; fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + Ok($output_fn(self.arguments, interpreter, $conversion_fn)?.into()) + } } + }; +} + +macro_rules! define_ident_concat_command { + ( + $command_name:literal => $command:ident: $output_fn:ident($conversion_fn:expr) + ) => { + #[derive(Clone)] + pub(crate) struct $command { + arguments: InterpretationStream, + } + + impl CommandDefinition for $command { + const COMMAND_NAME: &'static str = $command_name; + type OutputKind = OutputKindIdent; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + arguments: arguments.parse_all_for_interpretation()?, + }) + } - impl CommandInvocation for $command { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - $output_fn(self.arguments, interpreter, $conversion_fn) + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + $output_fn(self.arguments, interpreter, $conversion_fn).into() } } }; @@ -129,31 +154,31 @@ macro_rules! define_concat_command { // Concatenating type-conversion commands //======================================= -define_concat_command!("string" => StringCommand: concat_into_string(|s| s.to_string())); -define_concat_command!("ident" => IdentCommand: concat_into_ident(|s| s.to_string())); -define_concat_command!("ident_camel" => IdentCamelCommand: concat_into_ident(to_upper_camel_case)); -define_concat_command!("ident_snake" => IdentSnakeCommand: concat_into_ident(to_lower_snake_case)); -define_concat_command!("ident_upper_snake" => IdentUpperSnakeCommand: concat_into_ident(to_upper_snake_case)); -define_concat_command!("literal" => LiteralCommand: concat_into_literal(|s| s.to_string())); +define_literal_concat_command!("string" => StringCommand: concat_into_string(|s| s.to_string())); +define_ident_concat_command!("ident" => IdentCommand: concat_into_ident(|s| s.to_string())); +define_ident_concat_command!("ident_camel" => IdentCamelCommand: concat_into_ident(to_upper_camel_case)); +define_ident_concat_command!("ident_snake" => IdentSnakeCommand: concat_into_ident(to_lower_snake_case)); +define_ident_concat_command!("ident_upper_snake" => IdentUpperSnakeCommand: concat_into_ident(to_upper_snake_case)); +define_literal_concat_command!("literal" => LiteralCommand: concat_into_literal(|s| s.to_string())); //=========================== // String conversion commands //=========================== -define_concat_command!("upper" => UpperCommand: concat_into_string(to_uppercase)); -define_concat_command!("lower" => LowerCommand: concat_into_string(to_lowercase)); +define_literal_concat_command!("upper" => UpperCommand: concat_into_string(to_uppercase)); +define_literal_concat_command!("lower" => LowerCommand: concat_into_string(to_lowercase)); // Snake case is typically lower snake case in Rust, so default to that -define_concat_command!("snake" => SnakeCommand: concat_into_string(to_lower_snake_case)); -define_concat_command!("lower_snake" => LowerSnakeCommand: concat_into_string(to_lower_snake_case)); -define_concat_command!("upper_snake" => UpperSnakeCommand: concat_into_string(to_upper_snake_case)); +define_literal_concat_command!("snake" => SnakeCommand: concat_into_string(to_lower_snake_case)); +define_literal_concat_command!("lower_snake" => LowerSnakeCommand: concat_into_string(to_lower_snake_case)); +define_literal_concat_command!("upper_snake" => UpperSnakeCommand: concat_into_string(to_upper_snake_case)); // Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) // It can always be combined with other casing to get other versions -define_concat_command!("kebab" => KebabCommand: concat_into_string(to_lower_kebab_case)); +define_literal_concat_command!("kebab" => KebabCommand: concat_into_string(to_lower_kebab_case)); // Upper camel case is the more common casing in Rust, so default to that -define_concat_command!("camel" => CamelCommand: concat_into_string(to_upper_camel_case)); -define_concat_command!("lower_camel" => LowerCamelCommand: concat_into_string(to_lower_camel_case)); -define_concat_command!("upper_camel" => UpperCamelCommand: concat_into_string(to_upper_camel_case)); -define_concat_command!("capitalize" => CapitalizeCommand: concat_into_string(capitalize)); -define_concat_command!("decapitalize" => DecapitalizeCommand: concat_into_string(decapitalize)); -define_concat_command!("title" => TitleCommand: concat_into_string(title_case)); -define_concat_command!("insert_spaces" => InsertSpacesCommand: concat_into_string(insert_spaces_between_words)); +define_literal_concat_command!("camel" => CamelCommand: concat_into_string(to_upper_camel_case)); +define_literal_concat_command!("lower_camel" => LowerCamelCommand: concat_into_string(to_lower_camel_case)); +define_literal_concat_command!("upper_camel" => UpperCamelCommand: concat_into_string(to_upper_camel_case)); +define_literal_concat_command!("capitalize" => CapitalizeCommand: concat_into_string(capitalize)); +define_literal_concat_command!("decapitalize" => DecapitalizeCommand: concat_into_string(decapitalize)); +define_literal_concat_command!("title" => TitleCommand: concat_into_string(title_case)); +define_literal_concat_command!("insert_spaces" => InsertSpacesCommand: concat_into_string(insert_spaces_between_words)); diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 699bc567..f5e6ebb9 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -10,6 +10,7 @@ pub(crate) struct IfCommand { impl CommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -33,10 +34,8 @@ impl CommandDefinition for IfCommand { "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code }]", ) } -} -impl CommandInvocation for IfCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let evaluated_condition = self .condition .evaluate(interpreter)? @@ -51,7 +50,7 @@ impl CommandInvocation for IfCommand { InterpretedStream::new(self.nothing_span_range) }; - Ok(CommandOutput::Stream(output)) + Ok(output) } } @@ -64,6 +63,7 @@ pub(crate) struct WhileCommand { impl CommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -77,10 +77,8 @@ impl CommandDefinition for WhileCommand { "Expected [!while! (condition) { code }]", ) } -} -impl CommandInvocation for WhileCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(self.nothing_span_range); let mut iteration_count = 0; loop { @@ -104,6 +102,6 @@ impl CommandInvocation for WhileCommand { .interpret_as_tokens_into(interpreter, &mut output)?; } - Ok(CommandOutput::Stream(output)) + Ok(output) } } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 1a2a139b..692bc34d 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -10,6 +10,7 @@ pub(crate) struct SetCommand { impl CommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; + type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -23,14 +24,12 @@ impl CommandDefinition for SetCommand { "Expected [!set! #variable = ..]", ) } -} -impl CommandInvocation for SetCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { let result_tokens = self.arguments.interpret_as_tokens(interpreter)?; self.variable.set(interpreter, result_tokens)?; - Ok(CommandOutput::Empty) + Ok(()) } } @@ -44,6 +43,7 @@ pub(crate) struct ExtendCommand { impl CommandDefinition for ExtendCommand { const COMMAND_NAME: &'static str = "extend"; + type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -57,13 +57,11 @@ impl CommandDefinition for ExtendCommand { "Expected [!extend! #variable += ..]", ) } -} -impl CommandInvocation for ExtendCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { let output = self.arguments.interpret_as_tokens(interpreter)?; self.variable.get_mut(interpreter)?.extend(output); - Ok(CommandOutput::Empty) + Ok(()) } } @@ -75,6 +73,7 @@ pub(crate) struct RawCommand { impl CommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -82,14 +81,12 @@ impl CommandDefinition for RawCommand { token_stream: arguments.read_all_as_raw_token_stream(), }) } -} -impl CommandInvocation for RawCommand { - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::Stream(InterpretedStream::raw( + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { + Ok(InterpretedStream::raw( self.arguments_span_range, self.token_stream, - ))) + )) } } @@ -98,17 +95,16 @@ pub(crate) struct IgnoreCommand; impl CommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; + type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { // Avoid a syn parse error by reading all the tokens let _ = arguments.read_all_as_raw_token_stream(); Ok(Self) } -} -impl CommandInvocation for IgnoreCommand { - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::Empty) + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result<()> { + Ok(()) } } @@ -120,19 +116,16 @@ pub(crate) struct StreamCommand { impl CommandDefinition for StreamCommand { const COMMAND_NAME: &'static str = "stream"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } -} -impl CommandInvocation for StreamCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::Stream( - self.arguments.interpret_as_tokens(interpreter)?, - )) + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + self.arguments.interpret_as_tokens(interpreter) } } @@ -154,16 +147,15 @@ define_field_inputs! { impl CommandDefinition for ErrorCommand { const COMMAND_NAME: &'static str = "error"; + type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { Ok(Self { inputs: arguments.fully_parse_as()?, }) } -} -impl CommandInvocation for ErrorCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { let message = self.inputs.message.interpret(interpreter)?.value(); let error_span = match self.inputs.spans { diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index ca1abed0..bbc6644b 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -7,6 +7,7 @@ pub(crate) struct EvaluateCommand { impl CommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; + type OutputKind = OutputKindValue; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -18,14 +19,10 @@ impl CommandDefinition for EvaluateCommand { "Expected [!evaluate! ...] containing a valid preinterpret expression", ) } -} -impl CommandInvocation for EvaluateCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let expression = self.expression.start_expression_builder(interpreter)?; - Ok(CommandOutput::Stream( - expression.evaluate()?.into_interpreted_stream(), - )) + Ok(expression.evaluate()?.into_token_tree()) } } @@ -40,6 +37,7 @@ pub(crate) struct AssignCommand { impl CommandDefinition for AssignCommand { const COMMAND_NAME: &'static str = "assign"; + type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -61,10 +59,8 @@ impl CommandDefinition for AssignCommand { "Expected [!assign! #variable += ...] for + or some other operator supported in an expression", ) } -} -impl CommandInvocation for AssignCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { let Self { variable, operator, @@ -82,7 +78,7 @@ impl CommandInvocation for AssignCommand { let output = builder.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output)?; - Ok(CommandOutput::Empty) + Ok(()) } } @@ -95,6 +91,7 @@ pub(crate) struct RangeCommand { impl CommandDefinition for RangeCommand { const COMMAND_NAME: &'static str = "range"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -108,10 +105,8 @@ impl CommandDefinition for RangeCommand { "Expected a rust range expression such as [!range! 1..4]", ) } -} -impl CommandInvocation for RangeCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let range_span_range = self.range_limits.span_range(); let left = self @@ -124,8 +119,9 @@ impl CommandInvocation for RangeCommand { .evaluate(interpreter)? .expect_integer("The right side of the range must be an integer")? .try_into_i128()?; + if left > right { - return Ok(CommandOutput::Empty); + return Ok(InterpretedStream::new(range_span_range)); } let length = self @@ -151,7 +147,8 @@ impl CommandInvocation for RangeCommand { output.extend_raw_token_iter(iter) } }; - Ok(CommandOutput::Stream(output)) + + Ok(output) } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index cfe815af..94984745 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -5,6 +5,7 @@ pub(crate) struct EmptyCommand; impl CommandDefinition for EmptyCommand { const COMMAND_NAME: &'static str = "empty"; + type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { arguments.assert_empty( @@ -12,11 +13,9 @@ impl CommandDefinition for EmptyCommand { )?; Ok(Self) } -} -impl CommandInvocation for EmptyCommand { - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(CommandOutput::Empty) + fn execute(self: Box, _interpreter: &mut Interpreter) -> Result<()> { + Ok(()) } } @@ -27,22 +26,18 @@ pub(crate) struct IsEmptyCommand { impl CommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; + type OutputKind = OutputKindValue; fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } -} -impl CommandInvocation for IsEmptyCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_as_tokens(interpreter)?; - Ok(CommandOutput::Ident(Ident::new_bool( - interpreted.is_empty(), - output_span, - ))) + Ok(Ident::new_bool(interpreted.is_empty(), output_span).into()) } } @@ -53,21 +48,20 @@ pub(crate) struct LengthCommand { impl CommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; + type OutputKind = OutputKindValue; fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } -} -impl CommandInvocation for LengthCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_as_tokens(interpreter)?; let stream_length = interpreted.into_token_stream().into_iter().count(); let length_literal = Literal::usize_unsuffixed(stream_length).with_span(output_span); - Ok(CommandOutput::Literal(length_literal)) + Ok(length_literal.into()) } } @@ -78,21 +72,20 @@ pub(crate) struct GroupCommand { impl CommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } -} -impl CommandInvocation for GroupCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(self.arguments.span_range()); let group_span = self.arguments.span(); let inner = self.arguments.interpret_as_tokens(interpreter)?; output.push_new_group(inner, Delimiter::None, group_span); - Ok(CommandOutput::Stream(output)) + Ok(output) } } @@ -117,6 +110,7 @@ define_field_inputs! { impl CommandDefinition for IntersperseCommand { const COMMAND_NAME: &'static str = "intersperse"; + type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -124,10 +118,8 @@ impl CommandDefinition for IntersperseCommand { inputs: arguments.fully_parse_as()?, }) } -} -impl CommandInvocation for IntersperseCommand { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let items = self .inputs .items @@ -141,7 +133,7 @@ impl CommandInvocation for IntersperseCommand { let mut output = InterpretedStream::new(self.span_range); if items.is_empty() { - return Ok(CommandOutput::Stream(output)); + return Ok(output); } let mut appender = SeparatorAppender { @@ -172,7 +164,7 @@ impl CommandInvocation for IntersperseCommand { } } - Ok(CommandOutput::Stream(output)) + Ok(output) } } diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 08846ddb..2304f55f 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -90,6 +90,15 @@ impl EvaluationBoolean { PairedBinaryOperator::GreaterThan => operation.output(lhs & !rhs), } } + + pub(super) fn to_ident(&self) -> Ident { + Ident::new_bool(self.value, self.source_span.start()) + } + + #[allow(unused)] + pub(super) fn to_lit_bool(&self) -> LitBool { + LitBool::new(self.value, self.source_span.span()) + } } impl ToEvaluationOutput for bool { @@ -100,6 +109,6 @@ impl ToEvaluationOutput for bool { impl quote::ToTokens for EvaluationBoolean { fn to_tokens(&self, tokens: &mut TokenStream) { - LitBool::new(self.value, self.source_span.span()).to_tokens(tokens) + self.to_ident().to_tokens(tokens) } } diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index a84a821f..b2799c61 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -433,6 +433,12 @@ impl EvaluationOutput { } } + pub(crate) fn into_token_tree(self) -> TokenTree { + match self { + Self::Value(value) => value.into_token_tree(), + } + } + pub(crate) fn into_interpreted_stream(self) -> InterpretedStream { let value = self.into_value(); InterpretedStream::raw(value.source_span(), value.into_token_stream()) diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 4f9aad91..dd86d794 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -35,7 +35,8 @@ impl Parse for ExpressionInput { // before code blocks or .. in [!range!] so we can break on those. // These aren't valid inside expressions we support anyway, so it's good enough for now. let item = match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command => ExpressionItem::Command(input.parse()?), + PeekMatch::GroupedCommand => ExpressionItem::Command(input.parse()?), + PeekMatch::FlattenedCommand => ExpressionItem::Command(input.parse()?), PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), PeekMatch::InterpretationGroup(Delimiter::Brace | Delimiter::Bracket) => break, diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 925fcbcd..8b35a1cd 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -46,14 +46,17 @@ impl EvaluationFloat { } } } -} -impl quote::ToTokens for EvaluationFloat { - fn to_tokens(&self, tokens: &mut TokenStream) { + pub(super) fn to_literal(&self) -> Literal { self.value .to_unspanned_literal() .with_span(self.source_span.start()) - .to_tokens(tokens) + } +} + +impl quote::ToTokens for EvaluationFloat { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.to_literal().to_tokens(tokens) } } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 8032efda..478b1478 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -109,14 +109,17 @@ impl EvaluationInteger { } } } -} -impl quote::ToTokens for EvaluationInteger { - fn to_tokens(&self, tokens: &mut TokenStream) { + pub(super) fn to_literal(&self) -> Literal { self.value .to_unspanned_literal() .with_span(self.source_span.start()) - .to_tokens(tokens) + } +} + +impl quote::ToTokens for EvaluationInteger { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.to_literal().to_tokens(tokens) } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index fcf2fe63..9f6dbad9 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -7,6 +7,13 @@ pub(crate) enum EvaluationValue { } impl EvaluationValue { + pub(super) fn into_token_tree(self) -> TokenTree { + match self { + Self::Integer(int) => int.to_literal().into(), + Self::Float(float) => float.to_literal().into(), + Self::Boolean(bool) => bool.to_ident().into(), + } + } pub(super) fn for_literal_expression(expr: &ExprLit) -> Result { // https://docs.rs/syn/latest/syn/enum.Lit.html Ok(match &expr.lit { diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 803fca37..b6195e54 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,27 +1,114 @@ use crate::internal_prelude::*; -pub(crate) trait CommandDefinition: CommandInvocation + Clone { +pub(crate) enum CommandOutput { + Empty, + Literal(Literal), + Ident(Ident), + Stream(InterpretedStream), +} + +#[allow(unused)] +pub(crate) enum CommandOutputKind { + None, + /// LiteralOrBool + Value, + Ident, + StreamOrGroup, +} + +pub(crate) trait OutputKind { + type Output; + #[allow(unused)] + fn value() -> CommandOutputKind; + fn to_output_enum(output: Self::Output) -> CommandOutput; +} + +pub(crate) struct OutputKindNone; +impl OutputKind for OutputKindNone { + type Output = (); + + fn value() -> CommandOutputKind { + CommandOutputKind::None + } + + fn to_output_enum(_output: ()) -> CommandOutput { + CommandOutput::Empty + } +} + +pub(crate) struct OutputKindValue; +impl OutputKind for OutputKindValue { + type Output = TokenTree; + + fn value() -> CommandOutputKind { + CommandOutputKind::Value + } + + fn to_output_enum(output: TokenTree) -> CommandOutput { + match output { + TokenTree::Literal(literal) => CommandOutput::Literal(literal), + TokenTree::Ident(ident) => CommandOutput::Ident(ident), + _ => panic!("Value Output Commands should only output literals or idents"), + } + } +} + +pub(crate) struct OutputKindIdent; +impl OutputKind for OutputKindIdent { + type Output = Ident; + + fn value() -> CommandOutputKind { + CommandOutputKind::Ident + } + + fn to_output_enum(ident: Ident) -> CommandOutput { + CommandOutput::Ident(ident) + } +} + +pub(crate) struct OutputKindStreamOrGroup; +impl OutputKind for OutputKindStreamOrGroup { + type Output = InterpretedStream; + + fn value() -> CommandOutputKind { + CommandOutputKind::StreamOrGroup + } + + fn to_output_enum(output: InterpretedStream) -> CommandOutput { + CommandOutput::Stream(output) + } +} + +pub(crate) trait CommandDefinition: Sized { const COMMAND_NAME: &'static str; + type OutputKind: OutputKind; fn parse(arguments: CommandArguments) -> Result; + + fn execute( + self: Box, + interpreter: &mut Interpreter, + ) -> Result<::Output>; } -pub(crate) trait CommandInvocation { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; +impl CommandInvocation for C { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let output = ::execute(self, interpreter)?; + Ok(::to_output_enum(output)) + } } -pub(crate) enum CommandOutput { - Empty, - Literal(Literal), - Ident(Ident), - Stream(InterpretedStream), +//========================= + +pub(crate) trait CommandInvocation { + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; } pub(crate) trait ClonableCommandInvocation: CommandInvocation { fn clone_box(&self) -> Box; } -impl ClonableCommandInvocation for C { +impl ClonableCommandInvocation for C { fn clone_box(&self) -> Box { Box::new(self.clone()) } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 7d0135c6..ca562973 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -24,7 +24,8 @@ pub(crate) enum CommandStreamInput { impl Parse for CommandStreamInput { fn parse(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command => Self::Command(input.parse()?), + PeekMatch::GroupedCommand => Self::Command(input.parse()?), + PeekMatch::FlattenedCommand => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::InterpretationGroup(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 4d65c166..718880fa 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -16,7 +16,8 @@ pub(crate) enum CommandValueInput { impl Parse for CommandValueInput { fn parse(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command => Self::Command(input.parse()?), + PeekMatch::GroupedCommand => Self::Command(input.parse()?), + PeekMatch::FlattenedCommand => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::InterpretationGroup(Delimiter::Brace) => Self::Code(input.parse()?), diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 8c910ac9..cb5b9a4e 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -14,7 +14,8 @@ pub(crate) enum InterpretationItem { impl Parse for InterpretationItem { fn parse(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command => InterpretationItem::Command(input.parse()?), + PeekMatch::GroupedCommand => InterpretationItem::Command(input.parse()?), + PeekMatch::FlattenedCommand => InterpretationItem::Command(input.parse()?), PeekMatch::InterpretationGroup(_) => { InterpretationItem::InterpretationGroup(input.parse()?) } @@ -33,7 +34,8 @@ impl Parse for InterpretationItem { } pub(crate) enum PeekMatch { - Command, + GroupedCommand, + FlattenedCommand, GroupedVariable, FlattenedVariable, InterpretationGroup(Delimiter), @@ -48,7 +50,16 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa if let Some((_, next)) = next.punct_matching('!') { if let Some((_, next)) = next.ident() { if next.punct_matching('!').is_some() { - return PeekMatch::Command; + return PeekMatch::GroupedCommand; + } + } + if let Some((_, next)) = next.punct_matching('.') { + if let Some((_, next)) = next.punct_matching('.') { + if let Some((_, next)) = next.ident() { + if next.punct_matching('!').is_some() { + return PeekMatch::FlattenedCommand; + } + } } } } From d40b31e5d9bbd06863217584dfb422353fc80d2e Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 18 Jan 2025 23:57:39 +0000 Subject: [PATCH 036/126] feature: Added support for some !..command!s --- CHANGELOG.md | 2 + src/commands/concat_commands.rs | 14 +- src/commands/control_flow_commands.rs | 48 ++-- src/commands/core_commands.rs | 76 +++-- src/commands/expression_commands.rs | 32 ++- src/commands/token_commands.rs | 68 +++-- src/interpretation/command.rs | 318 +++++++++++++++++---- src/interpretation/command_stream_input.rs | 28 +- src/interpretation/interpreted_stream.rs | 12 + src/traits.rs | 1 + tests/tokens.rs | 2 +- 11 files changed, 458 insertions(+), 143 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83116cf3..fae6dbe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come * Consider [!..flattened! ] and getting rid of !group! + => Add separate ControlFlowCode type => Make the type of commands static => Disallow most command types in expressions => Get rid of [!empty!] and [!stream!] and replace with [!..group!] or [!group!] @@ -81,6 +82,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!split! { items: X, with: X, drop_trailing_empty?: true }]` * `[!intersperse! { items: X, with: X, add_trailing?: false, override_final_with?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` +* See comment on `!assign!` * Basic place parsing * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * In parse land: #x matches a single token, #..x consumes the rest of a stream diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index d353b11f..59d44e37 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -107,9 +107,12 @@ macro_rules! define_literal_concat_command { arguments: InterpretationStream, } - impl CommandDefinition for $command { - const COMMAND_NAME: &'static str = $command_name; + impl CommandType for $command { type OutputKind = OutputKindValue; + } + + impl ValueCommandDefinition for $command { + const COMMAND_NAME: &'static str = $command_name; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -133,9 +136,12 @@ macro_rules! define_ident_concat_command { arguments: InterpretationStream, } - impl CommandDefinition for $command { - const COMMAND_NAME: &'static str = $command_name; + impl CommandType for $command { type OutputKind = OutputKindIdent; + } + + impl IdentCommandDefinition for $command { + const COMMAND_NAME: &'static str = $command_name; fn parse(arguments: CommandArguments) -> Result { Ok(Self { diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index f5e6ebb9..fbb99019 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -5,12 +5,14 @@ pub(crate) struct IfCommand { condition: ExpressionInput, true_code: CommandCodeInput, false_code: Option, - nothing_span_range: SpanRange, } -impl CommandDefinition for IfCommand { +impl CommandType for IfCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -28,29 +30,31 @@ impl CommandDefinition for IfCommand { None } }, - nothing_span_range: arguments.full_span_range(), }) }, "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code }]", ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { let evaluated_condition = self .condition .evaluate(interpreter)? .expect_bool("An if condition must evaluate to a boolean")? .value(); - let output = if evaluated_condition { - self.true_code.interpret_as_tokens(interpreter)? + if evaluated_condition { + self.true_code + .interpret_as_tokens_into(interpreter, output)? } else if let Some(false_code) = self.false_code { - false_code.interpret_as_tokens(interpreter)? - } else { - InterpretedStream::new(self.nothing_span_range) - }; + false_code.interpret_as_tokens_into(interpreter, output)? + } - Ok(output) + Ok(()) } } @@ -58,12 +62,14 @@ impl CommandDefinition for IfCommand { pub(crate) struct WhileCommand { condition: ExpressionInput, loop_code: CommandCodeInput, - nothing_span_range: SpanRange, } -impl CommandDefinition for WhileCommand { +impl CommandType for WhileCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -71,15 +77,17 @@ impl CommandDefinition for WhileCommand { Ok(Self { condition: input.parse()?, loop_code: input.parse()?, - nothing_span_range: arguments.full_span_range(), }) }, "Expected [!while! (condition) { code }]", ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(self.nothing_span_range); + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { let mut iteration_count = 0; loop { let evaluated_condition = self @@ -99,9 +107,9 @@ impl CommandDefinition for WhileCommand { .check_iteration_count(&self.condition, iteration_count)?; self.loop_code .clone() - .interpret_as_tokens_into(interpreter, &mut output)?; + .interpret_as_tokens_into(interpreter, output)?; } - Ok(output) + Ok(()) } } diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 692bc34d..4487a867 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -8,9 +8,12 @@ pub(crate) struct SetCommand { arguments: InterpretationStream, } -impl CommandDefinition for SetCommand { - const COMMAND_NAME: &'static str = "set"; +impl CommandType for SetCommand { type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for SetCommand { + const COMMAND_NAME: &'static str = "set"; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -41,9 +44,12 @@ pub(crate) struct ExtendCommand { arguments: InterpretationStream, } -impl CommandDefinition for ExtendCommand { - const COMMAND_NAME: &'static str = "extend"; +impl CommandType for ExtendCommand { type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for ExtendCommand { + const COMMAND_NAME: &'static str = "extend"; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -59,6 +65,18 @@ impl CommandDefinition for ExtendCommand { } fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + // We'd like to do this to avoid double-passing the intrepreted tokens: + // self.arguments.interpret_as_tokens_into(interpreter, self.variable.get_mut(interpreter)?); + // But this doesn't work because the interpreter is mut borrowed twice. + // + // Conceptually this does protect us from issues... e.g. it prevents us + // from allowing: + // [!extend! #x += ..#x] + // Which is pretty non-sensical. + // + // In future, we could improve this by having the interpreter store + // a RefCell and erroring on self-reference, with a hint to use a [!buffer!] + // to break the self-reference / error. let output = self.arguments.interpret_as_tokens(interpreter)?; self.variable.get_mut(interpreter)?.extend(output); Ok(()) @@ -67,35 +85,41 @@ impl CommandDefinition for ExtendCommand { #[derive(Clone)] pub(crate) struct RawCommand { - arguments_span_range: SpanRange, token_stream: TokenStream, } -impl CommandDefinition for RawCommand { +impl CommandType for RawCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { - arguments_span_range: arguments.full_span_range(), token_stream: arguments.read_all_as_raw_token_stream(), }) } - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result { - Ok(InterpretedStream::raw( - self.arguments_span_range, - self.token_stream, - )) + fn execute( + self: Box, + _interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + output.extend_raw_token_iter(self.token_stream); + Ok(()) } } #[derive(Clone)] pub(crate) struct IgnoreCommand; -impl CommandDefinition for IgnoreCommand { - const COMMAND_NAME: &'static str = "ignore"; +impl CommandType for IgnoreCommand { type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for IgnoreCommand { + const COMMAND_NAME: &'static str = "ignore"; fn parse(arguments: CommandArguments) -> Result { // Avoid a syn parse error by reading all the tokens @@ -114,9 +138,12 @@ pub(crate) struct StreamCommand { arguments: InterpretationStream, } -impl CommandDefinition for StreamCommand { +impl CommandType for StreamCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for StreamCommand { const COMMAND_NAME: &'static str = "stream"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -124,8 +151,12 @@ impl CommandDefinition for StreamCommand { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - self.arguments.interpret_as_tokens(interpreter) + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + self.arguments.interpret_as_tokens_into(interpreter, output) } } @@ -134,6 +165,10 @@ pub(crate) struct ErrorCommand { inputs: ErrorInputs, } +impl CommandType for ErrorCommand { + type OutputKind = OutputKindNone; +} + define_field_inputs! { ErrorInputs { required: { @@ -145,9 +180,8 @@ define_field_inputs! { } } -impl CommandDefinition for ErrorCommand { +impl NoOutputCommandDefinition for ErrorCommand { const COMMAND_NAME: &'static str = "error"; - type OutputKind = OutputKindNone; fn parse(arguments: CommandArguments) -> Result { Ok(Self { diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index bbc6644b..5e4c9577 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -5,9 +5,12 @@ pub(crate) struct EvaluateCommand { expression: ExpressionInput, } -impl CommandDefinition for EvaluateCommand { - const COMMAND_NAME: &'static str = "evaluate"; +impl CommandType for EvaluateCommand { type OutputKind = OutputKindValue; +} + +impl ValueCommandDefinition for EvaluateCommand { + const COMMAND_NAME: &'static str = "evaluate"; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -35,9 +38,12 @@ pub(crate) struct AssignCommand { expression: ExpressionInput, } -impl CommandDefinition for AssignCommand { - const COMMAND_NAME: &'static str = "assign"; +impl CommandType for AssignCommand { type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for AssignCommand { + const COMMAND_NAME: &'static str = "assign"; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -89,9 +95,12 @@ pub(crate) struct RangeCommand { right: ExpressionInput, } -impl CommandDefinition for RangeCommand { +impl CommandType for RangeCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for RangeCommand { const COMMAND_NAME: &'static str = "range"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( @@ -106,7 +115,11 @@ impl CommandDefinition for RangeCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { let range_span_range = self.range_limits.span_range(); let left = self @@ -121,7 +134,7 @@ impl CommandDefinition for RangeCommand { .try_into_i128()?; if left > right { - return Ok(InterpretedStream::new(range_span_range)); + return Ok(()); } let length = self @@ -134,7 +147,6 @@ impl CommandDefinition for RangeCommand { .config() .check_iteration_count(&range_span_range, length)?; - let mut output = InterpretedStream::new(range_span_range); match self.range_limits { RangeLimits::HalfOpen(_) => { let iter = @@ -148,7 +160,7 @@ impl CommandDefinition for RangeCommand { } }; - Ok(output) + Ok(()) } } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 94984745..c7cb96e9 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -3,9 +3,12 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct EmptyCommand; -impl CommandDefinition for EmptyCommand { - const COMMAND_NAME: &'static str = "empty"; +impl CommandType for EmptyCommand { type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for EmptyCommand { + const COMMAND_NAME: &'static str = "empty"; fn parse(arguments: CommandArguments) -> Result { arguments.assert_empty( @@ -24,9 +27,12 @@ pub(crate) struct IsEmptyCommand { arguments: InterpretationStream, } -impl CommandDefinition for IsEmptyCommand { - const COMMAND_NAME: &'static str = "is_empty"; +impl CommandType for IsEmptyCommand { type OutputKind = OutputKindValue; +} + +impl ValueCommandDefinition for IsEmptyCommand { + const COMMAND_NAME: &'static str = "is_empty"; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -46,9 +52,12 @@ pub(crate) struct LengthCommand { arguments: InterpretationStream, } -impl CommandDefinition for LengthCommand { - const COMMAND_NAME: &'static str = "length"; +impl CommandType for LengthCommand { type OutputKind = OutputKindValue; +} + +impl ValueCommandDefinition for LengthCommand { + const COMMAND_NAME: &'static str = "length"; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -70,9 +79,12 @@ pub(crate) struct GroupCommand { arguments: InterpretationStream, } -impl CommandDefinition for GroupCommand { +impl CommandType for GroupCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { @@ -80,21 +92,29 @@ impl CommandDefinition for GroupCommand { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(self.arguments.span_range()); + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { let group_span = self.arguments.span(); - let inner = self.arguments.interpret_as_tokens(interpreter)?; - output.push_new_group(inner, Delimiter::None, group_span); - Ok(output) + output.push_grouped( + |inner| self.arguments.interpret_as_tokens_into(interpreter, inner), + Delimiter::None, + group_span, + ) } } #[derive(Clone)] pub(crate) struct IntersperseCommand { - span_range: SpanRange, inputs: IntersperseInputs, } +impl CommandType for IntersperseCommand { + type OutputKind = OutputKindStreaming; +} + define_field_inputs! { IntersperseInputs { required: { @@ -108,18 +128,20 @@ define_field_inputs! { } } -impl CommandDefinition for IntersperseCommand { +impl StreamingCommandDefinition for IntersperseCommand { const COMMAND_NAME: &'static str = "intersperse"; - type OutputKind = OutputKindStreamOrGroup; fn parse(arguments: CommandArguments) -> Result { Ok(Self { - span_range: arguments.full_span_range(), inputs: arguments.fully_parse_as()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { let items = self .inputs .items @@ -130,10 +152,8 @@ impl CommandDefinition for IntersperseCommand { None => false, }; - let mut output = InterpretedStream::new(self.span_range); - if items.is_empty() { - return Ok(output); + return Ok(()); } let mut appender = SeparatorAppender { @@ -154,17 +174,17 @@ impl CommandDefinition for IntersperseCommand { } else { RemainingItemCount::ExactlyOne }; - appender.add_separator(interpreter, remaining, &mut output)?; + appender.add_separator(interpreter, remaining, output)?; this_item = next_item; } None => { - appender.add_separator(interpreter, RemainingItemCount::None, &mut output)?; + appender.add_separator(interpreter, RemainingItemCount::None, output)?; break; } } } - Ok(output) + Ok(()) } } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index b6195e54..bd1f0709 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,38 +1,38 @@ use crate::internal_prelude::*; pub(crate) enum CommandOutput { - Empty, + None, Literal(Literal), Ident(Ident), - Stream(InterpretedStream), + FlattenedStream(InterpretedStream), + Grouped(InterpretedStream, Span), } #[allow(unused)] +#[derive(Clone, Copy)] pub(crate) enum CommandOutputKind { None, /// LiteralOrBool Value, Ident, - StreamOrGroup, + FlattenedStream, + GroupedStream(Span), } pub(crate) trait OutputKind { type Output; - #[allow(unused)] - fn value() -> CommandOutputKind; - fn to_output_enum(output: Self::Output) -> CommandOutput; + fn resolve(span: &DelimSpan, flattening: Option) -> Result; } pub(crate) struct OutputKindNone; impl OutputKind for OutputKindNone { type Output = (); - fn value() -> CommandOutputKind { - CommandOutputKind::None - } - - fn to_output_enum(_output: ()) -> CommandOutput { - CommandOutput::Empty + fn resolve(_: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(dots) => dots.err("This command has no output, so cannot be flattened with .."), + None => Ok(CommandOutputKind::None), + } } } @@ -40,15 +40,12 @@ pub(crate) struct OutputKindValue; impl OutputKind for OutputKindValue { type Output = TokenTree; - fn value() -> CommandOutputKind { - CommandOutputKind::Value - } - - fn to_output_enum(output: TokenTree) -> CommandOutput { - match output { - TokenTree::Literal(literal) => CommandOutput::Literal(literal), - TokenTree::Ident(ident) => CommandOutput::Ident(ident), - _ => panic!("Value Output Commands should only output literals or idents"), + fn resolve(_: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(dots) => { + dots.err("This command outputs a single value, so cannot be flattened with ..") + } + None => Ok(CommandOutputKind::Value), } } } @@ -57,51 +54,226 @@ pub(crate) struct OutputKindIdent; impl OutputKind for OutputKindIdent { type Output = Ident; - fn value() -> CommandOutputKind { - CommandOutputKind::Ident + fn resolve(_: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(dots) => { + dots.err("This command outputs a single ident, so cannot be flattened with ..") + } + None => Ok(CommandOutputKind::Ident), + } } +} + +pub(crate) struct OutputKindStreaming; +impl OutputKind for OutputKindStreaming { + type Output = InterpretedStream; - fn to_output_enum(ident: Ident) -> CommandOutput { - CommandOutput::Ident(ident) + fn resolve(span: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(_) => Ok(CommandOutputKind::FlattenedStream), + None => Ok(CommandOutputKind::GroupedStream(span.join())), + } } } -pub(crate) struct OutputKindStreamOrGroup; -impl OutputKind for OutputKindStreamOrGroup { - type Output = InterpretedStream; +pub(crate) trait CommandType { + type OutputKind: OutputKind; +} + +pub(crate) trait NoOutputCommandDefinition: + Sized + CommandType +{ + const COMMAND_NAME: &'static str; + fn parse(arguments: CommandArguments) -> Result; + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()>; +} - fn value() -> CommandOutputKind { - CommandOutputKind::StreamOrGroup +impl CommandInvocationAs for C { + fn execute_into( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + _: &mut InterpretedStream, + ) -> Result<()> { + self.execute(interpreter)?; + Ok(()) } - fn to_output_enum(output: InterpretedStream) -> CommandOutput { - CommandOutput::Stream(output) + fn execute_into_value( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result { + self.execute(interpreter)?; + Ok(CommandOutput::None) } } -pub(crate) trait CommandDefinition: Sized { +pub(crate) trait IdentCommandDefinition: + Sized + CommandType +{ const COMMAND_NAME: &'static str; - type OutputKind: OutputKind; + fn parse(arguments: CommandArguments) -> Result; + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; +} +impl CommandInvocationAs for C { + fn execute_into( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + output.push_ident(self.execute(interpreter)?); + Ok(()) + } + + fn execute_into_value( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result { + Ok(CommandOutput::Ident(self.execute(interpreter)?)) + } +} + +pub(crate) trait ValueCommandDefinition: + Sized + CommandType +{ + const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> Result; + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; +} + +impl CommandInvocationAs for C { + fn execute_into( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + output.push_raw_token_tree(self.execute(interpreter)?); + Ok(()) + } + fn execute_into_value( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result { + Ok(match self.execute(interpreter)? { + TokenTree::Literal(literal) => CommandOutput::Literal(literal), + TokenTree::Ident(ident) => CommandOutput::Ident(ident), + _ => panic!("Value Output Commands should only output literals or idents"), + }) + } +} + +pub(crate) trait StreamingCommandDefinition: + Sized + CommandType +{ + const COMMAND_NAME: &'static str; + fn parse(arguments: CommandArguments) -> Result; fn execute( self: Box, interpreter: &mut Interpreter, - ) -> Result<::Output>; + output: &mut InterpretedStream, + ) -> Result<()>; } -impl CommandInvocation for C { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { - let output = ::execute(self, interpreter)?; - Ok(::to_output_enum(output)) +impl CommandInvocationAs for C { + fn execute_into( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + match output_kind { + CommandOutputKind::FlattenedStream => self.execute(interpreter, output), + CommandOutputKind::GroupedStream(span) => output.push_grouped( + |inner| self.execute(interpreter, inner), + Delimiter::None, + span, + ), + _ => unreachable!(), + } + } + + fn execute_into_value( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result { + let mut output = InterpretedStream::new(SpanRange::ignored()); + self.execute(interpreter, &mut output)?; + Ok(match output_kind { + CommandOutputKind::FlattenedStream => CommandOutput::FlattenedStream(output), + CommandOutputKind::GroupedStream(span) => CommandOutput::Grouped(output, span), + _ => unreachable!(), + }) } } //========================= +// Using the trick for permitting multiple non-overlapping blanket +// implementations, conditioned on an associated type +pub(crate) trait CommandInvocationAs { + fn execute_into( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()>; + + fn execute_into_value( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result; +} + +impl> CommandInvocation for C { + fn execute_into( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + >::execute_into( + self, + output_kind, + interpreter, + output, + ) + } + + fn execute_into_value( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result { + >::execute_into_value( + self, + output_kind, + interpreter, + ) + } +} + pub(crate) trait CommandInvocation { - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; + fn execute_into( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()>; + + fn execute_into_value( + self: Box, + output_kind: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result; } pub(crate) trait ClonableCommandInvocation: CommandInvocation { @@ -139,22 +311,30 @@ macro_rules! define_command_kind { Ok(match self { $( Self::$command => Box::new( - <$command as CommandDefinition>::parse(arguments)? + $command::parse(arguments)? ), )* }) } + pub(crate) fn output_kind(&self, span: &DelimSpan, flattening: Option) -> Result { + match self { + $( + Self::$command => <$command as CommandType>::OutputKind::resolve(span, flattening), + )* + } + } + pub(crate) fn for_ident(ident: &Ident) -> Option { Some(match ident.to_string().as_ref() { $( - <$command as CommandDefinition>::COMMAND_NAME => Self::$command, + $command::COMMAND_NAME => Self::$command, )* _ => return None, }) } - const ALL_KIND_NAMES: &'static [&'static str] = &[$(<$command as CommandDefinition>::COMMAND_NAME,)*]; + const ALL_KIND_NAMES: &'static [&'static str] = &[$($command::COMMAND_NAME,)*]; pub(crate) fn list_all() -> String { // TODO improve to add an "and" at the end @@ -168,6 +348,7 @@ pub(crate) use define_command_kind; #[derive(Clone)] pub(crate) struct Command { invocation: Box, + output_kind: CommandOutputKind, source_group_span: DelimSpan, } @@ -176,9 +357,17 @@ impl Parse for Command { let content; let open_bracket = syn::bracketed!(content in input); content.parse::()?; + let flattening = if content.peek(Token![.]) { + Some(content.parse::()?) + } else { + None + }; let command_name = content.call(Ident::parse_any)?; - let command_kind = match CommandKind::for_ident(&command_name) { - Some(command_kind) => command_kind, + let (command_kind, output_kind) = match CommandKind::for_ident(&command_name) { + Some(command_kind) => { + let output_kind = command_kind.output_kind(&open_bracket.span, flattening)?; + (command_kind, output_kind) + } None => command_name.span().err( format!( "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", @@ -195,11 +384,23 @@ impl Parse for Command { ))?; Ok(Self { invocation, + output_kind, source_group_span: open_bracket.span, }) } } +impl Command { + pub(crate) fn output_kind(&self) -> CommandOutputKind { + self.output_kind + } + + /// Should only be used to swap valid kinds + pub(crate) unsafe fn set_output_kind(&mut self, output_kind: CommandOutputKind) { + self.output_kind = output_kind; + } +} + impl HasSpanRange for Command { fn span_range(&self) -> SpanRange { self.source_group_span.span_range() @@ -212,19 +413,8 @@ impl Interpret for Command { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - match self.invocation.execute(interpreter)? { - CommandOutput::Empty => {} - CommandOutput::Literal(literal) => { - output.push_literal(literal); - } - CommandOutput::Ident(ident) => { - output.push_ident(ident); - } - CommandOutput::Stream(stream) => { - output.extend(stream); - } - }; - Ok(()) + self.invocation + .execute_into(self.output_kind, interpreter, output) } } @@ -234,15 +424,21 @@ impl Express for Command { interpreter: &mut Interpreter, expression_stream: &mut ExpressionBuilder, ) -> Result<()> { - match self.invocation.execute(interpreter)? { - CommandOutput::Empty => {} + match self + .invocation + .execute_into_value(self.output_kind, interpreter)? + { + CommandOutput::None => {} CommandOutput::Literal(literal) => { expression_stream.push_literal(literal); } CommandOutput::Ident(ident) => { expression_stream.push_ident(ident); } - CommandOutput::Stream(stream) => { + CommandOutput::Grouped(stream, span) => { + expression_stream.push_grouped_interpreted_stream(stream, span); + } + CommandOutput::FlattenedStream(stream) => { expression_stream .push_grouped_interpreted_stream(stream, self.source_group_span.join()); } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index ca562973..9ceef7df 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -55,8 +55,32 @@ impl Interpret for CommandStreamInput { output: &mut InterpretedStream, ) -> Result<()> { match self { - CommandStreamInput::Command(command) => { - command.interpret_as_tokens_into(interpreter, output) + CommandStreamInput::Command(mut command) => { + match command.output_kind() { + CommandOutputKind::None + | CommandOutputKind::Value + | CommandOutputKind::Ident => { + command.err("The command does not output a stream") + } + CommandOutputKind::FlattenedStream => { + let span = command.span(); + let tokens = parse_as_stream_input( + command.interpret_as_tokens(interpreter)?, + || { + span.error("Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.") + }, + )?; + output.extend_raw_tokens(tokens); + Ok(()) + } + CommandOutputKind::GroupedStream(_) => { + unsafe { + // SAFETY: The kind change GroupedStream <=> FlattenedStream is valid + command.set_output_kind(CommandOutputKind::FlattenedStream); + } + command.interpret_as_tokens_into(interpreter, output) + } + } } CommandStreamInput::FlattenedVariable(variable) => { let tokens = parse_as_stream_input( diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index be1bacfa..11239417 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -81,6 +81,18 @@ impl InterpretedStream { self.push_raw_token_tree(punct.into()); } + pub(crate) fn push_grouped( + &mut self, + appender: impl FnOnce(&mut Self) -> Result<()>, + delimiter: Delimiter, + span: Span, + ) -> Result<()> { + let mut inner = Self::new(span.span_range()); + appender(&mut inner)?; + self.push_new_group(inner, delimiter, span); + Ok(()) + } + pub(crate) fn push_new_group( &mut self, inner_tokens: InterpretedStream, diff --git a/src/traits.rs b/src/traits.rs index 69a3662f..caa05df8 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -305,4 +305,5 @@ impl_auto_span_range! { syn::UnOp, syn::Type, syn::TypePath, + syn::token::DotDot, } diff --git a/tests/tokens.rs b/tests/tokens.rs index e2dd798b..87a2de71 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -220,7 +220,7 @@ fn complex_cases_for_intersperse_and_input_types() { }, "0_1_2_3"); // Grouped variable containing flattened command can be used for items assert_preinterpret_eq!({ - [!set! #items = [!range! 0..4]] + [!set! #items = [!..range! 0..4]] [!string! [!intersperse! { items: #items, separator: [_], From 55496db7f710b430eaa9b6420ff3b09b1b105945 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 19 Jan 2025 00:16:16 +0000 Subject: [PATCH 037/126] tweak: Handle control flow output as always flattened --- CHANGELOG.md | 5 +- src/commands/control_flow_commands.rs | 8 +- src/commands/core_commands.rs | 8 +- src/commands/expression_commands.rs | 4 +- src/commands/token_commands.rs | 8 +- src/interpretation/command.rs | 213 ++++++++++++++------- src/interpretation/command_stream_input.rs | 11 ++ 7 files changed, 168 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fae6dbe1..0516ea6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Consider [!..flattened! ] and getting rid of !group! - => Add separate ControlFlowCode type - => Make the type of commands static +* Add !interspersed! above +* Consider [!..flattened! ] etc => Disallow most command types in expressions => Get rid of [!empty!] and [!stream!] and replace with [!..group!] or [!group!] * Explore getting rid of lots of the span range stuff diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index fbb99019..00efe736 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -8,10 +8,10 @@ pub(crate) struct IfCommand { } impl CommandType for IfCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindControlFlow; } -impl StreamingCommandDefinition for IfCommand { +impl ControlFlowCommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; fn parse(arguments: CommandArguments) -> Result { @@ -65,10 +65,10 @@ pub(crate) struct WhileCommand { } impl CommandType for WhileCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindControlFlow; } -impl StreamingCommandDefinition for WhileCommand { +impl ControlFlowCommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; fn parse(arguments: CommandArguments) -> Result { diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 4487a867..022ec2c4 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -89,10 +89,10 @@ pub(crate) struct RawCommand { } impl CommandType for RawCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for RawCommand { +impl StreamCommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; fn parse(arguments: CommandArguments) -> Result { @@ -139,10 +139,10 @@ pub(crate) struct StreamCommand { } impl CommandType for StreamCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for StreamCommand { +impl StreamCommandDefinition for StreamCommand { const COMMAND_NAME: &'static str = "stream"; fn parse(arguments: CommandArguments) -> Result { diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 5e4c9577..35c086d1 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -96,10 +96,10 @@ pub(crate) struct RangeCommand { } impl CommandType for RangeCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for RangeCommand { +impl StreamCommandDefinition for RangeCommand { const COMMAND_NAME: &'static str = "range"; fn parse(arguments: CommandArguments) -> Result { diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index c7cb96e9..e2a5f885 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -80,10 +80,10 @@ pub(crate) struct GroupCommand { } impl CommandType for GroupCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for GroupCommand { +impl StreamCommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; fn parse(arguments: CommandArguments) -> Result { @@ -112,7 +112,7 @@ pub(crate) struct IntersperseCommand { } impl CommandType for IntersperseCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } define_field_inputs! { @@ -128,7 +128,7 @@ define_field_inputs! { } } -impl StreamingCommandDefinition for IntersperseCommand { +impl StreamCommandDefinition for IntersperseCommand { const COMMAND_NAME: &'static str = "intersperse"; fn parse(arguments: CommandArguments) -> Result { diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index bd1f0709..83d69600 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -5,7 +5,8 @@ pub(crate) enum CommandOutput { Literal(Literal), Ident(Ident), FlattenedStream(InterpretedStream), - Grouped(InterpretedStream, Span), + GroupedStream(InterpretedStream, Span), + ControlFlowCodeStream(InterpretedStream), } #[allow(unused)] @@ -17,6 +18,11 @@ pub(crate) enum CommandOutputKind { Ident, FlattenedStream, GroupedStream(Span), + ControlFlowFlattenedStream, +} + +pub(crate) trait CommandType { + type OutputKind: OutputKind; } pub(crate) trait OutputKind { @@ -24,6 +30,10 @@ pub(crate) trait OutputKind { fn resolve(span: &DelimSpan, flattening: Option) -> Result; } +//=============== +// OutputKindNone +//=============== + pub(crate) struct OutputKindNone; impl OutputKind for OutputKindNone { type Output = (); @@ -36,50 +46,6 @@ impl OutputKind for OutputKindNone { } } -pub(crate) struct OutputKindValue; -impl OutputKind for OutputKindValue { - type Output = TokenTree; - - fn resolve(_: &DelimSpan, flattening: Option) -> Result { - match flattening { - Some(dots) => { - dots.err("This command outputs a single value, so cannot be flattened with ..") - } - None => Ok(CommandOutputKind::Value), - } - } -} - -pub(crate) struct OutputKindIdent; -impl OutputKind for OutputKindIdent { - type Output = Ident; - - fn resolve(_: &DelimSpan, flattening: Option) -> Result { - match flattening { - Some(dots) => { - dots.err("This command outputs a single ident, so cannot be flattened with ..") - } - None => Ok(CommandOutputKind::Ident), - } - } -} - -pub(crate) struct OutputKindStreaming; -impl OutputKind for OutputKindStreaming { - type Output = InterpretedStream; - - fn resolve(span: &DelimSpan, flattening: Option) -> Result { - match flattening { - Some(_) => Ok(CommandOutputKind::FlattenedStream), - None => Ok(CommandOutputKind::GroupedStream(span.join())), - } - } -} - -pub(crate) trait CommandType { - type OutputKind: OutputKind; -} - pub(crate) trait NoOutputCommandDefinition: Sized + CommandType { @@ -109,22 +75,40 @@ impl CommandInvocationAs for C { } } -pub(crate) trait IdentCommandDefinition: - Sized + CommandType +//================ +// OutputKindValue +//================ + +pub(crate) struct OutputKindValue; +impl OutputKind for OutputKindValue { + type Output = TokenTree; + + fn resolve(_: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(dots) => { + dots.err("This command outputs a single value, so cannot be flattened with ..") + } + None => Ok(CommandOutputKind::Value), + } + } +} + +pub(crate) trait ValueCommandDefinition: + Sized + CommandType { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> Result; - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self: Box, _: CommandOutputKind, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.push_ident(self.execute(interpreter)?); + output.push_raw_token_tree(self.execute(interpreter)?); Ok(()) } @@ -133,26 +117,48 @@ impl CommandInvocationAs for C { _: CommandOutputKind, interpreter: &mut Interpreter, ) -> Result { - Ok(CommandOutput::Ident(self.execute(interpreter)?)) + Ok(match self.execute(interpreter)? { + TokenTree::Literal(literal) => CommandOutput::Literal(literal), + TokenTree::Ident(ident) => CommandOutput::Ident(ident), + _ => panic!("Value Output Commands should only output literals or idents"), + }) } } -pub(crate) trait ValueCommandDefinition: - Sized + CommandType +//================ +// OutputKindIdent +//================ + +pub(crate) struct OutputKindIdent; +impl OutputKind for OutputKindIdent { + type Output = Ident; + + fn resolve(_: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(dots) => { + dots.err("This command outputs a single ident, so cannot be flattened with ..") + } + None => Ok(CommandOutputKind::Ident), + } + } +} + +pub(crate) trait IdentCommandDefinition: + Sized + CommandType { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> Result; - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; + fn execute(self: Box, interpreter: &mut Interpreter) -> Result; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self: Box, _: CommandOutputKind, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.push_raw_token_tree(self.execute(interpreter)?); + output.push_ident(self.execute(interpreter)?); Ok(()) } @@ -161,16 +167,28 @@ impl CommandInvocationAs for C { _: CommandOutputKind, interpreter: &mut Interpreter, ) -> Result { - Ok(match self.execute(interpreter)? { - TokenTree::Literal(literal) => CommandOutput::Literal(literal), - TokenTree::Ident(ident) => CommandOutput::Ident(ident), - _ => panic!("Value Output Commands should only output literals or idents"), - }) + Ok(CommandOutput::Ident(self.execute(interpreter)?)) } } -pub(crate) trait StreamingCommandDefinition: - Sized + CommandType +//================= +// OutputKindStream +//================= + +pub(crate) struct OutputKindStream; +impl OutputKind for OutputKindStream { + type Output = InterpretedStream; + + fn resolve(span: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(_) => Ok(CommandOutputKind::FlattenedStream), + None => Ok(CommandOutputKind::GroupedStream(span.join())), + } + } +} + +pub(crate) trait StreamCommandDefinition: + Sized + CommandType { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> Result; @@ -181,7 +199,7 @@ pub(crate) trait StreamingCommandDefinition: ) -> Result<()>; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self: Box, output_kind: CommandOutputKind, @@ -208,12 +226,61 @@ impl CommandInvocationAs for self.execute(interpreter, &mut output)?; Ok(match output_kind { CommandOutputKind::FlattenedStream => CommandOutput::FlattenedStream(output), - CommandOutputKind::GroupedStream(span) => CommandOutput::Grouped(output, span), + CommandOutputKind::GroupedStream(span) => CommandOutput::GroupedStream(output, span), _ => unreachable!(), }) } } +//====================== +// OutputKindControlFlow +//====================== + +pub(crate) struct OutputKindControlFlow; +impl OutputKind for OutputKindControlFlow { + type Output = (); + + fn resolve(_: &DelimSpan, flattening: Option) -> Result { + match flattening { + Some(dots) => dots.err("This command is control flow, so is always flattened and cannot be flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), + None => Ok(CommandOutputKind::ControlFlowFlattenedStream), + } + } +} + +pub(crate) trait ControlFlowCommandDefinition: + Sized + CommandType +{ + const COMMAND_NAME: &'static str; + fn parse(arguments: CommandArguments) -> Result; + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()>; +} + +impl CommandInvocationAs for C { + fn execute_into( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + self.execute(interpreter, output) + } + + fn execute_into_value( + self: Box, + _: CommandOutputKind, + interpreter: &mut Interpreter, + ) -> Result { + let mut output = InterpretedStream::new(SpanRange::ignored()); + self.execute(interpreter, &mut output)?; + Ok(CommandOutput::ControlFlowCodeStream(output)) + } +} + //========================= // Using the trick for permitting multiple non-overlapping blanket @@ -422,7 +489,7 @@ impl Express for Command { fn add_to_expression( self, interpreter: &mut Interpreter, - expression_stream: &mut ExpressionBuilder, + builder: &mut ExpressionBuilder, ) -> Result<()> { match self .invocation @@ -430,17 +497,19 @@ impl Express for Command { { CommandOutput::None => {} CommandOutput::Literal(literal) => { - expression_stream.push_literal(literal); + builder.push_literal(literal); } CommandOutput::Ident(ident) => { - expression_stream.push_ident(ident); + builder.push_ident(ident); } - CommandOutput::Grouped(stream, span) => { - expression_stream.push_grouped_interpreted_stream(stream, span); + CommandOutput::GroupedStream(stream, span) => { + builder.push_grouped_interpreted_stream(stream, span); } CommandOutput::FlattenedStream(stream) => { - expression_stream - .push_grouped_interpreted_stream(stream, self.source_group_span.join()); + builder.push_grouped_interpreted_stream(stream, self.source_group_span.join()); + } + CommandOutput::ControlFlowCodeStream(stream) => { + builder.push_grouped_interpreted_stream(stream, self.source_group_span.join()); } }; Ok(()) diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 9ceef7df..4eed5146 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -80,6 +80,17 @@ impl Interpret for CommandStreamInput { } command.interpret_as_tokens_into(interpreter, output) } + CommandOutputKind::ControlFlowFlattenedStream => { + let span = command.span(); + let tokens = parse_as_stream_input( + command.interpret_as_tokens(interpreter)?, + || { + span.error("Expected output of control flow command to contain a single [ ... ] or transparent group.") + }, + )?; + output.extend_raw_tokens(tokens); + Ok(()) + } } } CommandStreamInput::FlattenedVariable(variable) => { From e1fbcd7325e09ceeca63f8df5061250cb2ff4e48 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 19 Jan 2025 18:12:04 +0000 Subject: [PATCH 038/126] refactor: Finished reworking commands in expressions --- CHANGELOG.md | 11 +- src/commands/expression_commands.rs | 4 +- src/commands/mod.rs | 60 +-- src/expressions/expression_stream.rs | 13 +- src/interpretation/command.rs | 384 ++++++++++-------- src/interpretation/command_stream_input.rs | 10 +- src/interpretation/interpretation_stream.rs | 10 +- src/interpretation/variable.rs | 46 +-- ...ix_me_comparison_operators_are_not_lazy.rs | 2 +- ...e_comparison_operators_are_not_lazy.stderr | 2 +- .../flattened_commands_in_expressions.rs | 7 + .../flattened_commands_in_expressions.stderr | 6 + .../flattened_variables_in_expressions.rs | 8 + .../flattened_variables_in_expressions.stderr | 6 + .../no_output_commands_in_expressions.rs | 7 + .../no_output_commands_in_expressions.stderr | 6 + tests/expressions.rs | 10 +- 17 files changed, 310 insertions(+), 282 deletions(-) create mode 100644 tests/compilation_failures/expressions/flattened_commands_in_expressions.rs create mode 100644 tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr create mode 100644 tests/compilation_failures/expressions/flattened_variables_in_expressions.rs create mode 100644 tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr create mode 100644 tests/compilation_failures/expressions/no_output_commands_in_expressions.rs create mode 100644 tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 0516ea6d..dd4f299f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,14 +25,14 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Add !interspersed! above * Consider [!..flattened! ] etc - => Disallow most command types in expressions => Get rid of [!empty!] and [!stream!] and replace with [!..group!] or [!group!] * Explore getting rid of lots of the span range stuff +* Add [!break!] and [!continue!] commands using a flag in the interpreter * Support `!else if!` in `!if!` * Support string & char literals (for comparisons & casts) in expressions * Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` +* See comment on `!assign!` * Other token stream commands * Add tests for CommandStreamInput (via `[!split! ]` or `[!intersperse! ]`?): => From `#x = Hello World` @@ -81,7 +81,6 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!split! { items: X, with: X, drop_trailing_empty?: true }]` * `[!intersperse! { items: X, with: X, add_trailing?: false, override_final_with?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` -* See comment on `!assign!` * Basic place parsing * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * In parse land: #x matches a single token, #..x consumes the rest of a stream @@ -99,7 +98,11 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` * `[!match!]` (with `#..x` as a catch-all) * Check all `#[allow(unused)]` and remove any which aren't needed -* Rework expression parsing +* Rework expression parsing, in order to: + * Fix comments in the expression files + * Enable lazy && and || + * Enable support for code blocks { .. } in expressions, + and remove hacks where expression parsing stops at {} or . * Work on book * Input paradigms: * Streams diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 35c086d1..29040928 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -77,9 +77,7 @@ impl NoOutputCommandDefinition for AssignCommand { let mut builder = ExpressionBuilder::new(); variable.add_to_expression(interpreter, &mut builder)?; builder.push_punct(operator); - builder.extend_with_interpreted_stream( - expression.evaluate(interpreter)?.into_interpreted_stream(), - ); + builder.extend_with_evaluation_output(expression.evaluate(interpreter)?); let output = builder.evaluate()?.into_interpreted_stream(); variable.set(interpreter, output)?; diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 2c3abcee..cc29c102 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -4,58 +4,8 @@ mod core_commands; mod expression_commands; mod token_commands; -use crate::internal_prelude::*; -use concat_commands::*; -use control_flow_commands::*; -use core_commands::*; -use expression_commands::*; -use token_commands::*; - -define_command_kind! { - // Core Commands - SetCommand, - ExtendCommand, - RawCommand, - IgnoreCommand, - StreamCommand, - ErrorCommand, - - // Concat & Type Convert Commands - StringCommand, - IdentCommand, - IdentCamelCommand, - IdentSnakeCommand, - IdentUpperSnakeCommand, - LiteralCommand, - - // Concat & String Convert Commands - UpperCommand, - LowerCommand, - SnakeCommand, - LowerSnakeCommand, - UpperSnakeCommand, - CamelCommand, - LowerCamelCommand, - UpperCamelCommand, - KebabCommand, - CapitalizeCommand, - DecapitalizeCommand, - TitleCommand, - InsertSpacesCommand, - - // Expression Commands - EvaluateCommand, - AssignCommand, - RangeCommand, - - // Control flow commands - IfCommand, - WhileCommand, - - // Token Commands - EmptyCommand, - IsEmptyCommand, - LengthCommand, - GroupCommand, - IntersperseCommand, -} +pub(crate) use concat_commands::*; +pub(crate) use control_flow_commands::*; +pub(crate) use core_commands::*; +pub(crate) use expression_commands::*; +pub(crate) use token_commands::*; diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index dd86d794..c233c544 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -215,21 +215,22 @@ impl ExpressionBuilder { self.interpreted_stream.push_punct(punct); } - pub(crate) fn push_grouped_interpreted_stream( + pub(crate) fn push_grouped( &mut self, - contents: InterpretedStream, + appender: impl FnOnce(&mut InterpretedStream) -> Result<()>, span: Span, - ) { + ) -> Result<()> { // Currently using Expr::Parse, it ignores transparent groups, which is // a little too permissive. // Instead, we use parentheses to ensure that the group has to be a valid // expression itself, without being flattened self.interpreted_stream - .push_new_group(contents, Delimiter::Parenthesis, span); + .push_grouped(appender, Delimiter::Parenthesis, span) } - pub(crate) fn extend_with_interpreted_stream(&mut self, contents: InterpretedStream) { - self.interpreted_stream.extend(contents); + pub(crate) fn extend_with_evaluation_output(&mut self, value: EvaluationOutput) { + self.interpreted_stream + .extend_raw_tokens(value.into_token_tree()); } pub(crate) fn push_expression_group( diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 83d69600..e14468d2 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,14 +1,5 @@ use crate::internal_prelude::*; -pub(crate) enum CommandOutput { - None, - Literal(Literal), - Ident(Ident), - FlattenedStream(InterpretedStream), - GroupedStream(InterpretedStream, Span), - ControlFlowCodeStream(InterpretedStream), -} - #[allow(unused)] #[derive(Clone, Copy)] pub(crate) enum CommandOutputKind { @@ -17,8 +8,8 @@ pub(crate) enum CommandOutputKind { Value, Ident, FlattenedStream, - GroupedStream(Span), - ControlFlowFlattenedStream, + GroupedStream, + ControlFlowCodeStream, } pub(crate) trait CommandType { @@ -27,7 +18,79 @@ pub(crate) trait CommandType { pub(crate) trait OutputKind { type Output; - fn resolve(span: &DelimSpan, flattening: Option) -> Result; + fn resolve(flattening: Option) -> Result; +} + +struct ExecutionContext<'a> { + interpreter: &'a mut Interpreter, + output_kind: CommandOutputKind, + delim_span: DelimSpan, +} + +trait CommandInvocation { + fn execute_into( + self: Box, + context: ExecutionContext, + output: &mut InterpretedStream, + ) -> Result<()>; + + fn execute_into_expression( + self: Box, + context: ExecutionContext, + builder: &mut ExpressionBuilder, + ) -> Result<()>; +} + +trait ClonableCommandInvocation: CommandInvocation { + fn clone_box(&self) -> Box; +} + +impl ClonableCommandInvocation for C { + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +// Using the trick for permitting multiple non-overlapping blanket +// implementations, conditioned on an associated type +trait CommandInvocationAs { + fn execute_into( + self: Box, + context: ExecutionContext, + output: &mut InterpretedStream, + ) -> Result<()>; + + fn execute_into_expression( + self: Box, + context: ExecutionContext, + builder: &mut ExpressionBuilder, + ) -> Result<()>; +} + +impl> CommandInvocation for C { + fn execute_into( + self: Box, + context: ExecutionContext, + output: &mut InterpretedStream, + ) -> Result<()> { + >::execute_into(self, context, output) + } + + fn execute_into_expression( + self: Box, + context: ExecutionContext, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + >::execute_into_expression( + self, context, builder, + ) + } } //=============== @@ -38,7 +101,7 @@ pub(crate) struct OutputKindNone; impl OutputKind for OutputKindNone { type Output = (); - fn resolve(_: &DelimSpan, flattening: Option) -> Result { + fn resolve(flattening: Option) -> Result { match flattening { Some(dots) => dots.err("This command has no output, so cannot be flattened with .."), None => Ok(CommandOutputKind::None), @@ -57,21 +120,21 @@ pub(crate) trait NoOutputCommandDefinition: impl CommandInvocationAs for C { fn execute_into( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, + context: ExecutionContext, _: &mut InterpretedStream, ) -> Result<()> { - self.execute(interpreter)?; + self.execute(context.interpreter)?; Ok(()) } - fn execute_into_value( + fn execute_into_expression( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result { - self.execute(interpreter)?; - Ok(CommandOutput::None) + context: ExecutionContext, + _: &mut ExpressionBuilder, + ) -> Result<()> { + context.delim_span + .join() + .err("Commands with no output cannot be used directly in expressions.\nConsider wrapping it inside a command such as [!group! ..] which returns an expression") } } @@ -83,7 +146,7 @@ pub(crate) struct OutputKindValue; impl OutputKind for OutputKindValue { type Output = TokenTree; - fn resolve(_: &DelimSpan, flattening: Option) -> Result { + fn resolve(flattening: Option) -> Result { match flattening { Some(dots) => { dots.err("This command outputs a single value, so cannot be flattened with ..") @@ -104,24 +167,24 @@ pub(crate) trait ValueCommandDefinition: impl CommandInvocationAs for C { fn execute_into( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, + context: ExecutionContext, output: &mut InterpretedStream, ) -> Result<()> { - output.push_raw_token_tree(self.execute(interpreter)?); + output.push_raw_token_tree(self.execute(context.interpreter)?); Ok(()) } - fn execute_into_value( + fn execute_into_expression( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result { - Ok(match self.execute(interpreter)? { - TokenTree::Literal(literal) => CommandOutput::Literal(literal), - TokenTree::Ident(ident) => CommandOutput::Ident(ident), + context: ExecutionContext, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + match self.execute(context.interpreter)? { + TokenTree::Literal(literal) => builder.push_literal(literal), + TokenTree::Ident(ident) => builder.push_ident(ident), _ => panic!("Value Output Commands should only output literals or idents"), - }) + } + Ok(()) } } @@ -133,7 +196,7 @@ pub(crate) struct OutputKindIdent; impl OutputKind for OutputKindIdent { type Output = Ident; - fn resolve(_: &DelimSpan, flattening: Option) -> Result { + fn resolve(flattening: Option) -> Result { match flattening { Some(dots) => { dots.err("This command outputs a single ident, so cannot be flattened with ..") @@ -154,20 +217,20 @@ pub(crate) trait IdentCommandDefinition: impl CommandInvocationAs for C { fn execute_into( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, + context: ExecutionContext, output: &mut InterpretedStream, ) -> Result<()> { - output.push_ident(self.execute(interpreter)?); + output.push_ident(self.execute(context.interpreter)?); Ok(()) } - fn execute_into_value( + fn execute_into_expression( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result { - Ok(CommandOutput::Ident(self.execute(interpreter)?)) + context: ExecutionContext, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + builder.push_ident(self.execute(context.interpreter)?); + Ok(()) } } @@ -179,10 +242,10 @@ pub(crate) struct OutputKindStream; impl OutputKind for OutputKindStream { type Output = InterpretedStream; - fn resolve(span: &DelimSpan, flattening: Option) -> Result { + fn resolve(flattening: Option) -> Result { match flattening { Some(_) => Ok(CommandOutputKind::FlattenedStream), - None => Ok(CommandOutputKind::GroupedStream(span.join())), + None => Ok(CommandOutputKind::GroupedStream), } } } @@ -202,33 +265,34 @@ pub(crate) trait StreamCommandDefinition: impl CommandInvocationAs for C { fn execute_into( self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, + context: ExecutionContext, output: &mut InterpretedStream, ) -> Result<()> { - match output_kind { - CommandOutputKind::FlattenedStream => self.execute(interpreter, output), - CommandOutputKind::GroupedStream(span) => output.push_grouped( - |inner| self.execute(interpreter, inner), + match context.output_kind { + CommandOutputKind::FlattenedStream => self.execute(context.interpreter, output), + CommandOutputKind::GroupedStream => output.push_grouped( + |inner| self.execute(context.interpreter, inner), Delimiter::None, - span, + context.delim_span.join(), ), _ => unreachable!(), } } - fn execute_into_value( + fn execute_into_expression( self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result { - let mut output = InterpretedStream::new(SpanRange::ignored()); - self.execute(interpreter, &mut output)?; - Ok(match output_kind { - CommandOutputKind::FlattenedStream => CommandOutput::FlattenedStream(output), - CommandOutputKind::GroupedStream(span) => CommandOutput::GroupedStream(output, span), - _ => unreachable!(), - }) + context: ExecutionContext, + builder: &mut ExpressionBuilder, + ) -> Result<()> { + if let CommandOutputKind::FlattenedStream = context.output_kind { + return context.delim_span + .join() + .err("Flattened commands cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"); + } + builder.push_grouped( + |output| self.execute(context.interpreter, output), + context.delim_span.join(), + ) } } @@ -240,10 +304,10 @@ pub(crate) struct OutputKindControlFlow; impl OutputKind for OutputKindControlFlow { type Output = (); - fn resolve(_: &DelimSpan, flattening: Option) -> Result { + fn resolve(flattening: Option) -> Result { match flattening { Some(dots) => dots.err("This command is control flow, so is always flattened and cannot be flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), - None => Ok(CommandOutputKind::ControlFlowFlattenedStream), + None => Ok(CommandOutputKind::ControlFlowCodeStream), } } } @@ -263,101 +327,25 @@ pub(crate) trait ControlFlowCommandDefinition: impl CommandInvocationAs for C { fn execute_into( self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, + context: ExecutionContext, output: &mut InterpretedStream, ) -> Result<()> { - self.execute(interpreter, output) - } - - fn execute_into_value( - self: Box, - _: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result { - let mut output = InterpretedStream::new(SpanRange::ignored()); - self.execute(interpreter, &mut output)?; - Ok(CommandOutput::ControlFlowCodeStream(output)) + self.execute(context.interpreter, output) } -} - -//========================= - -// Using the trick for permitting multiple non-overlapping blanket -// implementations, conditioned on an associated type -pub(crate) trait CommandInvocationAs { - fn execute_into( - self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()>; - - fn execute_into_value( - self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result; -} -impl> CommandInvocation for C { - fn execute_into( + fn execute_into_expression( self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, + context: ExecutionContext, + builder: &mut ExpressionBuilder, ) -> Result<()> { - >::execute_into( - self, - output_kind, - interpreter, - output, - ) - } - - fn execute_into_value( - self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result { - >::execute_into_value( - self, - output_kind, - interpreter, + builder.push_grouped( + |output| self.execute(context.interpreter, output), + context.delim_span.join(), ) } } -pub(crate) trait CommandInvocation { - fn execute_into( - self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()>; - - fn execute_into_value( - self: Box, - output_kind: CommandOutputKind, - interpreter: &mut Interpreter, - ) -> Result; -} - -pub(crate) trait ClonableCommandInvocation: CommandInvocation { - fn clone_box(&self) -> Box; -} - -impl ClonableCommandInvocation for C { - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } -} - -impl Clone for Box { - fn clone(&self) -> Self { - self.clone_box() - } -} +//========================= macro_rules! define_command_kind { ( @@ -374,7 +362,7 @@ macro_rules! define_command_kind { } impl CommandKind { - pub(crate) fn parse_invocation(&self, arguments: CommandArguments) -> Result> { + fn parse_invocation(&self, arguments: CommandArguments) -> Result> { Ok(match self { $( Self::$command => Box::new( @@ -384,10 +372,10 @@ macro_rules! define_command_kind { }) } - pub(crate) fn output_kind(&self, span: &DelimSpan, flattening: Option) -> Result { + pub(crate) fn output_kind(&self, flattening: Option) -> Result { match self { $( - Self::$command => <$command as CommandType>::OutputKind::resolve(span, flattening), + Self::$command => <$command as CommandType>::OutputKind::resolve(flattening), )* } } @@ -410,7 +398,55 @@ macro_rules! define_command_kind { } }; } -pub(crate) use define_command_kind; + +define_command_kind! { + // Core Commands + SetCommand, + ExtendCommand, + RawCommand, + IgnoreCommand, + StreamCommand, + ErrorCommand, + + // Concat & Type Convert Commands + StringCommand, + IdentCommand, + IdentCamelCommand, + IdentSnakeCommand, + IdentUpperSnakeCommand, + LiteralCommand, + + // Concat & String Convert Commands + UpperCommand, + LowerCommand, + SnakeCommand, + LowerSnakeCommand, + UpperSnakeCommand, + CamelCommand, + LowerCamelCommand, + UpperCamelCommand, + KebabCommand, + CapitalizeCommand, + DecapitalizeCommand, + TitleCommand, + InsertSpacesCommand, + + // Expression Commands + EvaluateCommand, + AssignCommand, + RangeCommand, + + // Control flow commands + IfCommand, + WhileCommand, + + // Token Commands + EmptyCommand, + IsEmptyCommand, + LengthCommand, + GroupCommand, + IntersperseCommand, +} #[derive(Clone)] pub(crate) struct Command { @@ -432,7 +468,7 @@ impl Parse for Command { let command_name = content.call(Ident::parse_any)?; let (command_kind, output_kind) = match CommandKind::for_ident(&command_name) { Some(command_kind) => { - let output_kind = command_kind.output_kind(&open_bracket.span, flattening)?; + let output_kind = command_kind.output_kind(flattening)?; (command_kind, output_kind) } None => command_name.span().err( @@ -480,8 +516,12 @@ impl Interpret for Command { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.invocation - .execute_into(self.output_kind, interpreter, output) + let context = ExecutionContext { + interpreter, + output_kind: self.output_kind, + delim_span: self.source_group_span, + }; + self.invocation.execute_into(context, output) } } @@ -491,27 +531,15 @@ impl Express for Command { interpreter: &mut Interpreter, builder: &mut ExpressionBuilder, ) -> Result<()> { - match self - .invocation - .execute_into_value(self.output_kind, interpreter)? - { - CommandOutput::None => {} - CommandOutput::Literal(literal) => { - builder.push_literal(literal); - } - CommandOutput::Ident(ident) => { - builder.push_ident(ident); - } - CommandOutput::GroupedStream(stream, span) => { - builder.push_grouped_interpreted_stream(stream, span); - } - CommandOutput::FlattenedStream(stream) => { - builder.push_grouped_interpreted_stream(stream, self.source_group_span.join()); - } - CommandOutput::ControlFlowCodeStream(stream) => { - builder.push_grouped_interpreted_stream(stream, self.source_group_span.join()); - } + let context = ExecutionContext { + interpreter, + output_kind: self.output_kind, + delim_span: self.source_group_span, }; - Ok(()) + // This is set up so that we can determine the exact expression + // structure at parse time, and in future refactor to parsing + // the expression ourselves, and then executing an expression AST + // rather than going via a syn::expression + self.invocation.execute_into_expression(context, builder) } } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 4eed5146..ce827ad0 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -73,14 +73,14 @@ impl Interpret for CommandStreamInput { output.extend_raw_tokens(tokens); Ok(()) } - CommandOutputKind::GroupedStream(_) => { + CommandOutputKind::GroupedStream => { unsafe { // SAFETY: The kind change GroupedStream <=> FlattenedStream is valid command.set_output_kind(CommandOutputKind::FlattenedStream); } command.interpret_as_tokens_into(interpreter, output) } - CommandOutputKind::ControlFlowFlattenedStream => { + CommandOutputKind::ControlFlowCodeStream => { let span = command.span(); let tokens = parse_as_stream_input( command.interpret_as_tokens(interpreter)?, @@ -95,7 +95,7 @@ impl Interpret for CommandStreamInput { } CommandStreamInput::FlattenedVariable(variable) => { let tokens = parse_as_stream_input( - variable.interpret_as_new_stream(interpreter)?, + variable.interpret_as_tokens(interpreter)?, || { variable.error(format!( "Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", @@ -107,8 +107,8 @@ impl Interpret for CommandStreamInput { Ok(()) } CommandStreamInput::GroupedVariable(variable) => { - let ungrouped_variable_contents = variable.interpret_as_new_stream(interpreter)?; - output.extend(ungrouped_variable_contents); + let group_contents = variable.interpret_ungrouped_contents(interpreter)?; + output.extend(group_contents); Ok(()) } CommandStreamInput::Code(code) => { diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index dabf517a..776344cf 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -140,11 +140,13 @@ impl Express for RawGroup { _: &mut Interpreter, expression_stream: &mut ExpressionBuilder, ) -> Result<()> { - expression_stream.push_grouped_interpreted_stream( - InterpretedStream::raw(self.source_delim_span.span_range(), self.content), + expression_stream.push_grouped( + |inner| { + inner.extend_raw_token_iter(self.content); + Ok(()) + }, self.source_delim_span.join(), - ); - Ok(()) + ) } } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 4a47a700..7e68183b 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -43,7 +43,7 @@ impl GroupedVariable { .ok_or_else(|| self.error(format!("The variable {} wasn't already set", self))) } - pub(super) fn interpret_as_new_stream( + pub(super) fn interpret_ungrouped_contents( &self, interpreter: &Interpreter, ) -> Result { @@ -52,13 +52,22 @@ impl GroupedVariable { Ok(cloned) } - pub(crate) fn substitute_into( + pub(crate) fn substitute_contents_into( + &self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + self.read_existing(interpreter)?.append_cloned_into(output); + Ok(()) + } + + pub(crate) fn substitute_grouped_into( &self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { output.push_new_group( - self.interpret_as_new_stream(interpreter)?, + self.interpret_ungrouped_contents(interpreter)?, Delimiter::None, self.span(), ); @@ -87,7 +96,7 @@ impl Interpret for &GroupedVariable { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.substitute_into(interpreter, output) + self.substitute_grouped_into(interpreter, output) } } @@ -97,11 +106,10 @@ impl Express for &GroupedVariable { interpreter: &mut Interpreter, expression_stream: &mut ExpressionBuilder, ) -> Result<()> { - expression_stream.push_grouped_interpreted_stream( - self.interpret_as_new_stream(interpreter)?, + expression_stream.push_grouped( + |inner| self.substitute_contents_into(interpreter, inner), self.span(), - ); - Ok(()) + ) } } @@ -147,15 +155,6 @@ impl Parse for FlattenedVariable { } impl FlattenedVariable { - pub(super) fn interpret_as_new_stream( - &self, - interpreter: &Interpreter, - ) -> Result { - let mut cloned = self.read_existing(interpreter)?.clone(); - cloned.set_span_range(self.span_range()); - Ok(cloned) - } - pub(crate) fn substitute_into( &self, interpreter: &mut Interpreter, @@ -197,13 +196,12 @@ impl Interpret for &FlattenedVariable { } impl Express for &FlattenedVariable { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionBuilder, - ) -> Result<()> { - expression_stream.extend_with_interpreted_stream(self.interpret_as_tokens(interpreter)?); - Ok(()) + fn add_to_expression(self, _: &mut Interpreter, _: &mut ExpressionBuilder) -> Result<()> { + // Just like with commands, we throw an error in the flattened case so + // that we can determine in future the exact structure of the expression + // at parse time. + self.flatten + .err("Flattened variables cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression") } } diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs index 7d0cba29..cf9e1327 100644 --- a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs +++ b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs @@ -3,7 +3,7 @@ use preinterpret::*; fn main() { preinterpret!{ [!set! #is_eager = false] - let _ = [!evaluate! false && ([!set! #is_eager = true] true)]; + let _ = [!evaluate! false && [!group! [!set! #is_eager = true] true]]; [!if! #is_eager { [!error! { message: "The && expression is not evaluated lazily" }] }] diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr index 92c1ef0e..688810a9 100644 --- a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr +++ b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr @@ -3,7 +3,7 @@ error: The && expression is not evaluated lazily | 4 | / preinterpret!{ 5 | | [!set! #is_eager = false] -6 | | let _ = [!evaluate! false && ([!set! #is_eager = true] true)]; +6 | | let _ = [!evaluate! false && [!group! [!set! #is_eager = true] true]]; 7 | | [!if! #is_eager { 8 | | [!error! { message: "The && expression is not evaluated lazily" }] 9 | | }] diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs b/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs new file mode 100644 index 00000000..ada3e9c3 --- /dev/null +++ b/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret! { + [!evaluate! 5 + [!..range! 1..2]] + }; +} diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr new file mode 100644 index 00000000..05d5138b --- /dev/null +++ b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr @@ -0,0 +1,6 @@ +error: Flattened commands cannot be used directly in expressions. + Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression + --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:25 + | +5 | [!evaluate! 5 + [!..range! 1..2]] + | ^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs new file mode 100644 index 00000000..b1c7cbe4 --- /dev/null +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret! { + [!set! #partial_sum = + 2] + [!evaluate! 5 #..partial_sum] + }; +} diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr new file mode 100644 index 00000000..42767567 --- /dev/null +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr @@ -0,0 +1,6 @@ +error: Flattened variables cannot be used directly in expressions. + Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression + --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:6:24 + | +6 | [!evaluate! 5 #..partial_sum] + | ^^ diff --git a/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs b/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs new file mode 100644 index 00000000..67fecaba --- /dev/null +++ b/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret! { + [!evaluate! 5 + [!set! #x = 2] 2] + }; +} diff --git a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr new file mode 100644 index 00000000..800497eb --- /dev/null +++ b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr @@ -0,0 +1,6 @@ +error: Commands with no output cannot be used directly in expressions. + Consider wrapping it inside a command such as [!group! ..] which returns an expression + --> tests/compilation_failures/expressions/no_output_commands_in_expressions.rs:5:25 + | +5 | [!evaluate! 5 + [!set! #x = 2] 2] + | ^^^^^^^^^^^^^^ diff --git a/tests/expressions.rs b/tests/expressions.rs index a46a661d..879b6ba5 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -50,10 +50,18 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!( { [!set! #partial_sum = + 2] - [!evaluate! 5 #..partial_sum] + // The [!group! ...] constructs an expression from tokens, + // which is then interpreted / executed. + [!evaluate! [!group! 5 #..partial_sum]] }, 7 ); + assert_preinterpret_eq!( + { + [!evaluate! 1 + [!range! 1..2]] + }, + 2 + ); } #[test] From 75aa1b5fb93e0329799be20539778672c023b028 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 19 Jan 2025 18:42:02 +0000 Subject: [PATCH 039/126] tweak: Improve range tests --- CHANGELOG.md | 9 ++---- src/commands/expression_commands.rs | 20 ++++++++---- src/traits.rs | 6 ++++ .../expressions/large_range.rs | 7 ++++ .../expressions/large_range.stderr | 5 +++ tests/expressions.rs | 32 ++++++++++++++++--- 6 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 tests/compilation_failures/expressions/large_range.rs create mode 100644 tests/compilation_failures/expressions/large_range.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index dd4f299f..38ab8707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,17 +32,12 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Support `!else if!` in `!if!` * Support string & char literals (for comparisons & casts) in expressions * Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` -* See comment on `!assign!` +* Implement comment on `!assign!` and !buffer! * Other token stream commands -* Add tests for CommandStreamInput (via `[!split! ]` or `[!intersperse! ]`?): - => From `#x = Hello World` - => Or `#..x = [Hello World]` - => But not `$x` - it has to be wrapped in `[]` - => Improve tests for `[!range!]` +* Add basic `!for!` loops * Add more tests * e.g. for various expressions * e.g. for long sums - * Add compile failure tests * Support `!for!` so we can use it for simple generation scenarios without needing macros at all: * Complexities: * Parsing `,` diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 29040928..8863d9e0 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -119,6 +119,7 @@ impl StreamCommandDefinition for RangeCommand { output: &mut InterpretedStream, ) -> Result<()> { let range_span_range = self.range_limits.span_range(); + let range_span = self.range_limits.span(); let left = self .left @@ -141,20 +142,17 @@ impl StreamCommandDefinition for RangeCommand { .ok_or_else(|| { range_span_range.error("The range is too large to be represented as a usize") })?; + interpreter .config() .check_iteration_count(&range_span_range, length)?; match self.range_limits { RangeLimits::HalfOpen(_) => { - let iter = - (left..right).map(|value| TokenTree::Literal(Literal::i128_unsuffixed(value))); - output.extend_raw_token_iter(iter) + output_range(left..right, range_span, output); } RangeLimits::Closed(_) => { - let iter = - (left..=right).map(|value| TokenTree::Literal(Literal::i128_unsuffixed(value))); - output.extend_raw_token_iter(iter) + output_range(left..=right, range_span, output); } }; @@ -162,6 +160,16 @@ impl StreamCommandDefinition for RangeCommand { } } +fn output_range(iter: impl Iterator, span: Span, output: &mut InterpretedStream) { + output.extend_raw_token_iter(iter.map(|value| { + let literal = Literal::i128_unsuffixed(value).with_span(span); + TokenTree::Literal(literal) + // We wrap it in a singleton group to ensure that negative + // numbers are treated as single items in other stream commands + .into_singleton_group(Delimiter::None) + })) +} + // A copy of syn::RangeLimits to avoid needing a `full` dependency on syn #[derive(Clone)] enum RangeLimits { diff --git a/src/traits.rs b/src/traits.rs index caa05df8..870dd17f 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -74,12 +74,18 @@ impl TokenStreamExt for TokenStream { pub(crate) trait TokenTreeExt: Sized { fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; + fn into_singleton_group(self, delimiter: Delimiter) -> Self; } impl TokenTreeExt for TokenTree { fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) } + + fn into_singleton_group(self, delimiter: Delimiter) -> Self { + let span = self.span(); + Self::group(self.into_token_stream(), delimiter, span) + } } pub(crate) trait ParserExt { diff --git a/tests/compilation_failures/expressions/large_range.rs b/tests/compilation_failures/expressions/large_range.rs new file mode 100644 index 00000000..b366f173 --- /dev/null +++ b/tests/compilation_failures/expressions/large_range.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!range! 0..10000000] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/large_range.stderr b/tests/compilation_failures/expressions/large_range.stderr new file mode 100644 index 00000000..a458c679 --- /dev/null +++ b/tests/compilation_failures/expressions/large_range.stderr @@ -0,0 +1,5 @@ +error: Iteration limit of 10000 exceeded + --> tests/compilation_failures/expressions/large_range.rs:5:19 + | +5 | [!range! 0..10000000] + | ^^ diff --git a/tests/expressions.rs b/tests/expressions.rs index 879b6ba5..5bce1f9c 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -79,13 +79,37 @@ fn assign_works() { #[test] fn range_works() { - assert_preinterpret_eq!([!string! [!range! -2..5]], "-2-101234"); + assert_preinterpret_eq!( + [!string![!intersperse! { + items: [!range! -2..5], + separator: [" "], + }]], + "-2 -1 0 1 2 3 4" + ); + assert_preinterpret_eq!( + [!string![!intersperse! { + items: [!range! -2..=5], + separator: [" "], + }]], + "-2 -1 0 1 2 3 4 5" + ); assert_preinterpret_eq!( { [!set! #x = 2] - [!string! [!range! (#x + #x)..=5]] + [!string! [!intersperse! { + items: [!range! (#x + #x)..=5], + separator: [" "], + }]] + }, + "4 5" + ); + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [!range! 8..=5], + separator: [" "], + }]] }, - "45" + "" ); - assert_preinterpret_eq!({ [!string! [!range! 8..=5]] }, ""); } From f4d7be4fd1d4f360ed2238bd32c11a07a41ae981 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 19 Jan 2025 18:58:03 +0000 Subject: [PATCH 040/126] feature: Remove superfluous stream and empty commands --- CHANGELOG.md | 8 ++--- src/commands/core_commands.rs | 28 ----------------- src/commands/token_commands.rs | 31 ++----------------- src/interpretation/command.rs | 2 -- .../tokens/empty_with_input.rs | 5 --- .../tokens/empty_with_input.stderr | 5 --- tests/core.rs | 2 +- tests/tokens.rs | 13 ++++---- 8 files changed, 14 insertions(+), 80 deletions(-) delete mode 100644 tests/compilation_failures/tokens/empty_with_input.rs delete mode 100644 tests/compilation_failures/tokens/empty_with_input.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ab8707..6e9fe60b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,6 @@ * Core commands: * `[!error! ...]` - * `[!stream! ...]` command which just returns its contents, but can be used in places where a single item is expected when parsing. * Expression commands: * `[!evaluate! ...]` * `[!assign! #x += ...]` for `+` and other supported operators @@ -15,18 +14,17 @@ * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` * `[!while! COND {}]` * Token-stream utility commands: - * `[!empty!]` * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. + * `[!..group!]` which just outputs its contents as-is, useful where the grammar + only takes a single item, but we want to output multiple tokens * `[!intersperse! { .. }]` which inserts separator tokens between each token tree in a stream. -I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!empty!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. +I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!..group!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. ### To come -* Consider [!..flattened! ] etc - => Get rid of [!empty!] and [!stream!] and replace with [!..group!] or [!group!] * Explore getting rid of lots of the span range stuff * Add [!break!] and [!continue!] commands using a flag in the interpreter * Support `!else if!` in `!if!` diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 022ec2c4..8b8ae98c 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -132,34 +132,6 @@ impl NoOutputCommandDefinition for IgnoreCommand { } } -/// This serves as a no-op command to take a stream when the grammar only allows a single item -#[derive(Clone)] -pub(crate) struct StreamCommand { - arguments: InterpretationStream, -} - -impl CommandType for StreamCommand { - type OutputKind = OutputKindStream; -} - -impl StreamCommandDefinition for StreamCommand { - const COMMAND_NAME: &'static str = "stream"; - - fn parse(arguments: CommandArguments) -> Result { - Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, - }) - } - - fn execute( - self: Box, - interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()> { - self.arguments.interpret_as_tokens_into(interpreter, output) - } -} - #[derive(Clone)] pub(crate) struct ErrorCommand { inputs: ErrorInputs, diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index e2a5f885..577e7da5 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -1,27 +1,5 @@ use crate::internal_prelude::*; -#[derive(Clone)] -pub(crate) struct EmptyCommand; - -impl CommandType for EmptyCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for EmptyCommand { - const COMMAND_NAME: &'static str = "empty"; - - fn parse(arguments: CommandArguments) -> Result { - arguments.assert_empty( - "The !empty! command does not take any arguments. Perhaps you want !is_empty! instead?", - )?; - Ok(Self) - } - - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result<()> { - Ok(()) - } -} - #[derive(Clone)] pub(crate) struct IsEmptyCommand { arguments: InterpretationStream, @@ -97,12 +75,9 @@ impl StreamCommandDefinition for GroupCommand { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - let group_span = self.arguments.span(); - output.push_grouped( - |inner| self.arguments.interpret_as_tokens_into(interpreter, inner), - Delimiter::None, - group_span, - ) + // The grouping happens automatically because a non-flattened + // stream command is outputted in a group. + self.arguments.interpret_as_tokens_into(interpreter, output) } } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index e14468d2..be955f09 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -405,7 +405,6 @@ define_command_kind! { ExtendCommand, RawCommand, IgnoreCommand, - StreamCommand, ErrorCommand, // Concat & Type Convert Commands @@ -441,7 +440,6 @@ define_command_kind! { WhileCommand, // Token Commands - EmptyCommand, IsEmptyCommand, LengthCommand, GroupCommand, diff --git a/tests/compilation_failures/tokens/empty_with_input.rs b/tests/compilation_failures/tokens/empty_with_input.rs deleted file mode 100644 index 932daedd..00000000 --- a/tests/compilation_failures/tokens/empty_with_input.rs +++ /dev/null @@ -1,5 +0,0 @@ -use preinterpret::*; - -fn main() { - preinterpret!([!empty! should not have arguments]); -} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/empty_with_input.stderr b/tests/compilation_failures/tokens/empty_with_input.stderr deleted file mode 100644 index 7fb6d15e..00000000 --- a/tests/compilation_failures/tokens/empty_with_input.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: The !empty! command does not take any arguments. Perhaps you want !is_empty! instead? - --> tests/compilation_failures/tokens/empty_with_input.rs:4:19 - | -4 | preinterpret!([!empty! should not have arguments]); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/core.rs b/tests/core.rs index 34b05b2a..587bf931 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -50,7 +50,7 @@ fn test_extend() { my_assert_eq!( { [!set! #i = 1] - [!set! #output = [!empty!]] + [!set! #output = [!..group!]] [!while! (#i <= 4) { [!extend! #output += #i] [!if! (#i <= 3) { diff --git a/tests/tokens.rs b/tests/tokens.rs index 87a2de71..419c6a30 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -14,19 +14,20 @@ fn test_tokens_compilation_failures() { } #[test] -fn test_empty_and_is_empty() { +fn test_flattened_group_and_is_empty() { assert_preinterpret_eq!({ - [!empty!] "hello" [!empty!] [!empty!] + [!..group!] "hello" [!..group!] [!..group!] }, "hello"); - assert_preinterpret_eq!([!is_empty! [!empty!]], true); - assert_preinterpret_eq!([!is_empty! [!empty!] [!empty!]], true); + assert_preinterpret_eq!([!is_empty!], true); + assert_preinterpret_eq!([!is_empty! [!..group!]], true); + assert_preinterpret_eq!([!is_empty! [!..group!] [!..group!]], true); assert_preinterpret_eq!([!is_empty! Not Empty], false); assert_preinterpret_eq!({ - [!set! #x = [!empty!]] + [!set! #x =] [!is_empty! #..x] }, true); assert_preinterpret_eq!({ - [!set! #x = [!empty!]] + [!set! #x =] [!set! #x = #x is no longer empty] [!is_empty! #x] }, false); From e988b62a320c529d6394294615656ab79e2e25e3 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 19 Jan 2025 20:34:04 +0000 Subject: [PATCH 041/126] refactor: Removed some redundant span ranges --- CHANGELOG.md | 5 +-- src/commands/expression_commands.rs | 4 +- src/commands/token_commands.rs | 22 ++++++---- src/expressions/evaluation_tree.rs | 5 --- src/expressions/expression_stream.rs | 3 +- src/interpretation/command_stream_input.rs | 4 +- src/interpretation/interpret_traits.rs | 4 +- src/interpretation/interpretation_stream.rs | 7 +++- src/interpretation/interpreted_stream.rs | 45 +++------------------ src/interpretation/variable.rs | 15 ++----- src/traits.rs | 17 +------- 11 files changed, 38 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e9fe60b..27fc8d11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,12 +25,11 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Explore getting rid of lots of the span range stuff -* Add [!break!] and [!continue!] commands using a flag in the interpreter +* Implement comment on `!assign!` and `!buffer!` * Support `!else if!` in `!if!` +* Add [!break!] and [!continue!] commands using a flag in the interpreter * Support string & char literals (for comparisons & casts) in expressions * Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` -* Implement comment on `!assign!` and !buffer! * Other token stream commands * Add basic `!for!` loops * Add more tests diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 8863d9e0..05942b0f 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -79,8 +79,8 @@ impl NoOutputCommandDefinition for AssignCommand { builder.push_punct(operator); builder.extend_with_evaluation_output(expression.evaluate(interpreter)?); - let output = builder.evaluate()?.into_interpreted_stream(); - variable.set(interpreter, output)?; + let output = builder.evaluate()?.into_token_tree(); + variable.set(interpreter, output.into())?; Ok(()) } diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 577e7da5..d64c289d 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -176,16 +176,22 @@ impl SeparatorAppender { remaining: RemainingItemCount, output: &mut InterpretedStream, ) -> Result<()> { - let extender = match self.separator(remaining) { - TrailingSeparator::Normal => self.separator.clone().interpret_as_tokens(interpreter)?, + match self.separator(remaining) { + TrailingSeparator::Normal => self + .separator + .clone() + .interpret_as_tokens_into(interpreter, output), TrailingSeparator::Final => match self.final_separator.take() { - Some(final_separator) => final_separator.interpret_as_tokens(interpreter)?, - None => self.separator.clone().interpret_as_tokens(interpreter)?, + Some(final_separator) => { + final_separator.interpret_as_tokens_into(interpreter, output) + } + None => self + .separator + .clone() + .interpret_as_tokens_into(interpreter, output), }, - TrailingSeparator::None => InterpretedStream::new(SpanRange::ignored()), - }; - output.extend(extender); - Ok(()) + TrailingSeparator::None => Ok(()), + } } fn separator(&self, remaining_item_count: RemainingItemCount) -> TrailingSeparator { diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index b2799c61..3a14a818 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -438,11 +438,6 @@ impl EvaluationOutput { Self::Value(value) => value.into_token_tree(), } } - - pub(crate) fn into_interpreted_stream(self) -> InterpretedStream { - let value = self.into_value(); - InterpretedStream::raw(value.source_span(), value.into_token_stream()) - } } impl From for EvaluationOutput { diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index c233c544..744d6049 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -196,9 +196,8 @@ pub(crate) struct ExpressionBuilder { impl ExpressionBuilder { pub(crate) fn new() -> Self { - let unused_span_range = Span::call_site().span_range(); Self { - interpreted_stream: InterpretedStream::new(unused_span_range), + interpreted_stream: InterpretedStream::new(), } } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index ce827ad0..410c8b84 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -107,9 +107,7 @@ impl Interpret for CommandStreamInput { Ok(()) } CommandStreamInput::GroupedVariable(variable) => { - let group_contents = variable.interpret_ungrouped_contents(interpreter)?; - output.extend(group_contents); - Ok(()) + variable.substitute_ungrouped_contents_into(interpreter, output) } CommandStreamInput::Code(code) => { let span = code.span(); diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index f6fd729b..9a2fb091 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -1,6 +1,6 @@ use crate::internal_prelude::*; -pub(crate) trait Interpret: Sized + HasSpanRange { +pub(crate) trait Interpret: Sized { fn interpret_as_tokens_into( self, interpreter: &mut Interpreter, @@ -8,7 +8,7 @@ pub(crate) trait Interpret: Sized + HasSpanRange { ) -> Result<()>; fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { - let mut output = InterpretedStream::new(self.span_range()); + let mut output = InterpretedStream::new(); self.interpret_as_tokens_into(interpreter, &mut output)?; Ok(output) } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 776344cf..02ac8df8 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -128,8 +128,11 @@ impl Interpret for RawGroup { _: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - let inner = InterpretedStream::raw(self.source_delim_span.span_range(), self.content); - output.push_new_group(inner, self.source_delimeter, self.source_delim_span.join()); + output.push_new_group( + InterpretedStream::raw(self.content), + self.source_delimeter, + self.source_delim_span.join(), + ); Ok(()) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 11239417..12822cd1 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -46,23 +46,18 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct InterpretedStream { - source_span_range: SpanRange, token_stream: TokenStream, } impl InterpretedStream { - pub(crate) fn new(source_span_range: SpanRange) -> Self { + pub(crate) fn new() -> Self { Self { - source_span_range, token_stream: TokenStream::new(), } } - pub(crate) fn raw(empty_span_range: SpanRange, token_stream: TokenStream) -> Self { - Self { - source_span_range: empty_span_range, - token_stream, - } + pub(crate) fn raw(token_stream: TokenStream) -> Self { + Self { token_stream } } pub(crate) fn extend(&mut self, interpreted_stream: InterpretedStream) { @@ -87,7 +82,7 @@ impl InterpretedStream { delimiter: Delimiter, span: Span, ) -> Result<()> { - let mut inner = Self::new(span.span_range()); + let mut inner = Self::new(); appender(&mut inner)?; self.push_new_group(inner, delimiter, span); Ok(()) @@ -122,9 +117,9 @@ impl InterpretedStream { pub(crate) fn parse_into_fields( self, parser: FieldsParseDefinition, + error_span_range: SpanRange, ) -> Result { - let source_span_range = self.source_span_range; - self.syn_parse(parser.create_syn_parser(source_span_range)) + self.syn_parse(parser.create_syn_parser(error_span_range)) } /// For a type `T` which implements `syn::Parse`, you can call this as `syn_parse(self, T::parse)`. @@ -133,23 +128,6 @@ impl InterpretedStream { parser.parse2(self.token_stream) } - #[allow(unused)] - pub(crate) fn into_singleton(self, error_message: &str) -> Result { - if self.is_empty() { - return self.source_span_range.err(error_message); - } - let mut iter = self.token_stream.into_iter(); - let first = iter.next().unwrap(); // No panic because we're not empty - match iter.next() { - Some(_) => self.source_span_range.err(error_message), - None => Ok(first), - } - } - - pub(crate) fn set_span_range(&mut self, span_range: SpanRange) { - self.source_span_range = span_range; - } - pub(crate) fn into_token_stream(self) -> TokenStream { self.token_stream } @@ -162,18 +140,7 @@ impl InterpretedStream { impl From for InterpretedStream { fn from(value: TokenTree) -> Self { InterpretedStream { - source_span_range: value.span_range(), token_stream: value.into(), } } } - -impl HasSpanRange for InterpretedStream { - fn span_range(&self) -> SpanRange { - if self.token_stream.is_empty() { - self.source_span_range - } else { - self.token_stream.span_range() - } - } -} diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 7e68183b..4a9fe415 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -43,16 +43,7 @@ impl GroupedVariable { .ok_or_else(|| self.error(format!("The variable {} wasn't already set", self))) } - pub(super) fn interpret_ungrouped_contents( - &self, - interpreter: &Interpreter, - ) -> Result { - let mut cloned = self.read_existing(interpreter)?.clone(); - cloned.set_span_range(self.span_range()); - Ok(cloned) - } - - pub(crate) fn substitute_contents_into( + pub(crate) fn substitute_ungrouped_contents_into( &self, interpreter: &mut Interpreter, output: &mut InterpretedStream, @@ -67,7 +58,7 @@ impl GroupedVariable { output: &mut InterpretedStream, ) -> Result<()> { output.push_new_group( - self.interpret_ungrouped_contents(interpreter)?, + self.read_existing(interpreter)?.clone(), Delimiter::None, self.span(), ); @@ -107,7 +98,7 @@ impl Express for &GroupedVariable { expression_stream: &mut ExpressionBuilder, ) -> Result<()> { expression_stream.push_grouped( - |inner| self.substitute_contents_into(interpreter, inner), + |inner| self.substitute_ungrouped_contents_into(interpreter, inner), self.span(), ) } diff --git a/src/traits.rs b/src/traits.rs index 870dd17f..c2a8f3d7 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -197,13 +197,7 @@ pub(crate) struct SpanRange { end: Span, } -#[allow(unused)] impl SpanRange { - /// For use where the span range is unused, but needs to be provided - pub(crate) fn ignored() -> Self { - Span::call_site().span_range() - } - pub(crate) fn new_between(start: Span, end: Span) -> Self { Self { start, end } } @@ -222,17 +216,10 @@ impl SpanRange { self.start } + #[allow(unused)] pub(crate) fn end(&self) -> Span { self.end } - - pub(crate) fn replace_start(self, start: Span) -> Self { - Self { start, ..self } - } - - pub(crate) fn replace_end(self, end: Span) -> Self { - Self { end, ..self } - } } // This is implemented so we can create an error from it using `Error::new_spanned(..)` @@ -274,7 +261,7 @@ impl HasSpanRange for DelimSpan { // We could use self.open() => self.close() here, but using // self.join() is better as it can be round-tripped to a span // as the whole span, rather than just the start or end. - SpanRange::new_between(self.join(), self.join()) + self.join().span_range() } } From c643dbca72156fa19fccc127ffc6a00a971d67ce Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 20 Jan 2025 10:02:56 +0000 Subject: [PATCH 042/126] feature: !extend! is more efficient --- CHANGELOG.md | 1 - src/commands/core_commands.rs | 21 ++--- src/interpretation/interpreted_stream.rs | 4 - src/interpretation/interpreter.rs | 85 ++++++++++++++++--- src/interpretation/variable.rs | 74 +++++++++------- .../core/extend_non_existing_variable.rs | 7 ++ .../core/extend_non_existing_variable.stderr | 5 ++ ...xtend_variable_and_then_extend_it_again.rs | 8 ++ ...d_variable_and_then_extend_it_again.stderr | 5 ++ .../core/extend_variable_and_then_read_it.rs | 8 ++ .../extend_variable_and_then_read_it.stderr | 5 ++ .../core/extend_variable_and_then_set_it.rs | 8 ++ .../extend_variable_and_then_set_it.stderr | 5 ++ tests/expressions.rs | 10 +++ 14 files changed, 181 insertions(+), 65 deletions(-) create mode 100644 tests/compilation_failures/core/extend_non_existing_variable.rs create mode 100644 tests/compilation_failures/core/extend_non_existing_variable.stderr create mode 100644 tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs create mode 100644 tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr create mode 100644 tests/compilation_failures/core/extend_variable_and_then_read_it.rs create mode 100644 tests/compilation_failures/core/extend_variable_and_then_read_it.stderr create mode 100644 tests/compilation_failures/core/extend_variable_and_then_set_it.rs create mode 100644 tests/compilation_failures/core/extend_variable_and_then_set_it.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 27fc8d11..14abfe5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,6 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Implement comment on `!assign!` and `!buffer!` * Support `!else if!` in `!if!` * Add [!break!] and [!continue!] commands using a flag in the interpreter * Support string & char literals (for comparisons & casts) in expressions diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 8b8ae98c..13282e8b 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -1,3 +1,5 @@ +use std::ops::DerefMut; + use crate::internal_prelude::*; #[derive(Clone)] @@ -65,20 +67,11 @@ impl NoOutputCommandDefinition for ExtendCommand { } fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { - // We'd like to do this to avoid double-passing the intrepreted tokens: - // self.arguments.interpret_as_tokens_into(interpreter, self.variable.get_mut(interpreter)?); - // But this doesn't work because the interpreter is mut borrowed twice. - // - // Conceptually this does protect us from issues... e.g. it prevents us - // from allowing: - // [!extend! #x += ..#x] - // Which is pretty non-sensical. - // - // In future, we could improve this by having the interpreter store - // a RefCell and erroring on self-reference, with a hint to use a [!buffer!] - // to break the self-reference / error. - let output = self.arguments.interpret_as_tokens(interpreter)?; - self.variable.get_mut(interpreter)?.extend(output); + let variable_data = self.variable.get_existing_for_mutation(interpreter)?; + self.arguments.interpret_as_tokens_into( + interpreter, + variable_data.get_mut(&self.variable)?.deref_mut(), + )?; Ok(()) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 12822cd1..f10ab0ab 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -60,10 +60,6 @@ impl InterpretedStream { Self { token_stream } } - pub(crate) fn extend(&mut self, interpreted_stream: InterpretedStream) { - self.token_stream.extend(interpreted_stream.token_stream); - } - pub(crate) fn push_literal(&mut self, literal: Literal) { self.push_raw_token_tree(literal.into()); } diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 50db11e2..12e901f5 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -1,31 +1,90 @@ use crate::internal_prelude::*; +use super::IsVariable; +use std::cell::*; +use std::collections::hash_map::Entry; +use std::rc::Rc; + pub(crate) struct Interpreter { config: InterpreterConfig, - variables: HashMap, + variable_data: HashMap, +} + +#[derive(Clone)] +pub(crate) struct VariableData { + value: Rc>, +} + +impl VariableData { + fn new(tokens: InterpretedStream) -> Self { + Self { + value: Rc::new(RefCell::new(tokens)), + } + } + + pub(crate) fn get<'d>( + &'d self, + variable: &impl IsVariable, + ) -> Result> { + self.value.try_borrow().map_err(|_| { + variable.error("The variable cannot be read if it is currently being modified") + }) + } + + pub(crate) fn get_mut<'d>( + &'d self, + variable: &impl IsVariable, + ) -> Result> { + self.value.try_borrow_mut().map_err(|_| { + variable + .error("The variable cannot be modified if it is already currently being modified") + }) + } + + pub(crate) fn set(&self, variable: &impl IsVariable, content: InterpretedStream) -> Result<()> { + *self.get_mut(variable)? = content; + Ok(()) + } + + pub(crate) fn cheap_clone(&self) -> Self { + Self { + value: self.value.clone(), + } + } } impl Interpreter { pub(crate) fn new() -> Self { Self { config: Default::default(), - variables: Default::default(), + variable_data: Default::default(), } } - pub(crate) fn set_variable(&mut self, name: String, tokens: InterpretedStream) { - self.variables.insert(name, tokens); - } - - pub(crate) fn get_variable(&self, name: &str) -> Option<&InterpretedStream> { - self.variables.get(name) + pub(super) fn set_variable( + &mut self, + variable: &impl IsVariable, + tokens: InterpretedStream, + ) -> Result<()> { + match self.variable_data.entry(variable.get_name()) { + Entry::Occupied(mut entry) => { + entry.get_mut().set(variable, tokens)?; + } + Entry::Vacant(entry) => { + entry.insert(VariableData::new(tokens)); + } + } + Ok(()) } - pub(crate) fn get_variable_mut<'i>( - &'i mut self, - name: &str, - ) -> Option<&'i mut InterpretedStream> { - self.variables.get_mut(name) + pub(crate) fn get_existing_variable_data( + &self, + variable: &impl IsVariable, + make_error: impl FnOnce() -> Error, + ) -> Result<&VariableData> { + self.variable_data + .get(&variable.get_name()) + .ok_or_else(make_error) } pub(crate) fn config(&self) -> &InterpreterConfig { diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 4a9fe415..f8947cc9 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -1,5 +1,9 @@ use crate::internal_prelude::*; +pub(crate) trait IsVariable: HasSpanRange { + fn get_name(&self) -> String; +} + #[derive(Clone)] pub(crate) struct GroupedVariable { marker: Token![#], @@ -21,26 +25,23 @@ impl Parse for GroupedVariable { } impl GroupedVariable { - pub(crate) fn variable_name(&self) -> String { - self.variable_name.to_string() - } - pub(crate) fn set( &self, interpreter: &mut Interpreter, value: InterpretedStream, ) -> Result<()> { - interpreter.set_variable(self.variable_name(), value); - Ok(()) + interpreter.set_variable(self, value) } - pub(crate) fn get_mut<'i>( + pub(crate) fn get_existing_for_mutation( &self, - interpreter: &'i mut Interpreter, - ) -> Result<&'i mut InterpretedStream> { - interpreter - .get_variable_mut(&self.variable_name()) - .ok_or_else(|| self.error(format!("The variable {} wasn't already set", self))) + interpreter: &Interpreter, + ) -> Result { + Ok(interpreter + .get_existing_variable_data(self, || { + self.error(format!("The variable {} wasn't already set", self)) + })? + .cheap_clone()) } pub(crate) fn substitute_ungrouped_contents_into( @@ -48,7 +49,9 @@ impl GroupedVariable { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.read_existing(interpreter)?.append_cloned_into(output); + self.read_existing(interpreter)? + .get(self)? + .append_cloned_into(output); Ok(()) } @@ -58,26 +61,28 @@ impl GroupedVariable { output: &mut InterpretedStream, ) -> Result<()> { output.push_new_group( - self.read_existing(interpreter)?.clone(), + self.read_existing(interpreter)?.get(self)?.clone(), Delimiter::None, self.span(), ); Ok(()) } - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i InterpretedStream> { - match self.read_option(interpreter) { - Some(token_stream) => Ok(token_stream), - None => self.span_range().err(format!( + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i VariableData> { + interpreter.get_existing_variable_data( + self, + || self.error(format!( "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", self, self, )), - } + ) } +} - fn read_option<'i>(&self, interpreter: &'i Interpreter) -> Option<&'i InterpretedStream> { - interpreter.get_variable(&self.variable_name.to_string()) +impl IsVariable for GroupedVariable { + fn get_name(&self) -> String { + self.variable_name.to_string() } } @@ -118,7 +123,7 @@ impl HasSpanRange for &GroupedVariable { impl core::fmt::Display for GroupedVariable { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "#..{}", self.variable_name) + write!(f, "#{}", self.variable_name) } } @@ -151,24 +156,21 @@ impl FlattenedVariable { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.read_existing(interpreter)?.append_cloned_into(output); + self.read_existing(interpreter)? + .get(self)? + .append_cloned_into(output); Ok(()) } - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i InterpretedStream> { - match self.read_option(interpreter) { - Some(token_stream) => Ok(token_stream), - None => self.span_range().err(format!( + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i VariableData> { + interpreter.get_existing_variable_data( + self, + || self.error(format!( "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", self, self, )), - } - } - - fn read_option<'i>(&self, interpreter: &'i Interpreter) -> Option<&'i InterpretedStream> { - let FlattenedVariable { variable_name, .. } = self; - interpreter.get_variable(&variable_name.to_string()) + ) } pub(crate) fn display_grouped_variable_token(&self) -> String { @@ -176,6 +178,12 @@ impl FlattenedVariable { } } +impl IsVariable for FlattenedVariable { + fn get_name(&self) -> String { + self.variable_name.to_string() + } +} + impl Interpret for &FlattenedVariable { fn interpret_as_tokens_into( self, diff --git a/tests/compilation_failures/core/extend_non_existing_variable.rs b/tests/compilation_failures/core/extend_non_existing_variable.rs new file mode 100644 index 00000000..2f53b841 --- /dev/null +++ b/tests/compilation_failures/core/extend_non_existing_variable.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!extend! #variable += 2] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_non_existing_variable.stderr b/tests/compilation_failures/core/extend_non_existing_variable.stderr new file mode 100644 index 00000000..17c75804 --- /dev/null +++ b/tests/compilation_failures/core/extend_non_existing_variable.stderr @@ -0,0 +1,5 @@ +error: The variable #variable wasn't already set + --> tests/compilation_failures/core/extend_non_existing_variable.rs:5:19 + | +5 | [!extend! #variable += 2] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs new file mode 100644 index 00000000..5ada7c6b --- /dev/null +++ b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!set! #variable = Hello] + [!extend! #variable += World [!extend! #variable += !]] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr new file mode 100644 index 00000000..0d5d52f0 --- /dev/null +++ b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr @@ -0,0 +1,5 @@ +error: The variable cannot be modified if it is already currently being modified + --> tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs:6:48 + | +6 | [!extend! #variable += World [!extend! #variable += !]] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/extend_variable_and_then_read_it.rs b/tests/compilation_failures/core/extend_variable_and_then_read_it.rs new file mode 100644 index 00000000..72881695 --- /dev/null +++ b/tests/compilation_failures/core/extend_variable_and_then_read_it.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!set! #variable = Hello] + [!extend! #variable += World #variable] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr new file mode 100644 index 00000000..5217d807 --- /dev/null +++ b/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr @@ -0,0 +1,5 @@ +error: The variable cannot be read if it is currently being modified + --> tests/compilation_failures/core/extend_variable_and_then_read_it.rs:6:38 + | +6 | [!extend! #variable += World #variable] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.rs b/tests/compilation_failures/core/extend_variable_and_then_set_it.rs new file mode 100644 index 00000000..c9dc4472 --- /dev/null +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!set! #variable = Hello] + [!extend! #variable += World [!set! #variable = Hello2]] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr new file mode 100644 index 00000000..f5f5475a --- /dev/null +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr @@ -0,0 +1,5 @@ +error: The variable cannot be modified if it is already currently being modified + --> tests/compilation_failures/core/extend_variable_and_then_set_it.rs:6:45 + | +6 | [!extend! #variable += World [!set! #variable = Hello2]] + | ^^^^^^^^^ diff --git a/tests/expressions.rs b/tests/expressions.rs index 5bce1f9c..d0b5c925 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -75,6 +75,16 @@ fn assign_works() { }, 12 ); + // Assign can reference itself in its expression, + // because the expression result is buffered. + assert_preinterpret_eq!( + { + [!set! #x = 2] // 10 + [!assign! #x += #x] + #x + }, + 4 + ); } #[test] From 7a87450d5dcf3295b0f56f8269c3ec4731582540 Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 20 Jan 2025 10:29:05 +0000 Subject: [PATCH 043/126] chore: Minor docs tweaks and renames --- CHANGELOG.md | 16 ++++++------- README.md | 25 ++++++++------------- src/commands/concat_commands.rs | 6 ++--- src/commands/control_flow_commands.rs | 9 +++----- src/commands/core_commands.rs | 6 ++--- src/commands/token_commands.rs | 22 ++++++------------ src/interpretation/command.rs | 2 +- src/interpretation/command_code_input.rs | 4 ++-- src/interpretation/command_stream_input.rs | 25 ++++++++++++--------- src/interpretation/command_value_input.rs | 8 +++---- src/interpretation/interpret_traits.rs | 8 +++---- src/interpretation/interpretation_item.rs | 10 ++++----- src/interpretation/interpretation_stream.rs | 14 +++++------- src/interpretation/variable.rs | 4 ++-- src/lib.rs | 2 +- 15 files changed, 71 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14abfe5a..6a39f6f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Core commands: * `[!error! ...]` + * `[!extend! #x += ...]` to performantly add extra characters to the stream * Expression commands: * `[!evaluate! ...]` * `[!assign! #x += ...]` for `+` and other supported operators @@ -17,7 +18,7 @@ * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. - * `[!..group!]` which just outputs its contents as-is, useful where the grammar + * `[!..group! ...]` which just outputs its contents as-is, useful where the grammar only takes a single item, but we want to output multiple tokens * `[!intersperse! { .. }]` which inserts separator tokens between each token tree in a stream. @@ -26,10 +27,11 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come * Support `!else if!` in `!if!` -* Add [!break!] and [!continue!] commands using a flag in the interpreter * Support string & char literals (for comparisons & casts) in expressions * Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` -* Other token stream commands +* Add [!loop!], [!break!] and [!continue!] commands using a flag in the interpreter +* `[!split! { items: X, with: X, drop_trailing_empty?: true }]` +* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` * Add basic `!for!` loops * Add more tests * e.g. for various expressions @@ -69,14 +71,11 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!for! [!SPLIT! #x,] in [Hello, World,]]` * Even though this might be more performant, I'm not too much of a fan of this, as it's hard to understand * Perhaps we can leave it to the `[!while_parse! #x[!OPTIONAL! ,] from #X]` style commands? -* `[!split! { items: X, with: X, drop_trailing_empty?: true }]` -* `[!intersperse! { items: X, with: X, add_trailing?: false, override_final_with?: X }]` for adding something between each item, where each `X` is a `CommandStreamInput` -* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` * Basic place parsing * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * In parse land: #x matches a single token, #..x consumes the rest of a stream * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! - * Explicit Punct, Idents, Literals + * Explicit raw Punct, Idents, Literals and Groups * `[!PARSER! ]` method to take a stream * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings * `[!OPTIONAL! ...]` and/or possibly `#(..)?` and `[!is_set! #x]`? @@ -85,7 +84,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Any binding could be set to an array, but it gets complicated fast with nested bindings. * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. * `#x` binding reads a token tree - * `#..x` binding reads the rest of the stream + * `#..x` binding reads the rest of the stream... until the following raw token stream is detected (if at all). It must be followed by one or more raw tokens, + or the end of the stream. * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` * `[!match!]` (with `#..x` as a catch-all) * Check all `#[allow(unused)]` and remove any which aren't needed diff --git a/README.md b/README.md index cebe90dd..99cd09c0 100644 --- a/README.md +++ b/README.md @@ -425,17 +425,23 @@ We could support a piped calling convention, such as the `[!pipe! ...]` special #### Possible extension: Better performance -Do some testing and ensure there are tools to avoid `N^2` performance when doing token manipulation, e.g. with: +We could tweak some commands to execute lazily: +* Replace `output: &mut InterpretedStream` with `output: &mut impl OutputSource` which could either write to the output or be buffered/streamed into other commands. +This would allow things like `[!zip! ...]` or `[!range! ...]` to execute lazily, assuming the consuming command such as `for` read lazily. -* `[!extend! #stream += new tokens...]` +Incremental parsing using a fork of syn ([see issue](https://github.com/dtolnay/syn/issues/1842)) would allow: + +* Cheaper conversions between variables and parsing. * `[!consume_from! #stream #x]` where `#x` is read as the first token tree from `#stream` * `[!consume_from! #stream ()]` where the parser is read greedily from `#stream` -We could consider tweaking some commands to execute lazily, i.e. change the infrastructure to operate over a `TokenIterable` which is either a `TokenStream` or `LazyTokenStream`. Things like `[!zip! ...]` or `[!range! ...]` could then output a `LazyTokenStream`. +Forking syn may also allow some parts to be made more performant. ### Possible extension: User-defined commands * `[!define! [!my_command! ] { }]` +* Some ability to define and re-use commands across multiple invocations + without being too expensive. Still unsure how to make this work. ### Possible extension: Further utility commands @@ -445,13 +451,6 @@ Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. -### Possible extension: Loop, Break - -These aren't the highest priority, as they can be simulated with `if` statements inside a `while` loop: - -* `[!loop! { ... }]` for `[!while! true { ... }]` -* `[!break!]` to break the inner-most loop - ### Possible extension: Goto _This probably isn't needed, if we have `while` and `for`_. @@ -475,12 +474,6 @@ preinterpret::preinterpret!{ When [eager expansion of macros returning literals](https://github.com/rust-lang/rust/issues/90765) is stabilized, it would be nice to include a command to do that, which could be used to include code, for example: `[!expand_literal_macros! include!("my-poem.txt")]`. -### Possible extension: Explicit parsing feature to enable syn - -The heavy `syn` library is (in basic preinterpret) only needed for literal parsing, and error conversion into compile errors. - -We could add a parsing feature to speed up compile times a lot for stacks which don't need the parsing functionality. - ## License Licensed under either of the [Apache License, Version 2.0](LICENSE-APACHE) diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index 59d44e37..f7f351f1 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -30,7 +30,7 @@ fn concat_into_string( conversion_fn: impl Fn(&str) -> String, ) -> Result { let output_span = input.span(); - let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); + let concatenated = concat_recursive(input.interpret_to_new_stream(interpreter)?); let string_literal = string_literal(&conversion_fn(&concatenated), output_span); Ok(string_literal) } @@ -41,7 +41,7 @@ fn concat_into_ident( conversion_fn: impl Fn(&str) -> String, ) -> Result { let output_span = input.span(); - let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); + let concatenated = concat_recursive(input.interpret_to_new_stream(interpreter)?); let ident = parse_ident(&conversion_fn(&concatenated), output_span)?; Ok(ident) } @@ -52,7 +52,7 @@ fn concat_into_literal( conversion_fn: impl Fn(&str) -> String, ) -> Result { let output_span = input.span(); - let concatenated = concat_recursive(input.interpret_as_tokens(interpreter)?); + let concatenated = concat_recursive(input.interpret_to_new_stream(interpreter)?); let literal = parse_literal(&conversion_fn(&concatenated), output_span)?; Ok(literal) } diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 00efe736..96c51fd0 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -48,10 +48,9 @@ impl ControlFlowCommandDefinition for IfCommand { .value(); if evaluated_condition { - self.true_code - .interpret_as_tokens_into(interpreter, output)? + self.true_code.interpret_into(interpreter, output)? } else if let Some(false_code) = self.false_code { - false_code.interpret_as_tokens_into(interpreter, output)? + false_code.interpret_into(interpreter, output)? } Ok(()) @@ -105,9 +104,7 @@ impl ControlFlowCommandDefinition for WhileCommand { interpreter .config() .check_iteration_count(&self.condition, iteration_count)?; - self.loop_code - .clone() - .interpret_as_tokens_into(interpreter, output)?; + self.loop_code.clone().interpret_into(interpreter, output)?; } Ok(()) diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 13282e8b..eeaf350d 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -31,7 +31,7 @@ impl NoOutputCommandDefinition for SetCommand { } fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { - let result_tokens = self.arguments.interpret_as_tokens(interpreter)?; + let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; self.variable.set(interpreter, result_tokens)?; Ok(()) @@ -68,7 +68,7 @@ impl NoOutputCommandDefinition for ExtendCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { let variable_data = self.variable.get_existing_for_mutation(interpreter)?; - self.arguments.interpret_as_tokens_into( + self.arguments.interpret_into( interpreter, variable_data.get_mut(&self.variable)?.deref_mut(), )?; @@ -159,7 +159,7 @@ impl NoOutputCommandDefinition for ErrorCommand { let error_span = match self.inputs.spans { Some(spans) => { - let error_span_stream = spans.interpret_as_tokens(interpreter)?; + let error_span_stream = spans.interpret_to_new_stream(interpreter)?; // Consider the case where preinterpret embeds in a declarative macro, and we have // an error like this: diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index d64c289d..1f2864ea 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -20,7 +20,7 @@ impl ValueCommandDefinition for IsEmptyCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); - let interpreted = self.arguments.interpret_as_tokens(interpreter)?; + let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; Ok(Ident::new_bool(interpreted.is_empty(), output_span).into()) } } @@ -45,7 +45,7 @@ impl ValueCommandDefinition for LengthCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); - let interpreted = self.arguments.interpret_as_tokens(interpreter)?; + let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; let stream_length = interpreted.into_token_stream().into_iter().count(); let length_literal = Literal::usize_unsuffixed(stream_length).with_span(output_span); Ok(length_literal.into()) @@ -77,7 +77,7 @@ impl StreamCommandDefinition for GroupCommand { ) -> Result<()> { // The grouping happens automatically because a non-flattened // stream command is outputted in a group. - self.arguments.interpret_as_tokens_into(interpreter, output) + self.arguments.interpret_into(interpreter, output) } } @@ -120,7 +120,7 @@ impl StreamCommandDefinition for IntersperseCommand { let items = self .inputs .items - .interpret_as_tokens(interpreter)? + .interpret_to_new_stream(interpreter)? .into_token_stream(); let add_trailing = match self.inputs.add_trailing { Some(add_trailing) => add_trailing.interpret(interpreter)?.value(), @@ -177,18 +177,10 @@ impl SeparatorAppender { output: &mut InterpretedStream, ) -> Result<()> { match self.separator(remaining) { - TrailingSeparator::Normal => self - .separator - .clone() - .interpret_as_tokens_into(interpreter, output), + TrailingSeparator::Normal => self.separator.clone().interpret_into(interpreter, output), TrailingSeparator::Final => match self.final_separator.take() { - Some(final_separator) => { - final_separator.interpret_as_tokens_into(interpreter, output) - } - None => self - .separator - .clone() - .interpret_as_tokens_into(interpreter, output), + Some(final_separator) => final_separator.interpret_into(interpreter, output), + None => self.separator.clone().interpret_into(interpreter, output), }, TrailingSeparator::None => Ok(()), } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index be955f09..d35b4d40 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -509,7 +509,7 @@ impl HasSpanRange for Command { } impl Interpret for Command { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index 667774ed..a39de5c0 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -26,11 +26,11 @@ impl HasSpanRange for CommandCodeInput { } impl Interpret for CommandCodeInput { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - self.inner.interpret_as_tokens_into(interpreter, output) + self.inner.interpret_into(interpreter, output) } } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 410c8b84..bcadef58 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -49,7 +49,7 @@ impl HasSpanRange for CommandStreamInput { } impl Interpret for CommandStreamInput { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, @@ -65,7 +65,7 @@ impl Interpret for CommandStreamInput { CommandOutputKind::FlattenedStream => { let span = command.span(); let tokens = parse_as_stream_input( - command.interpret_as_tokens(interpreter)?, + command.interpret_to_new_stream(interpreter)?, || { span.error("Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.") }, @@ -78,12 +78,12 @@ impl Interpret for CommandStreamInput { // SAFETY: The kind change GroupedStream <=> FlattenedStream is valid command.set_output_kind(CommandOutputKind::FlattenedStream); } - command.interpret_as_tokens_into(interpreter, output) + command.interpret_into(interpreter, output) } CommandOutputKind::ControlFlowCodeStream => { let span = command.span(); let tokens = parse_as_stream_input( - command.interpret_as_tokens(interpreter)?, + command.interpret_to_new_stream(interpreter)?, || { span.error("Expected output of control flow command to contain a single [ ... ] or transparent group.") }, @@ -95,7 +95,7 @@ impl Interpret for CommandStreamInput { } CommandStreamInput::FlattenedVariable(variable) => { let tokens = parse_as_stream_input( - variable.interpret_as_tokens(interpreter)?, + variable.interpret_to_new_stream(interpreter)?, || { variable.error(format!( "Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", @@ -111,15 +111,18 @@ impl Interpret for CommandStreamInput { } CommandStreamInput::Code(code) => { let span = code.span(); - let tokens = parse_as_stream_input(code.interpret_as_tokens(interpreter)?, || { - span.error("Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string()) - })?; + let tokens = parse_as_stream_input( + code.interpret_to_new_stream(interpreter)?, + || { + span.error("Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string()) + }, + )?; output.extend_raw_tokens(tokens); Ok(()) } - CommandStreamInput::ExplicitStream(group) => group - .into_content() - .interpret_as_tokens_into(interpreter, output), + CommandStreamInput::ExplicitStream(group) => { + group.into_content().interpret_into(interpreter, output) + } } } } diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 718880fa..94072348 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -50,14 +50,14 @@ impl, I: Parse> InterpretValue for Comma CommandValueInput::Value(_) => "value", }; let interpreted_stream = match self { - CommandValueInput::Command(command) => command.interpret_as_tokens(interpreter)?, + CommandValueInput::Command(command) => command.interpret_to_new_stream(interpreter)?, CommandValueInput::GroupedVariable(variable) => { - variable.interpret_as_tokens(interpreter)? + variable.interpret_to_new_stream(interpreter)? } CommandValueInput::FlattenedVariable(variable) => { - variable.interpret_as_tokens(interpreter)? + variable.interpret_to_new_stream(interpreter)? } - CommandValueInput::Code(code) => code.interpret_as_tokens(interpreter)?, + CommandValueInput::Code(code) => code.interpret_to_new_stream(interpreter)?, CommandValueInput::Value(value) => return value.interpret(interpreter), }; match interpreted_stream.syn_parse(I::parse) { diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index 9a2fb091..ad199ee2 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -1,15 +1,15 @@ use crate::internal_prelude::*; pub(crate) trait Interpret: Sized { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()>; - fn interpret_as_tokens(self, interpreter: &mut Interpreter) -> Result { + fn interpret_to_new_stream(self, interpreter: &mut Interpreter) -> Result { let mut output = InterpretedStream::new(); - self.interpret_as_tokens_into(interpreter, &mut output)?; + self.interpret_into(interpreter, &mut output)?; Ok(output) } } @@ -21,7 +21,7 @@ pub(crate) trait InterpretValue: Sized { } impl + HasSpanRange, I: ToTokens> Interpret for T { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index cb5b9a4e..0d1f09b2 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -98,23 +98,23 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa } impl Interpret for InterpretationItem { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { match self { InterpretationItem::Command(command_invocation) => { - command_invocation.interpret_as_tokens_into(interpreter, output)?; + command_invocation.interpret_into(interpreter, output)?; } InterpretationItem::GroupedVariable(variable) => { - variable.interpret_as_tokens_into(interpreter, output)?; + variable.interpret_into(interpreter, output)?; } InterpretationItem::FlattenedVariable(variable) => { - variable.interpret_as_tokens_into(interpreter, output)?; + variable.interpret_into(interpreter, output)?; } InterpretationItem::InterpretationGroup(group) => { - group.interpret_as_tokens_into(interpreter, output)?; + group.interpret_into(interpreter, output)?; } InterpretationItem::Punct(punct) => output.push_punct(punct), InterpretationItem::Ident(ident) => output.push_ident(ident), diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 02ac8df8..e274a4db 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -33,13 +33,13 @@ impl ContextualParse for InterpretationStream { } impl Interpret for InterpretationStream { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { for item in self.items { - item.interpret_as_tokens_into(interpreter, output)?; + item.interpret_into(interpreter, output)?; } Ok(()) } @@ -78,12 +78,12 @@ impl Parse for InterpretationGroup { } impl Interpret for InterpretationGroup { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - let inner = self.content.interpret_as_tokens(interpreter)?; + let inner = self.content.interpret_to_new_stream(interpreter)?; output.push_new_group(inner, self.source_delimiter, self.source_delim_span.join()); Ok(()) } @@ -123,11 +123,7 @@ impl Parse for RawGroup { } impl Interpret for RawGroup { - fn interpret_as_tokens_into( - self, - _: &mut Interpreter, - output: &mut InterpretedStream, - ) -> Result<()> { + fn interpret_into(self, _: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { output.push_new_group( InterpretedStream::raw(self.content), self.source_delimeter, diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index f8947cc9..e2d1b0c8 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -87,7 +87,7 @@ impl IsVariable for GroupedVariable { } impl Interpret for &GroupedVariable { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, @@ -185,7 +185,7 @@ impl IsVariable for FlattenedVariable { } impl Interpret for &FlattenedVariable { - fn interpret_as_tokens_into( + fn interpret_into( self, interpreter: &mut Interpreter, output: &mut InterpretedStream, diff --git a/src/lib.rs b/src/lib.rs index f3e7a2ed..998378c8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -548,7 +548,7 @@ fn preinterpret_internal(input: TokenStream) -> Result { let mut interpreter = Interpreter::new(); let interpretation_stream = InterpretationStream::parse_from_token_stream(input, Span::call_site().span_range())?; - let interpreted_stream = interpretation_stream.interpret_as_tokens(&mut interpreter)?; + let interpreted_stream = interpretation_stream.interpret_to_new_stream(&mut interpreter)?; Ok(interpreted_stream.into_token_stream()) } From 9134958bbab101b7d47c2d39d3d9731fd5e0a4d1 Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 20 Jan 2025 19:18:43 +0000 Subject: [PATCH 044/126] feature: Add !elif!, stream and char literals --- CHANGELOG.md | 74 ++++--- local-check-msrv.sh | 2 + regenerate-compilation-failures.sh | 5 +- src/commands/control_flow_commands.rs | 57 +++-- src/expressions/boolean.rs | 32 +-- src/expressions/char.rs | 104 +++++++++ src/expressions/evaluation_tree.rs | 6 + src/expressions/float.rs | 70 +++--- src/expressions/integer.rs | 267 ++++++++++------------- src/expressions/mod.rs | 4 + src/expressions/operations.rs | 39 ++-- src/expressions/string.rs | 88 ++++++++ src/expressions/value.rs | 38 +++- src/interpretation/interpreted_stream.rs | 168 ++++++++++---- src/traits.rs | 22 ++ style-check.sh | 2 + style-fix.sh | 2 + tests/control_flow.rs | 11 + tests/expressions.rs | 8 + tests/tokens.rs | 10 + 20 files changed, 686 insertions(+), 323 deletions(-) create mode 100644 src/expressions/char.rs create mode 100644 src/expressions/string.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a39f6f3..80a2624d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,16 +26,49 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Support `!else if!` in `!if!` -* Support string & char literals (for comparisons & casts) in expressions * Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` * Add [!loop!], [!break!] and [!continue!] commands using a flag in the interpreter -* `[!split! { items: X, with: X, drop_trailing_empty?: true }]` -* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` +* `[!split! { stream: X, separator: X, drop_empty?: false, drop_trailing_empty?: true, }]` and `[!comma_split! ...]` +* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` * Add basic `!for!` loops * Add more tests * e.g. for various expressions * e.g. for long sums +* Basic place parsing + * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` + * In parse land: #x matches a single token, #..x consumes the rest of a stream + * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! + * Explicit raw Punct, Idents, Literals and Groups + * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` + * `[!PARSER! ]` method to take a stream + * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings + * `[!OPTIONAL! ...]` and/or possibly `#(..)?` and `[!is_set! #x]`? + * Groups or `[!GROUP! ...]` + * Regarding `#(..)+` and `#(..)*`... + * Any binding could be set to an array, but it gets complicated fast with nested bindings. + * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. + * `#x` binding reads a token tree + * `#..x` binding reads the rest of the stream... until the following raw token stream is detected (if at all). It must be followed by one or more raw tokens, + or the end of the stream. + * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` + * `[!match!]` (with `#..x` as a catch-all) +* Check all `#[allow(unused)]` and remove any which aren't needed +* Rework expression parsing, in order to: + * Fix comments in the expression files + * Enable lazy && and || + * Enable support for code blocks { .. } in expressions, and remove hacks where expression parsing stops at {} or . +* Work on book + * Input paradigms: + * Streams + * StreamInput / ValueInput / CodeInput + * Including documenting expressions + * There are three main kinds of commands: + * Those taking a stream as-is + * Those taking some { fields } + * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` + +### Side notes on how to build !for! + * Support `!for!` so we can use it for simple generation scenarios without needing macros at all: * Complexities: * Parsing `,` @@ -71,38 +104,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!for! [!SPLIT! #x,] in [Hello, World,]]` * Even though this might be more performant, I'm not too much of a fan of this, as it's hard to understand * Perhaps we can leave it to the `[!while_parse! #x[!OPTIONAL! ,] from #X]` style commands? -* Basic place parsing - * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` - * In parse land: #x matches a single token, #..x consumes the rest of a stream - * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! - * Explicit raw Punct, Idents, Literals and Groups - * `[!PARSER! ]` method to take a stream - * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings - * `[!OPTIONAL! ...]` and/or possibly `#(..)?` and `[!is_set! #x]`? - * Groups or `[!GROUP! ...]` - * Regarding `#(..)+` and `#(..)*`... - * Any binding could be set to an array, but it gets complicated fast with nested bindings. - * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. - * `#x` binding reads a token tree - * `#..x` binding reads the rest of the stream... until the following raw token stream is detected (if at all). It must be followed by one or more raw tokens, - or the end of the stream. - * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` - * `[!match!]` (with `#..x` as a catch-all) -* Check all `#[allow(unused)]` and remove any which aren't needed -* Rework expression parsing, in order to: - * Fix comments in the expression files - * Enable lazy && and || - * Enable support for code blocks { .. } in expressions, - and remove hacks where expression parsing stops at {} or . -* Work on book - * Input paradigms: - * Streams - * StreamInput / ValueInput / CodeInput - * Including documenting expressions - * There are three main kinds of commands: - * Those taking a stream as-is - * Those taking some { fields } - * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` + # Major Version 0.2 diff --git a/local-check-msrv.sh b/local-check-msrv.sh index e3bd402c..3cce99fb 100755 --- a/local-check-msrv.sh +++ b/local-check-msrv.sh @@ -2,5 +2,7 @@ set -e +cd "$(dirname "$0")" + rustup install 1.61 rm Cargo.lock && rustup run 1.61 cargo check diff --git a/regenerate-compilation-failures.sh b/regenerate-compilation-failures.sh index ceefa46e..0b79f11e 100755 --- a/regenerate-compilation-failures.sh +++ b/regenerate-compilation-failures.sh @@ -2,4 +2,7 @@ set -e -TRYBUILD=overwrite cargo test \ No newline at end of file +TRYBUILD=overwrite cargo test + +# Remove the .wip folder created sometimes by try-build +rm -r ./wip \ No newline at end of file diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 96c51fd0..88f82ff1 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -4,7 +4,8 @@ use crate::internal_prelude::*; pub(crate) struct IfCommand { condition: ExpressionInput, true_code: CommandCodeInput, - false_code: Option, + else_ifs: Vec<(ExpressionInput, CommandCodeInput)>, + else_code: Option, } impl CommandType for IfCommand { @@ -17,22 +18,31 @@ impl ControlFlowCommandDefinition for IfCommand { fn parse(arguments: CommandArguments) -> Result { arguments.fully_parse_or_error( |input| { + let condition = input.parse()?; + let true_code = input.parse()?; + let mut else_ifs = Vec::new(); + let mut else_code = None; + while !input.is_empty() { + input.parse::()?; + if input.peek_ident_matching("elif") { + input.parse_ident_matching("elif")?; + input.parse::()?; + else_ifs.push((input.parse()?, input.parse()?)); + } else { + input.parse_ident_matching("else")?; + input.parse::()?; + else_code = Some(input.parse()?); + break; + } + } Ok(Self { - condition: input.parse()?, - true_code: input.parse()?, - false_code: { - if !input.is_empty() { - input.parse::()?; - input.parse::()?; - input.parse::()?; - Some(input.parse()?) - } else { - None - } - }, + condition, + true_code, + else_ifs, + else_code, }) }, - "Expected [!if! (condition) { true_code }] or [!if! (condition) { true_code } !else! { false_code }]", + "Expected [!if! ... { ... } !else if! ... { ... } !else! ... { ... }]", ) } @@ -48,9 +58,22 @@ impl ControlFlowCommandDefinition for IfCommand { .value(); if evaluated_condition { - self.true_code.interpret_into(interpreter, output)? - } else if let Some(false_code) = self.false_code { - false_code.interpret_into(interpreter, output)? + return self.true_code.interpret_into(interpreter, output); + } + + for (condition, code) in self.else_ifs { + let evaluated_condition = condition + .evaluate(interpreter)? + .expect_bool("An else if condition must evaluate to a boolean")? + .value(); + + if evaluated_condition { + return code.interpret_into(interpreter, output); + } + } + + if let Some(false_code) = self.else_code { + return false_code.interpret_into(interpreter, output); } Ok(()) diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 2304f55f..2c05b19e 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -31,23 +31,25 @@ impl EvaluationBoolean { UnaryOperator::Not => operation.output(!input), UnaryOperator::NoOp => operation.output(input), UnaryOperator::Cast(target) => match target { - ValueKind::Integer(IntegerKind::Untyped) => { + CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } - ValueKind::Integer(IntegerKind::I8) => operation.output(input as i8), - ValueKind::Integer(IntegerKind::I16) => operation.output(input as i16), - ValueKind::Integer(IntegerKind::I32) => operation.output(input as i32), - ValueKind::Integer(IntegerKind::I64) => operation.output(input as i64), - ValueKind::Integer(IntegerKind::I128) => operation.output(input as i128), - ValueKind::Integer(IntegerKind::Isize) => operation.output(input as isize), - ValueKind::Integer(IntegerKind::U8) => operation.output(input as u8), - ValueKind::Integer(IntegerKind::U16) => operation.output(input as u16), - ValueKind::Integer(IntegerKind::U32) => operation.output(input as u32), - ValueKind::Integer(IntegerKind::U64) => operation.output(input as u64), - ValueKind::Integer(IntegerKind::U128) => operation.output(input as u128), - ValueKind::Integer(IntegerKind::Usize) => operation.output(input as usize), - ValueKind::Float(_) => operation.err("This cast is not supported"), - ValueKind::Boolean => operation.output(self.value), + CastTarget::Integer(IntegerKind::I8) => operation.output(input as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(input as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(input as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(input as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(input as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(input as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(input as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(input as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(input as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(input as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(input as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(input as usize), + CastTarget::Float(_) | CastTarget::Char => { + operation.err("This cast is not supported") + } + CastTarget::Boolean => operation.output(self.value), }, } } diff --git a/src/expressions/char.rs b/src/expressions/char.rs new file mode 100644 index 00000000..b514f39c --- /dev/null +++ b/src/expressions/char.rs @@ -0,0 +1,104 @@ +use super::*; + +pub(crate) struct EvaluationChar { + pub(super) value: char, + pub(super) source_span: SpanRange, +} + +impl EvaluationChar { + pub(super) fn new(value: char, source_span: SpanRange) -> Self { + Self { value, source_span } + } + + pub(super) fn for_litchar(lit: &syn::LitChar) -> Self { + Self { + value: lit.value(), + source_span: lit.span().span_range(), + } + } + + pub(super) fn handle_unary_operation( + self, + operation: UnaryOperation, + ) -> Result { + let char = self.value; + match operation.operator { + UnaryOperator::NoOp => operation.output(char), + UnaryOperator::Neg | UnaryOperator::Not => { + operation.unsupported_for_value_type_err("char") + } + UnaryOperator::Cast(target) => match target { + CastTarget::Integer(IntegerKind::Untyped) => { + operation.output(UntypedInteger::from_fallback(char as FallbackInteger)) + } + CastTarget::Integer(IntegerKind::I8) => operation.output(char as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(char as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(char as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(char as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(char as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(char as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(char as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(char as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(char as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(char as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(char as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(char as usize), + CastTarget::Char => operation.output(char), + CastTarget::Boolean | CastTarget::Float(_) => { + operation.unsupported_for_value_type_err("char") + } + }, + } + } + + pub(super) fn handle_integer_binary_operation( + self, + _right: EvaluationInteger, + operation: BinaryOperation, + ) -> Result { + operation.unsupported_for_value_type_err("char") + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: &BinaryOperation, + ) -> Result { + let lhs = self.value; + let rhs = rhs.value; + match operation.paired_operator() { + PairedBinaryOperator::Addition + | PairedBinaryOperator::Subtraction + | PairedBinaryOperator::Multiplication + | PairedBinaryOperator::Division + | PairedBinaryOperator::LogicalAnd + | PairedBinaryOperator::LogicalOr + | PairedBinaryOperator::Remainder + | PairedBinaryOperator::BitXor + | PairedBinaryOperator::BitAnd + | PairedBinaryOperator::BitOr => operation.unsupported_for_value_type_err("char"), + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + pub(super) fn to_literal(&self) -> Literal { + Literal::character(self.value).with_span(self.source_span.start()) + } +} + +impl ToEvaluationOutput for char { + fn to_output(self, span: SpanRange) -> EvaluationOutput { + EvaluationValue::Char(EvaluationChar::new(self, span)).into() + } +} + +impl quote::ToTokens for EvaluationChar { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.to_literal().to_tokens(tokens) + } +} diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index 3a14a818..5cff6ca5 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -407,6 +407,12 @@ impl EvaluationOutput { }; EvaluationLiteralPair::Float(float_pair) } + (EvaluationValue::String(left), EvaluationValue::String(right)) => { + EvaluationLiteralPair::StringPair(left, right) + } + (EvaluationValue::Char(left), EvaluationValue::Char(right)) => { + EvaluationLiteralPair::CharPair(left, right) + } (left, right) => { return operator_span.err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 8b35a1cd..408b1a2b 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -149,27 +149,29 @@ impl UntypedFloat { UnaryOperator::Not => operation.unsupported_for_value_type_err("untyped float"), UnaryOperator::NoOp => operation.output(self), UnaryOperator::Cast(target) => match target { - ValueKind::Integer(IntegerKind::Untyped) => { + CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } - ValueKind::Integer(IntegerKind::I8) => operation.output(input as i8), - ValueKind::Integer(IntegerKind::I16) => operation.output(input as i16), - ValueKind::Integer(IntegerKind::I32) => operation.output(input as i32), - ValueKind::Integer(IntegerKind::I64) => operation.output(input as i64), - ValueKind::Integer(IntegerKind::I128) => operation.output(input as i128), - ValueKind::Integer(IntegerKind::Isize) => operation.output(input as isize), - ValueKind::Integer(IntegerKind::U8) => operation.output(input as u8), - ValueKind::Integer(IntegerKind::U16) => operation.output(input as u16), - ValueKind::Integer(IntegerKind::U32) => operation.output(input as u32), - ValueKind::Integer(IntegerKind::U64) => operation.output(input as u64), - ValueKind::Integer(IntegerKind::U128) => operation.output(input as u128), - ValueKind::Integer(IntegerKind::Usize) => operation.output(input as usize), - ValueKind::Float(FloatKind::Untyped) => { + CastTarget::Integer(IntegerKind::I8) => operation.output(input as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(input as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(input as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(input as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(input as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(input as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(input as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(input as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(input as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(input as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(input as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(input as usize), + CastTarget::Float(FloatKind::Untyped) => { operation.output(UntypedFloat::from_fallback(input as FallbackFloat)) } - ValueKind::Float(FloatKind::F32) => operation.output(input as f32), - ValueKind::Float(FloatKind::F64) => operation.output(input), - ValueKind::Boolean => operation.err("This cast is not supported"), + CastTarget::Float(FloatKind::F32) => operation.output(input as f32), + CastTarget::Float(FloatKind::F64) => operation.output(input), + CastTarget::Boolean | CastTarget::Char => { + operation.err("This cast is not supported") + } }, } } @@ -278,23 +280,23 @@ macro_rules! impl_float_operations { UnaryOperator::Not => operation.unsupported_for_value_type_err(stringify!($float_type)), UnaryOperator::NoOp => operation.output(self), UnaryOperator::Cast(target) => match target { - ValueKind::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), - ValueKind::Integer(IntegerKind::I8) => operation.output(self as i8), - ValueKind::Integer(IntegerKind::I16) => operation.output(self as i16), - ValueKind::Integer(IntegerKind::I32) => operation.output(self as i32), - ValueKind::Integer(IntegerKind::I64) => operation.output(self as i64), - ValueKind::Integer(IntegerKind::I128) => operation.output(self as i128), - ValueKind::Integer(IntegerKind::Isize) => operation.output(self as isize), - ValueKind::Integer(IntegerKind::U8) => operation.output(self as u8), - ValueKind::Integer(IntegerKind::U16) => operation.output(self as u16), - ValueKind::Integer(IntegerKind::U32) => operation.output(self as u32), - ValueKind::Integer(IntegerKind::U64) => operation.output(self as u64), - ValueKind::Integer(IntegerKind::U128) => operation.output(self as u128), - ValueKind::Integer(IntegerKind::Usize) => operation.output(self as usize), - ValueKind::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), - ValueKind::Float(FloatKind::F32) => operation.output(self as f32), - ValueKind::Float(FloatKind::F64) => operation.output(self as f64), - ValueKind::Boolean => operation.err("This cast is not supported"), + CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), + CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(self as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(self as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(self as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(self as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(self as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(self as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(self as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(self as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(self as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(self as usize), + CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), + CastTarget::Float(FloatKind::F32) => operation.output(self as f32), + CastTarget::Float(FloatKind::F64) => operation.output(self as f64), + CastTarget::Boolean | CastTarget::Char => operation.err("This cast is not supported"), } } } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 478b1478..510c98f3 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -282,27 +282,29 @@ impl UntypedInteger { UnaryOperator::Not => operation.unsupported_for_value_type_err("untyped integer"), UnaryOperator::NoOp => operation.output(self), UnaryOperator::Cast(target) => match target { - ValueKind::Integer(IntegerKind::Untyped) => { + CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } - ValueKind::Integer(IntegerKind::I8) => operation.output(input as i8), - ValueKind::Integer(IntegerKind::I16) => operation.output(input as i16), - ValueKind::Integer(IntegerKind::I32) => operation.output(input as i32), - ValueKind::Integer(IntegerKind::I64) => operation.output(input as i64), - ValueKind::Integer(IntegerKind::I128) => operation.output(input), - ValueKind::Integer(IntegerKind::Isize) => operation.output(input as isize), - ValueKind::Integer(IntegerKind::U8) => operation.output(input as u8), - ValueKind::Integer(IntegerKind::U16) => operation.output(input as u16), - ValueKind::Integer(IntegerKind::U32) => operation.output(input as u32), - ValueKind::Integer(IntegerKind::U64) => operation.output(input as u64), - ValueKind::Integer(IntegerKind::U128) => operation.output(input as u128), - ValueKind::Integer(IntegerKind::Usize) => operation.output(input as usize), - ValueKind::Float(FloatKind::Untyped) => { + CastTarget::Integer(IntegerKind::I8) => operation.output(input as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(input as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(input as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(input as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(input), + CastTarget::Integer(IntegerKind::Isize) => operation.output(input as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(input as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(input as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(input as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(input as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(input as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(input as usize), + CastTarget::Float(FloatKind::Untyped) => { operation.output(UntypedFloat::from_fallback(input as FallbackFloat)) } - ValueKind::Float(FloatKind::F32) => operation.output(input as f32), - ValueKind::Float(FloatKind::F64) => operation.output(input as f64), - ValueKind::Boolean => operation.err("This cast is not supported"), + CastTarget::Float(FloatKind::F32) => operation.output(input as f32), + CastTarget::Float(FloatKind::F64) => operation.output(input as f64), + CastTarget::Boolean | CastTarget::Char => { + operation.err("This cast is not supported") + } }, } } @@ -446,7 +448,7 @@ impl ToEvaluationOutput for UntypedInteger { } // We have to use a macro because we don't have checked xx traits :( -macro_rules! impl_signed_int_operations { +macro_rules! impl_int_operations_except_unary { ( $($integer_enum_variant:ident($integer_type:ident)),* $(,)? ) => {$( @@ -456,37 +458,6 @@ macro_rules! impl_signed_int_operations { } } - impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { - match operation.operator { - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Neg => operation.output(-self), - UnaryOperator::Not => { - operation.unsupported_for_value_type_err(stringify!($integer_type)) - }, - UnaryOperator::Cast(target) => match target { - ValueKind::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), - ValueKind::Integer(IntegerKind::I8) => operation.output(self as i8), - ValueKind::Integer(IntegerKind::I16) => operation.output(self as i16), - ValueKind::Integer(IntegerKind::I32) => operation.output(self as i32), - ValueKind::Integer(IntegerKind::I64) => operation.output(self as i64), - ValueKind::Integer(IntegerKind::I128) => operation.output(self as i128), - ValueKind::Integer(IntegerKind::Isize) => operation.output(self as isize), - ValueKind::Integer(IntegerKind::U8) => operation.output(self as u8), - ValueKind::Integer(IntegerKind::U16) => operation.output(self as u16), - ValueKind::Integer(IntegerKind::U32) => operation.output(self as u32), - ValueKind::Integer(IntegerKind::U64) => operation.output(self as u64), - ValueKind::Integer(IntegerKind::U128) => operation.output(self as u128), - ValueKind::Integer(IntegerKind::Usize) => operation.output(self as usize), - ValueKind::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), - ValueKind::Float(FloatKind::F32) => operation.output(self as f32), - ValueKind::Float(FloatKind::F64) => operation.output(self as f64), - ValueKind::Boolean => operation.err("This cast is not supported"), - } - } - } - } - impl HandleBinaryOperation for $integer_type { fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { let lhs = self; @@ -558,16 +529,8 @@ macro_rules! impl_signed_int_operations { )*}; } -macro_rules! impl_unsigned_int_operations { - ( - $($integer_enum_variant:ident($integer_type:ident)),* $(,)? - ) => {$( - impl ToEvaluationOutput for $integer_type { - fn to_output(self, span_range: SpanRange) -> EvaluationOutput { - EvaluationValue::Integer(EvaluationInteger::new(EvaluationIntegerValue::$integer_enum_variant(self), span_range)).into() - } - } - +macro_rules! impl_unsigned_unary_operations { + ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { match operation.operator { @@ -577,112 +540,116 @@ macro_rules! impl_unsigned_int_operations { operation.unsupported_for_value_type_err(stringify!($integer_type)) }, UnaryOperator::Cast(target) => match target { - ValueKind::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), - ValueKind::Integer(IntegerKind::I8) => operation.output(self as i8), - ValueKind::Integer(IntegerKind::I16) => operation.output(self as i16), - ValueKind::Integer(IntegerKind::I32) => operation.output(self as i32), - ValueKind::Integer(IntegerKind::I64) => operation.output(self as i64), - ValueKind::Integer(IntegerKind::I128) => operation.output(self as i128), - ValueKind::Integer(IntegerKind::Isize) => operation.output(self as isize), - ValueKind::Integer(IntegerKind::U8) => operation.output(self as u8), - ValueKind::Integer(IntegerKind::U16) => operation.output(self as u16), - ValueKind::Integer(IntegerKind::U32) => operation.output(self as u32), - ValueKind::Integer(IntegerKind::U64) => operation.output(self as u64), - ValueKind::Integer(IntegerKind::U128) => operation.output(self as u128), - ValueKind::Integer(IntegerKind::Usize) => operation.output(self as usize), - ValueKind::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), - ValueKind::Float(FloatKind::F32) => operation.output(self as f32), - ValueKind::Float(FloatKind::F64) => operation.output(self as f64), - ValueKind::Boolean => operation.err("This cast is not supported"), + CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), + CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(self as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(self as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(self as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(self as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(self as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(self as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(self as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(self as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(self as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(self as usize), + CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), + CastTarget::Float(FloatKind::F32) => operation.output(self as f32), + CastTarget::Float(FloatKind::F64) => operation.output(self as f64), + // Technically u8 => char is supported, but we can add it later + CastTarget::Boolean | CastTarget::Char => operation.err("This cast is not supported"), } } } } + )*}; +} - impl HandleBinaryOperation for $integer_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { - let lhs = self; - let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.operator.symbol(), rhs); - match operation.paired_operator() { - PairedBinaryOperator::Addition => operation.output_if_some(lhs.checked_add(rhs), overflow_error), - PairedBinaryOperator::Subtraction => operation.output_if_some(lhs.checked_sub(rhs), overflow_error), - PairedBinaryOperator::Multiplication => operation.output_if_some(lhs.checked_mul(rhs), overflow_error), - PairedBinaryOperator::Division => operation.output_if_some(lhs.checked_div(rhs), overflow_error), - PairedBinaryOperator::LogicalAnd - | PairedBinaryOperator::LogicalOr => operation.unsupported_for_value_type_err(stringify!($integer_type)), - PairedBinaryOperator::Remainder => operation.output_if_some(lhs.checked_rem(rhs), overflow_error), - PairedBinaryOperator::BitXor => operation.output(lhs ^ rhs), - PairedBinaryOperator::BitAnd => operation.output(lhs & rhs), - PairedBinaryOperator::BitOr => operation.output(lhs | rhs), - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), - } - } - - fn handle_integer_binary_operation( - self, - rhs: EvaluationInteger, - operation: &BinaryOperation, - ) -> Result { - let lhs = self; - match operation.integer_operator() { - IntegerBinaryOperator::ShiftLeft => { - match rhs.value { - EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), - EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U16(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U32(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U64(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U128(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::Usize(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I8(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I16(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I32(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I64(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), - } - }, - IntegerBinaryOperator::ShiftRight => { - match rhs.value { - EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs >> rhs.parse_fallback()?), - EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U16(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U32(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U64(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U128(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I8(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I16(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I32(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I64(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I128(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), - } +macro_rules! impl_signed_unary_operations { + ($($integer_type:ident),* $(,)?) => {$( + impl HandleUnaryOperation for $integer_type { + fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + match operation.operator { + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Neg => operation.output(-self), + UnaryOperator::Not => { + operation.unsupported_for_value_type_err(stringify!($integer_type)) }, + UnaryOperator::Cast(target) => match target { + CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), + CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(self as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(self as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(self as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(self as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(self as u8), + CastTarget::Integer(IntegerKind::U16) => operation.output(self as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(self as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(self as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(self as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(self as usize), + CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), + CastTarget::Float(FloatKind::F32) => operation.output(self as f32), + CastTarget::Float(FloatKind::F64) => operation.output(self as f64), + CastTarget::Boolean | CastTarget::Char => operation.err("This cast is not supported"), + } } } } )*}; } -impl_signed_int_operations!( - I8(i8), - I16(i16), - I32(i32), - I64(i64), - I128(i128), - Isize(isize) -); -impl_unsigned_int_operations!( +impl HandleUnaryOperation for u8 { + fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + match operation.operator { + UnaryOperator::NoOp => operation.output(self), + UnaryOperator::Neg | UnaryOperator::Not => { + operation.unsupported_for_value_type_err("u8") + } + UnaryOperator::Cast(target) => match target { + CastTarget::Integer(IntegerKind::Untyped) => { + operation.output(UntypedInteger::from_fallback(self as FallbackInteger)) + } + CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), + CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), + CastTarget::Integer(IntegerKind::I32) => operation.output(self as i32), + CastTarget::Integer(IntegerKind::I64) => operation.output(self as i64), + CastTarget::Integer(IntegerKind::I128) => operation.output(self as i128), + CastTarget::Integer(IntegerKind::Isize) => operation.output(self as isize), + CastTarget::Integer(IntegerKind::U8) => operation.output(self), + CastTarget::Integer(IntegerKind::U16) => operation.output(self as u16), + CastTarget::Integer(IntegerKind::U32) => operation.output(self as u32), + CastTarget::Integer(IntegerKind::U64) => operation.output(self as u64), + CastTarget::Integer(IntegerKind::U128) => operation.output(self as u128), + CastTarget::Integer(IntegerKind::Usize) => operation.output(self as usize), + CastTarget::Float(FloatKind::Untyped) => { + operation.output(UntypedFloat::from_fallback(self as FallbackFloat)) + } + CastTarget::Float(FloatKind::F32) => operation.output(self as f32), + CastTarget::Float(FloatKind::F64) => operation.output(self as f64), + CastTarget::Char => operation.output(self as char), + CastTarget::Boolean => operation.err("This cast is not supported"), + }, + } + } +} + +impl_int_operations_except_unary!( U8(u8), U16(u16), U32(u32), U64(u64), U128(u128), - Usize(usize) + Usize(usize), + I8(i8), + I16(i16), + I32(i32), + I64(i64), + I128(i128), + Isize(isize), ); + +// U8 has a char cast so is handled separately +impl_unsigned_unary_operations!(u16, u32, u64, u128, usize); +impl_signed_unary_operations!(i8, i16, i32, i64, i128, isize); diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index e5f9c086..72a68655 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -1,18 +1,22 @@ mod boolean; +mod char; mod evaluation_tree; mod expression_stream; mod float; mod integer; mod operations; +mod string; mod value; // Marked as use for expression sub-modules to use with a `use super::*` statement use crate::internal_prelude::*; use boolean::*; +use char::*; use evaluation_tree::*; use float::*; use integer::*; use operations::*; +use string::*; use value::*; pub(crate) use expression_stream::*; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 082caaf6..61b6108c 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -64,7 +64,7 @@ impl UnaryOperation { } pub(super) fn for_cast_expression(expr: &syn::ExprCast) -> Result { - fn extract_type(ty: &syn::Type) -> Result { + fn extract_type(ty: &syn::Type) -> Result { match ty { syn::Type::Group(group) => extract_type(&group.elem), syn::Type::Path(type_path) @@ -81,23 +81,24 @@ impl UnaryOperation { } }; match ident.to_string().as_str() { - "int" | "integer" => Ok(ValueKind::Integer(IntegerKind::Untyped)), - "u8" => Ok(ValueKind::Integer(IntegerKind::U8)), - "u16" => Ok(ValueKind::Integer(IntegerKind::U16)), - "u32" => Ok(ValueKind::Integer(IntegerKind::U32)), - "u64" => Ok(ValueKind::Integer(IntegerKind::U64)), - "u128" => Ok(ValueKind::Integer(IntegerKind::U128)), - "usize" => Ok(ValueKind::Integer(IntegerKind::Usize)), - "i8" => Ok(ValueKind::Integer(IntegerKind::I8)), - "i16" => Ok(ValueKind::Integer(IntegerKind::I16)), - "i32" => Ok(ValueKind::Integer(IntegerKind::I32)), - "i64" => Ok(ValueKind::Integer(IntegerKind::I64)), - "i128" => Ok(ValueKind::Integer(IntegerKind::I128)), - "isize" => Ok(ValueKind::Integer(IntegerKind::Isize)), - "float" => Ok(ValueKind::Float(FloatKind::Untyped)), - "f32" => Ok(ValueKind::Float(FloatKind::F32)), - "f64" => Ok(ValueKind::Float(FloatKind::F64)), - "bool" => Ok(ValueKind::Boolean), + "int" | "integer" => Ok(CastTarget::Integer(IntegerKind::Untyped)), + "u8" => Ok(CastTarget::Integer(IntegerKind::U8)), + "u16" => Ok(CastTarget::Integer(IntegerKind::U16)), + "u32" => Ok(CastTarget::Integer(IntegerKind::U32)), + "u64" => Ok(CastTarget::Integer(IntegerKind::U64)), + "u128" => Ok(CastTarget::Integer(IntegerKind::U128)), + "usize" => Ok(CastTarget::Integer(IntegerKind::Usize)), + "i8" => Ok(CastTarget::Integer(IntegerKind::I8)), + "i16" => Ok(CastTarget::Integer(IntegerKind::I16)), + "i32" => Ok(CastTarget::Integer(IntegerKind::I32)), + "i64" => Ok(CastTarget::Integer(IntegerKind::I64)), + "i128" => Ok(CastTarget::Integer(IntegerKind::I128)), + "isize" => Ok(CastTarget::Integer(IntegerKind::Isize)), + "float" => Ok(CastTarget::Float(FloatKind::Untyped)), + "f32" => Ok(CastTarget::Float(FloatKind::F32)), + "f64" => Ok(CastTarget::Float(FloatKind::F64)), + "bool" => Ok(CastTarget::Boolean), + "char" => Ok(CastTarget::Char), _ => ident .span() .span_range() @@ -160,7 +161,7 @@ pub(super) enum UnaryOperator { Neg, Not, NoOp, - Cast(ValueKind), + Cast(CastTarget), } impl UnaryOperator { diff --git a/src/expressions/string.rs b/src/expressions/string.rs new file mode 100644 index 00000000..efd9f8f9 --- /dev/null +++ b/src/expressions/string.rs @@ -0,0 +1,88 @@ +use super::*; + +pub(crate) struct EvaluationString { + pub(super) value: String, + pub(super) source_span: SpanRange, +} + +impl EvaluationString { + pub(super) fn new(value: String, source_span: SpanRange) -> Self { + Self { value, source_span } + } + + pub(super) fn for_litstr(lit: &syn::LitStr) -> Self { + Self { + value: lit.value(), + source_span: lit.span().span_range(), + } + } + + pub(super) fn handle_unary_operation( + self, + operation: UnaryOperation, + ) -> Result { + match operation.operator { + UnaryOperator::NoOp => operation.output(self.value), + UnaryOperator::Neg | UnaryOperator::Not | UnaryOperator::Cast(_) => { + operation.unsupported_for_value_type_err("string") + } + } + } + + pub(super) fn handle_integer_binary_operation( + self, + _right: EvaluationInteger, + operation: BinaryOperation, + ) -> Result { + operation.unsupported_for_value_type_err("string") + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: &BinaryOperation, + ) -> Result { + let lhs = self.value; + let rhs = rhs.value; + match operation.paired_operator() { + PairedBinaryOperator::Addition + | PairedBinaryOperator::Subtraction + | PairedBinaryOperator::Multiplication + | PairedBinaryOperator::Division + | PairedBinaryOperator::LogicalAnd + | PairedBinaryOperator::LogicalOr + | PairedBinaryOperator::Remainder + | PairedBinaryOperator::BitXor + | PairedBinaryOperator::BitAnd + | PairedBinaryOperator::BitOr => operation.unsupported_for_value_type_err("string"), + PairedBinaryOperator::Equal => operation.output(lhs == rhs), + PairedBinaryOperator::LessThan => operation.output(lhs < rhs), + PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), + PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), + PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), + PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + } + } + + pub(super) fn to_literal(&self) -> Literal { + Literal::string(&self.value).with_span(self.source_span.start()) + } +} + +impl ToEvaluationOutput for String { + fn to_output(self, span: SpanRange) -> EvaluationOutput { + EvaluationValue::String(EvaluationString::new(self, span)).into() + } +} + +impl ToEvaluationOutput for &str { + fn to_output(self, span: SpanRange) -> EvaluationOutput { + EvaluationValue::String(EvaluationString::new(self.to_string(), span)).into() + } +} + +impl quote::ToTokens for EvaluationString { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.to_literal().to_tokens(tokens) + } +} diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 9f6dbad9..a51b8b8c 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -4,6 +4,8 @@ pub(crate) enum EvaluationValue { Integer(EvaluationInteger), Float(EvaluationFloat), Boolean(EvaluationBoolean), + String(EvaluationString), + Char(EvaluationChar), } impl EvaluationValue { @@ -12,14 +14,19 @@ impl EvaluationValue { Self::Integer(int) => int.to_literal().into(), Self::Float(float) => float.to_literal().into(), Self::Boolean(bool) => bool.to_ident().into(), + Self::String(string) => string.to_literal().into(), + Self::Char(char) => char.to_literal().into(), } } + pub(super) fn for_literal_expression(expr: &ExprLit) -> Result { // https://docs.rs/syn/latest/syn/enum.Lit.html Ok(match &expr.lit { Lit::Int(lit) => Self::Integer(EvaluationInteger::for_litint(lit)?), Lit::Float(lit) => Self::Float(EvaluationFloat::for_litfloat(lit)?), Lit::Bool(lit) => Self::Boolean(EvaluationBoolean::for_litbool(lit)), + Lit::Str(lit) => Self::String(EvaluationString::for_litstr(lit)), + Lit::Char(lit) => Self::Char(EvaluationChar::for_litchar(lit)), other_literal => { return other_literal .span() @@ -33,6 +40,8 @@ impl EvaluationValue { Self::Integer(int) => int.value.describe_type(), Self::Float(float) => float.value.describe_type(), Self::Boolean(_) => "bool", + Self::String(_) => "string", + Self::Char(_) => "char", } } @@ -44,6 +53,8 @@ impl EvaluationValue { EvaluationValue::Integer(value) => value.handle_unary_operation(operation), EvaluationValue::Float(value) => value.handle_unary_operation(operation), EvaluationValue::Boolean(value) => value.handle_unary_operation(operation), + EvaluationValue::String(value) => value.handle_unary_operation(operation), + EvaluationValue::Char(value) => value.handle_unary_operation(operation), } } @@ -62,14 +73,20 @@ impl EvaluationValue { EvaluationValue::Boolean(value) => { value.handle_integer_binary_operation(right, operation) } + EvaluationValue::String(value) => { + value.handle_integer_binary_operation(right, operation) + } + EvaluationValue::Char(value) => value.handle_integer_binary_operation(right, operation), } } pub(super) fn source_span(&self) -> SpanRange { match self { - EvaluationValue::Integer(integer) => integer.source_span, - EvaluationValue::Float(float) => float.source_span, - EvaluationValue::Boolean(boolean) => boolean.source_span, + EvaluationValue::Integer(value) => value.source_span, + EvaluationValue::Float(value) => value.source_span, + EvaluationValue::Boolean(value) => value.source_span, + EvaluationValue::String(value) => value.source_span, + EvaluationValue::Char(value) => value.source_span, } } } @@ -83,24 +100,29 @@ impl HasSpanRange for EvaluationValue { impl quote::ToTokens for EvaluationValue { fn to_tokens(&self, tokens: &mut TokenStream) { match self { - Self::Integer(int) => int.to_tokens(tokens), - Self::Float(float) => float.to_tokens(tokens), - Self::Boolean(bool) => bool.to_tokens(tokens), + Self::Integer(value) => value.to_tokens(tokens), + Self::Float(value) => value.to_tokens(tokens), + Self::Boolean(value) => value.to_tokens(tokens), + Self::String(value) => value.to_tokens(tokens), + Self::Char(value) => value.to_tokens(tokens), } } } #[derive(Copy, Clone)] -pub(super) enum ValueKind { +pub(super) enum CastTarget { Integer(IntegerKind), Float(FloatKind), Boolean, + Char, } pub(super) enum EvaluationLiteralPair { Integer(EvaluationIntegerValuePair), Float(EvaluationFloatValuePair), BooleanPair(EvaluationBoolean, EvaluationBoolean), + StringPair(EvaluationString, EvaluationString), + CharPair(EvaluationChar, EvaluationChar), } impl EvaluationLiteralPair { @@ -112,6 +134,8 @@ impl EvaluationLiteralPair { Self::Integer(pair) => pair.handle_paired_binary_operation(&operation), Self::Float(pair) => pair.handle_paired_binary_operation(&operation), Self::BooleanPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), + Self::StringPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), + Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), } } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index f10ab0ab..b3471edc 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -1,51 +1,12 @@ use crate::internal_prelude::*; -// ====================================== -// How syn fits with preinterpret parsing -// ====================================== -// -// TLDR: This is discussed on this syn issue, where David Tolnay suggested -// forking syn to get what we want (i.e. a more versatile TokenBuffer): -// ==> https://github.com/dtolnay/syn/issues/1842 -// -// There are a few places where we support (or might wish to support) parsing -// as part of interpretation: -// * e.g. of a token stream in `CommandValueInput` -// * e.g. as part of a PARSER, from an InterpretedStream -// * e.g. of a variable, as part of incremental parsing (while_parse style loops) -// -// I spent quite a while considering whether this could be wrapping a -// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... -// -// Some commands want to performantly parse a variable or other token stream. -// -// Here we want variables to support: -// * Easy appending of tokens -// * Incremental parsing -// -// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to -// append to it, and freely convert it to a syn::ParseStream, possibly even storing -// a cursor position into it. -// -// Unfortunately this isn't at all possible: -// * TokenBuffer appending isn't a thing, you can only create one (recursively) from -// a TokenStream -// * TokenBuffer can't be converted to a ParseStream outside of the syn crate -// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be -// used against a fixed buffer. -// -// We could probably work around these limitations by sacrificing performance and transforming -// to TokenStream and back, but probably there's a better way. -// -// What we probably want is our own abstraction, likely a fork from `syn`, which supports -// converting a Cursor into an indexed based cursor, which can safely be stored separately -// from the TokenBuffer. -// -// We could use this abstraction for InterpretedStream; and our variables could store a -// tuple of (IndexCursor, PreinterpretTokenBuffer) - #[derive(Clone)] pub(crate) struct InterpretedStream { + /// Currently, even ~inside~ macro executions, round-tripping [`Delimiter::None`] groups to a TokenStream + /// breaks in rust-analyzer. This causes various spurious errors in the IDE. + /// See: https://github.com/dtolnay/syn/issues/1464 and https://github.com/rust-lang/rust-analyzer/issues/18211 + /// + /// In future, we may wish to internally use some kind of `TokenBuffer` type, which I've started on below. token_stream: TokenStream, } @@ -140,3 +101,122 @@ impl From for InterpretedStream { } } } + +// ====================================== +// How syn fits with preinterpret parsing +// ====================================== +// +// TLDR: This is discussed on this syn issue, where David Tolnay suggested +// forking syn to get what we want (i.e. a more versatile TokenBuffer): +// ==> https://github.com/dtolnay/syn/issues/1842 +// +// There are a few places where we support (or might wish to support) parsing +// as part of interpretation: +// * e.g. of a token stream in `CommandValueInput` +// * e.g. as part of a PARSER, from an InterpretedStream +// * e.g. of a variable, as part of incremental parsing (while_parse style loops) +// +// I spent quite a while considering whether this could be wrapping a +// `syn::parse::ParseBuffer<'a>` or `syn::buffer::Cursor<'a>`... +// +// Some commands want to performantly parse a variable or other token stream. +// +// Here we want variables to support: +// * Easy appending of tokens +// * Incremental parsing +// +// Ideally we'd want to be able to store a syn::TokenBuffer, and be able to +// append to it, and freely convert it to a syn::ParseStream, possibly even storing +// a cursor position into it. +// +// Unfortunately this isn't at all possible: +// * TokenBuffer appending isn't a thing, you can only create one (recursively) from +// a TokenStream +// * TokenBuffer can't be converted to a ParseStream outside of the syn crate +// * For performance, a cursor stores a pointer into a TokenBuffer, so it can only be +// used against a fixed buffer. +// +// We could probably work around these limitations by sacrificing performance and transforming +// to TokenStream and back, but probably there's a better way. +// +// What we probably want is our own abstraction, likely a fork from `syn`, which supports +// converting a Cursor into an indexed based cursor, which can safely be stored separately +// from the TokenBuffer. +// +// We could use this abstraction for InterpretedStream; and our variables could store a +// tuple of (IndexCursor, PreinterpretTokenBuffer) + +/// Inspired/ forked from [`syn::buffer::TokenBuffer`], in order to support appending tokens, +/// as per the issue here: https://github.com/dtolnay/syn/issues/1842 +/// +/// Syn is dual-licensed under MIT and Apache, and a subset of it is reproduced from version 2.0.96 +/// of syn, and then further edited as a derivative work as part of preinterpret, which is released +/// under the same licenses. +/// +/// LICENSE-MIT: https://github.com/dtolnay/syn/blob/2.0.96/LICENSE-MIT +/// LICENSE-APACHE: https://github.com/dtolnay/syn/blob/2.0.96/LICENSE-APACHE +#[allow(unused)] +mod token_buffer { + use super::*; + + /// Inspired by [`syn::buffer::Entry`] + /// Internal type which is used instead of `TokenTree` to represent a token tree + /// within a `TokenBuffer`. + enum TokenBufferEntry { + // Mimicking types from proc-macro. + // Group entries contain the offset to the matching End entry. + Group(Group, usize), + Ident(Ident), + Punct(Punct), + Literal(Literal), + // End entries contain the offset (negative) to the start of the buffer, and + // offset (negative) to the matching Group entry. + End(isize, isize), + } + + /// Inspired by [`syn::buffer::TokenBuffer`], but with the ability to append tokens. + pub(super) struct TokenBuffer { + entries: Vec, + } + + impl TokenBuffer { + pub(crate) fn new(tokens: impl IntoIterator) -> Self { + let mut entries = vec![]; + Self::recursive_new(&mut entries, tokens); + entries.push(TokenBufferEntry::End(-(entries.len() as isize), 0)); + Self { entries } + } + + pub(crate) fn append(&mut self, tokens: impl IntoIterator) { + self.entries.pop(); + Self::recursive_new(&mut self.entries, tokens); + self.entries + .push(TokenBufferEntry::End(-(self.entries.len() as isize), 0)); + } + + fn recursive_new( + entries: &mut Vec, + stream: impl IntoIterator, + ) { + for tt in stream { + match tt { + TokenTree::Ident(ident) => entries.push(TokenBufferEntry::Ident(ident)), + TokenTree::Punct(punct) => entries.push(TokenBufferEntry::Punct(punct)), + TokenTree::Literal(literal) => entries.push(TokenBufferEntry::Literal(literal)), + TokenTree::Group(group) => { + let group_start_index = entries.len(); + entries.push(TokenBufferEntry::End(0, 0)); // we replace this below + Self::recursive_new(entries, group.stream()); + let group_end_index = entries.len(); + let group_offset = group_end_index - group_start_index; + entries.push(TokenBufferEntry::End( + -(group_end_index as isize), + -(group_offset as isize), + )); + entries[group_start_index] = TokenBufferEntry::Group(group, group_offset); + } + } + } + } + } +} diff --git a/src/traits.rs b/src/traits.rs index c2a8f3d7..6b6dfd7f 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -11,10 +11,18 @@ impl IdentExt for Ident { } pub(crate) trait CursorExt: Sized { + fn ident_matching(self, content: &str) -> Option<(Ident, Self)>; fn punct_matching(self, char: char) -> Option<(Punct, Self)>; } impl CursorExt for Cursor<'_> { + fn ident_matching(self, content: &str) -> Option<(Ident, Self)> { + match self.ident() { + Some((ident, next)) if ident == content => Some((ident, next)), + _ => None, + } + } + fn punct_matching(self, char: char) -> Option<(Punct, Self)> { match self.punct() { Some((punct, next)) if punct.as_char() == char => Some((punct, next)), @@ -96,6 +104,8 @@ pub(crate) trait ParserExt { func: F, message: M, ) -> Result; + fn peek_ident_matching(&self, content: &str) -> bool; + fn parse_ident_matching(&self, content: &str) -> Result; } impl ParserExt for ParseBuffer<'_> { @@ -115,6 +125,18 @@ impl ParserExt for ParseBuffer<'_> { let error_span = self.span(); parse(self).map_err(|_| error_span.error(message)) } + + fn peek_ident_matching(&self, content: &str) -> bool { + self.cursor().ident_matching(content).is_some() + } + + fn parse_ident_matching(&self, content: &str) -> Result { + self.step(|cursor| { + cursor + .ident_matching(content) + .ok_or_else(|| cursor.span().error(format!("expected {}", content))) + }) + } } pub(crate) trait ContextualParse: Sized { diff --git a/style-check.sh b/style-check.sh index b5a96410..da8c7dd4 100755 --- a/style-check.sh +++ b/style-check.sh @@ -2,5 +2,7 @@ set -e +cd "$(dirname "$0")" + cargo fmt --check; cargo clippy --tests; \ No newline at end of file diff --git a/style-fix.sh b/style-fix.sh index f0fb61b8..6bba2585 100755 --- a/style-fix.sh +++ b/style-fix.sh @@ -2,5 +2,7 @@ set -e +cd "$(dirname "$0")" + cargo fmt; cargo clippy --fix --tests --allow-dirty --allow-staged; \ No newline at end of file diff --git a/tests/control_flow.rs b/tests/control_flow.rs index ad18c96a..f36898ce 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -36,6 +36,17 @@ fn test_if() { 0 [!if! false { + 1 }] }, 0); + assert_preinterpret_eq!({ + [!if! false { + 1 + } !elif! false { + 2 + } !elif! true { + 3 + } !else! { + 4 + }] + }, 3); } #[test] diff --git a/tests/expressions.rs b/tests/expressions.rs index d0b5c925..ae360279 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -62,6 +62,14 @@ fn test_basic_evaluate_works() { }, 2 ); + assert_preinterpret_eq!([!evaluate! "hello" == "world"], false); + assert_preinterpret_eq!([!evaluate! "hello" == "hello"], true); + assert_preinterpret_eq!([!evaluate! 'A' as u8 == 65], true); + assert_preinterpret_eq!([!evaluate! 65u8 as char == 'A'], true); + assert_preinterpret_eq!([!evaluate! 'A' == 'A'], true); + assert_preinterpret_eq!([!evaluate! 'A' == 'B'], false); + assert_preinterpret_eq!([!evaluate! 'A' < 'B'], true); + assert_preinterpret_eq!([!evaluate! "Zoo" > "Aardvark"], true); } #[test] diff --git a/tests/tokens.rs b/tests/tokens.rs index 419c6a30..b24d6556 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -275,6 +275,16 @@ fn complex_cases_for_intersperse_and_input_types() { separator: [_], }]] }, "01_01"); + // Control stream commands can be used, if they return a valid stream grouping + assert_preinterpret_eq!( + { + [!string![!intersperse! { + items: [!if! false { [0 1] } !else! { [2 3] }], + separator: [_], + }]] + }, + "2_3" + ); // All inputs can be variables // Inputs can be in any order assert_preinterpret_eq!({ From 033ba6ba94d34e1cb21963500c0028134570d231 Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 20 Jan 2025 22:11:05 +0000 Subject: [PATCH 045/126] feature: Add !for! and !settings! commands --- CHANGELOG.md | 3 +- regenerate-compilation-failures.sh | 4 +- src/commands/control_flow_commands.rs | 57 +++++++++++++++++++ src/commands/core_commands.rs | 36 ++++++++++++ src/interpretation/command.rs | 2 + src/interpretation/command_field_inputs.rs | 6 +- src/interpretation/interpreter.rs | 12 +++- src/interpretation/variable.rs | 20 +++++++ src/parsing/mod.rs | 4 ++ src/parsing/parse_place.rs | 22 +++++++ src/parsing/parse_traits.rs | 14 +++++ src/traits.rs | 1 + .../core/settings_update_iteration_limit.rs | 8 +++ .../settings_update_iteration_limit.stderr | 5 ++ tests/control_flow.rs | 12 ++++ 15 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 src/parsing/parse_place.rs create mode 100644 src/parsing/parse_traits.rs create mode 100644 tests/compilation_failures/core/settings_update_iteration_limit.rs create mode 100644 tests/compilation_failures/core/settings_update_iteration_limit.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 80a2624d..8e71c918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * Control flow commands: * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` * `[!while! COND {}]` + * `[!for! #x in [ ... ] { ... }]` * Token-stream utility commands: * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. @@ -30,7 +31,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Add [!loop!], [!break!] and [!continue!] commands using a flag in the interpreter * `[!split! { stream: X, separator: X, drop_empty?: false, drop_trailing_empty?: true, }]` and `[!comma_split! ...]` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` -* Add basic `!for!` loops +* Add casts of other integers to char, via char::from_u32(u32::try_from(x)) * Add more tests * e.g. for various expressions * e.g. for long sums diff --git a/regenerate-compilation-failures.sh b/regenerate-compilation-failures.sh index 0b79f11e..6defcd6d 100755 --- a/regenerate-compilation-failures.sh +++ b/regenerate-compilation-failures.sh @@ -5,4 +5,6 @@ set -e TRYBUILD=overwrite cargo test # Remove the .wip folder created sometimes by try-build -rm -r ./wip \ No newline at end of file +if [ -d "./wip" ]; then + rm -r ./wip +fi diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 88f82ff1..3f6db494 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -133,3 +133,60 @@ impl ControlFlowCommandDefinition for WhileCommand { Ok(()) } } + +#[derive(Clone)] +pub(crate) struct ForCommand { + parse_place: ParsePlace, + #[allow(unused)] + in_token: Token![in], + input: CommandStreamInput, + code_block: CommandCodeInput, +} + +impl CommandType for ForCommand { + type OutputKind = OutputKindControlFlow; +} + +impl ControlFlowCommandDefinition for ForCommand { + const COMMAND_NAME: &'static str = "for"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + parse_place: input.parse()?, + in_token: input.parse()?, + input: input.parse()?, + code_block: input.parse()?, + }) + }, + "Expected [!for! #x in [ ... ] { code }]", + ) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + let stream = self.input.interpret_to_new_stream(interpreter)?; + + let mut iteration_count = 0; + + for token in stream.into_token_stream() { + self.parse_place.handle_parse_from_stream( + InterpretedStream::raw(token.into_token_stream()), + interpreter, + )?; + self.code_block + .clone() + .interpret_into(interpreter, output)?; + iteration_count += 1; + interpreter + .config() + .check_iteration_count(&self.in_token, iteration_count)?; + } + + Ok(()) + } +} diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index eeaf350d..afacbce7 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -125,6 +125,42 @@ impl NoOutputCommandDefinition for IgnoreCommand { } } +#[derive(Clone)] +pub(crate) struct SettingsCommand { + inputs: SettingsInputs, +} + +impl CommandType for SettingsCommand { + type OutputKind = OutputKindNone; +} + +define_field_inputs! { + SettingsInputs { + required: {}, + optional: { + iteration_limit: CommandValueInput = DEFAULT_ITERATION_LIMIT ("The new iteration limit"), + } + } +} + +impl NoOutputCommandDefinition for SettingsCommand { + const COMMAND_NAME: &'static str = "settings"; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + inputs: arguments.fully_parse_as()?, + }) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + if let Some(limit) = self.inputs.iteration_limit { + let limit: usize = limit.interpret(interpreter)?.base10_parse()?; + interpreter.mut_config().set_iteration_limit(Some(limit)); + } + Ok(()) + } +} + #[derive(Clone)] pub(crate) struct ErrorCommand { inputs: ErrorInputs, diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index d35b4d40..cf769575 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -405,6 +405,7 @@ define_command_kind! { ExtendCommand, RawCommand, IgnoreCommand, + SettingsCommand, ErrorCommand, // Concat & Type Convert Commands @@ -438,6 +439,7 @@ define_command_kind! { // Control flow commands IfCommand, WhileCommand, + ForCommand, // Token Commands IsEmptyCommand, diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index e77b6c20..3dafb66e 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -3,12 +3,12 @@ macro_rules! define_field_inputs { $inputs_type:ident { required: { $( - $required_field:ident: $required_type:ty = $required_example:literal $(($required_description:literal))? + $required_field:ident: $required_type:ty = $required_example:tt $(($required_description:literal))? ),* $(,)? }$(,)? optional: { $( - $optional_field:ident: $optional_type:ty = $optional_example:literal $(($optional_description:literal))? + $optional_field:ident: $optional_type:ty = $optional_example:tt $(($optional_description:literal))? ),* $(,)? }$(,)? } @@ -62,6 +62,8 @@ macro_rules! define_field_inputs { content.parse::()?; } } + + #[allow(unused_mut)] let mut missing_fields: Vec = vec![]; $( diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 12e901f5..1cf52f3e 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -90,21 +90,31 @@ impl Interpreter { pub(crate) fn config(&self) -> &InterpreterConfig { &self.config } + + pub(crate) fn mut_config(&mut self) -> &mut InterpreterConfig { + &mut self.config + } } pub(crate) struct InterpreterConfig { iteration_limit: Option, } +pub(crate) const DEFAULT_ITERATION_LIMIT: usize = 10000; + impl Default for InterpreterConfig { fn default() -> Self { Self { - iteration_limit: Some(10000), + iteration_limit: Some(DEFAULT_ITERATION_LIMIT), } } } impl InterpreterConfig { + pub(crate) fn set_iteration_limit(&mut self, limit: Option) { + self.iteration_limit = limit; + } + pub(crate) fn check_iteration_count( &self, span_source: &impl HasSpanRange, diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index e2d1b0c8..3cc3cb2a 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -109,6 +109,26 @@ impl Express for &GroupedVariable { } } +impl HandleParse for GroupedVariable { + fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + let variable_contents = match input.parse::()? { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + InterpretedStream::raw(group.stream()) + } + TokenTree::Group(group) => { + return group + .delim_span() + .open() + .err("Expected a group with transparent delimiters"); + } + TokenTree::Ident(ident) => InterpretedStream::raw(ident.to_token_stream()), + TokenTree::Punct(punct) => InterpretedStream::raw(punct.to_token_stream()), + TokenTree::Literal(literal) => InterpretedStream::raw(literal.to_token_stream()), + }; + self.set(interpreter, variable_contents) + } +} + impl HasSpanRange for GroupedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs index 41689ae0..f66d370f 100644 --- a/src/parsing/mod.rs +++ b/src/parsing/mod.rs @@ -1,7 +1,11 @@ #[allow(unused)] mod building_blocks; mod fields; +mod parse_place; +mod parse_traits; #[allow(unused)] pub(crate) use building_blocks::*; pub(crate) use fields::*; +pub(crate) use parse_place::*; +pub(crate) use parse_traits::*; diff --git a/src/parsing/parse_place.rs b/src/parsing/parse_place.rs new file mode 100644 index 00000000..09cdba15 --- /dev/null +++ b/src/parsing/parse_place.rs @@ -0,0 +1,22 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) enum ParsePlace { + GroupedVariable(GroupedVariable), +} + +impl Parse for ParsePlace { + fn parse(input: ParseStream) -> Result { + Ok(Self::GroupedVariable(input.parse()?)) + } +} + +impl HandleParse for ParsePlace { + fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + match self { + Self::GroupedVariable(grouped_variable) => { + grouped_variable.handle_parse(input, interpreter) + } + } + } +} diff --git a/src/parsing/parse_traits.rs b/src/parsing/parse_traits.rs new file mode 100644 index 00000000..a15e5223 --- /dev/null +++ b/src/parsing/parse_traits.rs @@ -0,0 +1,14 @@ +use crate::internal_prelude::*; + +pub(crate) trait HandleParse { + fn handle_parse_from_stream( + &self, + input: InterpretedStream, + interpreter: &mut Interpreter, + ) -> Result<()> { + |input: ParseStream| -> Result<()> { self.handle_parse(input, interpreter) } + .parse2(input.into_token_stream()) + } + + fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; +} diff --git a/src/traits.rs b/src/traits.rs index 6b6dfd7f..d595f252 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -321,4 +321,5 @@ impl_auto_span_range! { syn::Type, syn::TypePath, syn::token::DotDot, + syn::token::In, } diff --git a/tests/compilation_failures/core/settings_update_iteration_limit.rs b/tests/compilation_failures/core/settings_update_iteration_limit.rs new file mode 100644 index 00000000..eceacad6 --- /dev/null +++ b/tests/compilation_failures/core/settings_update_iteration_limit.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + preinterpret!{ + [!settings! { iteration_limit: 5 }] + [!range! 0..100] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/core/settings_update_iteration_limit.stderr b/tests/compilation_failures/core/settings_update_iteration_limit.stderr new file mode 100644 index 00000000..1f0de354 --- /dev/null +++ b/tests/compilation_failures/core/settings_update_iteration_limit.stderr @@ -0,0 +1,5 @@ +error: Iteration limit of 5 exceeded + --> tests/compilation_failures/core/settings_update_iteration_limit.rs:6:19 + | +6 | [!range! 0..100] + | ^^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index f36898ce..53c4461f 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -57,3 +57,15 @@ fn test_while() { #x }, 5); } + +#[test] +fn test_for() { + assert_preinterpret_eq!( + { + [!string! [!for! #x in [!range! 65..70] { + [!evaluate! #x as u8 as char] + }]] + }, + "ABCDE" + ); +} From 425f8987c2338ef66e5469ce83ba579d17c33f96 Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 20 Jan 2025 23:57:42 +0000 Subject: [PATCH 046/126] feature: Add !loop!, !continue! and !break! --- CHANGELOG.md | 8 +- src/commands/control_flow_commands.rs | 133 ++++++++++++++++-- src/commands/core_commands.rs | 2 +- src/commands/expression_commands.rs | 4 +- src/interpretation/command.rs | 3 + src/interpretation/command_code_input.rs | 17 +++ src/interpretation/interpreter.rs | 87 +++++++++--- .../control_flow/break_outside_a_loop.rs | 5 + .../control_flow/break_outside_a_loop.stderr | 5 + .../control_flow/continue_outside_a_loop.rs | 5 + .../continue_outside_a_loop.stderr | 5 + .../control_flow/error_after_continue.rs | 19 +++ .../control_flow/error_after_continue.stderr | 13 ++ .../control_flow/while_infinite_loop.stderr | 3 +- .../settings_update_iteration_limit.stderr | 3 +- .../expressions/large_range.stderr | 3 +- tests/control_flow.rs | 24 ++++ 17 files changed, 292 insertions(+), 47 deletions(-) create mode 100644 tests/compilation_failures/control_flow/break_outside_a_loop.rs create mode 100644 tests/compilation_failures/control_flow/break_outside_a_loop.stderr create mode 100644 tests/compilation_failures/control_flow/continue_outside_a_loop.rs create mode 100644 tests/compilation_failures/control_flow/continue_outside_a_loop.stderr create mode 100644 tests/compilation_failures/control_flow/error_after_continue.rs create mode 100644 tests/compilation_failures/control_flow/error_after_continue.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e71c918..315276da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` * `[!while! COND {}]` * `[!for! #x in [ ... ] { ... }]` + * `[!loop! { ... }]` + * `[!continue!]` + * `[!break!]` * Token-stream utility commands: * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. @@ -27,11 +30,10 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Reconfiguring iteration limit, via `[!settings! { iteration_limit: 4000 }]` -* Add [!loop!], [!break!] and [!continue!] commands using a flag in the interpreter +* Accept `[!error! ]` as well * `[!split! { stream: X, separator: X, drop_empty?: false, drop_trailing_empty?: true, }]` and `[!comma_split! ...]` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` -* Add casts of other integers to char, via char::from_u32(u32::try_from(x)) +* Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Add more tests * e.g. for various expressions * e.g. for long sums diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 3f6db494..03efd72e 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -110,8 +110,10 @@ impl ControlFlowCommandDefinition for WhileCommand { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - let mut iteration_count = 0; + let mut iteration_counter = interpreter.start_iteration_counter(&self.condition); loop { + iteration_counter.increment_and_check()?; + let evaluated_condition = self .condition .clone() @@ -123,24 +125,74 @@ impl ControlFlowCommandDefinition for WhileCommand { break; } - iteration_count += 1; - interpreter - .config() - .check_iteration_count(&self.condition, iteration_count)?; - self.loop_code.clone().interpret_into(interpreter, output)?; + match self + .loop_code + .clone() + .interpret_loop_content_into(interpreter, output)? + { + LoopCondition::None => {} + LoopCondition::Continue => continue, + LoopCondition::Break => break, + } } Ok(()) } } +#[derive(Clone)] +pub(crate) struct LoopCommand { + loop_code: CommandCodeInput, +} + +impl CommandType for LoopCommand { + type OutputKind = OutputKindControlFlow; +} + +impl ControlFlowCommandDefinition for LoopCommand { + const COMMAND_NAME: &'static str = "loop"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + loop_code: input.parse()?, + }) + }, + "Expected [!loop! { ... }]", + ) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + let mut iteration_counter = interpreter.start_iteration_counter(&self.loop_code); + + loop { + iteration_counter.increment_and_check()?; + match self + .loop_code + .clone() + .interpret_loop_content_into(interpreter, output)? + { + LoopCondition::None => {} + LoopCondition::Continue => continue, + LoopCondition::Break => break, + } + } + Ok(()) + } +} + #[derive(Clone)] pub(crate) struct ForCommand { parse_place: ParsePlace, #[allow(unused)] in_token: Token![in], input: CommandStreamInput, - code_block: CommandCodeInput, + loop_code: CommandCodeInput, } impl CommandType for ForCommand { @@ -157,7 +209,7 @@ impl ControlFlowCommandDefinition for ForCommand { parse_place: input.parse()?, in_token: input.parse()?, input: input.parse()?, - code_block: input.parse()?, + loop_code: input.parse()?, }) }, "Expected [!for! #x in [ ... ] { code }]", @@ -171,22 +223,73 @@ impl ControlFlowCommandDefinition for ForCommand { ) -> Result<()> { let stream = self.input.interpret_to_new_stream(interpreter)?; - let mut iteration_count = 0; + let mut iteration_counter = interpreter.start_iteration_counter(&self.in_token); for token in stream.into_token_stream() { + iteration_counter.increment_and_check()?; self.parse_place.handle_parse_from_stream( InterpretedStream::raw(token.into_token_stream()), interpreter, )?; - self.code_block + match self + .loop_code .clone() - .interpret_into(interpreter, output)?; - iteration_count += 1; - interpreter - .config() - .check_iteration_count(&self.in_token, iteration_count)?; + .interpret_loop_content_into(interpreter, output)? + { + LoopCondition::None => {} + LoopCondition::Continue => continue, + LoopCondition::Break => break, + } } Ok(()) } } + +#[derive(Clone)] +pub(crate) struct ContinueCommand { + span: Span, +} + +impl CommandType for ContinueCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for ContinueCommand { + const COMMAND_NAME: &'static str = "continue"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.assert_empty("The !continue! command takes no arguments")?; + Ok(Self { + span: arguments.full_span_range().span(), + }) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + Err(interpreter.start_loop_action(self.span, LoopCondition::Continue)) + } +} + +#[derive(Clone)] +pub(crate) struct BreakCommand { + span: Span, +} + +impl CommandType for BreakCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for BreakCommand { + const COMMAND_NAME: &'static str = "break"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.assert_empty("The !break! command takes no arguments")?; + Ok(Self { + span: arguments.full_span_range().span(), + }) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + Err(interpreter.start_loop_action(self.span, LoopCondition::Break)) + } +} diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index afacbce7..013b518c 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -155,7 +155,7 @@ impl NoOutputCommandDefinition for SettingsCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { if let Some(limit) = self.inputs.iteration_limit { let limit: usize = limit.interpret(interpreter)?.base10_parse()?; - interpreter.mut_config().set_iteration_limit(Some(limit)); + interpreter.set_iteration_limit(Some(limit)); } Ok(()) } diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index 05942b0f..a5e612be 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -144,8 +144,8 @@ impl StreamCommandDefinition for RangeCommand { })?; interpreter - .config() - .check_iteration_count(&range_span_range, length)?; + .start_iteration_counter(&range_span_range) + .add_and_check(length)?; match self.range_limits { RangeLimits::HalfOpen(_) => { diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index cf769575..62f90f7d 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -440,6 +440,9 @@ define_command_kind! { IfCommand, WhileCommand, ForCommand, + LoopCommand, + ContinueCommand, + BreakCommand, // Token Commands IsEmptyCommand, diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index a39de5c0..66899bd3 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -25,6 +25,23 @@ impl HasSpanRange for CommandCodeInput { } } +impl CommandCodeInput { + pub(crate) fn interpret_loop_content_into( + self, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result { + match self.inner.interpret_into(interpreter, output) { + Ok(()) => Ok(LoopCondition::None), + Err(err) => match interpreter.outstanding_loop_condition() { + LoopCondition::None => Err(err), + LoopCondition::Break => Ok(LoopCondition::Break), + LoopCondition::Continue => Ok(LoopCondition::Continue), + }, + } + } +} + impl Interpret for CommandCodeInput { fn interpret_into( self, diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 1cf52f3e..e18568db 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -8,6 +8,15 @@ use std::rc::Rc; pub(crate) struct Interpreter { config: InterpreterConfig, variable_data: HashMap, + loop_condition: LoopCondition, +} + +#[derive(Clone, Copy)] +#[must_use] +pub(crate) enum LoopCondition { + None, + Continue, + Break, } #[derive(Clone)] @@ -58,6 +67,7 @@ impl Interpreter { Self { config: Default::default(), variable_data: Default::default(), + loop_condition: LoopCondition::None, } } @@ -87,44 +97,75 @@ impl Interpreter { .ok_or_else(make_error) } - pub(crate) fn config(&self) -> &InterpreterConfig { - &self.config + pub(crate) fn start_iteration_counter<'s, S: HasSpanRange>( + &self, + span_source: &'s S, + ) -> IterationCounter<'s, S> { + IterationCounter { + span_source, + count: 0, + iteration_limit: self.config.iteration_limit, + } + } + + pub(crate) fn set_iteration_limit(&mut self, limit: Option) { + self.config.iteration_limit = limit; + } + + /// Panics if called with [`LoopCondition::None`] + pub(crate) fn start_loop_action(&mut self, source: Span, action: LoopCondition) -> Error { + self.loop_condition = action; + match action { + LoopCondition::None => panic!("Not allowed"), + LoopCondition::Continue => { + source.error("The continue command is only allowed inside a loop") + } + LoopCondition::Break => source.error("The break command is only allowed inside a loop"), + } } - pub(crate) fn mut_config(&mut self) -> &mut InterpreterConfig { - &mut self.config + pub(crate) fn outstanding_loop_condition(&mut self) -> LoopCondition { + std::mem::replace(&mut self.loop_condition, LoopCondition::None) } } -pub(crate) struct InterpreterConfig { +pub(crate) struct IterationCounter<'a, S: HasSpanRange> { + span_source: &'a S, + count: usize, iteration_limit: Option, } -pub(crate) const DEFAULT_ITERATION_LIMIT: usize = 10000; - -impl Default for InterpreterConfig { - fn default() -> Self { - Self { - iteration_limit: Some(DEFAULT_ITERATION_LIMIT), - } +impl<'a, S: HasSpanRange> IterationCounter<'a, S> { + pub(crate) fn add_and_check(&mut self, count: usize) -> Result<()> { + self.count = self.count.wrapping_add(count); + self.check() } -} -impl InterpreterConfig { - pub(crate) fn set_iteration_limit(&mut self, limit: Option) { - self.iteration_limit = limit; + pub(crate) fn increment_and_check(&mut self) -> Result<()> { + self.count += 1; + self.check() } - pub(crate) fn check_iteration_count( - &self, - span_source: &impl HasSpanRange, - count: usize, - ) -> Result<()> { + pub(crate) fn check(&self) -> Result<()> { if let Some(limit) = self.iteration_limit { - if count > limit { - return span_source.err(format!("Iteration limit of {} exceeded", limit)); + if self.count > limit { + return self.span_source.err(format!("Iteration limit of {} exceeded.\nIf needed, the limit can be reconfigured with [!settings! {{ iteration_limit: X }}]", limit)); } } Ok(()) } } + +pub(crate) struct InterpreterConfig { + iteration_limit: Option, +} + +pub(crate) const DEFAULT_ITERATION_LIMIT: usize = 1000; + +impl Default for InterpreterConfig { + fn default() -> Self { + Self { + iteration_limit: Some(DEFAULT_ITERATION_LIMIT), + } + } +} diff --git a/tests/compilation_failures/control_flow/break_outside_a_loop.rs b/tests/compilation_failures/control_flow/break_outside_a_loop.rs new file mode 100644 index 00000000..a8844658 --- /dev/null +++ b/tests/compilation_failures/control_flow/break_outside_a_loop.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!(1 + [!break!] 2); +} \ No newline at end of file diff --git a/tests/compilation_failures/control_flow/break_outside_a_loop.stderr b/tests/compilation_failures/control_flow/break_outside_a_loop.stderr new file mode 100644 index 00000000..fb5f5429 --- /dev/null +++ b/tests/compilation_failures/control_flow/break_outside_a_loop.stderr @@ -0,0 +1,5 @@ +error: The break command is only allowed inside a loop + --> tests/compilation_failures/control_flow/break_outside_a_loop.rs:4:23 + | +4 | preinterpret!(1 + [!break!] 2); + | ^^^^^^^^^ diff --git a/tests/compilation_failures/control_flow/continue_outside_a_loop.rs b/tests/compilation_failures/control_flow/continue_outside_a_loop.rs new file mode 100644 index 00000000..f3275414 --- /dev/null +++ b/tests/compilation_failures/control_flow/continue_outside_a_loop.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!(1 + [!continue!] 2); +} \ No newline at end of file diff --git a/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr b/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr new file mode 100644 index 00000000..1512593e --- /dev/null +++ b/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr @@ -0,0 +1,5 @@ +error: The continue command is only allowed inside a loop + --> tests/compilation_failures/control_flow/continue_outside_a_loop.rs:4:23 + | +4 | preinterpret!(1 + [!continue!] 2); + | ^^^^^^^^^^^^ diff --git a/tests/compilation_failures/control_flow/error_after_continue.rs b/tests/compilation_failures/control_flow/error_after_continue.rs new file mode 100644 index 00000000..891942a4 --- /dev/null +++ b/tests/compilation_failures/control_flow/error_after_continue.rs @@ -0,0 +1,19 @@ +use preinterpret::*; + +fn main() { + preinterpret!( + [!set! #x = 0] + [!while! true { + [!assign! #x += 1] + [!if! #x == 3 { + [!continue!] + } !elif! #x >= 3 { + // This checks that the "continue" flag is consumed, + // and future errors propogate correctly. + [!error! { + message: "And now we error" + }] + }] + }] + ); +} \ No newline at end of file diff --git a/tests/compilation_failures/control_flow/error_after_continue.stderr b/tests/compilation_failures/control_flow/error_after_continue.stderr new file mode 100644 index 00000000..ea662970 --- /dev/null +++ b/tests/compilation_failures/control_flow/error_after_continue.stderr @@ -0,0 +1,13 @@ +error: And now we error + --> tests/compilation_failures/control_flow/error_after_continue.rs:4:5 + | +4 | / preinterpret!( +5 | | [!set! #x = 0] +6 | | [!while! true { +7 | | [!assign! #x += 1] +... | +17 | | }] +18 | | ); + | |_____^ + | + = note: this error originates in the macro `preinterpret` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/compilation_failures/control_flow/while_infinite_loop.stderr b/tests/compilation_failures/control_flow/while_infinite_loop.stderr index 9e6b5e43..50698898 100644 --- a/tests/compilation_failures/control_flow/while_infinite_loop.stderr +++ b/tests/compilation_failures/control_flow/while_infinite_loop.stderr @@ -1,4 +1,5 @@ -error: Iteration limit of 10000 exceeded +error: Iteration limit of 1000 exceeded. + If needed, the limit can be reconfigured with [!settings! { iteration_limit: X }] --> tests/compilation_failures/control_flow/while_infinite_loop.rs:4:28 | 4 | preinterpret!([!while! true {}]); diff --git a/tests/compilation_failures/core/settings_update_iteration_limit.stderr b/tests/compilation_failures/core/settings_update_iteration_limit.stderr index 1f0de354..2b365c76 100644 --- a/tests/compilation_failures/core/settings_update_iteration_limit.stderr +++ b/tests/compilation_failures/core/settings_update_iteration_limit.stderr @@ -1,4 +1,5 @@ -error: Iteration limit of 5 exceeded +error: Iteration limit of 5 exceeded. + If needed, the limit can be reconfigured with [!settings! { iteration_limit: X }] --> tests/compilation_failures/core/settings_update_iteration_limit.rs:6:19 | 6 | [!range! 0..100] diff --git a/tests/compilation_failures/expressions/large_range.stderr b/tests/compilation_failures/expressions/large_range.stderr index a458c679..8557a88c 100644 --- a/tests/compilation_failures/expressions/large_range.stderr +++ b/tests/compilation_failures/expressions/large_range.stderr @@ -1,4 +1,5 @@ -error: Iteration limit of 10000 exceeded +error: Iteration limit of 1000 exceeded. + If needed, the limit can be reconfigured with [!settings! { iteration_limit: X }] --> tests/compilation_failures/expressions/large_range.rs:5:19 | 5 | [!range! 0..10000000] diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 53c4461f..15911a33 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -69,3 +69,27 @@ fn test_for() { "ABCDE" ); } + +#[test] +fn test_loop_continue_and_break() { + assert_preinterpret_eq!( + { + [!set! #x = 0] + [!loop! { + [!assign! #x += 1] + [!if! #x >= 10 { [!break!] }] + }] + #x + }, + 10 + ); + assert_preinterpret_eq!( + { + [!string! [!for! #x in [!range! 65..75] { + [!if! #x % 2 == 0 { [!continue!] }] + [!evaluate! #x as u8 as char] + }]] + }, + "ACEGI" + ); +} From 03207a9674d69bae0600fc33c6207ba56619dc03 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 21 Jan 2025 09:37:33 +0000 Subject: [PATCH 047/126] fix: Fix msrv --- src/expressions/{char.rs => character.rs} | 0 src/expressions/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/expressions/{char.rs => character.rs} (100%) diff --git a/src/expressions/char.rs b/src/expressions/character.rs similarity index 100% rename from src/expressions/char.rs rename to src/expressions/character.rs diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 72a68655..ea2810ff 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -1,5 +1,5 @@ mod boolean; -mod char; +mod character; mod evaluation_tree; mod expression_stream; mod float; @@ -11,7 +11,7 @@ mod value; // Marked as use for expression sub-modules to use with a `use super::*` statement use crate::internal_prelude::*; use boolean::*; -use char::*; +use character::*; use evaluation_tree::*; use float::*; use integer::*; From 230ceb9f8da31b4c6e7c6d65fa4b042fb1f0abb8 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 21 Jan 2025 09:46:31 +0000 Subject: [PATCH 048/126] fix: Fix styling --- src/interpretation/interpreter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index e18568db..46c82bae 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -135,7 +135,7 @@ pub(crate) struct IterationCounter<'a, S: HasSpanRange> { iteration_limit: Option, } -impl<'a, S: HasSpanRange> IterationCounter<'a, S> { +impl IterationCounter<'_, S> { pub(crate) fn add_and_check(&mut self, count: usize) -> Result<()> { self.count = self.count.wrapping_add(count); self.check() From b87400913e5ca1a8d34aeb0b2f59f957375f8995 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 21 Jan 2025 10:45:06 +0000 Subject: [PATCH 049/126] tweak: Error has basic parse mode without fields --- CHANGELOG.md | 17 ++-- src/commands/concat_commands.rs | 89 +++++-------------- src/commands/core_commands.rs | 44 +++++++-- src/interpretation/command_field_inputs.rs | 14 ++- src/interpretation/interpreted_stream.rs | 41 +++++++++ src/traits.rs | 6 ++ .../complex/nested.stderr | 4 +- .../core/error_invalid_structure.rs | 5 ++ .../core/error_invalid_structure.stderr | 11 +++ .../core/error_no_fields.rs | 13 +++ .../core/error_no_fields.stderr | 15 ++++ 11 files changed, 174 insertions(+), 85 deletions(-) create mode 100644 tests/compilation_failures/core/error_invalid_structure.rs create mode 100644 tests/compilation_failures/core/error_invalid_structure.stderr create mode 100644 tests/compilation_failures/core/error_no_fields.rs create mode 100644 tests/compilation_failures/core/error_no_fields.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 315276da..ba425432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,13 +30,15 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = ### To come -* Accept `[!error! ]` as well * `[!split! { stream: X, separator: X, drop_empty?: false, drop_trailing_empty?: true, }]` and `[!comma_split! ...]` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Add more tests * e.g. for various expressions * e.g. for long sums +* Rework `Error` as: + * `ParseResult` with `ParseError::LowLevel(syn::Error)` | `ParseError::Contextual(syn::Error)` + * `ExecutionInterrupt` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` * Basic place parsing * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` * In parse land: #x matches a single token, #..x consumes the rest of a stream @@ -47,12 +49,15 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings * `[!OPTIONAL! ...]` and/or possibly `#(..)?` and `[!is_set! #x]`? * Groups or `[!GROUP! ...]` - * Regarding `#(..)+` and `#(..)*`... - * Any binding could be set to an array, but it gets complicated fast with nested bindings. - * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. - * `#x` binding reads a token tree + * `#x` binding reads a token tree and appends its contents into `#x` * `#..x` binding reads the rest of the stream... until the following raw token stream is detected (if at all). It must be followed by one or more raw tokens, or the end of the stream. + * `#+x` binding reads a token tree and appends it to `#x` as the full token tree + * `#..+x` reads a stream and appends it to `#x` + * `[!REPEATED! ...]` or `[!PUNCTUATED! ...]` also forbid `#x` bindings inside of them unless a `[!settings!]` has been overriden + * Regarding `#(..)+` and `#(..)*`... + * Any binding could be set to an array, but it gets complicated fast with nested bindings. + * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` * `[!match!]` (with `#..x` as a catch-all) * Check all `#[allow(unused)]` and remove any which aren't needed @@ -60,6 +65,8 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Fix comments in the expression files * Enable lazy && and || * Enable support for code blocks { .. } in expressions, and remove hacks where expression parsing stops at {} or . +* Make InterpretedStream an enum of either `Raw` or `Interpreted` including recursively (with `InterpretedTokenTree`) to fix rust-analyzer +* Push fork of syn::TokenBuffer to 0.4 to permit `[!parse_while! [!PARSE! ...] from #x { ... }]` * Work on book * Input paradigms: * Streams diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index f7f351f1..b8d018ce 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -4,35 +4,17 @@ use crate::internal_prelude::*; // Helpers //======== -fn string_literal(value: &str, span: Span) -> Literal { - let mut literal = Literal::string(value); - literal.set_span(span); - literal -} - -fn parse_literal(value: &str, span: Span) -> Result { - let mut literal = Literal::from_str(value) - .map_err(|err| span.error(format!("`{}` is not a valid literal: {:?}", value, err,)))?; - literal.set_span(span); - Ok(literal) -} - -fn parse_ident(value: &str, span: Span) -> Result { - let mut ident = parse_str::(value) - .map_err(|err| span.error(format!("`{}` is not a valid ident: {:?}", value, err,)))?; - ident.set_span(span); - Ok(ident) -} - fn concat_into_string( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, ) -> Result { let output_span = input.span(); - let concatenated = concat_recursive(input.interpret_to_new_stream(interpreter)?); - let string_literal = string_literal(&conversion_fn(&concatenated), output_span); - Ok(string_literal) + let concatenated = input + .interpret_to_new_stream(interpreter)? + .concat_recursive(); + let value = conversion_fn(&concatenated); + Ok(Literal::string(&value).with_span(output_span)) } fn concat_into_ident( @@ -41,8 +23,13 @@ fn concat_into_ident( conversion_fn: impl Fn(&str) -> String, ) -> Result { let output_span = input.span(); - let concatenated = concat_recursive(input.interpret_to_new_stream(interpreter)?); - let ident = parse_ident(&conversion_fn(&concatenated), output_span)?; + let concatenated = input + .interpret_to_new_stream(interpreter)? + .concat_recursive(); + let value = conversion_fn(&concatenated); + let ident = parse_str::(&value) + .map_err(|err| output_span.error(format!("`{}` is not a valid ident: {:?}", value, err,)))? + .with_span(output_span); Ok(ident) } @@ -52,52 +39,18 @@ fn concat_into_literal( conversion_fn: impl Fn(&str) -> String, ) -> Result { let output_span = input.span(); - let concatenated = concat_recursive(input.interpret_to_new_stream(interpreter)?); - let literal = parse_literal(&conversion_fn(&concatenated), output_span)?; + let concatenated = input + .interpret_to_new_stream(interpreter)? + .concat_recursive(); + let value = conversion_fn(&concatenated); + let literal = Literal::from_str(&value) + .map_err(|err| { + output_span.error(format!("`{}` is not a valid literal: {:?}", value, err,)) + })? + .with_span(output_span); Ok(literal) } -fn concat_recursive(arguments: InterpretedStream) -> String { - fn concat_recursive_internal(output: &mut String, arguments: TokenStream) { - for token_tree in arguments { - match token_tree { - TokenTree::Literal(literal) => match literal.content_if_string_like() { - Some(content) => output.push_str(&content), - None => output.push_str(&literal.to_string()), - }, - TokenTree::Group(group) => match group.delimiter() { - Delimiter::Parenthesis => { - output.push('('); - concat_recursive_internal(output, group.stream()); - output.push(')'); - } - Delimiter::Brace => { - output.push('{'); - concat_recursive_internal(output, group.stream()); - output.push('}'); - } - Delimiter::Bracket => { - output.push('['); - concat_recursive_internal(output, group.stream()); - output.push(']'); - } - Delimiter::None => { - concat_recursive_internal(output, group.stream()); - } - }, - TokenTree::Punct(punct) => { - output.push(punct.as_char()); - } - TokenTree::Ident(ident) => output.push_str(&ident.to_string()), - } - } - } - - let mut output = String::new(); - concat_recursive_internal(&mut output, arguments.into_token_stream()); - output -} - macro_rules! define_literal_concat_command { ( $command_name:literal => $command:ident: $output_fn:ident($conversion_fn:expr) diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 013b518c..2dc62f34 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -163,13 +163,19 @@ impl NoOutputCommandDefinition for SettingsCommand { #[derive(Clone)] pub(crate) struct ErrorCommand { - inputs: ErrorInputs, + inputs: EitherErrorInput, } impl CommandType for ErrorCommand { type OutputKind = OutputKindNone; } +#[derive(Clone)] +enum EitherErrorInput { + Fields(ErrorInputs), + JustMessage(InterpretationStream), +} + define_field_inputs! { ErrorInputs { required: { @@ -185,15 +191,41 @@ impl NoOutputCommandDefinition for ErrorCommand { const COMMAND_NAME: &'static str = "error"; fn parse(arguments: CommandArguments) -> Result { - Ok(Self { - inputs: arguments.fully_parse_as()?, - }) + arguments.fully_parse_or_error( + |input| { + if input.peek(syn::token::Brace) { + Ok(Self { + inputs: EitherErrorInput::Fields(input.parse()?), + }) + } else { + Ok(Self { + inputs: EitherErrorInput::JustMessage( + input.parse_with(arguments.full_span_range())?, + ), + }) + } + }, + format!( + "Expected [!error! \"Expected X, found: \" #world] or [!error! {}]", + ErrorInputs::fields_description() + ), + ) } fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { - let message = self.inputs.message.interpret(interpreter)?.value(); + let fields = match self.inputs { + EitherErrorInput::Fields(error_inputs) => error_inputs, + EitherErrorInput::JustMessage(stream) => { + let error_message = stream + .interpret_to_new_stream(interpreter)? + .concat_recursive(); + return Span::call_site().err(error_message); + } + }; + + let message = fields.message.interpret(interpreter)?.value(); - let error_span = match self.inputs.spans { + let error_span = match fields.spans { Some(spans) => { let error_span_stream = spans.interpret_to_new_stream(interpreter)?; diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index 3dafb66e..594c3974 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -96,9 +96,15 @@ macro_rules! define_field_inputs { impl ArgumentsContent for $inputs_type { fn error_message() -> String { + format!("Expected: {}", Self::fields_description()) + } + } + + impl $inputs_type { + fn fields_description() -> String { use std::fmt::Write; - let mut message = "Expected: {\n".to_string(); - let buffer = &mut message; + let mut buffer = String::new(); + buffer.write_str("{\n").unwrap(); $( $(writeln!(buffer, " // {}", $required_description).unwrap();)? writeln!(buffer, " {}: {},", stringify!($required_field), $required_example).unwrap(); @@ -107,8 +113,8 @@ macro_rules! define_field_inputs { $(writeln!(buffer, " // {}", $optional_description).unwrap();)? writeln!(buffer, " {}?: {},", stringify!($optional_field), $optional_example).unwrap(); )* - buffer.push('}'); - message + buffer.write_str("}").unwrap(); + buffer } } }; diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index b3471edc..2caa8dd9 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -92,6 +92,47 @@ impl InterpretedStream { pub(crate) fn append_cloned_into(&self, output: &mut InterpretedStream) { output.token_stream.extend(self.token_stream.clone()) } + + pub(crate) fn concat_recursive(self) -> String { + fn concat_recursive_internal(output: &mut String, token_stream: TokenStream) { + for token_tree in token_stream { + match token_tree { + TokenTree::Literal(literal) => match literal.content_if_string_like() { + Some(content) => output.push_str(&content), + None => output.push_str(&literal.to_string()), + }, + TokenTree::Group(group) => match group.delimiter() { + Delimiter::Parenthesis => { + output.push('('); + concat_recursive_internal(output, group.stream()); + output.push(')'); + } + Delimiter::Brace => { + output.push('{'); + concat_recursive_internal(output, group.stream()); + output.push('}'); + } + Delimiter::Bracket => { + output.push('['); + concat_recursive_internal(output, group.stream()); + output.push(']'); + } + Delimiter::None => { + concat_recursive_internal(output, group.stream()); + } + }, + TokenTree::Punct(punct) => { + output.push(punct.as_char()); + } + TokenTree::Ident(ident) => output.push_str(&ident.to_string()), + } + } + } + + let mut output = String::new(); + concat_recursive_internal(&mut output, self.into_token_stream()); + output + } } impl From for InterpretedStream { diff --git a/src/traits.rs b/src/traits.rs index d595f252..355d7329 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -2,12 +2,18 @@ use crate::internal_prelude::*; pub(crate) trait IdentExt: Sized { fn new_bool(value: bool, span: Span) -> Self; + fn with_span(self, span: Span) -> Self; } impl IdentExt for Ident { fn new_bool(value: bool, span: Span) -> Self { Ident::new(&value.to_string(), span) } + + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } } pub(crate) trait CursorExt: Sized { diff --git a/tests/compilation_failures/complex/nested.stderr b/tests/compilation_failures/complex/nested.stderr index 31fce6ce..05071229 100644 --- a/tests/compilation_failures/complex/nested.stderr +++ b/tests/compilation_failures/complex/nested.stderr @@ -1,10 +1,10 @@ error: required fields are missing: message - Occurred whilst parsing [!error! ..] - Expected: { + Occurred whilst parsing [!error! ..] - Expected [!error! "Expected X, found: " #world] or [!error! { // The error message to display message: "...", // An optional [token stream], to determine where to show the error message spans?: [$abc], - } + }] --> tests/compilation_failures/complex/nested.rs:7:26 | 7 | [!error! { diff --git a/tests/compilation_failures/core/error_invalid_structure.rs b/tests/compilation_failures/core/error_invalid_structure.rs new file mode 100644 index 00000000..f5b1aa27 --- /dev/null +++ b/tests/compilation_failures/core/error_invalid_structure.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!error! { }]); +} \ No newline at end of file diff --git a/tests/compilation_failures/core/error_invalid_structure.stderr b/tests/compilation_failures/core/error_invalid_structure.stderr new file mode 100644 index 00000000..468b1187 --- /dev/null +++ b/tests/compilation_failures/core/error_invalid_structure.stderr @@ -0,0 +1,11 @@ +error: required fields are missing: message + Occurred whilst parsing [!error! ..] - Expected [!error! "Expected X, found: " #world] or [!error! { + // The error message to display + message: "...", + // An optional [token stream], to determine where to show the error message + spans?: [$abc], + }] + --> tests/compilation_failures/core/error_invalid_structure.rs:4:28 + | +4 | preinterpret!([!error! { }]); + | ^^^ diff --git a/tests/compilation_failures/core/error_no_fields.rs b/tests/compilation_failures/core/error_no_fields.rs new file mode 100644 index 00000000..8c4aca90 --- /dev/null +++ b/tests/compilation_failures/core/error_no_fields.rs @@ -0,0 +1,13 @@ +use preinterpret::*; + +macro_rules! assert_literals_eq { + ($input1:literal, $input2:literal) => {preinterpret!{ + [!if! ($input1 != $input2) { + [!error! "Expected " $input1 " to equal " $input2] + }] + }}; +} + +fn main() { + assert_literals_eq!(102, 64); +} \ No newline at end of file diff --git a/tests/compilation_failures/core/error_no_fields.stderr b/tests/compilation_failures/core/error_no_fields.stderr new file mode 100644 index 00000000..332f8824 --- /dev/null +++ b/tests/compilation_failures/core/error_no_fields.stderr @@ -0,0 +1,15 @@ +error: Expected 102 to equal 64 + --> tests/compilation_failures/core/error_no_fields.rs:4:44 + | +4 | ($input1:literal, $input2:literal) => {preinterpret!{ + | ____________________________________________^ +5 | | [!if! ($input1 != $input2) { +6 | | [!error! "Expected " $input1 " to equal " $input2] +7 | | }] +8 | | }}; + | |_____^ +... +12 | assert_literals_eq!(102, 64); + | ---------------------------- in this macro invocation + | + = note: this error originates in the macro `preinterpret` which comes from the expansion of the macro `assert_literals_eq` (in Nightly builds, run with -Z macro-backtrace for more info) From 175de64b0b88f1f3dd2ce3b9073b4cfef45c2305 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 21 Jan 2025 12:18:52 +0000 Subject: [PATCH 050/126] tweak: Add comment in rust-analyzer --- src/interpretation/interpreted_stream.rs | 2 +- src/traits.rs | 4 ++-- tests/tokens.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 2caa8dd9..804ba6cf 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -4,7 +4,7 @@ use crate::internal_prelude::*; pub(crate) struct InterpretedStream { /// Currently, even ~inside~ macro executions, round-tripping [`Delimiter::None`] groups to a TokenStream /// breaks in rust-analyzer. This causes various spurious errors in the IDE. - /// See: https://github.com/dtolnay/syn/issues/1464 and https://github.com/rust-lang/rust-analyzer/issues/18211 + /// See: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 /// /// In future, we may wish to internally use some kind of `TokenBuffer` type, which I've started on below. token_stream: TokenStream, diff --git a/src/traits.rs b/src/traits.rs index 355d7329..20c353ea 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -92,8 +92,8 @@ pub(crate) trait TokenTreeExt: Sized { } impl TokenTreeExt for TokenTree { - fn group(inner_tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self { - TokenTree::Group(Group::new(delimeter, inner_tokens).with_span(span)) + fn group(inner_tokens: TokenStream, delimiter: Delimiter, span: Span) -> Self { + TokenTree::Group(Group::new(delimiter, inner_tokens).with_span(span)) } fn into_singleton_group(self, delimiter: Delimiter) -> Self { diff --git a/tests/tokens.rs b/tests/tokens.rs index b24d6556..0b58492a 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -238,9 +238,9 @@ fn complex_cases_for_intersperse_and_input_types() { // Flattened variable containing transparent group assert_preinterpret_eq!({ [!set! #items = 0 1 2 3] - [!set! #wrapped_items = #items] // [!GROUP! 0 1 2 3] + [!set! #wrapped_items = #items] // #items is "grouped variable" so outputs [!group! 0 1 2 3] [!string! [!intersperse! { - items: #..wrapped_items, // [!GROUP! 0 1 2 3] + items: #..wrapped_items, // #..wrapped_items returns its contents: [!group! 0 1 2 3] separator: [_], }]] }, "0_1_2_3"); @@ -249,7 +249,7 @@ fn complex_cases_for_intersperse_and_input_types() { [!set! #items = 0 1 2 3] [!string! [!intersperse! { items: { - #items + #items // #items is "grouped variable syntax" so outputs [!group! 0 1 2 3] }, separator: [_], }]] From 7b6460bcd82237f72bd5d5b7f684f611d0df3d00 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 21 Jan 2025 16:39:25 +0000 Subject: [PATCH 051/126] refactor: Keep interpreted token streams separate from TokenStream This aims to avoid https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 in most cases --- src/commands/control_flow_commands.rs | 8 +- src/commands/core_commands.rs | 7 +- src/commands/expression_commands.rs | 2 +- src/commands/token_commands.rs | 7 +- src/expressions/expression_stream.rs | 15 +- src/interpretation/command_stream_input.rs | 111 ++++---- src/interpretation/command_value_input.rs | 18 +- src/interpretation/interpret_traits.rs | 2 +- src/interpretation/interpretation_stream.rs | 2 +- src/interpretation/interpreted_stream.rs | 283 ++++++++++++++++---- src/lib.rs | 6 +- src/parsing/building_blocks.rs | 1 - src/parsing/mod.rs | 3 - src/parsing/parse_traits.rs | 10 +- 14 files changed, 327 insertions(+), 148 deletions(-) delete mode 100644 src/parsing/building_blocks.rs diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 03efd72e..02e0b4cf 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -225,12 +225,10 @@ impl ControlFlowCommandDefinition for ForCommand { let mut iteration_counter = interpreter.start_iteration_counter(&self.in_token); - for token in stream.into_token_stream() { + for token in stream.into_item_vec() { iteration_counter.increment_and_check()?; - self.parse_place.handle_parse_from_stream( - InterpretedStream::raw(token.into_token_stream()), - interpreter, - )?; + self.parse_place + .handle_parse_from_stream(token.into(), interpreter)?; match self .loop_code .clone() diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 2dc62f34..fc145d65 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -99,7 +99,7 @@ impl StreamCommandDefinition for RawCommand { _interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.extend_raw_token_iter(self.token_stream); + output.extend_raw_tokens(self.token_stream); Ok(()) } } @@ -255,9 +255,8 @@ impl NoOutputCommandDefinition for ErrorCommand { // transparent groups (as of Jan 2025), so gets it right without this flattening: // https://github.com/rust-lang/rust-analyzer/issues/18211 - let error_span_stream = error_span_stream - .into_token_stream() - .flatten_transparent_groups(); + let error_span_stream = + error_span_stream.into_token_stream_removing_any_transparent_groups(); if error_span_stream.is_empty() { Span::call_site().span_range() } else { diff --git a/src/commands/expression_commands.rs b/src/commands/expression_commands.rs index a5e612be..83d6b405 100644 --- a/src/commands/expression_commands.rs +++ b/src/commands/expression_commands.rs @@ -161,7 +161,7 @@ impl StreamCommandDefinition for RangeCommand { } fn output_range(iter: impl Iterator, span: Span, output: &mut InterpretedStream) { - output.extend_raw_token_iter(iter.map(|value| { + output.extend_raw_tokens(iter.map(|value| { let literal = Literal::i128_unsuffixed(value).with_span(span); TokenTree::Literal(literal) // We wrap it in a singleton group to ensure that negative diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index 1f2864ea..fb19f1c5 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -46,8 +46,7 @@ impl ValueCommandDefinition for LengthCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; - let stream_length = interpreted.into_token_stream().into_iter().count(); - let length_literal = Literal::usize_unsuffixed(stream_length).with_span(output_span); + let length_literal = Literal::usize_unsuffixed(interpreted.len()).with_span(output_span); Ok(length_literal.into()) } } @@ -121,7 +120,7 @@ impl StreamCommandDefinition for IntersperseCommand { .inputs .items .interpret_to_new_stream(interpreter)? - .into_token_stream(); + .into_item_vec(); let add_trailing = match self.inputs.add_trailing { Some(add_trailing) => add_trailing.interpret(interpreter)?.value(), None => false, @@ -140,7 +139,7 @@ impl StreamCommandDefinition for IntersperseCommand { let mut items = items.into_iter().peekable(); let mut this_item = items.next().unwrap(); // Safe to unwrap as non-empty loop { - output.push_raw_token_tree(this_item); + output.push_segment_item(this_item); let next_item = items.next(); match next_item { Some(next_item) => { diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 744d6049..6aabcdc6 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -219,17 +219,16 @@ impl ExpressionBuilder { appender: impl FnOnce(&mut InterpretedStream) -> Result<()>, span: Span, ) -> Result<()> { - // Currently using Expr::Parse, it ignores transparent groups, which is - // a little too permissive. - // Instead, we use parentheses to ensure that the group has to be a valid - // expression itself, without being flattened + // Currently using Expr::Parse, it ignores transparent groups, which is a little too permissive. + // Instead, we use parentheses to ensure that the group has to be a valid expression itself, without being flattened. + // This also works around the SAFETY issue in syn_parse below self.interpreted_stream .push_grouped(appender, Delimiter::Parenthesis, span) } pub(crate) fn extend_with_evaluation_output(&mut self, value: EvaluationOutput) { self.interpreted_stream - .extend_raw_tokens(value.into_token_tree()); + .extend_with_raw_tokens_from(value.into_token_tree()); } pub(crate) fn push_expression_group( @@ -254,7 +253,11 @@ impl ExpressionBuilder { // Because of the kind of expressions we're parsing (i.e. no {} allowed), // we can get by with parsing it as `Expr::parse` rather than with // `Expr::parse_without_eager_brace` or `Expr::parse_with_earlier_boundary_rule`. - let expression = Expr::parse.parse2(self.interpreted_stream.into_token_stream())?; + let expression = unsafe { + // RUST-ANALYZER SAFETY: We wrap commands and variables in `()` instead of none-delimited groups in expressions, + // so it doesn't matter that we can drop none-delimited groups + self.interpreted_stream.syn_parse(Expr::parse)? + }; EvaluationTree::build_from(&expression)?.evaluate() } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index bcadef58..d70a3b0b 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -62,17 +62,14 @@ impl Interpret for CommandStreamInput { | CommandOutputKind::Ident => { command.err("The command does not output a stream") } - CommandOutputKind::FlattenedStream => { - let span = command.span(); - let tokens = parse_as_stream_input( - command.interpret_to_new_stream(interpreter)?, - || { - span.error("Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.") - }, - )?; - output.extend_raw_tokens(tokens); - Ok(()) - } + CommandOutputKind::FlattenedStream => parse_as_stream_input( + command, + interpreter, + || { + "Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.".to_string() + }, + output, + ), CommandOutputKind::GroupedStream => { unsafe { // SAFETY: The kind change GroupedStream <=> FlattenedStream is valid @@ -80,46 +77,38 @@ impl Interpret for CommandStreamInput { } command.interpret_into(interpreter, output) } - CommandOutputKind::ControlFlowCodeStream => { - let span = command.span(); - let tokens = parse_as_stream_input( - command.interpret_to_new_stream(interpreter)?, - || { - span.error("Expected output of control flow command to contain a single [ ... ] or transparent group.") - }, - )?; - output.extend_raw_tokens(tokens); - Ok(()) - } + CommandOutputKind::ControlFlowCodeStream => parse_as_stream_input( + command, + interpreter, + || { + "Expected output of control flow command to contain a single [ ... ] or transparent group.".to_string() + }, + output, + ), } } - CommandStreamInput::FlattenedVariable(variable) => { - let tokens = parse_as_stream_input( - variable.interpret_to_new_stream(interpreter)?, - || { - variable.error(format!( + CommandStreamInput::FlattenedVariable(variable) => parse_as_stream_input( + &variable, + interpreter, + || { + format!( "Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", variable.display_grouped_variable_token(), - )) - }, - )?; - output.extend_raw_tokens(tokens); - Ok(()) - } + ) + }, + output, + ), CommandStreamInput::GroupedVariable(variable) => { variable.substitute_ungrouped_contents_into(interpreter, output) } - CommandStreamInput::Code(code) => { - let span = code.span(); - let tokens = parse_as_stream_input( - code.interpret_to_new_stream(interpreter)?, - || { - span.error("Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string()) - }, - )?; - output.extend_raw_tokens(tokens); - Ok(()) - } + CommandStreamInput::Code(code) => parse_as_stream_input( + code, + interpreter, + || { + "Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string() + }, + output, + ), CommandStreamInput::ExplicitStream(group) => { group.into_content().interpret_into(interpreter, output) } @@ -128,24 +117,18 @@ impl Interpret for CommandStreamInput { } fn parse_as_stream_input( - interpreted: InterpretedStream, - on_error: impl FnOnce() -> Error, -) -> Result { - fn get_group(interpreted: InterpretedStream) -> Option { - let mut token_iter = interpreted.into_token_stream().into_iter(); - let group = match token_iter.next()? { - TokenTree::Group(group) - if matches!(group.delimiter(), Delimiter::Bracket | Delimiter::None) => - { - Some(group) - } - _ => return None, - }; - if token_iter.next().is_some() { - return None; - } - group - } - let group = get_group(interpreted).ok_or_else(on_error)?; - Ok(group.stream()) + input: impl Interpret + HasSpanRange, + interpreter: &mut Interpreter, + error_message: impl FnOnce() -> String, + output: &mut InterpretedStream, +) -> Result<()> { + let span = input.span_range(); + input + .interpret_to_new_stream(interpreter)? + .unwrap_singleton_group( + |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None), + || span.error(error_message()), + )? + .append_into(output); + Ok(()) } diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 94072348..289c0d4a 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -60,13 +60,17 @@ impl, I: Parse> InterpretValue for Comma CommandValueInput::Code(code) => code.interpret_to_new_stream(interpreter)?, CommandValueInput::Value(value) => return value.interpret(interpreter), }; - match interpreted_stream.syn_parse(I::parse) { - Ok(value) => Ok(value), - Err(err) => Err(err.concat(&format!( - "\nOccurred whilst parsing the {} to a {}.", - descriptor, - std::any::type_name::() - ))), + unsafe { + // RUST-ANALYZER SAFETY: We only use I with simple parse functions so far which don't care about + // none-delimited groups + match interpreted_stream.syn_parse(I::parse) { + Ok(value) => Ok(value), + Err(err) => Err(err.concat(&format!( + "\nOccurred whilst parsing the {} to a {}.", + descriptor, + std::any::type_name::() + ))), + } } } } diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index ad199ee2..5a0c47f9 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -26,7 +26,7 @@ impl + HasSpanRange, I: ToTokens> Interp interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> Result<()> { - output.extend_raw_tokens(self.interpret(interpreter)?); + output.extend_with_raw_tokens_from(self.interpret(interpreter)?); Ok(()) } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index e274a4db..11ca6440 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -141,7 +141,7 @@ impl Express for RawGroup { ) -> Result<()> { expression_stream.push_grouped( |inner| { - inner.extend_raw_token_iter(self.content); + inner.extend_raw_tokens(self.content); Ok(()) }, self.source_delim_span.join(), diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 804ba6cf..303586bc 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -6,19 +6,48 @@ pub(crate) struct InterpretedStream { /// breaks in rust-analyzer. This causes various spurious errors in the IDE. /// See: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 /// + /// So instead of just storing a TokenStream here, we store a list of TokenStreams and interpreted groups. + /// This ensures any non-delimited groups are not round-tripped to a TokenStream. + /// /// In future, we may wish to internally use some kind of `TokenBuffer` type, which I've started on below. - token_stream: TokenStream, + segments: Vec, + token_length: usize, +} + +#[derive(Clone)] +// This was primarily implemented to avoid this issue: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 +// But it doesn't actually help because the `syn::parse` mechanism only operates on a TokenStream, +// so we have to convert back into a TokenStream. +enum InterpretedSegment { + TokenVec(Vec), // Cheaper than a TokenStream (probably) + InterpretedGroup(Delimiter, Span, InterpretedStream), +} + +pub(crate) enum InterpretedTokenTree { + TokenTree(TokenTree), + InterpretedGroup(Delimiter, Span, InterpretedStream), +} + +impl From for InterpretedStream { + fn from(value: InterpretedTokenTree) -> Self { + let mut new = Self::new(); + new.push_segment_item(value); + new + } } impl InterpretedStream { pub(crate) fn new() -> Self { Self { - token_stream: TokenStream::new(), + segments: vec![], + token_length: 0, } } pub(crate) fn raw(token_stream: TokenStream) -> Self { - Self { token_stream } + let mut new = Self::new(); + new.extend_raw_tokens(token_stream); + new } pub(crate) fn push_literal(&mut self, literal: Literal) { @@ -51,76 +80,236 @@ impl InterpretedStream { delimiter: Delimiter, span: Span, ) { - self.push_raw_token_tree(TokenTree::group(inner_tokens.token_stream, delimiter, span)); + self.segments.push(InterpretedSegment::InterpretedGroup( + delimiter, + span, + inner_tokens, + )); + self.token_length += 1; } - pub(crate) fn extend_raw_tokens(&mut self, tokens: impl ToTokens) { - tokens.to_tokens(&mut self.token_stream); + pub(crate) fn extend_with_raw_tokens_from(&mut self, tokens: impl ToTokens) { + self.extend_raw_tokens(tokens.into_token_stream()) } - pub(crate) fn extend_raw_token_iter(&mut self, tokens: impl IntoIterator) { - self.token_stream.extend(tokens); + pub(crate) fn push_segment_item(&mut self, segment_item: InterpretedTokenTree) { + match segment_item { + InterpretedTokenTree::TokenTree(token_tree) => { + self.push_raw_token_tree(token_tree); + } + InterpretedTokenTree::InterpretedGroup(delimiter, span, inner_tokens) => { + self.push_new_group(inner_tokens, delimiter, span); + } + } } pub(crate) fn push_raw_token_tree(&mut self, token_tree: TokenTree) { - self.token_stream.extend(iter::once(token_tree)); + self.extend_raw_tokens(iter::once(token_tree)); } - pub(crate) fn is_empty(&self) -> bool { - self.token_stream.is_empty() + pub(crate) fn extend_raw_tokens(&mut self, tokens: impl IntoIterator) { + if !matches!(self.segments.last(), Some(InterpretedSegment::TokenVec(_))) { + self.segments.push(InterpretedSegment::TokenVec(vec![])); + } + + match self.segments.last_mut() { + Some(InterpretedSegment::TokenVec(token_vec)) => { + let before_length = token_vec.len(); + token_vec.extend(tokens); + self.token_length += token_vec.len() - before_length; + } + _ => unreachable!(), + } } - #[allow(unused)] - pub(crate) fn parse_into_fields( - self, - parser: FieldsParseDefinition, - error_span_range: SpanRange, - ) -> Result { - self.syn_parse(parser.create_syn_parser(error_span_range)) + pub(crate) fn len(&self) -> usize { + self.token_length + } + + pub(crate) fn is_empty(&self) -> bool { + self.token_length == 0 } /// For a type `T` which implements `syn::Parse`, you can call this as `syn_parse(self, T::parse)`. /// For more complicated parsers, just pass the parsing function to this function. - pub(crate) fn syn_parse(self, parser: P) -> Result { - parser.parse2(self.token_stream) + /// + /// WARNING: With rust-analyzer, this loses transparent groups which have been inserted. + /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 + /// + /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. + pub(crate) unsafe fn syn_parse(self, parser: P) -> Result { + parser.parse2(self.into_token_stream()) } - pub(crate) fn into_token_stream(self) -> TokenStream { - self.token_stream + pub(crate) fn append_into(self, output: &mut InterpretedStream) { + output.segments.extend(self.segments); + output.token_length += self.token_length; } pub(crate) fn append_cloned_into(&self, output: &mut InterpretedStream) { - output.token_stream.extend(self.token_stream.clone()) + self.clone().append_into(output); + } + + /// WARNING: With rust-analyzer, this loses transparent groups which have been inserted. + /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 + /// + /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. + pub(crate) unsafe fn into_token_stream(self) -> TokenStream { + let mut output = TokenStream::new(); + self.append_to_token_stream(&mut output); + output + } + + unsafe fn append_to_token_stream(self, output: &mut TokenStream) { + for segment in self.segments { + match segment { + InterpretedSegment::TokenVec(vec) => { + output.extend(vec); + } + InterpretedSegment::InterpretedGroup(delimiter, span, inner) => { + output.extend(iter::once(TokenTree::Group( + Group::new(delimiter, inner.into_token_stream()).with_span(span), + ))) + } + } + } + } + + pub(crate) fn into_token_stream_removing_any_transparent_groups(self) -> TokenStream { + let mut output = TokenStream::new(); + self.append_to_token_stream_without_transparent_groups(&mut output); + output + } + + fn append_to_token_stream_without_transparent_groups(self, output: &mut TokenStream) { + for segment in self.segments { + match segment { + InterpretedSegment::TokenVec(vec) => { + for token in vec { + match token { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + output.extend(group.stream().flatten_transparent_groups()); + } + other => output.extend(iter::once(other)), + } + } + } + InterpretedSegment::InterpretedGroup(delimiter, span, interpreted_stream) => { + if delimiter == Delimiter::None { + interpreted_stream + .append_to_token_stream_without_transparent_groups(output); + } else { + let mut inner = TokenStream::new(); + interpreted_stream + .append_to_token_stream_without_transparent_groups(&mut inner); + output.extend(iter::once(TokenTree::Group( + Group::new(delimiter, inner).with_span(span), + ))); + } + } + } + } + } + + pub(crate) fn unwrap_singleton_group( + self, + check_group: impl FnOnce(Delimiter) -> bool, + create_error: impl FnOnce() -> Error, + ) -> Result { + let mut item_vec = self.into_item_vec(); + if item_vec.len() == 1 { + match item_vec.pop().unwrap() { + InterpretedTokenTree::InterpretedGroup(delimiter, _, inner) => { + if check_group(delimiter) { + return Ok(inner); + } + } + InterpretedTokenTree::TokenTree(TokenTree::Group(group)) => { + if check_group(group.delimiter()) { + return Ok(Self::raw(group.stream())); + } + } + _ => {} + } + } + Err(create_error()) + } + + pub(crate) fn into_item_vec(self) -> Vec { + let mut output = Vec::with_capacity(self.token_length); + for segment in self.segments { + match segment { + InterpretedSegment::TokenVec(vec) => { + output.extend(vec.into_iter().map(InterpretedTokenTree::TokenTree)); + } + InterpretedSegment::InterpretedGroup(delimiter, span, interpreted_stream) => { + output.push(InterpretedTokenTree::InterpretedGroup( + delimiter, + span, + interpreted_stream, + )); + } + } + } + output } pub(crate) fn concat_recursive(self) -> String { - fn concat_recursive_internal(output: &mut String, token_stream: TokenStream) { + fn wrap_delimiters( + output: &mut String, + delimiter: Delimiter, + inner: impl FnOnce(&mut String), + ) { + match delimiter { + Delimiter::Parenthesis => { + output.push('('); + inner(output); + output.push(')'); + } + Delimiter::Brace => { + output.push('{'); + inner(output); + output.push('}'); + } + Delimiter::Bracket => { + output.push('['); + inner(output); + output.push(']'); + } + Delimiter::None => { + inner(output); + } + } + } + + fn concat_recursive_interpreted_stream(output: &mut String, stream: InterpretedStream) { + for segment in stream.segments { + match segment { + InterpretedSegment::TokenVec(vec) => concat_recursive_token_stream(output, vec), + InterpretedSegment::InterpretedGroup(delimiter, _, interpreted_stream) => { + wrap_delimiters(output, delimiter, |output| { + concat_recursive_interpreted_stream(output, interpreted_stream); + }); + } + } + } + } + + fn concat_recursive_token_stream( + output: &mut String, + token_stream: impl IntoIterator, + ) { for token_tree in token_stream { match token_tree { TokenTree::Literal(literal) => match literal.content_if_string_like() { Some(content) => output.push_str(&content), None => output.push_str(&literal.to_string()), }, - TokenTree::Group(group) => match group.delimiter() { - Delimiter::Parenthesis => { - output.push('('); - concat_recursive_internal(output, group.stream()); - output.push(')'); - } - Delimiter::Brace => { - output.push('{'); - concat_recursive_internal(output, group.stream()); - output.push('}'); - } - Delimiter::Bracket => { - output.push('['); - concat_recursive_internal(output, group.stream()); - output.push(']'); - } - Delimiter::None => { - concat_recursive_internal(output, group.stream()); - } - }, + TokenTree::Group(group) => { + wrap_delimiters(output, group.delimiter(), |output| { + concat_recursive_token_stream(output, group.stream()); + }); + } TokenTree::Punct(punct) => { output.push(punct.as_char()); } @@ -130,16 +319,14 @@ impl InterpretedStream { } let mut output = String::new(); - concat_recursive_internal(&mut output, self.into_token_stream()); + concat_recursive_interpreted_stream(&mut output, self); output } } impl From for InterpretedStream { fn from(value: TokenTree) -> Self { - InterpretedStream { - token_stream: value.into(), - } + InterpretedStream::raw(value.into()) } } diff --git a/src/lib.rs b/src/lib.rs index 998378c8..adfb4a70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -549,7 +549,11 @@ fn preinterpret_internal(input: TokenStream) -> Result { let interpretation_stream = InterpretationStream::parse_from_token_stream(input, Span::call_site().span_range())?; let interpreted_stream = interpretation_stream.interpret_to_new_stream(&mut interpreter)?; - Ok(interpreted_stream.into_token_stream()) + unsafe { + // RUST-ANALYZER-SAFETY: This might drop transparent groups in the output of + // rust-analyzer. There's not much we can do here... + Ok(interpreted_stream.into_token_stream()) + } } // This is the recommended way to run the doc tests in the readme diff --git a/src/parsing/building_blocks.rs b/src/parsing/building_blocks.rs deleted file mode 100644 index db17e26e..00000000 --- a/src/parsing/building_blocks.rs +++ /dev/null @@ -1 +0,0 @@ -use crate::internal_prelude::*; diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs index f66d370f..52da7849 100644 --- a/src/parsing/mod.rs +++ b/src/parsing/mod.rs @@ -1,11 +1,8 @@ -#[allow(unused)] -mod building_blocks; mod fields; mod parse_place; mod parse_traits; #[allow(unused)] -pub(crate) use building_blocks::*; pub(crate) use fields::*; pub(crate) use parse_place::*; pub(crate) use parse_traits::*; diff --git a/src/parsing/parse_traits.rs b/src/parsing/parse_traits.rs index a15e5223..f24df362 100644 --- a/src/parsing/parse_traits.rs +++ b/src/parsing/parse_traits.rs @@ -6,8 +6,14 @@ pub(crate) trait HandleParse { input: InterpretedStream, interpreter: &mut Interpreter, ) -> Result<()> { - |input: ParseStream| -> Result<()> { self.handle_parse(input, interpreter) } - .parse2(input.into_token_stream()) + unsafe { + // RUST-ANALYZER-SAFETY: ...this isn't generally safe... + // We should only do this when we know that either the input or parser doesn't require + // analysis of nested None-delimited groups. + input.syn_parse(|input: ParseStream| -> Result<()> { + self.handle_parse(input, interpreter) + }) + } } fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; From 56e61e9daa9be7ab85ff7ee691956435244cb3cd Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 22 Jan 2025 18:23:09 +0000 Subject: [PATCH 052/126] feature: Add `let` and more destructurings --- CHANGELOG.md | 134 ++++--- src/commands/control_flow_commands.rs | 4 +- src/commands/core_commands.rs | 35 ++ src/destructuring/destructure_group.rs | 24 ++ src/destructuring/destructure_item.rs | 69 ++++ src/destructuring/destructure_segment.rs | 78 ++++ .../destructure_traits.rs} | 8 +- src/destructuring/destructure_variable.rs | 339 ++++++++++++++++++ src/{parsing => destructuring}/fields.rs | 0 src/destructuring/mod.rs | 14 + src/expressions/expression_stream.rs | 7 +- src/internal_prelude.rs | 4 +- src/interpretation/command.rs | 1 + src/interpretation/command_stream_input.rs | 6 +- src/interpretation/command_value_input.rs | 7 +- src/interpretation/interpretation_item.rs | 33 +- src/interpretation/interpreter.rs | 2 +- src/interpretation/variable.rs | 20 -- src/lib.rs | 2 +- src/parsing/mod.rs | 8 - src/parsing/parse_place.rs | 22 -- src/traits.rs | 70 ++++ tests/control_flow.rs | 8 + tests/core.rs | 48 +++ 24 files changed, 799 insertions(+), 144 deletions(-) create mode 100644 src/destructuring/destructure_group.rs create mode 100644 src/destructuring/destructure_item.rs create mode 100644 src/destructuring/destructure_segment.rs rename src/{parsing/parse_traits.rs => destructuring/destructure_traits.rs} (67%) create mode 100644 src/destructuring/destructure_variable.rs rename src/{parsing => destructuring}/fields.rs (100%) create mode 100644 src/destructuring/mod.rs delete mode 100644 src/parsing/mod.rs delete mode 100644 src/parsing/parse_place.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ba425432..2c801773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,19 +2,25 @@ ## 0.3.0 +### Variable Expansions + +* `#x` now outputs the contents of `x` in a transparent group. +* `#..x` outputs the contents of `x` "flattened" directly to the output stream. + ### New Commands * Core commands: - * `[!error! ...]` + * `[!error! ...]` to output a compile error * `[!extend! #x += ...]` to performantly add extra characters to the stream + * `[!let! = ...]` does destructuring/parsing (see next section) * Expression commands: - * `[!evaluate! ...]` - * `[!assign! #x += ...]` for `+` and other supported operators + * `[!evaluate! ]` + * `[!assign! #x += ]` for `+` and other supported operators * `[!range! 0..5]` outputs `0 1 2 3 4` * Control flow commands: - * `[!if! COND { ... }]` and `[!if! COND { ... } !else! { ... }]` - * `[!while! COND {}]` - * `[!for! #x in [ ... ] { ... }]` + * `[!if! { ... }]` and `[!if! { ... } !else! { ... }]` + * `[!while! { ... }]` + * `[!for! in [ ... ] { ... }]` * `[!loop! { ... }]` * `[!continue!]` * `[!break!]` @@ -24,14 +30,45 @@ * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. * `[!..group! ...]` which just outputs its contents as-is, useful where the grammar only takes a single item, but we want to output multiple tokens - * `[!intersperse! { .. }]` which inserts separator tokens between each token tree in a stream. + * `[!intersperse! { ... }]` which inserts separator tokens between each token tree in a stream. + +### Expressions + +Expressions can be evaluated with `[!evaluate! ...]` and are also used in the `if`, `for` and `while` loops. They operate on literals as values. + +Currently supported are: +* Integer, Float, Bool, String and Char literals +* The operators: `+ - * / % & | ^ || &&` +* The comparison operators: `== != < > <= >=` +* The shift operators: `>> <<` +* Casting with `as` including to untyped integers/floats with `as int` and `as float` +* () and none-delimited groups for precedence -I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = [!..group!]]`, but deemed it unhelpful, because then they have awkward edge-cases when embedding empty tokenstreams from declarative macros `($each_tt)*`. +Expressions behave intuitively as you'd expect from writing regular rust code, except they happen at compile time. + +### Destructuring + +Destructuring performs parsing of a token stream. It supports: + +* Explicit punctuation, idents, literals and groups +* Variable bindings: + * `#x` - Reads a token tree, writes a stream (opposite of #x) + * `#..x` - Reads a stream, writes a stream (opposite of #..x) + * `#>>x` - Reads a token tree, appends a token tree (opposite of for #y in #x) + * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) + * `#..>>x` - Reads a stream, appends a group (opposite of for #y in #x) + * `#..>>..x` - Reads a stream, appends a stream +* COMING SOON: Commands which don't output a value +* COMING SOON: Named destructurings which look like `(!name! ...)` ### To come -* `[!split! { stream: X, separator: X, drop_empty?: false, drop_trailing_empty?: true, }]` and `[!comma_split! ...]` +* `[!split! { stream: X, separator: X, drop_empty?: false, permit_trailing_separator?: true, }]` and `[!comma_split! ...]` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` + * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` +* `[!is_set! #x]` +* `[!debug!]` command to output an error with the current content of all variables. +* `[!str_split! { input: Value, separator: Value, }]` * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Add more tests * e.g. for various expressions @@ -39,34 +76,26 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * Rework `Error` as: * `ParseResult` with `ParseError::LowLevel(syn::Error)` | `ParseError::Contextual(syn::Error)` * `ExecutionInterrupt` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` -* Basic place parsing - * Introduce `[!let! #..x = Hello World]` does parsing and is equivalent to `[!set! #x = Hello World]` - * In parse land: #x matches a single token, #..x consumes the rest of a stream - * Auto-expand transparent groups, like syn. Maybe even using syn `TokenBuffer` / `Cursor`! - * Explicit raw Punct, Idents, Literals and Groups - * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` - * `[!PARSER! ]` method to take a stream - * `[!LITERAL! #x]` / `[!IDENT! #x]` bindings - * `[!OPTIONAL! ...]` and/or possibly `#(..)?` and `[!is_set! #x]`? - * Groups or `[!GROUP! ...]` - * `#x` binding reads a token tree and appends its contents into `#x` - * `#..x` binding reads the rest of the stream... until the following raw token stream is detected (if at all). It must be followed by one or more raw tokens, - or the end of the stream. - * `#+x` binding reads a token tree and appends it to `#x` as the full token tree - * `#..+x` reads a stream and appends it to `#x` - * `[!REPEATED! ...]` or `[!PUNCTUATED! ...]` also forbid `#x` bindings inside of them unless a `[!settings!]` has been overriden - * Regarding `#(..)+` and `#(..)*`... - * Any binding could be set to an array, but it gets complicated fast with nested bindings. - * Instead for now, we could push people towards capturing the input and parsing it with for loops and matches. - * `[!RAW!]` for e.g. `[!while_parse! [!RAW! from] from #X]` - * `[!match!]` (with `#..x` as a catch-all) +* Basic place destructuring + * `(!stream! ...)` method to take a group + * `(!literal! #x)` / `(!ident! #x)` bindings + * `(!optional! ...)` + * `(!group! ...)` + * Support commands, but only commands which don't output a value + * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden + * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` + * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much + * `(!raw! ...)` + * `(!match! ...)` (with `#..x` as a catch-all) * Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing, in order to: * Fix comments in the expression files * Enable lazy && and || * Enable support for code blocks { .. } in expressions, and remove hacks where expression parsing stops at {} or . -* Make InterpretedStream an enum of either `Raw` or `Interpreted` including recursively (with `InterpretedTokenTree`) to fix rust-analyzer -* Push fork of syn::TokenBuffer to 0.4 to permit `[!parse_while! [!PARSE! ...] from #x { ... }]` +* Pushed to 0.4 - fork of syn to: + * Fix issues in Rust Analyzer + * Improve performance + * Permit `[!parse_while! [!PARSE! ...] from #x { ... }]` * Work on book * Input paradigms: * Streams @@ -75,46 +104,7 @@ I considered disallowing commands like `[!set! #x =]` and requiring `[!set! #x = * There are three main kinds of commands: * Those taking a stream as-is * Those taking some { fields } - * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` - -### Side notes on how to build !for! - -* Support `!for!` so we can use it for simple generation scenarios without needing macros at all: - * Complexities: - * Parsing `,` - * When parsing `Punctuated` in syn, it typically needs to know how to parse a full X (e.g. X of an item) - * But ideally we want to defer the parsing of the next level... - * This means we may need to instead do something naive for now, e.g. split on `,` in the token stream - * Iterating over already parsed structure; e.g. if we parse a struct and want to iterate over the contents of the path of the type of the first field? - * Do we store such structured variables? - * Option 1 - Simple. For consumes 1 token tree per iteration. - * This means that we may need to pre-process the stream... - * Examples: - * `[!for! #x in [Hello World] {}]` - * Questions: - * How does this extend to parsing scenarios, such as a punctuated `,`? - * It doesn't explicitly... - * But `[!for! #x in [!split! [Hello, World,] on ,]]` could work, if `[!split!]` outputs transparent groups, and discards empty final items - * `[!comma_split! Hello, World,]` - * `[!split! { input: [Hello, World,], separator: [,], ignore_empty_at_end?: true, require_trailing_separator?: false, }]` - * How does this handle zipping / unzipping and un-grouping, and/or parsing a more complicated group? - * Proposal: The parsing operates over the content of a single token tree, possibly via explicitly ungrouping. - * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` - * To get this to work, we'd need to: - * Make it so that the `#x` binding consumes a single token tree, and auto-expands zero or more transparent group/s - * (Incidentally this is also how the syn Cursor type iteration works, roughly) - * And possibly have `#..x` consumes the remainder of the stream?? - * Make it so that `[!set! #x = ...]` wraps the `...` in a transparent group so it's consistent. - * And we can support basic parsing, via: - * Various groupings or `[!GROUP!]` for a transparent group - * Consuming various explicit tokens - * `[!OPTIONAL! ...]` - * Option 2 - we specify some stream-processor around each value - * `[!for! [!EACH! #x] in [Hello World]]` - * `[!for! [!SPLIT! #x,] in [Hello, World,]]` - * Even though this might be more performant, I'm not too much of a fan of this, as it's hard to understand - * Perhaps we can leave it to the `[!while_parse! #x[!OPTIONAL! ,] from #X]` style commands? - + * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` etc # Major Version 0.2 diff --git a/src/commands/control_flow_commands.rs b/src/commands/control_flow_commands.rs index 02e0b4cf..1fc8e60b 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/commands/control_flow_commands.rs @@ -188,7 +188,7 @@ impl ControlFlowCommandDefinition for LoopCommand { #[derive(Clone)] pub(crate) struct ForCommand { - parse_place: ParsePlace, + parse_place: DestructureUntil, #[allow(unused)] in_token: Token![in], input: CommandStreamInput, @@ -228,7 +228,7 @@ impl ControlFlowCommandDefinition for ForCommand { for token in stream.into_item_vec() { iteration_counter.increment_and_check()?; self.parse_place - .handle_parse_from_stream(token.into(), interpreter)?; + .handle_destructure_from_stream(token.into(), interpreter)?; match self .loop_code .clone() diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index fc145d65..75dfc263 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -38,6 +38,41 @@ impl NoOutputCommandDefinition for SetCommand { } } +#[derive(Clone)] +pub(crate) struct LetCommand { + destructuring: DestructureUntil, + #[allow(unused)] + equals: Token![=], + arguments: InterpretationStream, +} + +impl CommandType for LetCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for LetCommand { + const COMMAND_NAME: &'static str = "let"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + destructuring: input.parse()?, + equals: input.parse()?, + arguments: input.parse_with(arguments.full_span_range())?, + }) + }, + "Expected [!let! = ..]", + ) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; + self.destructuring + .handle_destructure_from_stream(result_tokens, interpreter) + } +} + #[derive(Clone)] pub(crate) struct ExtendCommand { variable: GroupedVariable, diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs new file mode 100644 index 00000000..67b848bd --- /dev/null +++ b/src/destructuring/destructure_group.rs @@ -0,0 +1,24 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct DestructureGroup { + delimiter: Delimiter, + inner: DestructureRemaining, +} + +impl Parse for DestructureGroup { + fn parse(input: ParseStream) -> Result { + let (delimiter, _, content) = input.parse_any_delimiter()?; + Ok(Self { + delimiter, + inner: content.parse()?, + }) + } +} + +impl HandleDestructure for DestructureGroup { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + let (_, inner) = input.parse_group_matching(self.delimiter)?; + self.inner.handle_destructure(&inner, interpreter) + } +} diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs new file mode 100644 index 00000000..4eda6d58 --- /dev/null +++ b/src/destructuring/destructure_item.rs @@ -0,0 +1,69 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) enum DestructureItem { + Variable(DestructureVariable), + ExactPunct(Punct), + ExactIdent(Ident), + ExactLiteral(Literal), + ExactGroup(DestructureGroup), +} + +impl DestructureItem { + /// We provide a stop condition so that some of the items can know when to stop consuming greedily - + /// notably the flattened command. This allows [!let! #..x = Hello => World] to parse as setting + /// `x` to `Hello => World` rather than having `#..x` peeking to see it is "up to =" and then only + /// parsing `Hello` into `x`. + pub(crate) fn parse_until(input: ParseStream) -> Result { + Ok(match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::GroupedCommand => { + return input + .span() + .err("Grouped commands are not currently supported in destructuring positions") + } + PeekMatch::FlattenedCommand => { + return input.span().err( + "Flattened commands are not currently supported in destructuring positions", + ) + } + PeekMatch::GroupedVariable + | PeekMatch::FlattenedVariable + | PeekMatch::AppendVariableDestructuring => { + Self::Variable(DestructureVariable::parse_until::(input)?) + } + PeekMatch::Group(_) => Self::ExactGroup(input.parse()?), + PeekMatch::NamedDestructuring => todo!(), + PeekMatch::Other => match input.parse::()? { + TokenTree::Group(_) => { + unreachable!("Should have been already handled by InterpretationGroup above") + } + TokenTree::Punct(punct) => Self::ExactPunct(punct), + TokenTree::Ident(ident) => Self::ExactIdent(ident), + TokenTree::Literal(literal) => Self::ExactLiteral(literal), + }, + }) + } +} + +impl HandleDestructure for DestructureItem { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + match self { + DestructureItem::Variable(variable) => { + variable.handle_destructure(input, interpreter)?; + } + DestructureItem::ExactPunct(punct) => { + input.parse_punct_matching(punct.as_char())?; + } + DestructureItem::ExactIdent(ident) => { + input.parse_ident_matching(&ident.to_string())?; + } + DestructureItem::ExactLiteral(literal) => { + input.parse_literal_matching(&literal.to_string())?; + } + DestructureItem::ExactGroup(group) => { + group.handle_destructure(input, interpreter)?; + } + } + Ok(()) + } +} diff --git a/src/destructuring/destructure_segment.rs b/src/destructuring/destructure_segment.rs new file mode 100644 index 00000000..984d919e --- /dev/null +++ b/src/destructuring/destructure_segment.rs @@ -0,0 +1,78 @@ +use crate::internal_prelude::*; + +pub(crate) trait StopCondition: Clone { + fn should_stop(input: ParseStream) -> bool; +} + +#[derive(Clone)] +pub(crate) struct UntilEmpty; +impl StopCondition for UntilEmpty { + fn should_stop(input: ParseStream) -> bool { + input.is_empty() + } +} + +pub(crate) trait PeekableToken: Clone { + fn peek(input: ParseStream) -> bool; +} + +// This is going through such pain to ensure we stay in the public API of syn +// We'd like to be able to use `input.peek::()` for some syn::token::Token +// but that's just not an API they support for some reason +macro_rules! impl_peekable_token { + ($(Token![$token:tt]),* $(,)?) => { + $( + impl PeekableToken for Token![$token] { + fn peek(input: ParseStream) -> bool { + input.peek(Token![$token]) + } + } + )* + }; +} + +impl_peekable_token! { + Token![=], + Token![in], +} + +#[derive(Clone)] +pub(crate) struct UntilToken { + token: PhantomData, +} +impl StopCondition for UntilToken { + fn should_stop(input: ParseStream) -> bool { + input.is_empty() || T::peek(input) + } +} + +pub(crate) type DestructureRemaining = DestructureSegment; +pub(crate) type DestructureUntil = DestructureSegment>; + +#[derive(Clone)] +pub(crate) struct DestructureSegment { + stop_condition: PhantomData, + inner: Vec, +} + +impl Parse for DestructureSegment { + fn parse(input: ParseStream) -> Result { + let mut inner = vec![]; + while !C::should_stop(input) { + inner.push(DestructureItem::parse_until::(input)?); + } + Ok(Self { + stop_condition: PhantomData, + inner, + }) + } +} + +impl HandleDestructure for DestructureSegment { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + for item in self.inner.iter() { + item.handle_destructure(input, interpreter)?; + } + Ok(()) + } +} diff --git a/src/parsing/parse_traits.rs b/src/destructuring/destructure_traits.rs similarity index 67% rename from src/parsing/parse_traits.rs rename to src/destructuring/destructure_traits.rs index f24df362..54e4ac8e 100644 --- a/src/parsing/parse_traits.rs +++ b/src/destructuring/destructure_traits.rs @@ -1,7 +1,7 @@ use crate::internal_prelude::*; -pub(crate) trait HandleParse { - fn handle_parse_from_stream( +pub(crate) trait HandleDestructure { + fn handle_destructure_from_stream( &self, input: InterpretedStream, interpreter: &mut Interpreter, @@ -11,10 +11,10 @@ pub(crate) trait HandleParse { // We should only do this when we know that either the input or parser doesn't require // analysis of nested None-delimited groups. input.syn_parse(|input: ParseStream| -> Result<()> { - self.handle_parse(input, interpreter) + self.handle_destructure(input, interpreter) }) } } - fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; } diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs new file mode 100644 index 00000000..92ce2419 --- /dev/null +++ b/src/destructuring/destructure_variable.rs @@ -0,0 +1,339 @@ +use crate::internal_prelude::*; + +/// We have the following write modes: +/// * `#x` - Reads a token tree, writes a stream (opposite of #x) +/// * `#..x` - Reads a stream, writes a stream (opposite of #..x) +/// +/// And the following append modes: +/// * `#>>x` - Reads a token tree, appends a token tree (opposite of for #y in #x) +/// * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) +/// * `#..>>x` - Reads a stream, appends a group (opposite of for #y in #x) +/// * `#..>>..x` - Reads a stream, appends a stream +#[derive(Clone)] +#[allow(unused)] +pub(crate) enum DestructureVariable { + /// #x - Reads a token tree, writes a stream (opposite of #x) + Grouped { marker: Token![#], name: Ident }, + /// #..x - Reads a stream, writes a stream (opposite of #..x) + Flattened { + marker: Token![#], + flatten: Token![..], + name: Ident, + until: ParseUntil, + }, + /// #>>x - Reads a token tree, appends the token tree (allows reading with !for! #y in #x) + GroupedAppendGrouped { + marker: Token![#], + append: Token![>>], + name: Ident, + }, + /// #>>..x - Reads a token tree, appends it flattened if its a group + GroupedAppendFlattened { + marker: Token![#], + append: Token![>>], + flattened_write: Token![..], + name: Ident, + }, + /// #..>>x - Reads a stream, appends a group (allows reading with !for! #y in #x) + FlattenedAppendGrouped { + marker: Token![#], + flattened_read: Token![..], + append: Token![>>], + name: Ident, + until: ParseUntil, + }, + /// #..>>..x - Reads a stream, appends a stream + FlattenedAppendFlattened { + marker: Token![#], + flattened_read: Token![..], + append: Token![>>], + flattened_write: Token![..], + name: Ident, + until: ParseUntil, + }, +} + +impl DestructureVariable { + pub(crate) fn parse_until(input: ParseStream) -> Result { + let marker = input.parse()?; + if input.peek(Token![..]) { + let flatten = input.parse()?; + if input.peek(Token![>>]) { + let append = input.parse()?; + if input.peek(Token![..]) { + let flattened_write = input.parse()?; + let name = input.parse()?; + let until = ParseUntil::peek_flatten_limit::(input)?; + return Ok(Self::FlattenedAppendFlattened { + marker, + flattened_read: flatten, + append, + flattened_write, + name, + until, + }); + } + let name = input.parse()?; + let until = ParseUntil::peek_flatten_limit::(input)?; + return Ok(Self::FlattenedAppendGrouped { + marker, + flattened_read: flatten, + append, + name, + until, + }); + } + let name = input.parse()?; + let until = ParseUntil::peek_flatten_limit::(input)?; + return Ok(Self::Flattened { + marker, + flatten, + name, + until, + }); + } + if input.peek(Token![>>]) { + let append = input.parse()?; + if input.peek(Token![..]) { + let flattened_write = input.parse()?; + let name = input.parse()?; + return Ok(Self::GroupedAppendFlattened { + marker, + append, + flattened_write, + name, + }); + } + let name = input.parse()?; + return Ok(Self::GroupedAppendGrouped { + marker, + append, + name, + }); + } + let name = input.parse()?; + Ok(Self::Grouped { marker, name }) + } +} + +impl IsVariable for DestructureVariable { + fn get_name(&self) -> String { + let name_ident = match self { + DestructureVariable::Grouped { name, .. } => name, + DestructureVariable::Flattened { name, .. } => name, + DestructureVariable::GroupedAppendGrouped { name, .. } => name, + DestructureVariable::GroupedAppendFlattened { name, .. } => name, + DestructureVariable::FlattenedAppendGrouped { name, .. } => name, + DestructureVariable::FlattenedAppendFlattened { name, .. } => name, + }; + name_ident.to_string() + } +} + +impl HasSpanRange for DestructureVariable { + fn span_range(&self) -> SpanRange { + let (marker, name) = match self { + DestructureVariable::Grouped { marker, name, .. } => (marker, name), + DestructureVariable::Flattened { marker, name, .. } => (marker, name), + DestructureVariable::GroupedAppendGrouped { marker, name, .. } => (marker, name), + DestructureVariable::GroupedAppendFlattened { marker, name, .. } => (marker, name), + DestructureVariable::FlattenedAppendGrouped { marker, name, .. } => (marker, name), + DestructureVariable::FlattenedAppendFlattened { marker, name, .. } => (marker, name), + }; + SpanRange::new_between(marker.span, name.span()) + } +} + +impl DestructureVariable { + fn get_variable_data(&self, interpreter: &mut Interpreter) -> Result { + let variable_data = interpreter + .get_existing_variable_data(self, || { + self.error(format!( + "The variable #{} wasn't already set", + self.get_name() + )) + })? + .cheap_clone(); + Ok(variable_data) + } +} + +impl HandleDestructure for DestructureVariable { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + match self { + DestructureVariable::Grouped { .. } => { + let content = input.parse::()?.into_interpreted(); + interpreter.set_variable(self, content)?; + } + DestructureVariable::Flattened { until, .. } => { + let mut content = InterpretedStream::new(); + until.handle_parse_into(input, &mut content)?; + interpreter.set_variable(self, content)?; + } + DestructureVariable::GroupedAppendGrouped { .. } => { + let variable_data = self.get_variable_data(interpreter)?; + input + .parse::()? + .push_as_token_tree(variable_data.get_mut(self)?.deref_mut()); + } + DestructureVariable::GroupedAppendFlattened { .. } => { + let variable_data = self.get_variable_data(interpreter)?; + input + .parse::()? + .flatten_into(variable_data.get_mut(self)?.deref_mut()); + } + DestructureVariable::FlattenedAppendGrouped { marker, until, .. } => { + let variable_data = self.get_variable_data(interpreter)?; + variable_data.get_mut(self)?.push_grouped( + |inner| until.handle_parse_into(input, inner), + Delimiter::None, + marker.span, + )?; + } + DestructureVariable::FlattenedAppendFlattened { until, .. } => { + let variable_data = self.get_variable_data(interpreter)?; + until.handle_parse_into(input, variable_data.get_mut(self)?.deref_mut())?; + } + } + Ok(()) + } +} + +enum ParsedTokenTree { + NoneGroup(Group), + Ident(Ident), + Punct(Punct), + Literal(Literal), +} + +impl ParsedTokenTree { + fn into_interpreted(self) -> InterpretedStream { + match self { + ParsedTokenTree::NoneGroup(group) => InterpretedStream::raw(group.stream()), + ParsedTokenTree::Ident(ident) => InterpretedStream::raw(ident.to_token_stream()), + ParsedTokenTree::Punct(punct) => InterpretedStream::raw(punct.to_token_stream()), + ParsedTokenTree::Literal(literal) => InterpretedStream::raw(literal.to_token_stream()), + } + } + + fn push_as_token_tree(self, output: &mut InterpretedStream) { + match self { + ParsedTokenTree::NoneGroup(group) => { + output.push_raw_token_tree(TokenTree::Group(group)) + } + ParsedTokenTree::Ident(ident) => output.push_ident(ident), + ParsedTokenTree::Punct(punct) => output.push_punct(punct), + ParsedTokenTree::Literal(literal) => output.push_literal(literal), + } + } + + fn flatten_into(self, output: &mut InterpretedStream) { + match self { + ParsedTokenTree::NoneGroup(group) => output.extend_raw_tokens(group.stream()), + ParsedTokenTree::Ident(ident) => output.push_ident(ident), + ParsedTokenTree::Punct(punct) => output.push_punct(punct), + ParsedTokenTree::Literal(literal) => output.push_literal(literal), + } + } +} + +impl Parse for ParsedTokenTree { + fn parse(input: ParseStream) -> Result { + Ok(match input.parse::()? { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + ParsedTokenTree::NoneGroup(group) + } + TokenTree::Group(group) => { + return group + .delim_span() + .open() + .err("Expected a group with transparent delimiters"); + } + TokenTree::Ident(ident) => ParsedTokenTree::Ident(ident), + TokenTree::Punct(punct) => ParsedTokenTree::Punct(punct), + TokenTree::Literal(literal) => ParsedTokenTree::Literal(literal), + }) + } +} + +#[derive(Clone)] +pub(crate) enum ParseUntil { + End, + Group(Delimiter), + Ident(Ident), + Punct(Punct), + Literal(Literal), +} + +impl ParseUntil { + /// Peeks the next token, to discover what we should parse next + fn peek_flatten_limit(input: ParseStream) -> Result { + if C::should_stop(input) { + return Ok(ParseUntil::End); + } + Ok(match detect_preinterpret_grammar(input.cursor()) { + PeekMatch::GroupedCommand + | PeekMatch::FlattenedCommand + | PeekMatch::GroupedVariable + | PeekMatch::FlattenedVariable + | PeekMatch::NamedDestructuring + | PeekMatch::AppendVariableDestructuring => { + return input + .span() + .err("This cannot follow a flattened destructure match"); + } + PeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), + PeekMatch::Other => match input.cursor().token_tree() { + Some((token_tree, _)) => match token_tree { + TokenTree::Group(group) => ParseUntil::Group(group.delimiter()), + TokenTree::Ident(ident) => ParseUntil::Ident(ident), + TokenTree::Punct(punct) => ParseUntil::Punct(punct), + TokenTree::Literal(literal) => ParseUntil::Literal(literal), + }, + None => ParseUntil::End, + }, + }) + } + + fn handle_parse_into(&self, input: ParseStream, output: &mut InterpretedStream) -> Result<()> { + match self { + ParseUntil::End => output.extend_raw_tokens(input.parse::()?), + ParseUntil::Group(delimiter) => { + while !input.is_empty() { + if input.peek_group_matching(*delimiter) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + ParseUntil::Ident(ident) => { + let content = ident.to_string(); + while !input.is_empty() { + if input.peek_ident_matching(&content) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + ParseUntil::Punct(punct) => { + let punct = punct.as_char(); + while !input.is_empty() { + if input.peek_punct_matching(punct) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + ParseUntil::Literal(literal) => { + let content = literal.to_string(); + while !input.is_empty() { + if input.peek_literal_matching(&content) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + } + Ok(()) + } +} diff --git a/src/parsing/fields.rs b/src/destructuring/fields.rs similarity index 100% rename from src/parsing/fields.rs rename to src/destructuring/fields.rs diff --git a/src/destructuring/mod.rs b/src/destructuring/mod.rs new file mode 100644 index 00000000..22e20fc3 --- /dev/null +++ b/src/destructuring/mod.rs @@ -0,0 +1,14 @@ +mod destructure_group; +mod destructure_item; +mod destructure_segment; +mod destructure_traits; +mod destructure_variable; +mod fields; + +pub(crate) use destructure_group::*; +pub(crate) use destructure_item::*; +pub(crate) use destructure_segment::*; +pub(crate) use destructure_traits::*; +pub(crate) use destructure_variable::*; +#[allow(unused)] +pub(crate) use fields::*; diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 6aabcdc6..531f7987 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -39,9 +39,10 @@ impl Parse for ExpressionInput { PeekMatch::FlattenedCommand => ExpressionItem::Command(input.parse()?), PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), - PeekMatch::InterpretationGroup(Delimiter::Brace | Delimiter::Bracket) => break, - PeekMatch::InterpretationGroup(_) => { - ExpressionItem::ExpressionGroup(input.parse()?) + PeekMatch::Group(Delimiter::Brace | Delimiter::Bracket) => break, + PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse()?), + PeekMatch::NamedDestructuring | PeekMatch::AppendVariableDestructuring => { + return Err(input.error("Destructuring is not supported in an expression")); } PeekMatch::Other => { if input.cursor().punct_matching('.').is_some() { diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 1ef1f529..2815ec8d 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -1,4 +1,6 @@ pub(crate) use core::iter; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::ops::DerefMut; pub(crate) use proc_macro2::extra::*; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; @@ -11,8 +13,8 @@ pub(crate) use syn::{ }; pub(crate) use crate::commands::*; +pub(crate) use crate::destructuring::*; pub(crate) use crate::expressions::*; pub(crate) use crate::interpretation::*; -pub(crate) use crate::parsing::*; pub(crate) use crate::string_conversion::*; pub(crate) use crate::traits::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 62f90f7d..6890e42e 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -402,6 +402,7 @@ macro_rules! define_command_kind { define_command_kind! { // Core Commands SetCommand, + LetCommand, ExtendCommand, RawCommand, IgnoreCommand, diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index d70a3b0b..e0942fc6 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -28,9 +28,9 @@ impl Parse for CommandStreamInput { PeekMatch::FlattenedCommand => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::InterpretationGroup(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), - PeekMatch::InterpretationGroup(Delimiter::Brace) => Self::Code(input.parse()?), - PeekMatch::InterpretationGroup(_) | PeekMatch::Other => input.span() + PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), + PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + PeekMatch::Group(_) | PeekMatch::AppendVariableDestructuring | PeekMatch::NamedDestructuring | PeekMatch::Other => input.span() .err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) } diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 289c0d4a..8591d0b0 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -20,8 +20,11 @@ impl Parse for CommandValueInput { PeekMatch::FlattenedCommand => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::InterpretationGroup(Delimiter::Brace) => Self::Code(input.parse()?), - PeekMatch::InterpretationGroup(_) | PeekMatch::Other => Self::Value(input.parse()?), + PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + PeekMatch::AppendVariableDestructuring | PeekMatch::NamedDestructuring => { + return input.span().err("Destructurings are not supported here") + } + PeekMatch::Group(_) | PeekMatch::Other => Self::Value(input.parse()?), }) } } diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 0d1f09b2..08449595 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -16,11 +16,12 @@ impl Parse for InterpretationItem { Ok(match detect_preinterpret_grammar(input.cursor()) { PeekMatch::GroupedCommand => InterpretationItem::Command(input.parse()?), PeekMatch::FlattenedCommand => InterpretationItem::Command(input.parse()?), - PeekMatch::InterpretationGroup(_) => { - InterpretationItem::InterpretationGroup(input.parse()?) - } + PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), + PeekMatch::AppendVariableDestructuring | PeekMatch::NamedDestructuring => { + return input.span().err("Destructurings are not supported here") + } PeekMatch::Other => match input.parse::()? { TokenTree::Group(_) => { unreachable!("Should have been already handled by InterpretationGroup above") @@ -38,7 +39,9 @@ pub(crate) enum PeekMatch { FlattenedCommand, GroupedVariable, FlattenedVariable, - InterpretationGroup(Delimiter), + AppendVariableDestructuring, + NamedDestructuring, + Group(Delimiter), Other, } @@ -64,6 +67,15 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa } } } + if delimiter == Delimiter::Parenthesis { + if let Some((_, next)) = next.punct_matching('!') { + if let Some((_, next)) = next.ident() { + if next.punct_matching('!').is_some() { + return PeekMatch::NamedDestructuring; + } + } + } + } // Ideally we'd like to detect $($tt)* substitutions from macros and interpret them as // a Raw (uninterpreted) group, because typically that's what a user would typically intend. @@ -76,7 +88,7 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa // So this isn't possible. It's unlikely to matter much, and a user can always do: // [!raw! $($tt)*] anyway. - return PeekMatch::InterpretationGroup(delimiter); + return PeekMatch::Group(delimiter); } if let Some((_, next)) = cursor.punct_matching('#') { if next.ident().is_some() { @@ -87,9 +99,20 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa if next.ident().is_some() { return PeekMatch::FlattenedVariable; } + if let Some((_, next)) = next.punct_matching('>') { + if next.punct_matching('>').is_some() { + return PeekMatch::AppendVariableDestructuring; + } + } + } + } + if let Some((_, next)) = next.punct_matching('>') { + if next.punct_matching('>').is_some() { + return PeekMatch::AppendVariableDestructuring; } } } + // This is rather annoying for our purposes, but the Cursor (and even `impl Parse on Punct`) // treats `'` specially, and there's no way to peek a ' token which isn't part of a lifetime. // Instead of dividing up specific cases here, we let the caller handle it by parsing as a diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 46c82bae..a15043d0 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -71,7 +71,7 @@ impl Interpreter { } } - pub(super) fn set_variable( + pub(crate) fn set_variable( &mut self, variable: &impl IsVariable, tokens: InterpretedStream, diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 3cc3cb2a..e2d1b0c8 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -109,26 +109,6 @@ impl Express for &GroupedVariable { } } -impl HandleParse for GroupedVariable { - fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - let variable_contents = match input.parse::()? { - TokenTree::Group(group) if group.delimiter() == Delimiter::None => { - InterpretedStream::raw(group.stream()) - } - TokenTree::Group(group) => { - return group - .delim_span() - .open() - .err("Expected a group with transparent delimiters"); - } - TokenTree::Ident(ident) => InterpretedStream::raw(ident.to_token_stream()), - TokenTree::Punct(punct) => InterpretedStream::raw(punct.to_token_stream()), - TokenTree::Literal(literal) => InterpretedStream::raw(literal.to_token_stream()), - }; - self.set(interpreter, variable_contents) - } -} - impl HasSpanRange for GroupedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) diff --git a/src/lib.rs b/src/lib.rs index adfb4a70..e8e9b983 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -511,10 +511,10 @@ //! Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in this crate by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. //! mod commands; +mod destructuring; mod expressions; mod internal_prelude; mod interpretation; -mod parsing; mod string_conversion; mod traits; diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs deleted file mode 100644 index 52da7849..00000000 --- a/src/parsing/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod fields; -mod parse_place; -mod parse_traits; - -#[allow(unused)] -pub(crate) use fields::*; -pub(crate) use parse_place::*; -pub(crate) use parse_traits::*; diff --git a/src/parsing/parse_place.rs b/src/parsing/parse_place.rs deleted file mode 100644 index 09cdba15..00000000 --- a/src/parsing/parse_place.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) enum ParsePlace { - GroupedVariable(GroupedVariable), -} - -impl Parse for ParsePlace { - fn parse(input: ParseStream) -> Result { - Ok(Self::GroupedVariable(input.parse()?)) - } -} - -impl HandleParse for ParsePlace { - fn handle_parse(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - match self { - Self::GroupedVariable(grouped_variable) => { - grouped_variable.handle_parse(input, interpreter) - } - } - } -} diff --git a/src/traits.rs b/src/traits.rs index 20c353ea..c70464ec 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -19,6 +19,8 @@ impl IdentExt for Ident { pub(crate) trait CursorExt: Sized { fn ident_matching(self, content: &str) -> Option<(Ident, Self)>; fn punct_matching(self, char: char) -> Option<(Punct, Self)>; + fn literal_matching(self, content: &str) -> Option<(Literal, Self)>; + fn group_matching(self, expected_delimiter: Delimiter) -> Option<(DelimSpan, Self, Self)>; } impl CursorExt for Cursor<'_> { @@ -35,6 +37,24 @@ impl CursorExt for Cursor<'_> { _ => None, } } + + fn literal_matching(self, content: &str) -> Option<(Literal, Self)> { + match self.literal() { + Some((literal, next)) if literal.to_string() == content => Some((literal, next)), + _ => None, + } + } + + fn group_matching(self, expected_delimiter: Delimiter) -> Option<(DelimSpan, Self, Self)> { + match self.any_group() { + Some((inner_cursor, delimiter, delim_span, next_outer_cursor)) + if delimiter == expected_delimiter => + { + Some((delim_span, inner_cursor, next_outer_cursor)) + } + _ => None, + } + } } pub(crate) trait LiteralExt: Sized { @@ -112,6 +132,12 @@ pub(crate) trait ParserExt { ) -> Result; fn peek_ident_matching(&self, content: &str) -> bool; fn parse_ident_matching(&self, content: &str) -> Result; + fn peek_punct_matching(&self, punct: char) -> bool; + fn parse_punct_matching(&self, content: char) -> Result; + fn peek_literal_matching(&self, content: &str) -> bool; + fn parse_literal_matching(&self, content: &str) -> Result; + fn peek_group_matching(&self, delimiter: Delimiter) -> bool; + fn parse_group_matching(&self, delimiter: Delimiter) -> Result<(DelimSpan, ParseBuffer)>; } impl ParserExt for ParseBuffer<'_> { @@ -143,6 +169,50 @@ impl ParserExt for ParseBuffer<'_> { .ok_or_else(|| cursor.span().error(format!("expected {}", content))) }) } + + fn peek_punct_matching(&self, punct: char) -> bool { + self.cursor().punct_matching(punct).is_some() + } + + fn parse_punct_matching(&self, punct: char) -> Result { + self.step(|cursor| { + cursor + .punct_matching(punct) + .ok_or_else(|| cursor.span().error(format!("expected {}", punct))) + }) + } + + fn peek_literal_matching(&self, content: &str) -> bool { + self.cursor().literal_matching(content).is_some() + } + + fn parse_literal_matching(&self, content: &str) -> Result { + self.step(|cursor| { + cursor + .literal_matching(content) + .ok_or_else(|| cursor.span().error(format!("expected {}", content))) + }) + } + + fn peek_group_matching(&self, delimiter: Delimiter) -> bool { + self.cursor().group_matching(delimiter).is_some() + } + + fn parse_group_matching( + &self, + expected_delimiter: Delimiter, + ) -> Result<(DelimSpan, ParseBuffer)> { + let (delimiter, delim_span, inner_stream) = self.parse_any_delimiter()?; + if delimiter != expected_delimiter { + return delim_span.open().err(match expected_delimiter { + Delimiter::Parenthesis => "Expected (", + Delimiter::Brace => "Expected {", + Delimiter::Bracket => "Expected [", + Delimiter::None => "Expected start of transparent group", + }); + } + Ok((delim_span, inner_stream)) + } } pub(crate) trait ContextualParse: Sized { diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 15911a33..89876304 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -68,6 +68,14 @@ fn test_for() { }, "ABCDE" ); + assert_preinterpret_eq!( + { + [!string! [!for! (#x,) in [(a,) (b,) (c,)] { + #x + }]] + }, + "abc" + ); } #[test] diff --git a/tests/core.rs b/tests/core.rs index 587bf931..aae8be33 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -29,6 +29,54 @@ fn test_set() { }, "Hello World!"); } +#[test] +fn test_let() { + my_assert_eq!({ + [!let! = ] + [!string! #inner] + }, "Beautiful"); + my_assert_eq!({ + [!let! #..inner = ] + [!string! #inner] + }, ""); + my_assert_eq!({ + [!let! #..x = Hello => World] + [!string! #x] + }, "Hello=>World"); + my_assert_eq!({ + [!let! Hello #..x!! = Hello => World!!] + [!string! #x] + }, "=>World"); + my_assert_eq!({ + [!let! Hello #..x World = Hello => World] + [!string! #x] + }, "=>"); + my_assert_eq!({ + [!let! Hello #..x World = Hello And Welcome To The Wonderful World] + [!string! #x] + }, "AndWelcomeToTheWonderful"); + my_assert_eq!({ + [!let! Hello #..x "World"! = Hello World And Welcome To The Wonderful "World"!] + [!string! #x] + }, "WorldAndWelcomeToTheWonderful"); + my_assert_eq!({ + [!let! #..x (#..y) = Why Hello (World)] + [!string! "#x = " #x "; #y = " #y] + }, "#x = WhyHello; #y = World"); + my_assert_eq!({ + [!set! #x =] + [!let! + #>>x // Matches one tt and appends it: Why + #..>>x // Matches stream until (, appends it grouped: [!group! Hello Everyone] + ( + #>>..x // Matches one tt and appends it flattened: This is an exciting adventure + #..>>..x // Matches stream and appends it flattened: do you agree ? + ) + = Why Hello Everyone ([!group! This is an exciting adventure] do you agree?)] + [!string! [!intersperse! { items: #x, separator: [_] } ]] + }, "Why_HelloEveryone_This_is_an_exciting_adventure_do_you_agree_?"); +} + #[test] fn test_raw() { my_assert_eq!( From 93c7fa03ab55b11094e3d8165343a5b4c59513c9 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 22 Jan 2025 20:22:58 +0000 Subject: [PATCH 053/126] feature: Added support for named destructurers --- CHANGELOG.md | 32 ++-- src/destructuring/destructure_item.rs | 39 +++-- src/destructuring/destructure_segment.rs | 6 +- src/destructuring/destructure_variable.rs | 38 +++-- src/destructuring/destructurers.rs | 189 +++++++++++++++++++++ src/destructuring/mod.rs | 4 + src/destructuring/named_destructurers.rs | 124 ++++++++++++++ src/expressions/expression_stream.rs | 26 +-- src/interpretation/command.rs | 4 +- src/interpretation/command_arguments.rs | 5 +- src/interpretation/command_stream_input.rs | 6 +- src/interpretation/command_value_input.rs | 12 +- src/interpretation/interpretation_item.rs | 54 +++--- src/traits.rs | 23 +++ 14 files changed, 462 insertions(+), 100 deletions(-) create mode 100644 src/destructuring/destructurers.rs create mode 100644 src/destructuring/named_destructurers.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c801773..891830f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,11 +58,20 @@ Destructuring performs parsing of a token stream. It supports: * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) * `#..>>x` - Reads a stream, appends a group (opposite of for #y in #x) * `#..>>..x` - Reads a stream, appends a stream -* COMING SOON: Commands which don't output a value -* COMING SOON: Named destructurings which look like `(!name! ...)` +* Commands which don't output a value, like `[!set! ...]` or `[!extend! ...]` +* Named destructurings: + * `(!stream! ...)` (TODO - decide if this is a good name) + * `(!ident! ...)` + * `(!punct! ...)` + * `(!literal! ...)` + * `(!group! ...)` ### To come +* Refactoring & testing + * Move destructuring commands to separate file and tests + * Add tests for `(!stream! ...)`, `(!group! ...)`, `(!ident! ...)`, `(!punct! ...)` including matching `'`, `(!literal! ...)` + * Add compile error tests for all the destructuring errors * `[!split! { stream: X, separator: X, drop_empty?: false, permit_trailing_separator?: true, }]` and `[!comma_split! ...]` * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` @@ -76,26 +85,25 @@ Destructuring performs parsing of a token stream. It supports: * Rework `Error` as: * `ParseResult` with `ParseError::LowLevel(syn::Error)` | `ParseError::Contextual(syn::Error)` * `ExecutionInterrupt` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` -* Basic place destructuring - * `(!stream! ...)` method to take a group - * `(!literal! #x)` / `(!ident! #x)` bindings +* Place destructuring * `(!optional! ...)` - * `(!group! ...)` - * Support commands, but only commands which don't output a value * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` - * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much * `(!raw! ...)` * `(!match! ...)` (with `#..x` as a catch-all) + * `(!fields! ...)` and `(!subfields! ...)` + * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much * Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing, in order to: * Fix comments in the expression files * Enable lazy && and || * Enable support for code blocks { .. } in expressions, and remove hacks where expression parsing stops at {} or . -* Pushed to 0.4 - fork of syn to: - * Fix issues in Rust Analyzer - * Improve performance - * Permit `[!parse_while! [!PARSE! ...] from #x { ... }]` +* Pushed to 0.4: + * Fork of syn to: + * Fix issues in Rust Analyzer + * Improve performance + * Permit `[!parse_while! [!PARSE! ...] from #x { ... }]` + * Further syn parsings (e.g. item, fields, etc) * Work on book * Input paradigms: * Streams diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs index 4eda6d58..962b079f 100644 --- a/src/destructuring/destructure_item.rs +++ b/src/destructuring/destructure_item.rs @@ -2,7 +2,9 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum DestructureItem { + NoneOutputCommand(Command), Variable(DestructureVariable), + Destructurer(Destructurer), ExactPunct(Punct), ExactIdent(Ident), ExactLiteral(Literal), @@ -16,15 +18,18 @@ impl DestructureItem { /// parsing `Hello` into `x`. pub(crate) fn parse_until(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand => { + PeekMatch::GroupedCommand(Some(command_kind)) + if matches!(command_kind.output_kind(None), Ok(CommandOutputKind::None)) => + { + Self::NoneOutputCommand(input.parse()?) + } + PeekMatch::GroupedCommand(_) => return input.span().err( + "Grouped commands returning a value are not supported in destructuring positions", + ), + PeekMatch::FlattenedCommand(_) => { return input .span() - .err("Grouped commands are not currently supported in destructuring positions") - } - PeekMatch::FlattenedCommand => { - return input.span().err( - "Flattened commands are not currently supported in destructuring positions", - ) + .err("Flattened commands are not supported in destructuring positions") } PeekMatch::GroupedVariable | PeekMatch::FlattenedVariable @@ -32,15 +37,11 @@ impl DestructureItem { Self::Variable(DestructureVariable::parse_until::(input)?) } PeekMatch::Group(_) => Self::ExactGroup(input.parse()?), - PeekMatch::NamedDestructuring => todo!(), - PeekMatch::Other => match input.parse::()? { - TokenTree::Group(_) => { - unreachable!("Should have been already handled by InterpretationGroup above") - } - TokenTree::Punct(punct) => Self::ExactPunct(punct), - TokenTree::Ident(ident) => Self::ExactIdent(ident), - TokenTree::Literal(literal) => Self::ExactLiteral(literal), - }, + PeekMatch::Destructurer(_) => Self::Destructurer(input.parse()?), + PeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), + PeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), + PeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), + PeekMatch::End => return input.span().err("Unexpected end"), }) } } @@ -51,6 +52,12 @@ impl HandleDestructure for DestructureItem { DestructureItem::Variable(variable) => { variable.handle_destructure(input, interpreter)?; } + DestructureItem::NoneOutputCommand(command) => { + let _ = command.clone().interpret_to_new_stream(interpreter)?; + } + DestructureItem::Destructurer(destructurer) => { + destructurer.handle_destructure(input, interpreter)?; + } DestructureItem::ExactPunct(punct) => { input.parse_punct_matching(punct.as_char())?; } diff --git a/src/destructuring/destructure_segment.rs b/src/destructuring/destructure_segment.rs index 984d919e..d2567bfd 100644 --- a/src/destructuring/destructure_segment.rs +++ b/src/destructuring/destructure_segment.rs @@ -5,8 +5,8 @@ pub(crate) trait StopCondition: Clone { } #[derive(Clone)] -pub(crate) struct UntilEmpty; -impl StopCondition for UntilEmpty { +pub(crate) struct UntilEnd; +impl StopCondition for UntilEnd { fn should_stop(input: ParseStream) -> bool { input.is_empty() } @@ -46,7 +46,7 @@ impl StopCondition for UntilToken { } } -pub(crate) type DestructureRemaining = DestructureSegment; +pub(crate) type DestructureRemaining = DestructureSegment; pub(crate) type DestructureUntil = DestructureSegment>; #[derive(Clone)] diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index 92ce2419..173abbf7 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -54,6 +54,16 @@ pub(crate) enum DestructureVariable { } impl DestructureVariable { + pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> Result { + let variable: DestructureVariable = Self::parse_until::(input)?; + if variable.is_flattened_input() { + return variable + .span_range() + .err("A flattened input variable is not supported here"); + } + Ok(variable) + } + pub(crate) fn parse_until(input: ParseStream) -> Result { let marker = input.parse()?; if input.peek(Token![..]) { @@ -145,6 +155,15 @@ impl HasSpanRange for DestructureVariable { } impl DestructureVariable { + pub(crate) fn is_flattened_input(&self) -> bool { + matches!( + self, + DestructureVariable::Flattened { .. } + | DestructureVariable::FlattenedAppendGrouped { .. } + | DestructureVariable::FlattenedAppendFlattened { .. } + ) + } + fn get_variable_data(&self, interpreter: &mut Interpreter) -> Result { let variable_data = interpreter .get_existing_variable_data(self, || { @@ -272,26 +291,21 @@ impl ParseUntil { return Ok(ParseUntil::End); } Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand - | PeekMatch::FlattenedCommand + PeekMatch::GroupedCommand(_) + | PeekMatch::FlattenedCommand(_) | PeekMatch::GroupedVariable | PeekMatch::FlattenedVariable - | PeekMatch::NamedDestructuring + | PeekMatch::Destructurer(_) | PeekMatch::AppendVariableDestructuring => { return input .span() .err("This cannot follow a flattened destructure match"); } PeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), - PeekMatch::Other => match input.cursor().token_tree() { - Some((token_tree, _)) => match token_tree { - TokenTree::Group(group) => ParseUntil::Group(group.delimiter()), - TokenTree::Ident(ident) => ParseUntil::Ident(ident), - TokenTree::Punct(punct) => ParseUntil::Punct(punct), - TokenTree::Literal(literal) => ParseUntil::Literal(literal), - }, - None => ParseUntil::End, - }, + PeekMatch::Ident(ident) => ParseUntil::Ident(ident), + PeekMatch::Literal(literal) => ParseUntil::Literal(literal), + PeekMatch::Punct(punct) => ParseUntil::Punct(punct), + PeekMatch::End => ParseUntil::End, }) } diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs new file mode 100644 index 00000000..7408e707 --- /dev/null +++ b/src/destructuring/destructurers.rs @@ -0,0 +1,189 @@ +use crate::internal_prelude::*; + +pub(crate) trait DestructurerDefinition: Clone { + const DESTRUCTURER_NAME: &'static str; + fn parse(arguments: DestructurerArguments) -> Result; + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; +} + +#[derive(Clone)] +pub(crate) struct DestructurerArguments<'a> { + parse_stream: ParseStream<'a>, + destructurer_name: Ident, + full_span_range: SpanRange, +} + +#[allow(unused)] +impl<'a> DestructurerArguments<'a> { + pub(crate) fn new( + parse_stream: ParseStream<'a>, + destructurer_name: Ident, + span_range: SpanRange, + ) -> Self { + Self { + parse_stream, + destructurer_name, + full_span_range: span_range, + } + } + + pub(crate) fn full_span_range(&self) -> SpanRange { + self.full_span_range + } + + /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message + pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> Result<()> { + if self.parse_stream.is_empty() { + Ok(()) + } else { + self.full_span_range.err(error_message) + } + } + + pub(crate) fn fully_parse_no_error_override(&self) -> Result { + self.parse_stream.parse() + } + + pub(crate) fn fully_parse_as(&self) -> Result { + self.fully_parse_or_error(T::parse, T::error_message()) + } + + pub(crate) fn fully_parse_or_error( + &self, + parse_function: impl FnOnce(ParseStream) -> Result, + error_message: impl std::fmt::Display, + ) -> Result { + let parsed = parse_function(self.parse_stream).or_else(|error| { + // In future, when the diagnostic API is stable, + // we can add this context directly onto the command ident... + // Rather than just selectively adding it to the inner-most error. + let error_string = error.to_string(); + + // We avoid adding this additional context if it's already been added in an + // inner error, because that's likely the correct error to show. + if error_string.contains("\nOccurred whilst parsing") { + return Err(error); + } + error.span().err(format!( + "{}\nOccurred whilst parsing (!{}! ..) - {}", + error_string, self.destructurer_name, error_message, + )) + })?; + + self.assert_empty(error_message)?; + + Ok(parsed) + } +} + +#[derive(Clone)] +pub(crate) struct Destructurer { + instance: NamedDestructurer, + #[allow(unused)] + source_group_span: DelimSpan, +} + +impl Parse for Destructurer { + fn parse(input: ParseStream) -> Result { + let content; + let open_bracket = syn::parenthesized!(content in input); + content.parse::()?; + let destructurer_name = content.call(Ident::parse_any)?; + let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { + Some(destructurer_kind) => destructurer_kind, + None => destructurer_name.span().err( + format!( + "Expected `(!! ...)`, for one of: {}.\nIf this wasn't intended to be a named destructuring, you can work around this with (!raw! (!{} ... ))", + DestructurerKind::list_all(), + destructurer_name, + ), + )?, + }; + content.parse::()?; + let instance = destructurer_kind.parse_instance(DestructurerArguments::new( + &content, + destructurer_name, + open_bracket.span.span_range(), + ))?; + Ok(Self { + instance, + source_group_span: open_bracket.span, + }) + } +} + +impl HandleDestructure for Destructurer { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + self.instance.handle_destructure(input, interpreter) + } +} + +macro_rules! define_destructurers { + ( + $( + $destructurer:ident, + )* + ) => { + #[allow(clippy::enum_variant_names)] + #[derive(Clone, Copy)] + pub(crate) enum DestructurerKind { + $( + $destructurer, + )* + } + + impl DestructurerKind { + fn parse_instance(&self, arguments: DestructurerArguments) -> Result { + Ok(match self { + $( + Self::$destructurer => NamedDestructurer::$destructurer( + $destructurer::parse(arguments)? + ), + )* + }) + } + + pub(crate) fn for_ident(ident: &Ident) -> Option { + Some(match ident.to_string().as_ref() { + $( + $destructurer::DESTRUCTURER_NAME => Self::$destructurer, + )* + _ => return None, + }) + } + + const ALL_KIND_NAMES: &'static [&'static str] = &[$($destructurer::DESTRUCTURER_NAME,)*]; + + pub(crate) fn list_all() -> String { + // TODO improve to add an "and" at the end + Self::ALL_KIND_NAMES.join(", ") + } + } + + #[derive(Clone)] + #[allow(clippy::enum_variant_names)] + pub(crate) enum NamedDestructurer { + $( + $destructurer($destructurer), + )* + } + + impl NamedDestructurer { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + match self { + $( + Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), + )* + } + } + } + }; +} + +define_destructurers! { + StreamDestructurer, + IdentDestructurer, + LiteralDestructurer, + PunctDestructurer, + GroupDestructurer, +} diff --git a/src/destructuring/mod.rs b/src/destructuring/mod.rs index 22e20fc3..11327c0b 100644 --- a/src/destructuring/mod.rs +++ b/src/destructuring/mod.rs @@ -3,12 +3,16 @@ mod destructure_item; mod destructure_segment; mod destructure_traits; mod destructure_variable; +mod destructurers; mod fields; +mod named_destructurers; pub(crate) use destructure_group::*; pub(crate) use destructure_item::*; pub(crate) use destructure_segment::*; pub(crate) use destructure_traits::*; pub(crate) use destructure_variable::*; +pub(crate) use destructurers::*; #[allow(unused)] pub(crate) use fields::*; +pub(crate) use named_destructurers::*; diff --git a/src/destructuring/named_destructurers.rs b/src/destructuring/named_destructurers.rs new file mode 100644 index 00000000..f7600f02 --- /dev/null +++ b/src/destructuring/named_destructurers.rs @@ -0,0 +1,124 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct StreamDestructurer { + inner: DestructureRemaining, +} + +impl DestructurerDefinition for StreamDestructurer { + const DESTRUCTURER_NAME: &'static str = "stream"; + + fn parse(arguments: DestructurerArguments) -> Result { + Ok(Self { + inner: arguments.fully_parse_no_error_override()?, + }) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + self.inner.handle_destructure(input, interpreter) + } +} + +#[derive(Clone)] +pub(crate) struct IdentDestructurer { + variable: DestructureVariable, +} + +impl DestructurerDefinition for IdentDestructurer { + const DESTRUCTURER_NAME: &'static str = "ident"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + variable: DestructureVariable::parse_only_unflattened_input(input)?, + }) + }, + "Expected (!ident! #x) or (!ident #>>x)", + ) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + if input.cursor().ident().is_some() { + self.variable.handle_destructure(input, interpreter) + } else { + Err(input.error("Expected an ident")) + } + } +} + +#[derive(Clone)] +pub(crate) struct LiteralDestructurer { + variable: DestructureVariable, +} + +impl DestructurerDefinition for LiteralDestructurer { + const DESTRUCTURER_NAME: &'static str = "literal"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + variable: DestructureVariable::parse_only_unflattened_input(input)?, + }) + }, + "Expected (!literal! #x) or (!literal! #>>x)", + ) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + if input.cursor().literal().is_some() { + self.variable.handle_destructure(input, interpreter) + } else { + Err(input.error("Expected a literal")) + } + } +} + +#[derive(Clone)] +pub(crate) struct PunctDestructurer { + variable: DestructureVariable, +} + +impl DestructurerDefinition for PunctDestructurer { + const DESTRUCTURER_NAME: &'static str = "punct"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + variable: DestructureVariable::parse_only_unflattened_input(input)?, + }) + }, + "Expected (!punct! #x) or (!punct! #>>x)", + ) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + if input.cursor().any_punct().is_some() { + self.variable.handle_destructure(input, interpreter) + } else { + Err(input.error("Expected a punct")) + } + } +} + +#[derive(Clone)] +pub(crate) struct GroupDestructurer { + inner: DestructureRemaining, +} + +impl DestructurerDefinition for GroupDestructurer { + const DESTRUCTURER_NAME: &'static str = "group"; + + fn parse(arguments: DestructurerArguments) -> Result { + Ok(Self { + inner: arguments.fully_parse_no_error_override()?, + }) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + let (_, inner) = input.parse_group_matching(Delimiter::None)?; + self.inner.handle_destructure(&inner, interpreter) + } +} diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 531f7987..4ab3a2c0 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -35,30 +35,20 @@ impl Parse for ExpressionInput { // before code blocks or .. in [!range!] so we can break on those. // These aren't valid inside expressions we support anyway, so it's good enough for now. let item = match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand => ExpressionItem::Command(input.parse()?), - PeekMatch::FlattenedCommand => ExpressionItem::Command(input.parse()?), + PeekMatch::GroupedCommand(_) => ExpressionItem::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => ExpressionItem::Command(input.parse()?), PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), PeekMatch::Group(Delimiter::Brace | Delimiter::Bracket) => break, PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse()?), - PeekMatch::NamedDestructuring | PeekMatch::AppendVariableDestructuring => { + PeekMatch::Destructurer(_) | PeekMatch::AppendVariableDestructuring => { return Err(input.error("Destructuring is not supported in an expression")); } - PeekMatch::Other => { - if input.cursor().punct_matching('.').is_some() { - break; - } - match input.parse::()? { - TokenTree::Group(_) => { - unreachable!( - "Should have been already handled by InterpretationGroup above" - ) - } - TokenTree::Punct(punct) => ExpressionItem::Punct(punct), - TokenTree::Ident(ident) => ExpressionItem::Ident(ident), - TokenTree::Literal(literal) => ExpressionItem::Literal(literal), - } - } + PeekMatch::Punct(punct) if punct.as_char() == '.' => break, + PeekMatch::Punct(_) => ExpressionItem::Punct(input.parse_any_punct()?), + PeekMatch::Ident(_) => ExpressionItem::Ident(input.parse_any_ident()?), + PeekMatch::Literal(_) => ExpressionItem::Literal(input.parse()?), + PeekMatch::End => return input.span().err("Expected an expression"), }; items.push(item); } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 6890e42e..ddd7721c 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,7 +1,7 @@ use crate::internal_prelude::*; #[allow(unused)] -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum CommandOutputKind { None, /// LiteralOrBool @@ -306,7 +306,7 @@ impl OutputKind for OutputKindControlFlow { fn resolve(flattening: Option) -> Result { match flattening { - Some(dots) => dots.err("This command is control flow, so is always flattened and cannot be flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), + Some(dots) => dots.err("This command is control flow, so is always flattened and cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), None => Ok(CommandOutputKind::ControlFlowCodeStream), } } diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index de07e9c3..2aba1640 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -6,8 +6,6 @@ pub(crate) struct CommandArguments<'a> { command_name: Ident, /// The span range of the original stream, before tokens were consumed full_span_range: SpanRange, - /// The span of the last item consumed (or the full span range if no items have been consumed yet) - latest_item_span_range: SpanRange, } impl<'a> CommandArguments<'a> { @@ -20,7 +18,6 @@ impl<'a> CommandArguments<'a> { parse_stream, command_name, full_span_range: span_range, - latest_item_span_range: span_range, } } @@ -33,7 +30,7 @@ impl<'a> CommandArguments<'a> { if self.parse_stream.is_empty() { Ok(()) } else { - self.latest_item_span_range.err(error_message) + self.full_span_range.err(error_message) } } diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index e0942fc6..0d9b3075 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -24,13 +24,13 @@ pub(crate) enum CommandStreamInput { impl Parse for CommandStreamInput { fn parse(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand => Self::Command(input.parse()?), - PeekMatch::FlattenedCommand => Self::Command(input.parse()?), + PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), - PeekMatch::Group(_) | PeekMatch::AppendVariableDestructuring | PeekMatch::NamedDestructuring | PeekMatch::Other => input.span() + _ => input.span() .err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) } diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 8591d0b0..7e4a1e52 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -16,15 +16,19 @@ pub(crate) enum CommandValueInput { impl Parse for CommandValueInput { fn parse(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand => Self::Command(input.parse()?), - PeekMatch::FlattenedCommand => Self::Command(input.parse()?), + PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), - PeekMatch::AppendVariableDestructuring | PeekMatch::NamedDestructuring => { + PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { return input.span().err("Destructurings are not supported here") } - PeekMatch::Group(_) | PeekMatch::Other => Self::Value(input.parse()?), + PeekMatch::Group(_) + | PeekMatch::Punct(_) + | PeekMatch::Literal(_) + | PeekMatch::Ident(_) + | PeekMatch::End => Self::Value(input.parse()?), }) } } diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 08449595..1e042157 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -14,35 +14,35 @@ pub(crate) enum InterpretationItem { impl Parse for InterpretationItem { fn parse(input: ParseStream) -> Result { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand => InterpretationItem::Command(input.parse()?), - PeekMatch::FlattenedCommand => InterpretationItem::Command(input.parse()?), + PeekMatch::GroupedCommand(_) => InterpretationItem::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => InterpretationItem::Command(input.parse()?), PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), - PeekMatch::AppendVariableDestructuring | PeekMatch::NamedDestructuring => { + PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { return input.span().err("Destructurings are not supported here") } - PeekMatch::Other => match input.parse::()? { - TokenTree::Group(_) => { - unreachable!("Should have been already handled by InterpretationGroup above") - } - TokenTree::Punct(punct) => InterpretationItem::Punct(punct), - TokenTree::Ident(ident) => InterpretationItem::Ident(ident), - TokenTree::Literal(literal) => InterpretationItem::Literal(literal), - }, + PeekMatch::Punct(_) => InterpretationItem::Punct(input.parse_any_punct()?), + PeekMatch::Ident(_) => InterpretationItem::Ident(input.parse_any_ident()?), + PeekMatch::Literal(_) => InterpretationItem::Literal(input.parse()?), + PeekMatch::End => return input.span().err("Expected some item"), }) } } +#[allow(unused)] pub(crate) enum PeekMatch { - GroupedCommand, - FlattenedCommand, + GroupedCommand(Option), + FlattenedCommand(Option), GroupedVariable, FlattenedVariable, AppendVariableDestructuring, - NamedDestructuring, + Destructurer(Option), Group(Delimiter), - Other, + Ident(Ident), + Punct(Punct), + Literal(Literal), + End, } pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMatch { @@ -51,16 +51,16 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa if let Some((next, delimiter, _, _)) = cursor.any_group() { if delimiter == Delimiter::Bracket { if let Some((_, next)) = next.punct_matching('!') { - if let Some((_, next)) = next.ident() { + if let Some((ident, next)) = next.ident() { if next.punct_matching('!').is_some() { - return PeekMatch::GroupedCommand; + return PeekMatch::GroupedCommand(CommandKind::for_ident(&ident)); } } if let Some((_, next)) = next.punct_matching('.') { if let Some((_, next)) = next.punct_matching('.') { - if let Some((_, next)) = next.ident() { + if let Some((ident, next)) = next.ident() { if next.punct_matching('!').is_some() { - return PeekMatch::FlattenedCommand; + return PeekMatch::FlattenedCommand(CommandKind::for_ident(&ident)); } } } @@ -69,9 +69,9 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa } if delimiter == Delimiter::Parenthesis { if let Some((_, next)) = next.punct_matching('!') { - if let Some((_, next)) = next.ident() { + if let Some((ident, next)) = next.ident() { if next.punct_matching('!').is_some() { - return PeekMatch::NamedDestructuring; + return PeekMatch::Destructurer(DestructurerKind::for_ident(&ident)); } } } @@ -113,11 +113,13 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa } } - // This is rather annoying for our purposes, but the Cursor (and even `impl Parse on Punct`) - // treats `'` specially, and there's no way to peek a ' token which isn't part of a lifetime. - // Instead of dividing up specific cases here, we let the caller handle it by parsing as a - // TokenTree if they need to, which doesn't have these limitations. - PeekMatch::Other + match cursor.token_tree() { + Some((TokenTree::Ident(ident), _)) => PeekMatch::Ident(ident), + Some((TokenTree::Punct(punct), _)) => PeekMatch::Punct(punct), + Some((TokenTree::Literal(literal), _)) => PeekMatch::Literal(literal), + Some((TokenTree::Group(_), _)) => unreachable!("Already covered above"), + None => PeekMatch::End, + } } impl Interpret for InterpretationItem { diff --git a/src/traits.rs b/src/traits.rs index c70464ec..8b013519 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -17,6 +17,8 @@ impl IdentExt for Ident { } pub(crate) trait CursorExt: Sized { + /// Because syn doesn't parse ' as a punct (not aligned with the TokenTree abstraction) + fn any_punct(self) -> Option<(Punct, Self)>; fn ident_matching(self, content: &str) -> Option<(Ident, Self)>; fn punct_matching(self, char: char) -> Option<(Punct, Self)>; fn literal_matching(self, content: &str) -> Option<(Literal, Self)>; @@ -24,6 +26,13 @@ pub(crate) trait CursorExt: Sized { } impl CursorExt for Cursor<'_> { + fn any_punct(self) -> Option<(Punct, Self)> { + match self.token_tree() { + Some((TokenTree::Punct(punct), next)) => Some((punct, next)), + _ => None, + } + } + fn ident_matching(self, content: &str) -> Option<(Ident, Self)> { match self.ident() { Some((ident, next)) if ident == content => Some((ident, next)), @@ -130,6 +139,8 @@ pub(crate) trait ParserExt { func: F, message: M, ) -> Result; + fn parse_any_ident(&self) -> Result; + fn parse_any_punct(&self) -> Result; fn peek_ident_matching(&self, content: &str) -> bool; fn parse_ident_matching(&self, content: &str) -> Result; fn peek_punct_matching(&self, punct: char) -> bool; @@ -158,6 +169,18 @@ impl ParserExt for ParseBuffer<'_> { parse(self).map_err(|_| error_span.error(message)) } + fn parse_any_ident(&self) -> Result { + Ident::parse_any(self) + } + + fn parse_any_punct(&self) -> Result { + // Annoyingly, ' behaves weirdly in syn, so we need to handle it + match self.parse::()? { + TokenTree::Punct(punct) => Ok(punct), + _ => self.span().err("expected punctuation"), + } + } + fn peek_ident_matching(&self, content: &str) -> bool { self.cursor().ident_matching(content).is_some() } From 975a61d30413a8db8ef358784b6c492cbaa36272 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 23 Jan 2025 00:16:59 +0000 Subject: [PATCH 054/126] tests: Add some more destructuring tests --- CHANGELOG.md | 20 +- regenerate-compilation-failures.sh | 2 +- src/commands/concat_commands.rs | 6 +- src/commands/core_commands.rs | 68 ++--- src/commands/destructuring_commands.rs | 36 +++ src/commands/mod.rs | 2 + src/destructuring/destructurer.rs | 189 ++++++++++++ src/destructuring/destructurers.rs | 275 ++++++++---------- src/destructuring/mod.rs | 4 +- src/destructuring/named_destructurers.rs | 124 -------- src/interpretation/command.rs | 1 + src/interpretation/interpreted_stream.rs | 135 ++++++--- src/traits.rs | 8 +- .../destructuring/append_before_set.rs | 5 + .../destructuring/append_before_set.stderr | 5 + .../double_flattened_variable.rs | 5 + .../double_flattened_variable.stderr | 6 + tests/control_flow.rs | 41 +-- tests/core.rs | 62 +--- tests/destructuring.rs | 136 +++++++++ 20 files changed, 681 insertions(+), 449 deletions(-) create mode 100644 src/commands/destructuring_commands.rs create mode 100644 src/destructuring/destructurer.rs delete mode 100644 src/destructuring/named_destructurers.rs create mode 100644 tests/compilation_failures/destructuring/append_before_set.rs create mode 100644 tests/compilation_failures/destructuring/append_before_set.stderr create mode 100644 tests/compilation_failures/destructuring/double_flattened_variable.rs create mode 100644 tests/compilation_failures/destructuring/double_flattened_variable.stderr create mode 100644 tests/destructuring.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 891830f4..1d6e9878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,15 +68,11 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* Refactoring & testing - * Move destructuring commands to separate file and tests - * Add tests for `(!stream! ...)`, `(!group! ...)`, `(!ident! ...)`, `(!punct! ...)` including matching `'`, `(!literal! ...)` - * Add compile error tests for all the destructuring errors -* `[!split! { stream: X, separator: X, drop_empty?: false, permit_trailing_separator?: true, }]` and `[!comma_split! ...]` -* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), trim_to_shortest?: false }]` with `InterpretValue>>` +* Add compile error tests for all the destructuring errors +* `[!split! { stream: [...], separator: [...], drop_empty?: false, permit_trailing_separator?: true, }]` and `[!comma_split! ...]` +* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), error_on_length_mismatch?: true }]` with `InterpretValue>>` * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` * `[!is_set! #x]` -* `[!debug!]` command to output an error with the current content of all variables. * `[!str_split! { input: Value, separator: Value, }]` * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Add more tests @@ -84,15 +80,17 @@ Destructuring performs parsing of a token stream. It supports: * e.g. for long sums * Rework `Error` as: * `ParseResult` with `ParseError::LowLevel(syn::Error)` | `ParseError::Contextual(syn::Error)` - * `ExecutionInterrupt` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` -* Place destructuring - * `(!optional! ...)` + * `ExecutionResult` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` +* Destructurers * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` * `(!raw! ...)` - * `(!match! ...)` (with `#..x` as a catch-all) * `(!fields! ...)` and `(!subfields! ...)` + * Add ability to fork (copy on write?) / revert the interpreter state and can then add: + * `(!optional! ...)` + * `(!match! ...)` (with `#..x` as a catch-all) * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much +* `[!match! ...]` command - similar to the match destructurer, but a command... * Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing, in order to: * Fix comments in the expression files diff --git a/regenerate-compilation-failures.sh b/regenerate-compilation-failures.sh index 6defcd6d..f1a3fd31 100755 --- a/regenerate-compilation-failures.sh +++ b/regenerate-compilation-failures.sh @@ -2,7 +2,7 @@ set -e -TRYBUILD=overwrite cargo test +TRYBUILD=overwrite cargo test compilation_failures # Remove the .wip folder created sometimes by try-build if [ -d "./wip" ]; then diff --git a/src/commands/concat_commands.rs b/src/commands/concat_commands.rs index b8d018ce..4d9a26d0 100644 --- a/src/commands/concat_commands.rs +++ b/src/commands/concat_commands.rs @@ -12,7 +12,7 @@ fn concat_into_string( let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? - .concat_recursive(); + .concat_recursive(&ConcatBehaviour::standard()); let value = conversion_fn(&concatenated); Ok(Literal::string(&value).with_span(output_span)) } @@ -25,7 +25,7 @@ fn concat_into_ident( let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? - .concat_recursive(); + .concat_recursive(&ConcatBehaviour::standard()); let value = conversion_fn(&concatenated); let ident = parse_str::(&value) .map_err(|err| output_span.error(format!("`{}` is not a valid ident: {:?}", value, err,)))? @@ -41,7 +41,7 @@ fn concat_into_literal( let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? - .concat_recursive(); + .concat_recursive(&ConcatBehaviour::standard()); let value = conversion_fn(&concatenated); let literal = Literal::from_str(&value) .map_err(|err| { diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 75dfc263..56e0299d 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -1,5 +1,3 @@ -use std::ops::DerefMut; - use crate::internal_prelude::*; #[derive(Clone)] @@ -33,46 +31,10 @@ impl NoOutputCommandDefinition for SetCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; self.variable.set(interpreter, result_tokens)?; - Ok(()) } } -#[derive(Clone)] -pub(crate) struct LetCommand { - destructuring: DestructureUntil, - #[allow(unused)] - equals: Token![=], - arguments: InterpretationStream, -} - -impl CommandType for LetCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for LetCommand { - const COMMAND_NAME: &'static str = "let"; - - fn parse(arguments: CommandArguments) -> Result { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - destructuring: input.parse()?, - equals: input.parse()?, - arguments: input.parse_with(arguments.full_span_range())?, - }) - }, - "Expected [!let! = ..]", - ) - } - - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { - let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; - self.destructuring - .handle_destructure_from_stream(result_tokens, interpreter) - } -} - #[derive(Clone)] pub(crate) struct ExtendCommand { variable: GroupedVariable, @@ -253,7 +215,7 @@ impl NoOutputCommandDefinition for ErrorCommand { EitherErrorInput::JustMessage(stream) => { let error_message = stream .interpret_to_new_stream(interpreter)? - .concat_recursive(); + .concat_recursive(&ConcatBehaviour::standard()); return Span::call_site().err(error_message); } }; @@ -304,3 +266,31 @@ impl NoOutputCommandDefinition for ErrorCommand { error_span.err(message) } } + +#[derive(Clone)] +pub(crate) struct DebugCommand { + inner: InterpretationStream, +} + +impl CommandType for DebugCommand { + type OutputKind = OutputKindValue; +} + +impl ValueCommandDefinition for DebugCommand { + const COMMAND_NAME: &'static str = "debug"; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + inner: arguments.parse_all_for_interpretation()?, + }) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + let span = self.inner.span(); + let debug_string = self + .inner + .interpret_to_new_stream(interpreter)? + .concat_recursive(&ConcatBehaviour::debug()); + Ok(Literal::string(&debug_string).with_span(span).into()) + } +} diff --git a/src/commands/destructuring_commands.rs b/src/commands/destructuring_commands.rs new file mode 100644 index 00000000..58129a4b --- /dev/null +++ b/src/commands/destructuring_commands.rs @@ -0,0 +1,36 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct LetCommand { + destructuring: DestructureUntil, + #[allow(unused)] + equals: Token![=], + arguments: InterpretationStream, +} + +impl CommandType for LetCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for LetCommand { + const COMMAND_NAME: &'static str = "let"; + + fn parse(arguments: CommandArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + destructuring: input.parse()?, + equals: input.parse()?, + arguments: input.parse_with(arguments.full_span_range())?, + }) + }, + "Expected [!let! = ...]", + ) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; + self.destructuring + .handle_destructure_from_stream(result_tokens, interpreter) + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index cc29c102..97d4f826 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,11 +1,13 @@ mod concat_commands; mod control_flow_commands; mod core_commands; +mod destructuring_commands; mod expression_commands; mod token_commands; pub(crate) use concat_commands::*; pub(crate) use control_flow_commands::*; pub(crate) use core_commands::*; +pub(crate) use destructuring_commands::*; pub(crate) use expression_commands::*; pub(crate) use token_commands::*; diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs new file mode 100644 index 00000000..7408e707 --- /dev/null +++ b/src/destructuring/destructurer.rs @@ -0,0 +1,189 @@ +use crate::internal_prelude::*; + +pub(crate) trait DestructurerDefinition: Clone { + const DESTRUCTURER_NAME: &'static str; + fn parse(arguments: DestructurerArguments) -> Result; + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; +} + +#[derive(Clone)] +pub(crate) struct DestructurerArguments<'a> { + parse_stream: ParseStream<'a>, + destructurer_name: Ident, + full_span_range: SpanRange, +} + +#[allow(unused)] +impl<'a> DestructurerArguments<'a> { + pub(crate) fn new( + parse_stream: ParseStream<'a>, + destructurer_name: Ident, + span_range: SpanRange, + ) -> Self { + Self { + parse_stream, + destructurer_name, + full_span_range: span_range, + } + } + + pub(crate) fn full_span_range(&self) -> SpanRange { + self.full_span_range + } + + /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message + pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> Result<()> { + if self.parse_stream.is_empty() { + Ok(()) + } else { + self.full_span_range.err(error_message) + } + } + + pub(crate) fn fully_parse_no_error_override(&self) -> Result { + self.parse_stream.parse() + } + + pub(crate) fn fully_parse_as(&self) -> Result { + self.fully_parse_or_error(T::parse, T::error_message()) + } + + pub(crate) fn fully_parse_or_error( + &self, + parse_function: impl FnOnce(ParseStream) -> Result, + error_message: impl std::fmt::Display, + ) -> Result { + let parsed = parse_function(self.parse_stream).or_else(|error| { + // In future, when the diagnostic API is stable, + // we can add this context directly onto the command ident... + // Rather than just selectively adding it to the inner-most error. + let error_string = error.to_string(); + + // We avoid adding this additional context if it's already been added in an + // inner error, because that's likely the correct error to show. + if error_string.contains("\nOccurred whilst parsing") { + return Err(error); + } + error.span().err(format!( + "{}\nOccurred whilst parsing (!{}! ..) - {}", + error_string, self.destructurer_name, error_message, + )) + })?; + + self.assert_empty(error_message)?; + + Ok(parsed) + } +} + +#[derive(Clone)] +pub(crate) struct Destructurer { + instance: NamedDestructurer, + #[allow(unused)] + source_group_span: DelimSpan, +} + +impl Parse for Destructurer { + fn parse(input: ParseStream) -> Result { + let content; + let open_bracket = syn::parenthesized!(content in input); + content.parse::()?; + let destructurer_name = content.call(Ident::parse_any)?; + let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { + Some(destructurer_kind) => destructurer_kind, + None => destructurer_name.span().err( + format!( + "Expected `(!! ...)`, for one of: {}.\nIf this wasn't intended to be a named destructuring, you can work around this with (!raw! (!{} ... ))", + DestructurerKind::list_all(), + destructurer_name, + ), + )?, + }; + content.parse::()?; + let instance = destructurer_kind.parse_instance(DestructurerArguments::new( + &content, + destructurer_name, + open_bracket.span.span_range(), + ))?; + Ok(Self { + instance, + source_group_span: open_bracket.span, + }) + } +} + +impl HandleDestructure for Destructurer { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + self.instance.handle_destructure(input, interpreter) + } +} + +macro_rules! define_destructurers { + ( + $( + $destructurer:ident, + )* + ) => { + #[allow(clippy::enum_variant_names)] + #[derive(Clone, Copy)] + pub(crate) enum DestructurerKind { + $( + $destructurer, + )* + } + + impl DestructurerKind { + fn parse_instance(&self, arguments: DestructurerArguments) -> Result { + Ok(match self { + $( + Self::$destructurer => NamedDestructurer::$destructurer( + $destructurer::parse(arguments)? + ), + )* + }) + } + + pub(crate) fn for_ident(ident: &Ident) -> Option { + Some(match ident.to_string().as_ref() { + $( + $destructurer::DESTRUCTURER_NAME => Self::$destructurer, + )* + _ => return None, + }) + } + + const ALL_KIND_NAMES: &'static [&'static str] = &[$($destructurer::DESTRUCTURER_NAME,)*]; + + pub(crate) fn list_all() -> String { + // TODO improve to add an "and" at the end + Self::ALL_KIND_NAMES.join(", ") + } + } + + #[derive(Clone)] + #[allow(clippy::enum_variant_names)] + pub(crate) enum NamedDestructurer { + $( + $destructurer($destructurer), + )* + } + + impl NamedDestructurer { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + match self { + $( + Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), + )* + } + } + } + }; +} + +define_destructurers! { + StreamDestructurer, + IdentDestructurer, + LiteralDestructurer, + PunctDestructurer, + GroupDestructurer, +} diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index 7408e707..0d7ecbe4 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -1,189 +1,154 @@ use crate::internal_prelude::*; -pub(crate) trait DestructurerDefinition: Clone { - const DESTRUCTURER_NAME: &'static str; - fn parse(arguments: DestructurerArguments) -> Result; - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; -} - #[derive(Clone)] -pub(crate) struct DestructurerArguments<'a> { - parse_stream: ParseStream<'a>, - destructurer_name: Ident, - full_span_range: SpanRange, +pub(crate) struct StreamDestructurer { + inner: DestructureRemaining, } -#[allow(unused)] -impl<'a> DestructurerArguments<'a> { - pub(crate) fn new( - parse_stream: ParseStream<'a>, - destructurer_name: Ident, - span_range: SpanRange, - ) -> Self { - Self { - parse_stream, - destructurer_name, - full_span_range: span_range, - } - } +impl DestructurerDefinition for StreamDestructurer { + const DESTRUCTURER_NAME: &'static str = "stream"; - pub(crate) fn full_span_range(&self) -> SpanRange { - self.full_span_range + fn parse(arguments: DestructurerArguments) -> Result { + Ok(Self { + inner: arguments.fully_parse_no_error_override()?, + }) } - /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message - pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> Result<()> { - if self.parse_stream.is_empty() { - Ok(()) - } else { - self.full_span_range.err(error_message) - } + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + self.inner.handle_destructure(input, interpreter) } +} - pub(crate) fn fully_parse_no_error_override(&self) -> Result { - self.parse_stream.parse() - } +#[derive(Clone)] +pub(crate) struct IdentDestructurer { + variable: Option, +} - pub(crate) fn fully_parse_as(&self) -> Result { - self.fully_parse_or_error(T::parse, T::error_message()) +impl DestructurerDefinition for IdentDestructurer { + const DESTRUCTURER_NAME: &'static str = "ident"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + if input.is_empty() { + Ok(Self { variable: None }) + } else { + Ok(Self { + variable: Some(DestructureVariable::parse_only_unflattened_input(input)?), + }) + } + }, + "Expected (!ident! #x) or (!ident #>>x) or (!ident!)", + ) } - pub(crate) fn fully_parse_or_error( - &self, - parse_function: impl FnOnce(ParseStream) -> Result, - error_message: impl std::fmt::Display, - ) -> Result { - let parsed = parse_function(self.parse_stream).or_else(|error| { - // In future, when the diagnostic API is stable, - // we can add this context directly onto the command ident... - // Rather than just selectively adding it to the inner-most error. - let error_string = error.to_string(); - - // We avoid adding this additional context if it's already been added in an - // inner error, because that's likely the correct error to show. - if error_string.contains("\nOccurred whilst parsing") { - return Err(error); + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + if input.cursor().ident().is_some() { + match &self.variable { + Some(variable) => variable.handle_destructure(input, interpreter), + None => { + let _ = input.parse_any_ident()?; + Ok(()) + } } - error.span().err(format!( - "{}\nOccurred whilst parsing (!{}! ..) - {}", - error_string, self.destructurer_name, error_message, - )) - })?; - - self.assert_empty(error_message)?; - - Ok(parsed) + } else { + Err(input.error("Expected an ident")) + } } } #[derive(Clone)] -pub(crate) struct Destructurer { - instance: NamedDestructurer, - #[allow(unused)] - source_group_span: DelimSpan, +pub(crate) struct LiteralDestructurer { + variable: Option, } -impl Parse for Destructurer { - fn parse(input: ParseStream) -> Result { - let content; - let open_bracket = syn::parenthesized!(content in input); - content.parse::()?; - let destructurer_name = content.call(Ident::parse_any)?; - let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { - Some(destructurer_kind) => destructurer_kind, - None => destructurer_name.span().err( - format!( - "Expected `(!! ...)`, for one of: {}.\nIf this wasn't intended to be a named destructuring, you can work around this with (!raw! (!{} ... ))", - DestructurerKind::list_all(), - destructurer_name, - ), - )?, - }; - content.parse::()?; - let instance = destructurer_kind.parse_instance(DestructurerArguments::new( - &content, - destructurer_name, - open_bracket.span.span_range(), - ))?; - Ok(Self { - instance, - source_group_span: open_bracket.span, - }) +impl DestructurerDefinition for LiteralDestructurer { + const DESTRUCTURER_NAME: &'static str = "literal"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + if input.is_empty() { + Ok(Self { variable: None }) + } else { + Ok(Self { + variable: Some(DestructureVariable::parse_only_unflattened_input(input)?), + }) + } + }, + "Expected (!literal! #x) or (!literal! #>>x) or (!literal!)", + ) } -} -impl HandleDestructure for Destructurer { fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - self.instance.handle_destructure(input, interpreter) + if input.cursor().literal().is_some() { + match &self.variable { + Some(variable) => variable.handle_destructure(input, interpreter), + None => { + let _ = input.parse::()?; + Ok(()) + } + } + } else { + Err(input.error("Expected a literal")) + } } } -macro_rules! define_destructurers { - ( - $( - $destructurer:ident, - )* - ) => { - #[allow(clippy::enum_variant_names)] - #[derive(Clone, Copy)] - pub(crate) enum DestructurerKind { - $( - $destructurer, - )* - } - - impl DestructurerKind { - fn parse_instance(&self, arguments: DestructurerArguments) -> Result { - Ok(match self { - $( - Self::$destructurer => NamedDestructurer::$destructurer( - $destructurer::parse(arguments)? - ), - )* - }) - } - - pub(crate) fn for_ident(ident: &Ident) -> Option { - Some(match ident.to_string().as_ref() { - $( - $destructurer::DESTRUCTURER_NAME => Self::$destructurer, - )* - _ => return None, - }) - } - - const ALL_KIND_NAMES: &'static [&'static str] = &[$($destructurer::DESTRUCTURER_NAME,)*]; - - pub(crate) fn list_all() -> String { - // TODO improve to add an "and" at the end - Self::ALL_KIND_NAMES.join(", ") - } - } +#[derive(Clone)] +pub(crate) struct PunctDestructurer { + variable: Option, +} - #[derive(Clone)] - #[allow(clippy::enum_variant_names)] - pub(crate) enum NamedDestructurer { - $( - $destructurer($destructurer), - )* - } +impl DestructurerDefinition for PunctDestructurer { + const DESTRUCTURER_NAME: &'static str = "punct"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + if input.is_empty() { + Ok(Self { variable: None }) + } else { + Ok(Self { + variable: Some(DestructureVariable::parse_only_unflattened_input(input)?), + }) + } + }, + "Expected (!punct! #x) or (!punct! #>>x) or (!punct!)", + ) + } - impl NamedDestructurer { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - match self { - $( - Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), - )* + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + if input.cursor().any_punct().is_some() { + match &self.variable { + Some(variable) => variable.handle_destructure(input, interpreter), + None => { + let _ = input.parse_any_punct()?; + Ok(()) } } + } else { + Err(input.error("Expected a punct")) } - }; + } } -define_destructurers! { - StreamDestructurer, - IdentDestructurer, - LiteralDestructurer, - PunctDestructurer, - GroupDestructurer, +#[derive(Clone)] +pub(crate) struct GroupDestructurer { + inner: DestructureRemaining, +} + +impl DestructurerDefinition for GroupDestructurer { + const DESTRUCTURER_NAME: &'static str = "group"; + + fn parse(arguments: DestructurerArguments) -> Result { + Ok(Self { + inner: arguments.fully_parse_no_error_override()?, + }) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + let (_, inner) = input.parse_group_matching(Delimiter::None)?; + self.inner.handle_destructure(&inner, interpreter) + } } diff --git a/src/destructuring/mod.rs b/src/destructuring/mod.rs index 11327c0b..79efd71c 100644 --- a/src/destructuring/mod.rs +++ b/src/destructuring/mod.rs @@ -3,16 +3,16 @@ mod destructure_item; mod destructure_segment; mod destructure_traits; mod destructure_variable; +mod destructurer; mod destructurers; mod fields; -mod named_destructurers; pub(crate) use destructure_group::*; pub(crate) use destructure_item::*; pub(crate) use destructure_segment::*; pub(crate) use destructure_traits::*; pub(crate) use destructure_variable::*; +pub(crate) use destructurer::*; pub(crate) use destructurers::*; #[allow(unused)] pub(crate) use fields::*; -pub(crate) use named_destructurers::*; diff --git a/src/destructuring/named_destructurers.rs b/src/destructuring/named_destructurers.rs deleted file mode 100644 index f7600f02..00000000 --- a/src/destructuring/named_destructurers.rs +++ /dev/null @@ -1,124 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) struct StreamDestructurer { - inner: DestructureRemaining, -} - -impl DestructurerDefinition for StreamDestructurer { - const DESTRUCTURER_NAME: &'static str = "stream"; - - fn parse(arguments: DestructurerArguments) -> Result { - Ok(Self { - inner: arguments.fully_parse_no_error_override()?, - }) - } - - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - self.inner.handle_destructure(input, interpreter) - } -} - -#[derive(Clone)] -pub(crate) struct IdentDestructurer { - variable: DestructureVariable, -} - -impl DestructurerDefinition for IdentDestructurer { - const DESTRUCTURER_NAME: &'static str = "ident"; - - fn parse(arguments: DestructurerArguments) -> Result { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - variable: DestructureVariable::parse_only_unflattened_input(input)?, - }) - }, - "Expected (!ident! #x) or (!ident #>>x)", - ) - } - - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - if input.cursor().ident().is_some() { - self.variable.handle_destructure(input, interpreter) - } else { - Err(input.error("Expected an ident")) - } - } -} - -#[derive(Clone)] -pub(crate) struct LiteralDestructurer { - variable: DestructureVariable, -} - -impl DestructurerDefinition for LiteralDestructurer { - const DESTRUCTURER_NAME: &'static str = "literal"; - - fn parse(arguments: DestructurerArguments) -> Result { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - variable: DestructureVariable::parse_only_unflattened_input(input)?, - }) - }, - "Expected (!literal! #x) or (!literal! #>>x)", - ) - } - - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - if input.cursor().literal().is_some() { - self.variable.handle_destructure(input, interpreter) - } else { - Err(input.error("Expected a literal")) - } - } -} - -#[derive(Clone)] -pub(crate) struct PunctDestructurer { - variable: DestructureVariable, -} - -impl DestructurerDefinition for PunctDestructurer { - const DESTRUCTURER_NAME: &'static str = "punct"; - - fn parse(arguments: DestructurerArguments) -> Result { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - variable: DestructureVariable::parse_only_unflattened_input(input)?, - }) - }, - "Expected (!punct! #x) or (!punct! #>>x)", - ) - } - - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - if input.cursor().any_punct().is_some() { - self.variable.handle_destructure(input, interpreter) - } else { - Err(input.error("Expected a punct")) - } - } -} - -#[derive(Clone)] -pub(crate) struct GroupDestructurer { - inner: DestructureRemaining, -} - -impl DestructurerDefinition for GroupDestructurer { - const DESTRUCTURER_NAME: &'static str = "group"; - - fn parse(arguments: DestructurerArguments) -> Result { - Ok(Self { - inner: arguments.fully_parse_no_error_override()?, - }) - } - - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { - let (_, inner) = input.parse_group_matching(Delimiter::None)?; - self.inner.handle_destructure(&inner, interpreter) - } -} diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index ddd7721c..50c47f73 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -408,6 +408,7 @@ define_command_kind! { IgnoreCommand, SettingsCommand, ErrorCommand, + DebugCommand, // Concat & Type Convert Commands StringCommand, diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 303586bc..1e5af856 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -254,41 +254,24 @@ impl InterpretedStream { output } - pub(crate) fn concat_recursive(self) -> String { - fn wrap_delimiters( + pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> String { + fn concat_recursive_interpreted_stream( + behaviour: &ConcatBehaviour, output: &mut String, - delimiter: Delimiter, - inner: impl FnOnce(&mut String), + stream: InterpretedStream, ) { - match delimiter { - Delimiter::Parenthesis => { - output.push('('); - inner(output); - output.push(')'); - } - Delimiter::Brace => { - output.push('{'); - inner(output); - output.push('}'); - } - Delimiter::Bracket => { - output.push('['); - inner(output); - output.push(']'); - } - Delimiter::None => { - inner(output); - } - } - } - - fn concat_recursive_interpreted_stream(output: &mut String, stream: InterpretedStream) { for segment in stream.segments { match segment { - InterpretedSegment::TokenVec(vec) => concat_recursive_token_stream(output, vec), + InterpretedSegment::TokenVec(vec) => { + concat_recursive_token_stream(behaviour, output, vec) + } InterpretedSegment::InterpretedGroup(delimiter, _, interpreted_stream) => { - wrap_delimiters(output, delimiter, |output| { - concat_recursive_interpreted_stream(output, interpreted_stream); + behaviour.wrap_delimiters(output, delimiter, |output| { + concat_recursive_interpreted_stream( + behaviour, + output, + interpreted_stream, + ); }); } } @@ -296,18 +279,19 @@ impl InterpretedStream { } fn concat_recursive_token_stream( + behaviour: &ConcatBehaviour, output: &mut String, token_stream: impl IntoIterator, ) { - for token_tree in token_stream { + for (n, token_tree) in token_stream.into_iter().enumerate() { + behaviour.before_nth_token_tree(output, n); match token_tree { - TokenTree::Literal(literal) => match literal.content_if_string_like() { - Some(content) => output.push_str(&content), - None => output.push_str(&literal.to_string()), - }, + TokenTree::Literal(literal) => { + behaviour.handle_literal(output, literal); + } TokenTree::Group(group) => { - wrap_delimiters(output, group.delimiter(), |output| { - concat_recursive_token_stream(output, group.stream()); + behaviour.wrap_delimiters(output, group.delimiter(), |output| { + concat_recursive_token_stream(behaviour, output, group.stream()); }); } TokenTree::Punct(punct) => { @@ -319,11 +303,86 @@ impl InterpretedStream { } let mut output = String::new(); - concat_recursive_interpreted_stream(&mut output, self); + concat_recursive_interpreted_stream(behaviour, &mut output, self); output } } +pub(crate) struct ConcatBehaviour<'a> { + pub(crate) between_token_trees: Option<&'a str>, + pub(crate) output_transparent_group_as_command: bool, + pub(crate) unwrap_contents_of_string_like_literals: bool, +} + +impl ConcatBehaviour<'_> { + pub(crate) fn standard() -> Self { + Self { + between_token_trees: None, + output_transparent_group_as_command: false, + unwrap_contents_of_string_like_literals: true, + } + } + + pub(crate) fn debug() -> Self { + Self { + between_token_trees: Some(" "), + output_transparent_group_as_command: true, + unwrap_contents_of_string_like_literals: false, + } + } + + fn before_nth_token_tree(&self, output: &mut String, n: usize) { + if let Some(between) = self.between_token_trees { + if n > 0 { + output.push_str(between); + } + } + } + + fn handle_literal(&self, output: &mut String, literal: Literal) { + match literal.content_if_string_like() { + Some(content) if self.unwrap_contents_of_string_like_literals => { + output.push_str(&content) + } + _ => output.push_str(&literal.to_string()), + } + } + + fn wrap_delimiters( + &self, + output: &mut String, + delimiter: Delimiter, + inner: impl FnOnce(&mut String), + ) { + match delimiter { + Delimiter::Parenthesis => { + output.push('('); + inner(output); + output.push(')'); + } + Delimiter::Brace => { + output.push('{'); + inner(output); + output.push('}'); + } + Delimiter::Bracket => { + output.push('['); + inner(output); + output.push(']'); + } + Delimiter::None => { + if self.output_transparent_group_as_command { + output.push_str("[!group! "); + inner(output); + output.push(']'); + } else { + inner(output); + } + } + } + } +} + impl From for InterpretedStream { fn from(value: TokenTree) -> Self { InterpretedStream::raw(value.into()) diff --git a/src/traits.rs b/src/traits.rs index 8b013519..b7770918 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -41,7 +41,13 @@ impl CursorExt for Cursor<'_> { } fn punct_matching(self, char: char) -> Option<(Punct, Self)> { - match self.punct() { + // self.punct() is a little more efficient, but can't match ' + let matcher = if char == '\'' { + self.any_punct() + } else { + self.punct() + }; + match matcher { Some((punct, next)) if punct.as_char() == char => Some((punct, next)), _ => None, } diff --git a/tests/compilation_failures/destructuring/append_before_set.rs b/tests/compilation_failures/destructuring/append_before_set.rs new file mode 100644 index 00000000..0f3e1c7c --- /dev/null +++ b/tests/compilation_failures/destructuring/append_before_set.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! #>>x = Hello]); +} \ No newline at end of file diff --git a/tests/compilation_failures/destructuring/append_before_set.stderr b/tests/compilation_failures/destructuring/append_before_set.stderr new file mode 100644 index 00000000..236367af --- /dev/null +++ b/tests/compilation_failures/destructuring/append_before_set.stderr @@ -0,0 +1,5 @@ +error: The variable #x wasn't already set + --> tests/compilation_failures/destructuring/append_before_set.rs:4:26 + | +4 | preinterpret!([!let! #>>x = Hello]); + | ^^^^ diff --git a/tests/compilation_failures/destructuring/double_flattened_variable.rs b/tests/compilation_failures/destructuring/double_flattened_variable.rs new file mode 100644 index 00000000..4177821e --- /dev/null +++ b/tests/compilation_failures/destructuring/double_flattened_variable.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! #..x #..y = Hello World]); +} \ No newline at end of file diff --git a/tests/compilation_failures/destructuring/double_flattened_variable.stderr b/tests/compilation_failures/destructuring/double_flattened_variable.stderr new file mode 100644 index 00000000..8f63bc32 --- /dev/null +++ b/tests/compilation_failures/destructuring/double_flattened_variable.stderr @@ -0,0 +1,6 @@ +error: This cannot follow a flattened destructure match + Occurred whilst parsing [!let! ..] - Expected [!let! = ...] + --> tests/compilation_failures/destructuring/double_flattened_variable.rs:4:31 + | +4 | preinterpret!([!let! #..x #..y = Hello World]); + | ^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 89876304..26977503 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -58,26 +58,6 @@ fn test_while() { }, 5); } -#[test] -fn test_for() { - assert_preinterpret_eq!( - { - [!string! [!for! #x in [!range! 65..70] { - [!evaluate! #x as u8 as char] - }]] - }, - "ABCDE" - ); - assert_preinterpret_eq!( - { - [!string! [!for! (#x,) in [(a,) (b,) (c,)] { - #x - }]] - }, - "abc" - ); -} - #[test] fn test_loop_continue_and_break() { assert_preinterpret_eq!( @@ -101,3 +81,24 @@ fn test_loop_continue_and_break() { "ACEGI" ); } + +#[test] +fn test_for() { + assert_preinterpret_eq!( + { + [!string! [!for! #x in [!range! 65..70] { + [!evaluate! #x as u8 as char] + }]] + }, + "ABCDE" + ); + assert_preinterpret_eq!( + { + [!string! [!for! (#x,) in [(a,) (b,) (c,)] { + #x + [!if! [!string! #x] == "b" { [!break!] }] + }]] + }, + "ab" + ); +} diff --git a/tests/core.rs b/tests/core.rs index aae8be33..b0bc6021 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,6 +1,6 @@ use preinterpret::preinterpret; -macro_rules! my_assert_eq { +macro_rules! assert_preinterpret_eq { ($input:tt, $($output:tt)*) => { assert_eq!(preinterpret!($input), $($output)*); }; @@ -16,11 +16,11 @@ fn test_core_compilation_failures() { #[test] fn test_set() { - my_assert_eq!({ + assert_preinterpret_eq!({ [!set! #output = "Hello World!"] #output }, "Hello World!"); - my_assert_eq!({ + assert_preinterpret_eq!({ [!set! #hello = "Hello"] [!set! #world = "World"] [!set! #output = #hello " " #world "!"] @@ -29,57 +29,9 @@ fn test_set() { }, "Hello World!"); } -#[test] -fn test_let() { - my_assert_eq!({ - [!let! = ] - [!string! #inner] - }, "Beautiful"); - my_assert_eq!({ - [!let! #..inner = ] - [!string! #inner] - }, ""); - my_assert_eq!({ - [!let! #..x = Hello => World] - [!string! #x] - }, "Hello=>World"); - my_assert_eq!({ - [!let! Hello #..x!! = Hello => World!!] - [!string! #x] - }, "=>World"); - my_assert_eq!({ - [!let! Hello #..x World = Hello => World] - [!string! #x] - }, "=>"); - my_assert_eq!({ - [!let! Hello #..x World = Hello And Welcome To The Wonderful World] - [!string! #x] - }, "AndWelcomeToTheWonderful"); - my_assert_eq!({ - [!let! Hello #..x "World"! = Hello World And Welcome To The Wonderful "World"!] - [!string! #x] - }, "WorldAndWelcomeToTheWonderful"); - my_assert_eq!({ - [!let! #..x (#..y) = Why Hello (World)] - [!string! "#x = " #x "; #y = " #y] - }, "#x = WhyHello; #y = World"); - my_assert_eq!({ - [!set! #x =] - [!let! - #>>x // Matches one tt and appends it: Why - #..>>x // Matches stream until (, appends it grouped: [!group! Hello Everyone] - ( - #>>..x // Matches one tt and appends it flattened: This is an exciting adventure - #..>>..x // Matches stream and appends it flattened: do you agree ? - ) - = Why Hello Everyone ([!group! This is an exciting adventure] do you agree?)] - [!string! [!intersperse! { items: #x, separator: [_] } ]] - }, "Why_HelloEveryone_This_is_an_exciting_adventure_do_you_agree_?"); -} - #[test] fn test_raw() { - my_assert_eq!( + assert_preinterpret_eq!( { [!string! [!raw! #variable and [!command!] are not interpreted or error]] }, "#variableand[!command!]arenotinterpretedorerror" ); @@ -87,7 +39,7 @@ fn test_raw() { #[test] fn test_extend() { - my_assert_eq!( + assert_preinterpret_eq!( { [!set! #variable = "Hello"] [!extend! #variable += " World!"] @@ -95,7 +47,7 @@ fn test_extend() { }, "Hello World!" ); - my_assert_eq!( + assert_preinterpret_eq!( { [!set! #i = 1] [!set! #output = [!..group!]] @@ -114,7 +66,7 @@ fn test_extend() { #[test] fn test_ignore() { - my_assert_eq!({ + assert_preinterpret_eq!({ true [!ignore! this is all just ignored!] }, true); diff --git a/tests/destructuring.rs b/tests/destructuring.rs new file mode 100644 index 00000000..a73f8391 --- /dev/null +++ b/tests/destructuring.rs @@ -0,0 +1,136 @@ +use preinterpret::preinterpret; + +macro_rules! assert_preinterpret_eq { + ($input:tt, $($output:tt)*) => { + assert_eq!(preinterpret!($input), $($output)*); + }; +} + +#[test] +#[cfg_attr(miri, ignore = "incompatible with miri")] +fn test_destructuring_compilation_failures() { + let t = trybuild::TestCases::new(); + // In particular, the "error" command is tested here. + t.compile_fail("tests/compilation_failures/destructuring/*.rs"); +} + +#[test] +fn test_variable_parsing() { + assert_preinterpret_eq!({ + [!let! = ] + [!string! #inner] + }, "Beautiful"); + assert_preinterpret_eq!({ + [!let! #..inner = ] + [!string! #inner] + }, ""); + assert_preinterpret_eq!({ + [!let! #..x = Hello => World] + [!string! #x] + }, "Hello=>World"); + assert_preinterpret_eq!({ + [!let! Hello #..x!! = Hello => World!!] + [!string! #x] + }, "=>World"); + assert_preinterpret_eq!({ + [!let! Hello #..x World = Hello => World] + [!string! #x] + }, "=>"); + assert_preinterpret_eq!({ + [!let! Hello #..x World = Hello And Welcome To The Wonderful World] + [!string! #x] + }, "AndWelcomeToTheWonderful"); + assert_preinterpret_eq!({ + [!let! Hello #..x "World"! = Hello World And Welcome To The Wonderful "World"!] + [!string! #x] + }, "WorldAndWelcomeToTheWonderful"); + assert_preinterpret_eq!({ + [!let! #..x (#..y) = Why Hello (World)] + [!string! "#x = " #x "; #y = " #y] + }, "#x = WhyHello; #y = World"); + assert_preinterpret_eq!({ + [!set! #x =] + [!let! + #>>x // Matches one tt and appends it: Why + #..>>x // Matches stream until (, appends it grouped: [!group! Hello Everyone] + ( + #>>..x // Matches one tt and appends it flattened: This is an exciting adventure + #..>>..x // Matches stream and appends it flattened: do you agree ? + ) + = Why Hello Everyone ([!group! This is an exciting adventure] do you agree?)] + [!string! [!intersperse! { items: #x, separator: [_] } ]] + }, "Why_HelloEveryone_This_is_an_exciting_adventure_do_you_agree_?"); +} + +#[test] +fn test_stream_destructurer() { + // It's not very exciting + preinterpret!([!let! (!stream! Hello World) = Hello World]); + preinterpret!([!let! Hello (!stream! World) = Hello World]); + preinterpret!([!let! (!stream! Hello (!stream! World)) = Hello World]); +} + +#[test] +fn test_ident_destructurer() { + assert_preinterpret_eq!({ + [!let! The "quick" (!ident! #x) fox "jumps" = The "quick" brown fox "jumps"] + [!string! #x] + }, "brown"); + assert_preinterpret_eq!({ + [!set! #x =] + [!let! The quick (!ident! #>>x) fox jumps (!ident! #>>x) the lazy dog = The quick brown fox jumps over the lazy dog] + [!string! [!intersperse! { items: #x, separator: ["_"] }]] + }, "brown_over"); +} + +#[test] +fn test_literal_destructurer() { + assert_preinterpret_eq!({ + [!let! The "quick" (!literal! #x) fox "jumps" = The "quick" "brown" fox "jumps"] + #x + }, "brown"); + // Lots of literals + assert_preinterpret_eq!({ + [!set! #x =] + [!let! (!literal!) (!literal!) (!literal!) (!literal! #>>x) (!literal!) (!literal! #>>x) (!literal! #>>x) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] + [!debug! #..x] + }, "'c' 0b1010 r#\"123\"#"); +} + +#[test] +fn test_punct_destructurer() { + assert_preinterpret_eq!({ + [!let! The "quick" brown fox "jumps" (!punct! #x) = The "quick" brown fox "jumps"!] + [!debug! #..x] + }, "!"); + // Test for ' which is treated weirdly by syn / rustc + assert_preinterpret_eq!({ + [!let! The "quick" fox isn 't brown and doesn (!punct! #x) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] + [!debug! #..x] + }, "'"); + // Lots of punctuation, most of it ignored + assert_preinterpret_eq!({ + [!set! #x =] + [!let! (!punct!) (!punct!) (!punct!) (!punct!) (!punct! #>>x) (!punct!) (!punct!) (!punct!) (!punct!) (!punct!) (!punct! #>>x) (!punct!) (!punct!) (!punct!) = # ! $$ % ^ & * + = | @ : ;] + [!debug! #..x] + }, "% |"); +} + +#[test] +fn test_group_destructurer() { + assert_preinterpret_eq!({ + [!let! The "quick" (!group! brown #x) "jumps" = The "quick" [!group! brown fox] "jumps"] + [!debug! #..x] + }, "fox"); + assert_preinterpret_eq!({ + [!set! #x = "hello" "world"] + [!let! I said (!group! #..y)! = I said #x!] + [!debug! #..y] + }, "\"hello\" \"world\""); + // ... which is equivalent to this: + assert_preinterpret_eq!({ + [!set! #x = "hello" "world"] + [!let! I said #y! = I said #x!] + [!debug! #..y] + }, "\"hello\" \"world\""); +} From 5cb35a96db47a1db5179bad52d6a827d74d05dbd Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 23 Jan 2025 00:33:04 +0000 Subject: [PATCH 055/126] feature: Added void command --- CHANGELOG.md | 13 ++++++++----- src/commands/core_commands.rs | 24 ++++++++++++++++++++++++ src/interpretation/command.rs | 1 + tests/core.rs | 14 ++++++++++++-- tests/destructuring.rs | 8 ++++++++ 5 files changed, 53 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d6e9878..4f392d64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ * Core commands: * `[!error! ...]` to output a compile error * `[!extend! #x += ...]` to performantly add extra characters to the stream - * `[!let! = ...]` does destructuring/parsing (see next section) + * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. + * `[!void! ...]` interprets its arguments but then ignores any outputs. It can be used inside destructurings. * Expression commands: * `[!evaluate! ]` * `[!assign! #x += ]` for `+` and other supported operators @@ -31,6 +32,8 @@ * `[!..group! ...]` which just outputs its contents as-is, useful where the grammar only takes a single item, but we want to output multiple tokens * `[!intersperse! { ... }]` which inserts separator tokens between each token tree in a stream. +* Destructuring commands: + * `[!let! = ...]` does destructuring/parsing (see next section). Note `[!let! #..x = ...]` is equivalent to `[!set! #x = ...]` ### Expressions @@ -52,11 +55,11 @@ Destructuring performs parsing of a token stream. It supports: * Explicit punctuation, idents, literals and groups * Variable bindings: - * `#x` - Reads a token tree, writes a stream (opposite of #x) - * `#..x` - Reads a stream, writes a stream (opposite of #..x) - * `#>>x` - Reads a token tree, appends a token tree (opposite of for #y in #x) + * `#x` - Reads a token tree, writes a stream (opposite of `#x`) + * `#..x` - Reads a stream, writes a stream (opposite of `#..x`) + * `#>>x` - Reads a token tree, appends a token tree (can be read back with `!for! #y in #x { ... }`) * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) - * `#..>>x` - Reads a stream, appends a group (opposite of for #y in #x) + * `#..>>x` - Reads a stream, appends a group (can be read back with `!for! #y in #x { ... }`) * `#..>>..x` - Reads a stream, appends a stream * Commands which don't output a value, like `[!set! ...]` or `[!extend! ...]` * Named destructurings: diff --git a/src/commands/core_commands.rs b/src/commands/core_commands.rs index 56e0299d..e8adbc10 100644 --- a/src/commands/core_commands.rs +++ b/src/commands/core_commands.rs @@ -122,6 +122,30 @@ impl NoOutputCommandDefinition for IgnoreCommand { } } +#[derive(Clone)] +pub(crate) struct VoidCommand { + inner: InterpretationStream, +} + +impl CommandType for VoidCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for VoidCommand { + const COMMAND_NAME: &'static str = "void"; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + inner: arguments.parse_all_for_interpretation()?, + }) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + let _ = self.inner.interpret_to_new_stream(interpreter)?; + Ok(()) + } +} + #[derive(Clone)] pub(crate) struct SettingsCommand { inputs: SettingsInputs, diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 50c47f73..d7917534 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -406,6 +406,7 @@ define_command_kind! { ExtendCommand, RawCommand, IgnoreCommand, + VoidCommand, SettingsCommand, ErrorCommand, DebugCommand, diff --git a/tests/core.rs b/tests/core.rs index b0bc6021..bd46e285 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -67,7 +67,17 @@ fn test_extend() { #[test] fn test_ignore() { assert_preinterpret_eq!({ - true - [!ignore! this is all just ignored!] + [!set! #x = false] + [!ignore! [!set! #x = true] nothing is interpreted. Everything is ignored...] + #x + }, false); +} + +#[test] +fn test_void() { + assert_preinterpret_eq!({ + [!set! #x = false] + [!void! [!set! #x = true] things _are_ interpreted, but the result is ignored...] + #x }, true); } diff --git a/tests/destructuring.rs b/tests/destructuring.rs index a73f8391..2b1e2c2c 100644 --- a/tests/destructuring.rs +++ b/tests/destructuring.rs @@ -134,3 +134,11 @@ fn test_group_destructurer() { [!debug! #..y] }, "\"hello\" \"world\""); } + +#[test] +fn test_none_output_commands_mid_parse() { + assert_preinterpret_eq!({ + [!let! The "quick" (!literal! #x) fox [!let! #y = #x] (!ident! #x) = The "quick" "brown" fox jumps] + [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] + }, "#x = jumps; #y = \"brown\""); +} From c525242144142be315f1c76bed5033a63a248611 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 23 Jan 2025 23:55:08 +0000 Subject: [PATCH 056/126] feature: Add split commands and raw destructurers --- CHANGELOG.md | 12 +- src/commands/token_commands.rs | 141 +++++++++++++++++++++++ src/destructuring/destructure_raw.rs | 98 ++++++++++++++++ src/destructuring/destructurer.rs | 2 + src/destructuring/destructurers.rs | 51 ++++++++ src/destructuring/mod.rs | 2 + src/interpretation/command.rs | 2 + src/interpretation/interpreted_stream.rs | 62 ++++++++-- tests/destructuring.rs | 25 ++++ tests/tokens.rs | 77 +++++++++++++ 10 files changed, 455 insertions(+), 17 deletions(-) create mode 100644 src/destructuring/destructure_raw.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f392d64..715c7c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ * `[!..group! ...]` which just outputs its contents as-is, useful where the grammar only takes a single item, but we want to output multiple tokens * `[!intersperse! { ... }]` which inserts separator tokens between each token tree in a stream. + * `[!split! ...]` + * `[!comma_split! ...]` * Destructuring commands: * `[!let! = ...]` does destructuring/parsing (see next section). Note `[!let! #..x = ...]` is equivalent to `[!set! #x = ...]` @@ -68,11 +70,12 @@ Destructuring performs parsing of a token stream. It supports: * `(!punct! ...)` * `(!literal! ...)` * `(!group! ...)` + * `(!raw! ...)` + * `(!content! ...)` ### To come -* Add compile error tests for all the destructuring errors -* `[!split! { stream: [...], separator: [...], drop_empty?: false, permit_trailing_separator?: true, }]` and `[!comma_split! ...]` +* Add compile error tests for all the standard destructuring errors * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), error_on_length_mismatch?: true }]` with `InterpretValue>>` * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` * `[!is_set! #x]` @@ -85,12 +88,11 @@ Destructuring performs parsing of a token stream. It supports: * `ParseResult` with `ParseError::LowLevel(syn::Error)` | `ParseError::Contextual(syn::Error)` * `ExecutionResult` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` * Destructurers - * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden - * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` - * `(!raw! ...)` * `(!fields! ...)` and `(!subfields! ...)` * Add ability to fork (copy on write?) / revert the interpreter state and can then add: * `(!optional! ...)` + * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden + * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` * `(!match! ...)` (with `#..x` as a catch-all) * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much * `[!match! ...]` command - similar to the match destructurer, but a command... diff --git a/src/commands/token_commands.rs b/src/commands/token_commands.rs index fb19f1c5..79c25d1c 100644 --- a/src/commands/token_commands.rs +++ b/src/commands/token_commands.rs @@ -217,3 +217,144 @@ enum TrailingSeparator { Final, None, } + +#[derive(Clone)] +pub(crate) struct SplitCommand { + inputs: SplitInputs, +} + +impl CommandType for SplitCommand { + type OutputKind = OutputKindStream; +} + +define_field_inputs! { + SplitInputs { + required: { + stream: CommandStreamInput = "[...] or #var or [!cmd! ...]", + separator: CommandStreamInput = "[::]" ("The token/s to split if they match"), + }, + optional: { + drop_empty_start: CommandValueInput = "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), + drop_empty_middle: CommandValueInput = "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), + drop_empty_end: CommandValueInput = "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), + } + } +} + +impl StreamCommandDefinition for SplitCommand { + const COMMAND_NAME: &'static str = "split"; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + inputs: arguments.fully_parse_as()?, + }) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + let output_span = self.inputs.stream.span(); + let stream = self.inputs.stream.interpret_to_new_stream(interpreter)?; + let separator = self.inputs.separator.interpret_to_new_stream(interpreter)?; + + let drop_empty_start = match self.inputs.drop_empty_start { + Some(value) => value.interpret(interpreter)?.value(), + None => false, + }; + let drop_empty_middle = match self.inputs.drop_empty_middle { + Some(value) => value.interpret(interpreter)?.value(), + None => false, + }; + let drop_empty_end = match self.inputs.drop_empty_end { + Some(value) => value.interpret(interpreter)?.value(), + None => true, + }; + + handle_split( + stream, + output, + output_span, + separator.into_raw_destructure_stream(), + drop_empty_start, + drop_empty_middle, + drop_empty_end, + ) + } +} + +fn handle_split( + input: InterpretedStream, + output: &mut InterpretedStream, + output_span: Span, + separator: RawDestructureStream, + drop_empty_start: bool, + drop_empty_middle: bool, + drop_empty_end: bool, +) -> Result<()> { + unsafe { + // RUST-ANALYZER SAFETY: This is as safe as we can get. + // Typically the separator won't contain none-delimited groups, so we're OK + input.syn_parse(move |input: ParseStream| -> Result<()> { + let mut current_item = InterpretedStream::new(); + let mut drop_empty_next = drop_empty_start; + while !input.is_empty() { + let separator_fork = input.fork(); + if separator.handle_destructure(&separator_fork).is_err() { + current_item.push_raw_token_tree(input.parse()?); + continue; + } + input.advance_to(&separator_fork); + if !(current_item.is_empty() && drop_empty_next) { + let complete_item = + core::mem::replace(&mut current_item, InterpretedStream::new()); + output.push_new_group(complete_item, Delimiter::None, output_span); + } + drop_empty_next = drop_empty_middle; + } + if !(current_item.is_empty() && drop_empty_end) { + output.push_new_group(current_item, Delimiter::None, output_span); + } + Ok(()) + })?; + } + Ok(()) +} + +#[derive(Clone)] +pub(crate) struct CommaSplitCommand { + input: InterpretationStream, +} + +impl CommandType for CommaSplitCommand { + type OutputKind = OutputKindStream; +} + +impl StreamCommandDefinition for CommaSplitCommand { + const COMMAND_NAME: &'static str = "comma_split"; + + fn parse(arguments: CommandArguments) -> Result { + Ok(Self { + input: arguments.parse_all_for_interpretation()?, + }) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> Result<()> { + let output_span = self.input.span(); + let stream = self.input.interpret_to_new_stream(interpreter)?; + let separator = { + let mut stream = RawDestructureStream::empty(); + stream.push_item(RawDestructureItem::Punct( + Punct::new(',', Spacing::Alone).with_span(output_span), + )); + stream + }; + + handle_split(stream, output, output_span, separator, false, false, true) + } +} diff --git a/src/destructuring/destructure_raw.rs b/src/destructuring/destructure_raw.rs new file mode 100644 index 00000000..d7d21471 --- /dev/null +++ b/src/destructuring/destructure_raw.rs @@ -0,0 +1,98 @@ +use crate::internal_prelude::*; + +/// This is an *exact* destructuring, which must match item-by-item. +#[derive(Clone)] +pub(crate) enum RawDestructureItem { + Punct(Punct), + Ident(Ident), + Literal(Literal), + Group(RawDestructureGroup), +} + +impl RawDestructureItem { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> Result<()> { + match self { + RawDestructureItem::Punct(punct) => { + input.parse_punct_matching(punct.as_char())?; + } + RawDestructureItem::Ident(ident) => { + input.parse_ident_matching(&ident.to_string())?; + } + RawDestructureItem::Literal(literal) => { + input.parse_literal_matching(&literal.to_string())?; + } + RawDestructureItem::Group(group) => { + group.handle_destructure(input)?; + } + } + Ok(()) + } + + pub(crate) fn new_from_token_tree(token_tree: TokenTree) -> Self { + match token_tree { + TokenTree::Punct(punct) => Self::Punct(punct), + TokenTree::Ident(ident) => Self::Ident(ident), + TokenTree::Literal(literal) => Self::Literal(literal), + TokenTree::Group(group) => Self::Group(RawDestructureGroup::new_from_group(&group)), + } + } +} + +#[derive(Clone)] +pub(crate) struct RawDestructureStream { + inner: Vec, +} + +impl RawDestructureStream { + pub(crate) fn empty() -> Self { + Self { inner: vec![] } + } + + pub(crate) fn new_from_token_stream(tokens: impl IntoIterator) -> Self { + let mut new = Self::empty(); + new.append_from_token_stream(tokens); + new + } + + pub(crate) fn append_from_token_stream(&mut self, tokens: impl IntoIterator) { + for token_tree in tokens { + self.inner + .push(RawDestructureItem::new_from_token_tree(token_tree)); + } + } + + pub(crate) fn push_item(&mut self, item: RawDestructureItem) { + self.inner.push(item); + } + + pub(crate) fn handle_destructure(&self, input: ParseStream) -> Result<()> { + for item in self.inner.iter() { + item.handle_destructure(input)?; + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct RawDestructureGroup { + delimiter: Delimiter, + inner: RawDestructureStream, +} + +impl RawDestructureGroup { + pub(crate) fn new(delimiter: Delimiter, inner: RawDestructureStream) -> Self { + Self { delimiter, inner } + } + + pub(crate) fn new_from_group(group: &Group) -> Self { + Self { + delimiter: group.delimiter(), + inner: RawDestructureStream::new_from_token_stream(group.stream()), + } + } + + pub(crate) fn handle_destructure(&self, input: ParseStream) -> Result<()> { + let (_, inner) = input.parse_group_matching(self.delimiter)?; + self.inner.handle_destructure(&inner) + } +} diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index 7408e707..652f4450 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -186,4 +186,6 @@ define_destructurers! { LiteralDestructurer, PunctDestructurer, GroupDestructurer, + RawDestructurer, + ContentDestructurer, } diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index 0d7ecbe4..3c4b2839 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -152,3 +152,54 @@ impl DestructurerDefinition for GroupDestructurer { self.inner.handle_destructure(&inner, interpreter) } } + +#[derive(Clone)] +pub(crate) struct RawDestructurer { + stream: RawDestructureStream, +} + +impl DestructurerDefinition for RawDestructurer { + const DESTRUCTURER_NAME: &'static str = "raw"; + + fn parse(arguments: DestructurerArguments) -> Result { + let token_stream: TokenStream = arguments.fully_parse_no_error_override()?; + Ok(Self { + stream: RawDestructureStream::new_from_token_stream(token_stream), + }) + } + + fn handle_destructure(&self, input: ParseStream, _: &mut Interpreter) -> Result<()> { + self.stream.handle_destructure(input) + } +} + +#[derive(Clone)] +pub(crate) struct ContentDestructurer { + stream: InterpretationStream, +} + +impl DestructurerDefinition for ContentDestructurer { + const DESTRUCTURER_NAME: &'static str = "content"; + + fn parse(arguments: DestructurerArguments) -> Result { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + stream: InterpretationStream::parse_with_context( + input, + arguments.full_span_range(), + )?, + }) + }, + "Expected (!content! ... interpretable input ...)", + ) + } + + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + self.stream + .clone() + .interpret_to_new_stream(interpreter)? + .into_raw_destructure_stream() + .handle_destructure(input) + } +} diff --git a/src/destructuring/mod.rs b/src/destructuring/mod.rs index 79efd71c..0a2d15a5 100644 --- a/src/destructuring/mod.rs +++ b/src/destructuring/mod.rs @@ -1,5 +1,6 @@ mod destructure_group; mod destructure_item; +mod destructure_raw; mod destructure_segment; mod destructure_traits; mod destructure_variable; @@ -9,6 +10,7 @@ mod fields; pub(crate) use destructure_group::*; pub(crate) use destructure_item::*; +pub(crate) use destructure_raw::*; pub(crate) use destructure_segment::*; pub(crate) use destructure_traits::*; pub(crate) use destructure_variable::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index d7917534..ead048f3 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -452,6 +452,8 @@ define_command_kind! { LengthCommand, GroupCommand, IntersperseCommand, + SplitCommand, + CommaSplitCommand, } #[derive(Clone)] diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 1e5af856..aed21831 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -254,25 +254,52 @@ impl InterpretedStream { output } + pub(crate) fn into_raw_destructure_stream(self) -> RawDestructureStream { + let mut output = RawDestructureStream::empty(); + for segment in self.segments { + match segment { + InterpretedSegment::TokenVec(vec) => { + output.append_from_token_stream(vec); + } + InterpretedSegment::InterpretedGroup(delimiter, _, inner) => { + output.push_item(RawDestructureItem::Group(RawDestructureGroup::new( + delimiter, + inner.into_raw_destructure_stream(), + ))); + } + } + } + output + } + pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> String { fn concat_recursive_interpreted_stream( behaviour: &ConcatBehaviour, output: &mut String, stream: InterpretedStream, ) { + let mut n = 0; for segment in stream.segments { match segment { InterpretedSegment::TokenVec(vec) => { - concat_recursive_token_stream(behaviour, output, vec) + n += vec.len(); + concat_recursive_token_stream(behaviour, output, vec); } InterpretedSegment::InterpretedGroup(delimiter, _, interpreted_stream) => { - behaviour.wrap_delimiters(output, delimiter, |output| { - concat_recursive_interpreted_stream( - behaviour, - output, - interpreted_stream, - ); - }); + behaviour.before_nth_token_tree(output, n); + behaviour.wrap_delimiters( + output, + delimiter, + interpreted_stream.is_empty(), + |output| { + concat_recursive_interpreted_stream( + behaviour, + output, + interpreted_stream, + ); + }, + ); + n += 1; } } } @@ -290,9 +317,15 @@ impl InterpretedStream { behaviour.handle_literal(output, literal); } TokenTree::Group(group) => { - behaviour.wrap_delimiters(output, group.delimiter(), |output| { - concat_recursive_token_stream(behaviour, output, group.stream()); - }); + let inner = group.stream(); + behaviour.wrap_delimiters( + output, + group.delimiter(), + inner.is_empty(), + |output| { + concat_recursive_token_stream(behaviour, output, inner); + }, + ); } TokenTree::Punct(punct) => { output.push(punct.as_char()); @@ -352,6 +385,7 @@ impl ConcatBehaviour<'_> { &self, output: &mut String, delimiter: Delimiter, + is_empty: bool, inner: impl FnOnce(&mut String), ) { match delimiter { @@ -372,7 +406,11 @@ impl ConcatBehaviour<'_> { } Delimiter::None => { if self.output_transparent_group_as_command { - output.push_str("[!group! "); + if is_empty { + output.push_str("[!group!"); + } else { + output.push_str("[!group! "); + } inner(output); output.push(']'); } else { diff --git a/tests/destructuring.rs b/tests/destructuring.rs index 2b1e2c2c..3bf9e6f3 100644 --- a/tests/destructuring.rs +++ b/tests/destructuring.rs @@ -142,3 +142,28 @@ fn test_none_output_commands_mid_parse() { [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] }, "#x = jumps; #y = \"brown\""); } + +#[test] +fn test_raw_destructurer() { + assert_preinterpret_eq!({ + [!set! #x = true] + [!let! The (!raw! #x) = The [!raw! #] x] + #x + }, true); +} + +#[test] +fn test_content_destructurer() { + // Content works + assert_preinterpret_eq!({ + [!set! #x = true] + [!let! The (!content! #..x) = The true] + #x + }, true); + // Content is evaluated at destructuring time + assert_preinterpret_eq!({ + [!set! #x =] + [!let! The #>>..x fox is #>>..x. It 's super (!content! #..x). = The brown fox is brown. It 's super brown brown.] + true + }, true); +} diff --git a/tests/tokens.rs b/tests/tokens.rs index 0b58492a..ac5866b4 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -313,3 +313,80 @@ fn complex_cases_for_intersperse_and_input_types() { #x }, "EXECUTED"); } + +#[test] +fn test_split() { + // Double separators are allowed + assert_preinterpret_eq!( + { + [!debug! [!..split! { + stream: [A::B::C], + separator: [::], + }]] + }, + "[!group! A] [!group! B] [!group! C]" + ); + // Trailing separator is ignored by default + assert_preinterpret_eq!( + { + [!debug! [!..split! { + stream: [Pizza, Mac and Cheese, Hamburger,], + separator: [,], + }]] + }, + "[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]" + ); + // By default, empty groups are included except at the end + assert_preinterpret_eq!( + { + [!debug! [!..split! { + stream: [::A::B::::C::], + separator: [::], + }]] + }, + "[!group!] [!group! A] [!group! B] [!group!] [!group! C]" + ); + // Stream and separator are both interpreted + assert_preinterpret_eq!({ + [!set! #x = ;] + [!debug! [!..split! { + stream: [;A;;B;C;D #..x E;], + separator: #x, + drop_empty_start: true, + drop_empty_middle: true, + drop_empty_end: true, + }]] + }, "[!group! A] [!group! B] [!group! C] [!group! D] [!group! E]"); + // Drop empty false works + assert_preinterpret_eq!({ + [!set! #x = ;] + [!debug! [!..split! { + stream: [;A;;B;C;D #..x E;], + separator: #x, + drop_empty_start: false, + drop_empty_middle: false, + drop_empty_end: false, + }]] + }, "[!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]"); + // Drop empty middle works + assert_preinterpret_eq!( + { + [!debug! [!..split! { + stream: [;A;;B;;;;E;], + separator: [;], + drop_empty_start: false, + drop_empty_middle: true, + drop_empty_end: false, + }]] + }, + "[!group!] [!group! A] [!group! B] [!group! E] [!group!]" + ); +} + +#[test] +fn test_comma_split() { + assert_preinterpret_eq!( + { [!debug! [!..comma_split! Pizza, Mac and Cheese, Hamburger,]] }, + "[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]" + ); +} From 7a19622ea821dfc3979719c3248eb1885ff6e55c Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 24 Jan 2025 00:08:06 +0000 Subject: [PATCH 057/126] refactor: Minor tweaks --- CHANGELOG.md | 2 ++ src/destructuring/destructurer.rs | 2 +- src/interpretation/command.rs | 2 +- src/interpretation/command_field_inputs.rs | 2 +- src/interpretation/variable.rs | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 715c7c0c..4a90a87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ Destructuring performs parsing of a token stream. It supports: * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` +* Change `(!content! ...)` to unwrap none groups, to be more permissive * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Add more tests * e.g. for various expressions @@ -106,6 +107,7 @@ Destructuring performs parsing of a token stream. It supports: * Fix issues in Rust Analyzer * Improve performance * Permit `[!parse_while! [!PARSE! ...] from #x { ... }]` + * Fix `any_punct()` to ignore none groups * Further syn parsings (e.g. item, fields, etc) * Work on book * Input paradigms: diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index 652f4450..7857ca1c 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -88,7 +88,7 @@ impl Parse for Destructurer { let content; let open_bracket = syn::parenthesized!(content in input); content.parse::()?; - let destructurer_name = content.call(Ident::parse_any)?; + let destructurer_name = content.parse_any_ident()?; let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { Some(destructurer_kind) => destructurer_kind, None => destructurer_name.span().err( diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index ead048f3..e7987431 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -473,7 +473,7 @@ impl Parse for Command { } else { None }; - let command_name = content.call(Ident::parse_any)?; + let command_name = content.parse_any_ident()?; let (command_kind, output_kind) = match CommandKind::for_ident(&command_name) { Some(command_kind) => { let output_kind = command_kind.output_kind(flattening)?; diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index 594c3974..922dcbe8 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -37,7 +37,7 @@ macro_rules! define_field_inputs { let brace = syn::braced!(content in input); while !content.is_empty() { - let ident = content.call(Ident::parse_any)?; + let ident = content.parse_any_ident()?; content.parse::()?; match ident.to_string().as_str() { $( diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index e2d1b0c8..4cbbdff9 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -16,7 +16,7 @@ impl Parse for GroupedVariable { |input| { Ok(Self { marker: input.parse()?, - variable_name: input.call(Ident::parse_any)?, + variable_name: input.parse_any_ident()?, }) }, "Expected #variable", @@ -142,7 +142,7 @@ impl Parse for FlattenedVariable { Ok(Self { marker: input.parse()?, flatten: input.parse()?, - variable_name: input.call(Ident::parse_any)?, + variable_name: input.parse_any_ident()?, }) }, "Expected #..variable", From 356b3a86e881b052f5032ffd6483b0f39fe5dc19 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 24 Jan 2025 19:17:28 +0000 Subject: [PATCH 058/126] refactor: Added distinct result types --- CHANGELOG.md | 4 +- src/destructuring/destructure_group.rs | 10 +- src/destructuring/destructure_item.rs | 21 +- src/destructuring/destructure_raw.rs | 6 +- src/destructuring/destructure_segment.rs | 8 +- src/destructuring/destructure_traits.rs | 12 +- src/destructuring/destructure_variable.rs | 34 +- src/destructuring/destructurer.rs | 71 +-- src/destructuring/destructurers.rs | 58 ++- src/destructuring/fields.rs | 30 +- src/expressions/boolean.rs | 6 +- src/expressions/character.rs | 6 +- src/expressions/evaluation_tree.rs | 30 +- src/expressions/expression_stream.rs | 54 ++- src/expressions/float.rs | 32 +- src/expressions/integer.rs | 43 +- src/expressions/operations.rs | 84 ++-- src/expressions/string.rs | 6 +- src/expressions/value.rs | 10 +- src/extensions/errors_and_spans.rs | 171 +++++++ src/extensions/mod.rs | 7 + src/extensions/parsing.rs | 220 +++++++++ src/extensions/tokens.rs | 107 +++++ src/internal_prelude.rs | 12 +- src/interpretation/command.rs | 102 ++--- src/interpretation/command_arguments.rs | 42 +- src/interpretation/command_code_input.rs | 27 +- src/interpretation/command_field_inputs.rs | 17 +- src/interpretation/command_stream_input.rs | 22 +- src/interpretation/command_value_input.rs | 38 +- .../commands/concat_commands.rs | 17 +- .../commands/control_flow_commands.rs | 74 +-- .../commands/core_commands.rs | 42 +- .../commands/destructuring_commands.rs | 8 +- .../commands/expression_commands.rs | 28 +- src/{ => interpretation}/commands/mod.rs | 0 .../commands/token_commands.rs | 33 +- src/interpretation/interpret_traits.rs | 13 +- src/interpretation/interpretation_item.rs | 20 +- src/interpretation/interpretation_stream.rs | 33 +- src/interpretation/interpreted_stream.rs | 17 +- src/interpretation/interpreter.rs | 61 +-- src/interpretation/mod.rs | 1 + src/interpretation/variable.rs | 32 +- src/lib.rs | 21 +- src/misc/errors.rs | 107 +++++ src/misc/mod.rs | 7 + src/misc/parse_traits.rs | 17 + src/{ => misc}/string_conversion.rs | 0 src/traits.rs | 430 ------------------ .../complex/nested.stderr | 2 +- .../control_flow/break_outside_a_loop.stderr | 2 +- .../continue_outside_a_loop.stderr | 2 +- .../core/error_invalid_structure.stderr | 2 +- .../core/extend_flattened_variable.stderr | 2 +- .../core/set_flattened_variable.stderr | 2 +- .../double_flattened_variable.stderr | 2 +- .../expressions/braces.stderr | 2 +- .../expressions/inner_braces.stderr | 2 +- 59 files changed, 1266 insertions(+), 1003 deletions(-) create mode 100644 src/extensions/errors_and_spans.rs create mode 100644 src/extensions/mod.rs create mode 100644 src/extensions/parsing.rs create mode 100644 src/extensions/tokens.rs rename src/{ => interpretation}/commands/concat_commands.rs (92%) rename src/{ => interpretation}/commands/control_flow_commands.rs (76%) rename src/{ => interpretation}/commands/core_commands.rs (90%) rename src/{ => interpretation}/commands/destructuring_commands.rs (82%) rename src/{ => interpretation}/commands/expression_commands.rs (88%) rename src/{ => interpretation}/commands/mod.rs (100%) rename src/{ => interpretation}/commands/token_commands.rs (94%) create mode 100644 src/misc/errors.rs create mode 100644 src/misc/mod.rs create mode 100644 src/misc/parse_traits.rs rename src/{ => misc}/string_conversion.rs (100%) delete mode 100644 src/traits.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a90a87b..d5de79d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,7 @@ Destructuring performs parsing of a token stream. It supports: ### To come +* Add our own ParseBuffer / ParseStream types so we can e.g. override `parse` * Add compile error tests for all the standard destructuring errors * `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), error_on_length_mismatch?: true }]` with `InterpretValue>>` * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` @@ -85,9 +86,6 @@ Destructuring performs parsing of a token stream. It supports: * Add more tests * e.g. for various expressions * e.g. for long sums -* Rework `Error` as: - * `ParseResult` with `ParseError::LowLevel(syn::Error)` | `ParseError::Contextual(syn::Error)` - * `ExecutionResult` with `ExecutionInterrupt::Err(syn::Error)` | `ExecutionInterrupt::ControlFlow(..)` * Destructurers * `(!fields! ...)` and `(!subfields! ...)` * Add ability to fork (copy on write?) / revert the interpreter state and can then add: diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs index 67b848bd..21486902 100644 --- a/src/destructuring/destructure_group.rs +++ b/src/destructuring/destructure_group.rs @@ -7,17 +7,21 @@ pub(crate) struct DestructureGroup { } impl Parse for DestructureGroup { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, _, content) = input.parse_any_delimiter()?; Ok(Self { delimiter, - inner: content.parse()?, + inner: content.parse_v2()?, }) } } impl HandleDestructure for DestructureGroup { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { let (_, inner) = input.parse_group_matching(self.delimiter)?; self.inner.handle_destructure(&inner, interpreter) } diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs index 962b079f..65d2e418 100644 --- a/src/destructuring/destructure_item.rs +++ b/src/destructuring/destructure_item.rs @@ -16,38 +16,41 @@ impl DestructureItem { /// notably the flattened command. This allows [!let! #..x = Hello => World] to parse as setting /// `x` to `Hello => World` rather than having `#..x` peeking to see it is "up to =" and then only /// parsing `Hello` into `x`. - pub(crate) fn parse_until(input: ParseStream) -> Result { + pub(crate) fn parse_until(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { PeekMatch::GroupedCommand(Some(command_kind)) if matches!(command_kind.output_kind(None), Ok(CommandOutputKind::None)) => { - Self::NoneOutputCommand(input.parse()?) + Self::NoneOutputCommand(input.parse_v2()?) } - PeekMatch::GroupedCommand(_) => return input.span().err( + PeekMatch::GroupedCommand(_) => return input.parse_err( "Grouped commands returning a value are not supported in destructuring positions", ), PeekMatch::FlattenedCommand(_) => { return input - .span() - .err("Flattened commands are not supported in destructuring positions") + .parse_err("Flattened commands are not supported in destructuring positions") } PeekMatch::GroupedVariable | PeekMatch::FlattenedVariable | PeekMatch::AppendVariableDestructuring => { Self::Variable(DestructureVariable::parse_until::(input)?) } - PeekMatch::Group(_) => Self::ExactGroup(input.parse()?), - PeekMatch::Destructurer(_) => Self::Destructurer(input.parse()?), + PeekMatch::Group(_) => Self::ExactGroup(input.parse_v2()?), + PeekMatch::Destructurer(_) => Self::Destructurer(input.parse_v2()?), PeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), PeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), PeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), - PeekMatch::End => return input.span().err("Unexpected end"), + PeekMatch::End => return input.parse_err("Unexpected end"), }) } } impl HandleDestructure for DestructureItem { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { match self { DestructureItem::Variable(variable) => { variable.handle_destructure(input, interpreter)?; diff --git a/src/destructuring/destructure_raw.rs b/src/destructuring/destructure_raw.rs index d7d21471..72878cdc 100644 --- a/src/destructuring/destructure_raw.rs +++ b/src/destructuring/destructure_raw.rs @@ -10,7 +10,7 @@ pub(crate) enum RawDestructureItem { } impl RawDestructureItem { - pub(crate) fn handle_destructure(&self, input: ParseStream) -> Result<()> { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { match self { RawDestructureItem::Punct(punct) => { input.parse_punct_matching(punct.as_char())?; @@ -65,7 +65,7 @@ impl RawDestructureStream { self.inner.push(item); } - pub(crate) fn handle_destructure(&self, input: ParseStream) -> Result<()> { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { for item in self.inner.iter() { item.handle_destructure(input)?; } @@ -91,7 +91,7 @@ impl RawDestructureGroup { } } - pub(crate) fn handle_destructure(&self, input: ParseStream) -> Result<()> { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { let (_, inner) = input.parse_group_matching(self.delimiter)?; self.inner.handle_destructure(&inner) } diff --git a/src/destructuring/destructure_segment.rs b/src/destructuring/destructure_segment.rs index d2567bfd..ed395aac 100644 --- a/src/destructuring/destructure_segment.rs +++ b/src/destructuring/destructure_segment.rs @@ -56,7 +56,7 @@ pub(crate) struct DestructureSegment { } impl Parse for DestructureSegment { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { let mut inner = vec![]; while !C::should_stop(input) { inner.push(DestructureItem::parse_until::(input)?); @@ -69,7 +69,11 @@ impl Parse for DestructureSegment { } impl HandleDestructure for DestructureSegment { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { for item in self.inner.iter() { item.handle_destructure(input, interpreter)?; } diff --git a/src/destructuring/destructure_traits.rs b/src/destructuring/destructure_traits.rs index 54e4ac8e..1349ff3d 100644 --- a/src/destructuring/destructure_traits.rs +++ b/src/destructuring/destructure_traits.rs @@ -5,16 +5,18 @@ pub(crate) trait HandleDestructure { &self, input: InterpretedStream, interpreter: &mut Interpreter, - ) -> Result<()> { + ) -> ExecutionResult<()> { unsafe { // RUST-ANALYZER-SAFETY: ...this isn't generally safe... // We should only do this when we know that either the input or parser doesn't require // analysis of nested None-delimited groups. - input.syn_parse(|input: ParseStream| -> Result<()> { - self.handle_destructure(input, interpreter) - }) + input.syn_parse(|input| self.handle_destructure(input, interpreter)) } } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()>; } diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index 173abbf7..54fde891 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -54,17 +54,17 @@ pub(crate) enum DestructureVariable { } impl DestructureVariable { - pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> Result { + pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> ParseResult { let variable: DestructureVariable = Self::parse_until::(input)?; if variable.is_flattened_input() { return variable .span_range() - .err("A flattened input variable is not supported here"); + .parse_err("A flattened input variable is not supported here"); } Ok(variable) } - pub(crate) fn parse_until(input: ParseStream) -> Result { + pub(crate) fn parse_until(input: ParseStream) -> ParseResult { let marker = input.parse()?; if input.peek(Token![..]) { let flatten = input.parse()?; @@ -164,7 +164,7 @@ impl DestructureVariable { ) } - fn get_variable_data(&self, interpreter: &mut Interpreter) -> Result { + fn get_variable_data(&self, interpreter: &mut Interpreter) -> ExecutionResult { let variable_data = interpreter .get_existing_variable_data(self, || { self.error(format!( @@ -178,10 +178,14 @@ impl DestructureVariable { } impl HandleDestructure for DestructureVariable { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { match self { DestructureVariable::Grouped { .. } => { - let content = input.parse::()?.into_interpreted(); + let content = input.parse_v2::()?.into_interpreted(); interpreter.set_variable(self, content)?; } DestructureVariable::Flattened { until, .. } => { @@ -192,13 +196,13 @@ impl HandleDestructure for DestructureVariable { DestructureVariable::GroupedAppendGrouped { .. } => { let variable_data = self.get_variable_data(interpreter)?; input - .parse::()? + .parse_v2::()? .push_as_token_tree(variable_data.get_mut(self)?.deref_mut()); } DestructureVariable::GroupedAppendFlattened { .. } => { let variable_data = self.get_variable_data(interpreter)?; input - .parse::()? + .parse_v2::()? .flatten_into(variable_data.get_mut(self)?.deref_mut()); } DestructureVariable::FlattenedAppendGrouped { marker, until, .. } => { @@ -257,7 +261,7 @@ impl ParsedTokenTree { } impl Parse for ParsedTokenTree { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { Ok(match input.parse::()? { TokenTree::Group(group) if group.delimiter() == Delimiter::None => { ParsedTokenTree::NoneGroup(group) @@ -266,7 +270,7 @@ impl Parse for ParsedTokenTree { return group .delim_span() .open() - .err("Expected a group with transparent delimiters"); + .parse_err("Expected a group with transparent delimiters"); } TokenTree::Ident(ident) => ParsedTokenTree::Ident(ident), TokenTree::Punct(punct) => ParsedTokenTree::Punct(punct), @@ -286,7 +290,7 @@ pub(crate) enum ParseUntil { impl ParseUntil { /// Peeks the next token, to discover what we should parse next - fn peek_flatten_limit(input: ParseStream) -> Result { + fn peek_flatten_limit(input: ParseStream) -> ParseResult { if C::should_stop(input) { return Ok(ParseUntil::End); } @@ -299,7 +303,7 @@ impl ParseUntil { | PeekMatch::AppendVariableDestructuring => { return input .span() - .err("This cannot follow a flattened destructure match"); + .parse_err("This cannot follow a flattened destructure match"); } PeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), PeekMatch::Ident(ident) => ParseUntil::Ident(ident), @@ -309,7 +313,11 @@ impl ParseUntil { }) } - fn handle_parse_into(&self, input: ParseStream, output: &mut InterpretedStream) -> Result<()> { + fn handle_parse_into( + &self, + input: ParseStream, + output: &mut InterpretedStream, + ) -> ExecutionResult<()> { match self { ParseUntil::End => output.extend_raw_tokens(input.parse::()?), ParseUntil::Group(delimiter) => { diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index 7857ca1c..68db02f8 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -2,8 +2,12 @@ use crate::internal_prelude::*; pub(crate) trait DestructurerDefinition: Clone { const DESTRUCTURER_NAME: &'static str; - fn parse(arguments: DestructurerArguments) -> Result; - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()>; + fn parse(arguments: DestructurerArguments) -> ParseResult; + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()>; } #[derive(Clone)] @@ -32,43 +36,41 @@ impl<'a> DestructurerArguments<'a> { } /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message - pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> Result<()> { + pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> ParseResult<()> { if self.parse_stream.is_empty() { Ok(()) } else { - self.full_span_range.err(error_message) + self.full_span_range.parse_err(error_message) } } - pub(crate) fn fully_parse_no_error_override(&self) -> Result { - self.parse_stream.parse() + pub(crate) fn fully_parse_no_error_override(&self) -> ParseResult { + self.parse_stream.parse_v2() } - pub(crate) fn fully_parse_as(&self) -> Result { + pub(crate) fn fully_parse_as(&self) -> ParseResult { self.fully_parse_or_error(T::parse, T::error_message()) } pub(crate) fn fully_parse_or_error( &self, - parse_function: impl FnOnce(ParseStream) -> Result, + parse_function: impl FnOnce(ParseStream) -> ParseResult, error_message: impl std::fmt::Display, - ) -> Result { - let parsed = parse_function(self.parse_stream).or_else(|error| { - // In future, when the diagnostic API is stable, - // we can add this context directly onto the command ident... - // Rather than just selectively adding it to the inner-most error. - let error_string = error.to_string(); - - // We avoid adding this additional context if it's already been added in an - // inner error, because that's likely the correct error to show. - if error_string.contains("\nOccurred whilst parsing") { - return Err(error); - } - error.span().err(format!( - "{}\nOccurred whilst parsing (!{}! ..) - {}", - error_string, self.destructurer_name, error_message, - )) - })?; + ) -> ParseResult { + // In future, when the diagnostic API is stable, + // we can add this context directly onto the command ident... + // Rather than just selectively adding it to the inner-most error. + // + // For now though, we can add additional context to the error message. + // But we can avoid adding this additional context if it's already been added in an + // inner error, because that's likely the correct local context to show. + let parsed = + parse_function(self.parse_stream).add_context_if_error_and_no_context(|| { + format!( + "Occurred whilst parsing (!{}! ...) - {}", + self.destructurer_name, error_message, + ) + })?; self.assert_empty(error_message)?; @@ -84,9 +86,8 @@ pub(crate) struct Destructurer { } impl Parse for Destructurer { - fn parse(input: ParseStream) -> Result { - let content; - let open_bracket = syn::parenthesized!(content in input); + fn parse(input: ParseStream) -> ParseResult { + let (delim_span, content) = input.parse_group_matching(Delimiter::Parenthesis)?; content.parse::()?; let destructurer_name = content.parse_any_ident()?; let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { @@ -103,17 +104,21 @@ impl Parse for Destructurer { let instance = destructurer_kind.parse_instance(DestructurerArguments::new( &content, destructurer_name, - open_bracket.span.span_range(), + delim_span.join().span_range(), ))?; Ok(Self { instance, - source_group_span: open_bracket.span, + source_group_span: delim_span, }) } } impl HandleDestructure for Destructurer { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { self.instance.handle_destructure(input, interpreter) } } @@ -133,7 +138,7 @@ macro_rules! define_destructurers { } impl DestructurerKind { - fn parse_instance(&self, arguments: DestructurerArguments) -> Result { + fn parse_instance(&self, arguments: DestructurerArguments) -> ParseResult { Ok(match self { $( Self::$destructurer => NamedDestructurer::$destructurer( @@ -169,7 +174,7 @@ macro_rules! define_destructurers { } impl NamedDestructurer { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> ExecutionResult<()> { match self { $( Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index 3c4b2839..ed610798 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -8,13 +8,17 @@ pub(crate) struct StreamDestructurer { impl DestructurerDefinition for StreamDestructurer { const DESTRUCTURER_NAME: &'static str = "stream"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { Ok(Self { inner: arguments.fully_parse_no_error_override()?, }) } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { self.inner.handle_destructure(input, interpreter) } } @@ -27,7 +31,7 @@ pub(crate) struct IdentDestructurer { impl DestructurerDefinition for IdentDestructurer { const DESTRUCTURER_NAME: &'static str = "ident"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { if input.is_empty() { @@ -42,7 +46,11 @@ impl DestructurerDefinition for IdentDestructurer { ) } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { if input.cursor().ident().is_some() { match &self.variable { Some(variable) => variable.handle_destructure(input, interpreter), @@ -52,7 +60,7 @@ impl DestructurerDefinition for IdentDestructurer { } } } else { - Err(input.error("Expected an ident")) + input.parse_err("Expected an ident")? } } } @@ -65,7 +73,7 @@ pub(crate) struct LiteralDestructurer { impl DestructurerDefinition for LiteralDestructurer { const DESTRUCTURER_NAME: &'static str = "literal"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { if input.is_empty() { @@ -80,7 +88,11 @@ impl DestructurerDefinition for LiteralDestructurer { ) } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { if input.cursor().literal().is_some() { match &self.variable { Some(variable) => variable.handle_destructure(input, interpreter), @@ -90,7 +102,7 @@ impl DestructurerDefinition for LiteralDestructurer { } } } else { - Err(input.error("Expected a literal")) + input.parse_err("Expected a literal")? } } } @@ -103,7 +115,7 @@ pub(crate) struct PunctDestructurer { impl DestructurerDefinition for PunctDestructurer { const DESTRUCTURER_NAME: &'static str = "punct"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { if input.is_empty() { @@ -118,7 +130,11 @@ impl DestructurerDefinition for PunctDestructurer { ) } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { if input.cursor().any_punct().is_some() { match &self.variable { Some(variable) => variable.handle_destructure(input, interpreter), @@ -128,7 +144,7 @@ impl DestructurerDefinition for PunctDestructurer { } } } else { - Err(input.error("Expected a punct")) + input.parse_err("Expected a punct")? } } } @@ -141,13 +157,17 @@ pub(crate) struct GroupDestructurer { impl DestructurerDefinition for GroupDestructurer { const DESTRUCTURER_NAME: &'static str = "group"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { Ok(Self { inner: arguments.fully_parse_no_error_override()?, }) } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { let (_, inner) = input.parse_group_matching(Delimiter::None)?; self.inner.handle_destructure(&inner, interpreter) } @@ -161,14 +181,14 @@ pub(crate) struct RawDestructurer { impl DestructurerDefinition for RawDestructurer { const DESTRUCTURER_NAME: &'static str = "raw"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { let token_stream: TokenStream = arguments.fully_parse_no_error_override()?; Ok(Self { stream: RawDestructureStream::new_from_token_stream(token_stream), }) } - fn handle_destructure(&self, input: ParseStream, _: &mut Interpreter) -> Result<()> { + fn handle_destructure(&self, input: ParseStream, _: &mut Interpreter) -> ExecutionResult<()> { self.stream.handle_destructure(input) } } @@ -181,7 +201,7 @@ pub(crate) struct ContentDestructurer { impl DestructurerDefinition for ContentDestructurer { const DESTRUCTURER_NAME: &'static str = "content"; - fn parse(arguments: DestructurerArguments) -> Result { + fn parse(arguments: DestructurerArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { @@ -195,7 +215,11 @@ impl DestructurerDefinition for ContentDestructurer { ) } - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> Result<()> { + fn handle_destructure( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + ) -> ExecutionResult<()> { self.stream .clone() .interpret_to_new_stream(interpreter)? diff --git a/src/destructuring/fields.rs b/src/destructuring/fields.rs index ae3a5b58..2f5f2a48 100644 --- a/src/destructuring/fields.rs +++ b/src/destructuring/fields.rs @@ -16,7 +16,7 @@ impl FieldsParseDefinition { } } - pub(crate) fn add_required_field( + pub(crate) fn add_required_field( self, field_name: &str, example: &str, @@ -26,7 +26,7 @@ impl FieldsParseDefinition { self.add_field(field_name, example, explanation, true, F::parse, set) } - pub(crate) fn add_optional_field( + pub(crate) fn add_optional_field( self, field_name: &str, example: &str, @@ -42,7 +42,7 @@ impl FieldsParseDefinition { example: &str, explanation: Option<&str>, is_required: bool, - parse: impl Fn(syn::parse::ParseStream) -> Result + 'static, + parse: impl Fn(syn::parse::ParseStream) -> ParseResult + 'static, set: impl Fn(&mut T, F) + 'static, ) -> Self { if self @@ -71,16 +71,15 @@ impl FieldsParseDefinition { pub(crate) fn create_syn_parser( self, error_span_range: SpanRange, - ) -> impl FnOnce(syn::parse::ParseStream) -> Result { + ) -> impl FnOnce(ParseStream) -> ParseResult { fn inner( - input: syn::parse::ParseStream, + input: ParseStream, new_builder: T, field_definitions: &FieldDefinitions, error_span_range: SpanRange, - ) -> Result { + ) -> ParseResult { let mut builder = new_builder; - let content; - let _ = syn::braced!(content in input); + let (_, content) = input.parse_group_matching(Delimiter::Brace)?; let mut required_field_names: BTreeSet<_> = field_definitions .0 @@ -98,7 +97,7 @@ impl FieldsParseDefinition { let field_name = content.parse::()?; let field_name_value = field_name.to_string(); if !seen_field_names.insert(field_name_value.clone()) { - return field_name.err("Duplicate field name"); + return field_name.parse_err("Duplicate field name"); } required_field_names.remove(field_name_value.as_str()); let _ = content.parse::()?; @@ -113,7 +112,7 @@ impl FieldsParseDefinition { } if !required_field_names.is_empty() { - return error_span_range.err(format!( + return error_span_range.parse_err(format!( "Missing required fields: {missing_fields:?}", missing_fields = required_field_names, )); @@ -128,14 +127,7 @@ impl FieldsParseDefinition { &self.field_definitions, error_span_range, ) - .map_err(|error| { - // Sadly error combination is just buggy - the two outputted - // compile_error! invocations are back to back which causes a rustc - // parse error. Instead, let's do this. - error - .concat("\n") - .concat(&self.field_definitions.error_message()) - }) + .add_context_if_error_and_no_context(|| self.field_definitions.error_message()) } } } @@ -180,5 +172,5 @@ struct FieldParseDefinition { example: String, explanation: Option, #[allow(clippy::type_complexity)] - parse_and_set: Box Result<()>>, + parse_and_set: Box ParseResult<()>>, } diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 2c05b19e..cdfa300e 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -24,7 +24,7 @@ impl EvaluationBoolean { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> Result { + ) -> ExecutionResult { let input = self.value; match operation.operator { UnaryOperator::Neg => operation.unsupported_for_value_type_err("boolean"), @@ -58,7 +58,7 @@ impl EvaluationBoolean { self, _right: EvaluationInteger, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { operation.unsupported_for_value_type_err("boolean") @@ -70,7 +70,7 @@ impl EvaluationBoolean { self, rhs: Self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation.paired_operator() { diff --git a/src/expressions/character.rs b/src/expressions/character.rs index b514f39c..7de7ba2c 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -20,7 +20,7 @@ impl EvaluationChar { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> Result { + ) -> ExecutionResult { let char = self.value; match operation.operator { UnaryOperator::NoOp => operation.output(char), @@ -55,7 +55,7 @@ impl EvaluationChar { self, _right: EvaluationInteger, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { operation.unsupported_for_value_type_err("char") } @@ -63,7 +63,7 @@ impl EvaluationChar { self, rhs: Self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation.paired_operator() { diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index 5cff6ca5..64b64302 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -7,11 +7,11 @@ pub(super) struct EvaluationTree { } impl EvaluationTree { - pub(super) fn build_from(expression: &Expr) -> Result { + pub(super) fn build_from(expression: &Expr) -> ExecutionResult { EvaluationTreeBuilder::new(expression).build() } - pub(super) fn evaluate(mut self) -> Result { + pub(super) fn evaluate(mut self) -> ExecutionResult { loop { let EvaluationNode { result_placement, content } = self.evaluation_stack.pop() .expect("The builder should ensure that the stack is non-empty and has a final element of a RootResult which results in a return below."); @@ -60,7 +60,7 @@ impl<'a> EvaluationTreeBuilder<'a> { /// Attempts to construct a preinterpret expression tree from a syn [Expr]. /// It tries to align with the [rustc expression] building approach. /// [rustc expression]: https://doc.rust-lang.org/reference/expressions.html - fn build(mut self) -> Result { + fn build(mut self) -> ExecutionResult { while let Some((expression, placement)) = self.work_stack.pop() { match expression { Expr::Binary(expr) => { @@ -103,9 +103,9 @@ impl<'a> EvaluationTreeBuilder<'a> { ); } other_expression => { - return other_expression - .span_range() - .err("This expression is not supported in preinterpret expressions"); + return other_expression.execution_err( + "This expression is not supported in preinterpret expressions", + ); } } } @@ -186,7 +186,7 @@ enum EvaluationNodeContent { } impl EvaluationNodeContent { - fn evaluate(self) -> Result { + fn evaluate(self) -> ExecutionResult { match self { Self::Literal(literal) => Ok(EvaluationOutput::Value(literal)), Self::Operator(operator) => operator.evaluate(), @@ -238,7 +238,7 @@ impl EvaluationOutput { operator: PairedBinaryOperator, right: EvaluationOutput, operator_span: SpanRange, - ) -> Result { + ) -> ExecutionResult { let left_lit = self.into_value(); let right_lit = right.into_value(); Ok(match (left_lit, right_lit) { @@ -363,7 +363,7 @@ impl EvaluationOutput { EvaluationIntegerValuePair::Isize(lhs, rhs) } (left_value, right_value) => { - return operator_span.err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + return operator_span.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); } }; EvaluationLiteralPair::Integer(integer_pair) @@ -402,7 +402,7 @@ impl EvaluationOutput { EvaluationFloatValuePair::F64(lhs, rhs) } (left_value, right_value) => { - return operator_span.err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + return operator_span.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); } }; EvaluationLiteralPair::Float(float_pair) @@ -414,22 +414,22 @@ impl EvaluationOutput { EvaluationLiteralPair::CharPair(left, right) } (left, right) => { - return operator_span.err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); + return operator_span.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); } }) } - pub(crate) fn expect_integer(self, error_message: &str) -> Result { + pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { match self.into_value() { EvaluationValue::Integer(value) => Ok(value), - other => other.source_span().err(error_message), + other => other.source_span().execution_err(error_message), } } - pub(crate) fn expect_bool(self, error_message: &str) -> Result { + pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { match self.into_value() { EvaluationValue::Boolean(value) => Ok(value), - other => other.source_span().err(error_message), + other => other.source_span().execution_err(error_message), } } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index 4ab3a2c0..d194218b 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -5,9 +5,12 @@ pub(crate) trait Express: Sized { self, interpreter: &mut Interpreter, builder: &mut ExpressionBuilder, - ) -> Result<()>; + ) -> ExecutionResult<()>; - fn start_expression_builder(self, interpreter: &mut Interpreter) -> Result { + fn start_expression_builder( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { let mut output = ExpressionBuilder::new(); self.add_to_expression(interpreter, &mut output)?; Ok(output) @@ -27,7 +30,7 @@ pub(crate) struct ExpressionInput { } impl Parse for ExpressionInput { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { let mut items = Vec::new(); while !input.is_empty() { // Until we create a proper ExpressionInput parser which builds up a syntax tree @@ -35,25 +38,29 @@ impl Parse for ExpressionInput { // before code blocks or .. in [!range!] so we can break on those. // These aren't valid inside expressions we support anyway, so it's good enough for now. let item = match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => ExpressionItem::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => ExpressionItem::Command(input.parse()?), - PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), + PeekMatch::GroupedCommand(_) => ExpressionItem::Command(input.parse_v2()?), + PeekMatch::FlattenedCommand(_) => ExpressionItem::Command(input.parse_v2()?), + PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse_v2()?), + PeekMatch::FlattenedVariable => { + ExpressionItem::FlattenedVariable(input.parse_v2()?) + } PeekMatch::Group(Delimiter::Brace | Delimiter::Bracket) => break, - PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse()?), + PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse_v2()?), PeekMatch::Destructurer(_) | PeekMatch::AppendVariableDestructuring => { - return Err(input.error("Destructuring is not supported in an expression")); + return input + .span() + .parse_err("Destructuring is not supported in an expression"); } PeekMatch::Punct(punct) if punct.as_char() == '.' => break, PeekMatch::Punct(_) => ExpressionItem::Punct(input.parse_any_punct()?), PeekMatch::Ident(_) => ExpressionItem::Ident(input.parse_any_ident()?), - PeekMatch::Literal(_) => ExpressionItem::Literal(input.parse()?), - PeekMatch::End => return input.span().err("Expected an expression"), + PeekMatch::Literal(_) => ExpressionItem::Literal(input.parse_v2()?), + PeekMatch::End => return input.span().parse_err("Expected an expression"), }; items.push(item); } if items.is_empty() { - return input.span().err("Expected an expression"); + return input.span().parse_err("Expected an expression"); } Ok(Self { items }) } @@ -73,7 +80,7 @@ impl Express for ExpressionInput { self, interpreter: &mut Interpreter, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { for item in self.items { item.add_to_expression(interpreter, builder)?; } @@ -82,7 +89,10 @@ impl Express for ExpressionInput { } impl ExpressionInput { - pub(crate) fn evaluate(self, interpreter: &mut Interpreter) -> Result { + pub(crate) fn evaluate( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { self.start_expression_builder(interpreter)?.evaluate() } } @@ -119,7 +129,7 @@ impl Express for ExpressionItem { self, interpreter: &mut Interpreter, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { match self { ExpressionItem::Command(command_invocation) => { command_invocation.add_to_expression(interpreter, builder)?; @@ -149,12 +159,12 @@ pub(crate) struct ExpressionGroup { } impl Parse for ExpressionGroup { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_delimiter()?; Ok(Self { source_delimiter: delimiter, source_delim_span: delim_span, - content: content.parse()?, + content: content.parse_v2()?, }) } } @@ -164,7 +174,7 @@ impl Express for ExpressionGroup { self, interpreter: &mut Interpreter, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { builder.push_expression_group( self.content.start_expression_builder(interpreter)?, self.source_delimiter, @@ -207,9 +217,9 @@ impl ExpressionBuilder { pub(crate) fn push_grouped( &mut self, - appender: impl FnOnce(&mut InterpretedStream) -> Result<()>, + appender: impl FnOnce(&mut InterpretedStream) -> ExecutionResult<()>, span: Span, - ) -> Result<()> { + ) -> ExecutionResult<()> { // Currently using Expr::Parse, it ignores transparent groups, which is a little too permissive. // Instead, we use parentheses to ensure that the group has to be a valid expression itself, without being flattened. // This also works around the SAFETY issue in syn_parse below @@ -232,7 +242,7 @@ impl ExpressionBuilder { .push_new_group(contents.interpreted_stream, delimiter, span); } - pub(crate) fn evaluate(self) -> Result { + pub(crate) fn evaluate(self) -> ExecutionResult { // Parsing into a rust expression is overkill here. // // In future we could choose to implement a subset of the grammar which we actually can use/need. @@ -247,7 +257,7 @@ impl ExpressionBuilder { let expression = unsafe { // RUST-ANALYZER SAFETY: We wrap commands and variables in `()` instead of none-delimited groups in expressions, // so it doesn't matter that we can drop none-delimited groups - self.interpreted_stream.syn_parse(Expr::parse)? + self.interpreted_stream.syn_parse(::parse)? }; EvaluationTree::build_from(&expression)?.evaluate() diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 408b1a2b..197fff89 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -11,7 +11,7 @@ impl EvaluationFloat { Self { value, source_span } } - pub(super) fn for_litfloat(lit: &syn::LitFloat) -> Result { + pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ExecutionResult { Ok(Self { source_span: lit.span().span_range(), value: EvaluationFloatValue::for_litfloat(lit)?, @@ -21,7 +21,7 @@ impl EvaluationFloat { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> Result { + ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => input.handle_unary_operation(&operation), EvaluationFloatValue::F32(input) => input.handle_unary_operation(&operation), @@ -33,7 +33,7 @@ impl EvaluationFloat { self, right: EvaluationInteger, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => { input.handle_integer_binary_operation(right, &operation) @@ -70,7 +70,7 @@ impl EvaluationFloatValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::F32(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -86,13 +86,13 @@ pub(super) enum EvaluationFloatValue { } impl EvaluationFloatValue { - pub(super) fn for_litfloat(lit: &syn::LitFloat) -> Result { + pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ExecutionResult { Ok(match lit.suffix() { "" => Self::Untyped(UntypedFloat::new_from_lit_float(lit)), "f32" => Self::F32(lit.base10_parse()?), "f64" => Self::F64(lit.base10_parse()?), suffix => { - return lit.span().err(format!( + return lit.span().execution_err(format!( "The literal suffix {suffix} is not supported in preinterpret expressions" )); } @@ -142,7 +142,7 @@ impl UntypedFloat { pub(super) fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> Result { + ) -> ExecutionResult { let input = self.parse_fallback()?; match operation.operator { UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), @@ -180,7 +180,7 @@ impl UntypedFloat { self, _rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { operation.unsupported_for_value_type_err("untyped float") @@ -192,7 +192,7 @@ impl UntypedFloat { self, rhs: Self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; match operation.paired_operator() { @@ -224,9 +224,9 @@ impl UntypedFloat { Self::new_from_literal(Literal::f64_unsuffixed(value)) } - fn parse_fallback(&self) -> Result { + fn parse_fallback(&self) -> ExecutionResult { self.0.base10_digits().parse().map_err(|err| { - self.0.span().error(format!( + self.0.span().execution_error(format!( "Could not parse as the default inferred type {}: {}", core::any::type_name::(), err @@ -234,13 +234,13 @@ impl UntypedFloat { }) } - pub(super) fn parse_as(&self) -> Result + pub(super) fn parse_as(&self) -> ExecutionResult where N: FromStr, N::Err: core::fmt::Display, { self.0.base10_digits().parse().map_err(|err| { - self.0.span().error(format!( + self.0.span().execution_error(format!( "Could not parse as {}: {}", core::any::type_name::(), err @@ -274,7 +274,7 @@ macro_rules! impl_float_operations { } impl HandleUnaryOperation for $float_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { UnaryOperator::Neg => operation.output(-self), UnaryOperator::Not => operation.unsupported_for_value_type_err(stringify!($float_type)), @@ -303,7 +303,7 @@ macro_rules! impl_float_operations { } impl HandleBinaryOperation for $float_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { // Unlike integer arithmetic, float arithmetic does not overflow // and instead falls back to NaN or infinity. In future we could // allow trapping on these codes, but for now this is good enough @@ -336,7 +336,7 @@ macro_rules! impl_float_operations { self, _rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { operation.unsupported_for_value_type_err(stringify!($float_type)) diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 510c98f3..9c91ffae 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -10,14 +10,14 @@ impl EvaluationInteger { Self { value, source_span } } - pub(super) fn for_litint(lit: &syn::LitInt) -> Result { + pub(super) fn for_litint(lit: &syn::LitInt) -> ExecutionResult { Ok(Self { source_span: lit.span().span_range(), value: EvaluationIntegerValue::for_litint(lit)?, }) } - pub(crate) fn try_into_i128(self) -> Result { + pub(crate) fn try_into_i128(self) -> ExecutionResult { let option_of_fallback = match self.value { EvaluationIntegerValue::Untyped(x) => x.parse_fallback().ok(), EvaluationIntegerValue::U8(x) => Some(x.into()), @@ -37,14 +37,14 @@ impl EvaluationInteger { Some(value) => Ok(value), None => self .source_span - .err("The integer does not fit in a i128".to_string()), + .execution_err("The integer does not fit in a i128".to_string()), } } pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> Result { + ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), EvaluationIntegerValue::U8(input) => input.handle_unary_operation(&operation), @@ -66,7 +66,7 @@ impl EvaluationInteger { self, right: EvaluationInteger, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => { input.handle_integer_binary_operation(right, &operation) @@ -143,7 +143,7 @@ impl EvaluationIntegerValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::U8(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -196,7 +196,7 @@ pub(super) enum EvaluationIntegerValue { } impl EvaluationIntegerValue { - pub(super) fn for_litint(lit: &syn::LitInt) -> Result { + pub(super) fn for_litint(lit: &syn::LitInt) -> ExecutionResult { Ok(match lit.suffix() { "" => Self::Untyped(UntypedInteger::new_from_lit_int(lit)), "u8" => Self::U8(lit.base10_parse()?), @@ -212,7 +212,7 @@ impl EvaluationIntegerValue { "i128" => Self::I128(lit.base10_parse()?), "isize" => Self::Isize(lit.base10_parse()?), suffix => { - return lit.span().err(format!( + return lit.span().execution_err(format!( "The literal suffix {suffix} is not supported in preinterpret expressions" )); } @@ -275,7 +275,7 @@ impl UntypedInteger { pub(super) fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> Result { + ) -> ExecutionResult { let input = self.parse_fallback()?; match operation.operator { UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), @@ -313,7 +313,7 @@ impl UntypedInteger { self, rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft => match rhs.value { @@ -357,7 +357,7 @@ impl UntypedInteger { self, rhs: Self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; let overflow_error = || { @@ -408,9 +408,9 @@ impl UntypedInteger { Self::new_from_literal(Literal::i128_unsuffixed(value)) } - pub(super) fn parse_fallback(&self) -> Result { + pub(super) fn parse_fallback(&self) -> ExecutionResult { self.0.base10_digits().parse().map_err(|err| { - self.0.span().error(format!( + self.0.span().execution_error(format!( "Could not parse as the default inferred type {}: {}", core::any::type_name::(), err @@ -418,13 +418,13 @@ impl UntypedInteger { }) } - pub(super) fn parse_as(&self) -> Result + pub(super) fn parse_as(&self) -> ExecutionResult where N: FromStr, N::Err: core::fmt::Display, { self.0.base10_digits().parse().map_err(|err| { - self.0.span().error(format!( + self.0.span().execution_error(format!( "Could not parse as {}: {}", core::any::type_name::(), err @@ -459,7 +459,7 @@ macro_rules! impl_int_operations_except_unary { } impl HandleBinaryOperation for $integer_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> Result { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { let lhs = self; let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.operator.symbol(), rhs); match operation.paired_operator() { @@ -486,7 +486,7 @@ macro_rules! impl_int_operations_except_unary { self, rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self; match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft => { @@ -532,7 +532,7 @@ macro_rules! impl_int_operations_except_unary { macro_rules! impl_unsigned_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self), UnaryOperator::Neg @@ -568,7 +568,7 @@ macro_rules! impl_unsigned_unary_operations { macro_rules! impl_signed_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self), UnaryOperator::Neg => operation.output(-self), @@ -601,7 +601,10 @@ macro_rules! impl_signed_unary_operations { } impl HandleUnaryOperation for u8 { - fn handle_unary_operation(self, operation: &UnaryOperation) -> Result { + fn handle_unary_operation( + self, + operation: &UnaryOperation, + ) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self), UnaryOperator::Neg | UnaryOperator::Not => { diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 61b6108c..f712eb72 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -13,7 +13,7 @@ pub(super) enum EvaluationOperator { } impl EvaluationOperator { - pub(super) fn evaluate(self) -> Result { + pub(super) fn evaluate(self) -> ExecutionResult { const OPERATOR_INPUT_EXPECT_STR: &str = "Handling children on the stack ordering should ensure the parent input is always set when the parent is evaluated"; match self { @@ -40,14 +40,14 @@ pub(super) struct UnaryOperation { } impl UnaryOperation { - fn error(&self, error_message: &str) -> syn::Error { - self.operator_span.error(error_message) + fn error(&self, error_message: &str) -> ExecutionInterrupt { + self.operator_span.execution_error(error_message) } pub(super) fn unsupported_for_value_type_err( &self, value_type: &'static str, - ) -> Result { + ) -> ExecutionResult { Err(self.error(&format!( "The {} operator is not supported for {} values", self.operator.symbol(), @@ -55,16 +55,19 @@ impl UnaryOperation { ))) } - pub(super) fn err(&self, error_message: &'static str) -> Result { + pub(super) fn err(&self, error_message: &'static str) -> ExecutionResult { Err(self.error(error_message)) } - pub(super) fn output(&self, output_value: impl ToEvaluationOutput) -> Result { + pub(super) fn output( + &self, + output_value: impl ToEvaluationOutput, + ) -> ExecutionResult { Ok(output_value.to_output(self.span_for_output)) } - pub(super) fn for_cast_expression(expr: &syn::ExprCast) -> Result { - fn extract_type(ty: &syn::Type) -> Result { + pub(super) fn for_cast_expression(expr: &syn::ExprCast) -> ExecutionResult { + fn extract_type(ty: &syn::Type) -> ExecutionResult { match ty { syn::Type::Group(group) => extract_type(&group.elem), syn::Type::Path(type_path) @@ -75,9 +78,9 @@ impl UnaryOperation { let ident = match type_path.path.get_ident() { Some(ident) => ident, None => { - return type_path - .span_range() - .err("This type is not supported in preinterpret cast expressions") + return type_path.execution_err( + "This type is not supported in preinterpret cast expressions", + ) } }; match ident.to_string().as_str() { @@ -99,15 +102,13 @@ impl UnaryOperation { "f64" => Ok(CastTarget::Float(FloatKind::F64)), "bool" => Ok(CastTarget::Boolean), "char" => Ok(CastTarget::Char), - _ => ident - .span() - .span_range() - .err("This type is not supported in preinterpret cast expressions"), + _ => ident.execution_err( + "This type is not supported in preinterpret cast expressions", + ), } } other => other - .span_range() - .err("This type is not supported in preinterpret cast expressions"), + .execution_err("This type is not supported in preinterpret cast expressions"), } } @@ -118,7 +119,7 @@ impl UnaryOperation { }) } - pub(super) fn for_group_expression(expr: &syn::ExprGroup) -> Result { + pub(super) fn for_group_expression(expr: &syn::ExprGroup) -> ExecutionResult { Ok(Self { span_for_output: expr.group_token.span.span_range(), operator_span: expr.group_token.span.span_range(), @@ -126,7 +127,7 @@ impl UnaryOperation { }) } - pub(super) fn for_paren_expression(expr: &syn::ExprParen) -> Result { + pub(super) fn for_paren_expression(expr: &syn::ExprParen) -> ExecutionResult { Ok(Self { span_for_output: expr.paren_token.span.span_range(), operator_span: expr.paren_token.span.span_range(), @@ -134,14 +135,14 @@ impl UnaryOperation { }) } - pub(super) fn for_unary_expression(expr: &syn::ExprUnary) -> Result { + pub(super) fn for_unary_expression(expr: &syn::ExprUnary) -> ExecutionResult { let operator = match &expr.op { UnOp::Neg(_) => UnaryOperator::Neg, UnOp::Not(_) => UnaryOperator::Not, other_unary_op => { - return other_unary_op - .span_range() - .err("This unary operator is not supported in preinterpret expressions"); + return other_unary_op.execution_err( + "This unary operator is not supported in preinterpret expressions", + ); } }; Ok(Self { @@ -151,7 +152,7 @@ impl UnaryOperation { }) } - pub(super) fn evaluate(self, input: EvaluationOutput) -> Result { + pub(super) fn evaluate(self, input: EvaluationOutput) -> ExecutionResult { input.into_value().handle_unary_operation(self) } } @@ -176,7 +177,10 @@ impl UnaryOperator { } pub(super) trait HandleUnaryOperation: Sized { - fn handle_unary_operation(self, operation: &UnaryOperation) -> Result; + fn handle_unary_operation( + self, + operation: &UnaryOperation, + ) -> ExecutionResult; } pub(super) struct BinaryOperation { @@ -186,14 +190,14 @@ pub(super) struct BinaryOperation { } impl BinaryOperation { - fn error(&self, error_message: &str) -> syn::Error { - self.operator_span.error(error_message) + fn error(&self, error_message: &str) -> ExecutionInterrupt { + self.operator_span.execution_error(error_message) } pub(super) fn unsupported_for_value_type_err( &self, value_type: &'static str, - ) -> Result { + ) -> ExecutionResult { Err(self.error(&format!( "The {} operator is not supported for {} values", self.operator.symbol(), @@ -201,7 +205,10 @@ impl BinaryOperation { ))) } - pub(super) fn output(&self, output_value: impl ToEvaluationOutput) -> Result { + pub(super) fn output( + &self, + output_value: impl ToEvaluationOutput, + ) -> ExecutionResult { Ok(output_value.to_output(self.span_for_output)) } @@ -209,14 +216,14 @@ impl BinaryOperation { &self, output_value: Option, error_message: impl FnOnce() -> String, - ) -> Result { + ) -> ExecutionResult { match output_value { Some(output_value) => self.output(output_value), - None => Err(self.operator_span.error(error_message())), + None => self.operator_span.execution_err(error_message()), } } - pub(super) fn for_binary_expression(expr: &syn::ExprBinary) -> Result { + pub(super) fn for_binary_expression(expr: &syn::ExprBinary) -> ExecutionResult { let operator = match &expr.op { syn::BinOp::Add(_) => BinaryOperator::Paired(PairedBinaryOperator::Addition), syn::BinOp::Sub(_) => BinaryOperator::Paired(PairedBinaryOperator::Subtraction), @@ -238,8 +245,7 @@ impl BinaryOperation { syn::BinOp::Gt(_) => BinaryOperator::Paired(PairedBinaryOperator::GreaterThan), other_binary_operation => { return other_binary_operation - .span_range() - .err("This operation is not supported in preinterpret expressions") + .execution_err("This operation is not supported in preinterpret expressions") } }; Ok(Self { @@ -249,7 +255,11 @@ impl BinaryOperation { }) } - fn evaluate(self, left: EvaluationOutput, right: EvaluationOutput) -> Result { + fn evaluate( + self, + left: EvaluationOutput, + right: EvaluationOutput, + ) -> ExecutionResult { match self.operator { BinaryOperator::Paired(operator) => { let value_pair = left.expect_value_pair(operator, right, self.operator_span)?; @@ -356,11 +366,11 @@ pub(super) trait HandleBinaryOperation: Sized { self, rhs: Self, operation: &BinaryOperation, - ) -> Result; + ) -> ExecutionResult; fn handle_integer_binary_operation( self, rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> Result; + ) -> ExecutionResult; } diff --git a/src/expressions/string.rs b/src/expressions/string.rs index efd9f8f9..b708ef3b 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -20,7 +20,7 @@ impl EvaluationString { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> Result { + ) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self.value), UnaryOperator::Neg | UnaryOperator::Not | UnaryOperator::Cast(_) => { @@ -33,7 +33,7 @@ impl EvaluationString { self, _right: EvaluationInteger, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { operation.unsupported_for_value_type_err("string") } @@ -41,7 +41,7 @@ impl EvaluationString { self, rhs: Self, operation: &BinaryOperation, - ) -> Result { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation.paired_operator() { diff --git a/src/expressions/value.rs b/src/expressions/value.rs index a51b8b8c..7dc78925 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -19,7 +19,7 @@ impl EvaluationValue { } } - pub(super) fn for_literal_expression(expr: &ExprLit) -> Result { + pub(super) fn for_literal_expression(expr: &ExprLit) -> ExecutionResult { // https://docs.rs/syn/latest/syn/enum.Lit.html Ok(match &expr.lit { Lit::Int(lit) => Self::Integer(EvaluationInteger::for_litint(lit)?), @@ -30,7 +30,7 @@ impl EvaluationValue { other_literal => { return other_literal .span() - .err("This literal is not supported in preinterpret expressions"); + .execution_err("This literal is not supported in preinterpret expressions"); } }) } @@ -48,7 +48,7 @@ impl EvaluationValue { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> Result { + ) -> ExecutionResult { match self { EvaluationValue::Integer(value) => value.handle_unary_operation(operation), EvaluationValue::Float(value) => value.handle_unary_operation(operation), @@ -62,7 +62,7 @@ impl EvaluationValue { self, right: EvaluationInteger, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match self { EvaluationValue::Integer(value) => { value.handle_integer_binary_operation(right, operation) @@ -129,7 +129,7 @@ impl EvaluationLiteralPair { pub(super) fn handle_paired_binary_operation( self, operation: BinaryOperation, - ) -> Result { + ) -> ExecutionResult { match self { Self::Integer(pair) => pair.handle_paired_binary_operation(&operation), Self::Float(pair) => pair.handle_paired_binary_operation(&operation), diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs new file mode 100644 index 00000000..549ecb57 --- /dev/null +++ b/src/extensions/errors_and_spans.rs @@ -0,0 +1,171 @@ +use crate::internal_prelude::*; + +pub(crate) trait SynErrorExt: Sized { + fn concat(self, extra: &str) -> Self; +} + +impl SynErrorExt for syn::Error { + fn concat(self, extra: &str) -> Self { + let mut message = self.to_string(); + message.push_str(extra); + Self::new(self.span(), message) + } +} + +pub(crate) trait SpanErrorExt: Sized { + fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { + Err(self.error(message).into()) + } + + fn execution_err(&self, message: impl std::fmt::Display) -> ExecutionResult { + Err(self.error(message).into()) + } + + fn err(&self, message: impl std::fmt::Display) -> syn::Result { + Err(self.error(message)) + } + + fn error(&self, message: impl std::fmt::Display) -> syn::Error; + + fn execution_error(&self, message: impl std::fmt::Display) -> ExecutionInterrupt { + ExecutionInterrupt::Error(self.error(message)) + } + + #[allow(unused)] + fn parse_error(&self, message: impl std::fmt::Display) -> ParseError { + ParseError::Standard(self.error(message)) + } +} + +impl SpanErrorExt for T { + fn error(&self, message: impl std::fmt::Display) -> syn::Error { + self.span_range().create_error(message) + } +} + +pub(crate) trait HasSpanRange { + fn span_range(&self) -> SpanRange; + + fn span(&self) -> Span { + self.span_range().span() + } +} + +/// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] +/// and falls back to the span of the first token when not available. +/// +/// Instead, [`syn::Error`] uses a trick involving a span range. This effectively +/// allows capturing this trick when we're not immediately creating an error. +/// +/// When [`proc_macro::Span::join`] is stabilised and [`syn::spanned`] works, +/// we can swap [`SpanRange`] contents for [`Span`] (or even remove it and [`HasSpanRange`]). +#[derive(Copy, Clone)] +pub(crate) struct SpanRange { + start: Span, + end: Span, +} + +impl SpanRange { + pub(crate) fn new_between(start: Span, end: Span) -> Self { + Self { start, end } + } + + fn create_error(&self, message: impl std::fmt::Display) -> syn::Error { + syn::Error::new_spanned(self, message) + } + + /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) + /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) + pub(crate) fn span(&self) -> Span { + ::span(self) + } + + pub(crate) fn start(&self) -> Span { + self.start + } + + #[allow(unused)] + pub(crate) fn end(&self) -> Span { + self.end + } +} + +// This is implemented so we can create an error from it using `Error::new_spanned(..)` +impl ToTokens for SpanRange { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend([ + TokenTree::Punct(Punct::new('<', Spacing::Alone).with_span(self.start)), + TokenTree::Punct(Punct::new('>', Spacing::Alone).with_span(self.end)), + ]); + } +} + +impl HasSpanRange for SpanRange { + fn span_range(&self) -> SpanRange { + *self + } +} + +impl HasSpanRange for Span { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(*self, *self) + } +} + +impl HasSpanRange for TokenTree { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl HasSpanRange for Group { + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl HasSpanRange for DelimSpan { + fn span_range(&self) -> SpanRange { + // We could use self.open() => self.close() here, but using + // self.join() is better as it can be round-tripped to a span + // as the whole span, rather than just the start or end. + self.join().span_range() + } +} + +impl HasSpanRange for T { + fn span_range(&self) -> SpanRange { + let mut iter = self.into_token_stream().into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + SpanRange { start, end } + } +} + +/// This should only be used for syn built-ins or when there isn't a better +/// span range available +pub(crate) trait AutoSpanRange {} + +macro_rules! impl_auto_span_range { + ($($ty:ty),* $(,)?) => { + $( + impl AutoSpanRange for $ty {} + )* + }; +} + +impl_auto_span_range! { + TokenStream, + Ident, + Punct, + Literal, + syn::Expr, + syn::ExprBinary, + syn::ExprUnary, + syn::BinOp, + syn::UnOp, + syn::Type, + syn::TypePath, + syn::token::DotDot, + syn::token::In, +} diff --git a/src/extensions/mod.rs b/src/extensions/mod.rs new file mode 100644 index 00000000..346db9de --- /dev/null +++ b/src/extensions/mod.rs @@ -0,0 +1,7 @@ +mod errors_and_spans; +mod parsing; +mod tokens; + +pub(crate) use errors_and_spans::*; +pub(crate) use parsing::*; +pub(crate) use tokens::*; diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs new file mode 100644 index 00000000..2623aea2 --- /dev/null +++ b/src/extensions/parsing.rs @@ -0,0 +1,220 @@ +use crate::internal_prelude::*; + +pub(crate) trait TokenStreamParseExt: Sized { + fn parse_with>( + self, + parser: impl FnOnce(ParseStream) -> Result, + ) -> Result; +} + +impl TokenStreamParseExt for TokenStream { + fn parse_with>( + self, + parser: impl FnOnce(ParseStream) -> Result, + ) -> Result { + let mut result = None; + let parse_result = (|input: ParseStream| -> SynResult<()> { + result = Some(parser(input)); + match &result { + // Some fallback error to ensure that we don't go down the unexpected branch inside parse2 + Some(Err(_)) => Err(SynError::new(Span::call_site(), "")), + _ => Ok(()), + } + }) + .parse2(self); + + match (result, parse_result) { + (Some(Ok(value)), Ok(())) => Ok(value), + (Some(Err(error)), _) => Err(error), + // If the inner result was Ok, but the parse result was an error, this indicates that the parse2 + // hit the "unexpected" path, indicating that some parse buffer (i.e. group) wasn't fully consumed. + // So we propagate this error. + (Some(Ok(_)), Err(error)) => Err(error.into()), + (None, _) => unreachable!(), + } + } +} + +pub(crate) trait CursorExt: Sized { + /// Because syn doesn't parse ' as a punct (not aligned with the TokenTree abstraction) + fn any_punct(self) -> Option<(Punct, Self)>; + fn ident_matching(self, content: &str) -> Option<(Ident, Self)>; + fn punct_matching(self, char: char) -> Option<(Punct, Self)>; + fn literal_matching(self, content: &str) -> Option<(Literal, Self)>; + fn group_matching(self, expected_delimiter: Delimiter) -> Option<(DelimSpan, Self, Self)>; +} + +impl CursorExt for Cursor<'_> { + fn any_punct(self) -> Option<(Punct, Self)> { + match self.token_tree() { + Some((TokenTree::Punct(punct), next)) => Some((punct, next)), + _ => None, + } + } + + fn ident_matching(self, content: &str) -> Option<(Ident, Self)> { + match self.ident() { + Some((ident, next)) if ident == content => Some((ident, next)), + _ => None, + } + } + + fn punct_matching(self, char: char) -> Option<(Punct, Self)> { + // self.punct() is a little more efficient, but can't match ' + let matcher = if char == '\'' { + self.any_punct() + } else { + self.punct() + }; + match matcher { + Some((punct, next)) if punct.as_char() == char => Some((punct, next)), + _ => None, + } + } + + fn literal_matching(self, content: &str) -> Option<(Literal, Self)> { + match self.literal() { + Some((literal, next)) if literal.to_string() == content => Some((literal, next)), + _ => None, + } + } + + fn group_matching(self, expected_delimiter: Delimiter) -> Option<(DelimSpan, Self, Self)> { + match self.any_group() { + Some((inner_cursor, delimiter, delim_span, next_outer_cursor)) + if delimiter == expected_delimiter => + { + Some((delim_span, inner_cursor, next_outer_cursor)) + } + _ => None, + } + } +} + +pub(crate) trait ParserExt { + fn parse_v2(&self) -> ParseResult; + fn parse_with(&self, context: T::Context) -> ParseResult; + fn parse_all_for_interpretation( + &self, + span_range: SpanRange, + ) -> ParseResult; + fn try_parse_or_message ParseResult, M: std::fmt::Display>( + &self, + func: F, + message: M, + ) -> ParseResult; + fn parse_any_ident(&self) -> ParseResult; + fn parse_any_punct(&self) -> ParseResult; + fn peek_ident_matching(&self, content: &str) -> bool; + fn parse_ident_matching(&self, content: &str) -> ParseResult; + fn peek_punct_matching(&self, punct: char) -> bool; + fn parse_punct_matching(&self, content: char) -> ParseResult; + fn peek_literal_matching(&self, content: &str) -> bool; + fn parse_literal_matching(&self, content: &str) -> ParseResult; + fn peek_group_matching(&self, delimiter: Delimiter) -> bool; + fn parse_group_matching(&self, delimiter: Delimiter) -> ParseResult<(DelimSpan, ParseBuffer)>; + fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult; + fn parse_error(&self, message: impl std::fmt::Display) -> ParseError; +} + +impl ParserExt for ParseBuffer<'_> { + fn parse_v2(&self) -> ParseResult { + T::parse(self) + } + + fn parse_with(&self, context: T::Context) -> ParseResult { + T::parse_with_context(self, context) + } + + fn parse_all_for_interpretation( + &self, + span_range: SpanRange, + ) -> ParseResult { + self.parse_with(span_range) + } + + fn try_parse_or_message ParseResult, M: std::fmt::Display>( + &self, + parse: F, + message: M, + ) -> ParseResult { + let error_span = self.span(); + parse(self).map_err(|_| error_span.error(message).into()) + } + + fn parse_any_ident(&self) -> ParseResult { + Ok(Ident::parse_any(self)?) + } + + fn parse_any_punct(&self) -> ParseResult { + // Annoyingly, ' behaves weirdly in syn, so we need to handle it + match self.parse::()? { + TokenTree::Punct(punct) => Ok(punct), + _ => self.span().parse_err("expected punctuation"), + } + } + + fn peek_ident_matching(&self, content: &str) -> bool { + self.cursor().ident_matching(content).is_some() + } + + fn parse_ident_matching(&self, content: &str) -> ParseResult { + Ok(self.step(|cursor| { + cursor + .ident_matching(content) + .ok_or_else(|| cursor.span().error(format!("expected {}", content))) + })?) + } + + fn peek_punct_matching(&self, punct: char) -> bool { + self.cursor().punct_matching(punct).is_some() + } + + fn parse_punct_matching(&self, punct: char) -> ParseResult { + Ok(self.step(|cursor| { + cursor + .punct_matching(punct) + .ok_or_else(|| cursor.span().error(format!("expected {}", punct))) + })?) + } + + fn peek_literal_matching(&self, content: &str) -> bool { + self.cursor().literal_matching(content).is_some() + } + + fn parse_literal_matching(&self, content: &str) -> ParseResult { + Ok(self.step(|cursor| { + cursor + .literal_matching(content) + .ok_or_else(|| cursor.span().error(format!("expected {}", content))) + })?) + } + + fn peek_group_matching(&self, delimiter: Delimiter) -> bool { + self.cursor().group_matching(delimiter).is_some() + } + + fn parse_group_matching( + &self, + expected_delimiter: Delimiter, + ) -> ParseResult<(DelimSpan, ParseBuffer)> { + let (delimiter, delim_span, inner_stream) = self.parse_any_delimiter()?; + if delimiter != expected_delimiter { + return delim_span.open().parse_err(match expected_delimiter { + Delimiter::Parenthesis => "Expected (", + Delimiter::Brace => "Expected {", + Delimiter::Bracket => "Expected [", + Delimiter::None => "Expected start of transparent group", + }); + } + Ok((delim_span, inner_stream)) + } + + fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { + Err(self.parse_error(message)) + } + + fn parse_error(&self, message: impl std::fmt::Display) -> ParseError { + self.span().parse_error(message) + } +} diff --git a/src/extensions/tokens.rs b/src/extensions/tokens.rs new file mode 100644 index 00000000..997a0358 --- /dev/null +++ b/src/extensions/tokens.rs @@ -0,0 +1,107 @@ +use crate::internal_prelude::*; + +pub(crate) trait TokenStreamExt: Sized { + #[allow(unused)] + fn push(&mut self, token: TokenTree); + fn flatten_transparent_groups(self) -> Self; +} + +impl TokenStreamExt for TokenStream { + fn push(&mut self, token: TokenTree) { + self.extend(iter::once(token)); + } + + fn flatten_transparent_groups(self) -> Self { + let mut output = TokenStream::new(); + for token in self { + match token { + TokenTree::Group(group) if group.delimiter() == Delimiter::None => { + output.extend(group.stream().flatten_transparent_groups()); + } + other => output.extend(iter::once(other)), + } + } + output + } +} + +pub(crate) trait TokenTreeExt: Sized { + fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; + fn into_singleton_group(self, delimiter: Delimiter) -> Self; +} + +impl TokenTreeExt for TokenTree { + fn group(inner_tokens: TokenStream, delimiter: Delimiter, span: Span) -> Self { + TokenTree::Group(Group::new(delimiter, inner_tokens).with_span(span)) + } + + fn into_singleton_group(self, delimiter: Delimiter) -> Self { + let span = self.span(); + Self::group(self.into_token_stream(), delimiter, span) + } +} + +pub(crate) trait IdentExt: Sized { + fn new_bool(value: bool, span: Span) -> Self; + fn with_span(self, span: Span) -> Self; +} + +impl IdentExt for Ident { + fn new_bool(value: bool, span: Span) -> Self { + Ident::new(&value.to_string(), span) + } + + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +pub(crate) trait LiteralExt: Sized { + #[allow(unused)] + fn content_if_string(&self) -> Option; + fn content_if_string_like(&self) -> Option; +} + +impl LiteralExt for Literal { + fn content_if_string(&self) -> Option { + match parse_str::(&self.to_string()).unwrap() { + Lit::Str(lit_str) => Some(lit_str.value()), + _ => None, + } + } + + fn content_if_string_like(&self) -> Option { + match parse_str::(&self.to_string()).unwrap() { + Lit::Str(lit_str) => Some(lit_str.value()), + Lit::Char(lit_char) => Some(lit_char.value().to_string()), + Lit::CStr(lit_cstr) => Some(lit_cstr.value().to_string_lossy().to_string()), + _ => None, + } + } +} + +pub(crate) trait WithSpanExt { + fn with_span(self, span: Span) -> Self; +} + +impl WithSpanExt for Literal { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +impl WithSpanExt for Punct { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} + +impl WithSpanExt for Group { + fn with_span(mut self, span: Span) -> Self { + self.set_span(span); + self + } +} diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 2815ec8d..d5494f3c 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -7,14 +7,12 @@ pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; pub(crate) use syn::buffer::Cursor; pub(crate) use syn::ext::IdentExt as SynIdentExt; -pub(crate) use syn::parse::{discouraged::*, Parse, ParseBuffer, ParseStream, Parser}; -pub(crate) use syn::{ - parse_str, Error, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Result, Token, UnOp, -}; +pub(crate) use syn::parse::{discouraged::*, Parse as SynParse, ParseBuffer, ParseStream, Parser}; +pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Token, UnOp}; +pub(crate) use syn::{Error as SynError, Result as SynResult}; -pub(crate) use crate::commands::*; pub(crate) use crate::destructuring::*; pub(crate) use crate::expressions::*; +pub(crate) use crate::extensions::*; pub(crate) use crate::interpretation::*; -pub(crate) use crate::string_conversion::*; -pub(crate) use crate::traits::*; +pub(crate) use crate::misc::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index e7987431..7b3c8c40 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -1,3 +1,4 @@ +use super::commands::*; use crate::internal_prelude::*; #[allow(unused)] @@ -18,7 +19,7 @@ pub(crate) trait CommandType { pub(crate) trait OutputKind { type Output; - fn resolve(flattening: Option) -> Result; + fn resolve(flattening: Option) -> ParseResult; } struct ExecutionContext<'a> { @@ -32,13 +33,13 @@ trait CommandInvocation { self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()>; + ) -> ExecutionResult<()>; fn execute_into_expression( self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()>; + ) -> ExecutionResult<()>; } trait ClonableCommandInvocation: CommandInvocation { @@ -64,13 +65,13 @@ trait CommandInvocationAs { self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()>; + ) -> ExecutionResult<()>; fn execute_into_expression( self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()>; + ) -> ExecutionResult<()>; } impl> CommandInvocation for C { @@ -78,7 +79,7 @@ impl> CommandInvocation for self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { >::execute_into(self, context, output) } @@ -86,7 +87,7 @@ impl> CommandInvocation for self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { >::execute_into_expression( self, context, builder, ) @@ -101,9 +102,11 @@ pub(crate) struct OutputKindNone; impl OutputKind for OutputKindNone { type Output = (); - fn resolve(flattening: Option) -> Result { + fn resolve(flattening: Option) -> ParseResult { match flattening { - Some(dots) => dots.err("This command has no output, so cannot be flattened with .."), + Some(dots) => { + dots.parse_err("This command has no output, so cannot be flattened with ..") + } None => Ok(CommandOutputKind::None), } } @@ -113,8 +116,8 @@ pub(crate) trait NoOutputCommandDefinition: Sized + CommandType { const COMMAND_NAME: &'static str; - fn parse(arguments: CommandArguments) -> Result; - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()>; + fn parse(arguments: CommandArguments) -> ParseResult; + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()>; } impl CommandInvocationAs for C { @@ -122,7 +125,7 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, _: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.execute(context.interpreter)?; Ok(()) } @@ -131,10 +134,10 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, _: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { context.delim_span .join() - .err("Commands with no output cannot be used directly in expressions.\nConsider wrapping it inside a command such as [!group! ..] which returns an expression") + .execution_err("Commands with no output cannot be used directly in expressions.\nConsider wrapping it inside a command such as [!group! ..] which returns an expression") } } @@ -146,11 +149,10 @@ pub(crate) struct OutputKindValue; impl OutputKind for OutputKindValue { type Output = TokenTree; - fn resolve(flattening: Option) -> Result { + fn resolve(flattening: Option) -> ParseResult { match flattening { - Some(dots) => { - dots.err("This command outputs a single value, so cannot be flattened with ..") - } + Some(dots) => dots + .parse_err("This command outputs a single value, so cannot be flattened with .."), None => Ok(CommandOutputKind::Value), } } @@ -160,8 +162,8 @@ pub(crate) trait ValueCommandDefinition: Sized + CommandType { const COMMAND_NAME: &'static str; - fn parse(arguments: CommandArguments) -> Result; - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; + fn parse(arguments: CommandArguments) -> ParseResult; + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult; } impl CommandInvocationAs for C { @@ -169,7 +171,7 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { output.push_raw_token_tree(self.execute(context.interpreter)?); Ok(()) } @@ -178,7 +180,7 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { match self.execute(context.interpreter)? { TokenTree::Literal(literal) => builder.push_literal(literal), TokenTree::Ident(ident) => builder.push_ident(ident), @@ -196,11 +198,10 @@ pub(crate) struct OutputKindIdent; impl OutputKind for OutputKindIdent { type Output = Ident; - fn resolve(flattening: Option) -> Result { + fn resolve(flattening: Option) -> ParseResult { match flattening { - Some(dots) => { - dots.err("This command outputs a single ident, so cannot be flattened with ..") - } + Some(dots) => dots + .parse_err("This command outputs a single ident, so cannot be flattened with .."), None => Ok(CommandOutputKind::Ident), } } @@ -210,8 +211,8 @@ pub(crate) trait IdentCommandDefinition: Sized + CommandType { const COMMAND_NAME: &'static str; - fn parse(arguments: CommandArguments) -> Result; - fn execute(self: Box, interpreter: &mut Interpreter) -> Result; + fn parse(arguments: CommandArguments) -> ParseResult; + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult; } impl CommandInvocationAs for C { @@ -219,7 +220,7 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { output.push_ident(self.execute(context.interpreter)?); Ok(()) } @@ -228,7 +229,7 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { builder.push_ident(self.execute(context.interpreter)?); Ok(()) } @@ -242,7 +243,7 @@ pub(crate) struct OutputKindStream; impl OutputKind for OutputKindStream { type Output = InterpretedStream; - fn resolve(flattening: Option) -> Result { + fn resolve(flattening: Option) -> ParseResult { match flattening { Some(_) => Ok(CommandOutputKind::FlattenedStream), None => Ok(CommandOutputKind::GroupedStream), @@ -254,12 +255,12 @@ pub(crate) trait StreamCommandDefinition: Sized + CommandType { const COMMAND_NAME: &'static str; - fn parse(arguments: CommandArguments) -> Result; + fn parse(arguments: CommandArguments) -> ParseResult; fn execute( self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()>; + ) -> ExecutionResult<()>; } impl CommandInvocationAs for C { @@ -267,7 +268,7 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { match context.output_kind { CommandOutputKind::FlattenedStream => self.execute(context.interpreter, output), CommandOutputKind::GroupedStream => output.push_grouped( @@ -283,11 +284,11 @@ impl CommandInvocationAs for C { self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { if let CommandOutputKind::FlattenedStream = context.output_kind { return context.delim_span .join() - .err("Flattened commands cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"); + .execution_err("Flattened commands cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"); } builder.push_grouped( |output| self.execute(context.interpreter, output), @@ -304,9 +305,9 @@ pub(crate) struct OutputKindControlFlow; impl OutputKind for OutputKindControlFlow { type Output = (); - fn resolve(flattening: Option) -> Result { + fn resolve(flattening: Option) -> ParseResult { match flattening { - Some(dots) => dots.err("This command is control flow, so is always flattened and cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), + Some(dots) => dots.parse_err("This command is control flow, so is always flattened and cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), None => Ok(CommandOutputKind::ControlFlowCodeStream), } } @@ -316,12 +317,12 @@ pub(crate) trait ControlFlowCommandDefinition: Sized + CommandType { const COMMAND_NAME: &'static str; - fn parse(arguments: CommandArguments) -> Result; + fn parse(arguments: CommandArguments) -> ParseResult; fn execute( self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()>; + ) -> ExecutionResult<()>; } impl CommandInvocationAs for C { @@ -329,7 +330,7 @@ impl CommandInvocationAs self: Box, context: ExecutionContext, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.execute(context.interpreter, output) } @@ -337,7 +338,7 @@ impl CommandInvocationAs self: Box, context: ExecutionContext, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { builder.push_grouped( |output| self.execute(context.interpreter, output), context.delim_span.join(), @@ -362,7 +363,7 @@ macro_rules! define_command_kind { } impl CommandKind { - fn parse_invocation(&self, arguments: CommandArguments) -> Result> { + fn parse_invocation(&self, arguments: CommandArguments) -> ParseResult> { Ok(match self { $( Self::$command => Box::new( @@ -372,7 +373,7 @@ macro_rules! define_command_kind { }) } - pub(crate) fn output_kind(&self, flattening: Option) -> Result { + pub(crate) fn output_kind(&self, flattening: Option) -> ParseResult { match self { $( Self::$command => <$command as CommandType>::OutputKind::resolve(flattening), @@ -464,9 +465,8 @@ pub(crate) struct Command { } impl Parse for Command { - fn parse(input: ParseStream) -> Result { - let content; - let open_bracket = syn::bracketed!(content in input); + fn parse(input: ParseStream) -> ParseResult { + let (delim_span, content) = input.parse_group_matching(Delimiter::Bracket)?; content.parse::()?; let flattening = if content.peek(Token![.]) { Some(content.parse::()?) @@ -491,12 +491,12 @@ impl Parse for Command { let invocation = command_kind.parse_invocation(CommandArguments::new( &content, command_name, - open_bracket.span.span_range(), + delim_span.span_range(), ))?; Ok(Self { invocation, output_kind, - source_group_span: open_bracket.span, + source_group_span: delim_span, }) } } @@ -523,7 +523,7 @@ impl Interpret for Command { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let context = ExecutionContext { interpreter, output_kind: self.output_kind, @@ -538,7 +538,7 @@ impl Express for Command { self, interpreter: &mut Interpreter, builder: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { let context = ExecutionContext { interpreter, output_kind: self.output_kind, diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index 2aba1640..191e710f 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -26,46 +26,44 @@ impl<'a> CommandArguments<'a> { } /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message - pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> Result<()> { + pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> ParseResult<()> { if self.parse_stream.is_empty() { Ok(()) } else { - self.full_span_range.err(error_message) + self.full_span_range.parse_err(error_message) } } - pub(crate) fn fully_parse_as(&self) -> Result { + pub(crate) fn fully_parse_as(&self) -> ParseResult { self.fully_parse_or_error(T::parse, T::error_message()) } pub(crate) fn fully_parse_or_error( &self, - parse_function: impl FnOnce(ParseStream) -> Result, + parse_function: impl FnOnce(ParseStream) -> ParseResult, error_message: impl std::fmt::Display, - ) -> Result { - let parsed = parse_function(self.parse_stream).or_else(|error| { - // In future, when the diagnostic API is stable, - // we can add this context directly onto the command ident... - // Rather than just selectively adding it to the inner-most error. - let error_string = error.to_string(); - - // We avoid adding this additional context if it's already been added in an - // inner error, because that's likely the correct error to show. - if error_string.contains("\nOccurred whilst parsing") { - return Err(error); - } - error.span().err(format!( - "{}\nOccurred whilst parsing [!{}! ..] - {}", - error_string, self.command_name, error_message, - )) - })?; + ) -> ParseResult { + // In future, when the diagnostic API is stable, + // we can add this context directly onto the command ident... + // Rather than just selectively adding it to the inner-most error. + // + // For now though, we can add additional context to the error message. + // But we can avoid adding this additional context if it's already been added in an + // inner error, because that's likely the correct local context to show. + let parsed = + parse_function(self.parse_stream).add_context_if_error_and_no_context(|| { + format!( + "Occurred whilst parsing [!{}! ...] - {}", + self.command_name, error_message, + ) + })?; self.assert_empty(error_message)?; Ok(parsed) } - pub(crate) fn parse_all_for_interpretation(&self) -> Result { + pub(crate) fn parse_all_for_interpretation(&self) -> ParseResult { self.parse_stream .parse_all_for_interpretation(self.full_span_range) } diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index 66899bd3..d9ac32d1 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -8,14 +8,10 @@ pub(crate) struct CommandCodeInput { } impl Parse for CommandCodeInput { - fn parse(input: ParseStream) -> Result { - let content; - let bracket = syn::braced!(content in input); - let inner = content.parse_with(bracket.span.span_range())?; - Ok(Self { - delim_span: bracket.span, - inner, - }) + fn parse(input: ParseStream) -> ParseResult { + let (delim_span, content) = input.parse_group_matching(Delimiter::Brace)?; + let inner = content.parse_with(delim_span.join().span_range())?; + Ok(Self { delim_span, inner }) } } @@ -30,14 +26,13 @@ impl CommandCodeInput { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result { + ) -> ExecutionResult> { match self.inner.interpret_into(interpreter, output) { - Ok(()) => Ok(LoopCondition::None), - Err(err) => match interpreter.outstanding_loop_condition() { - LoopCondition::None => Err(err), - LoopCondition::Break => Ok(LoopCondition::Break), - LoopCondition::Continue => Ok(LoopCondition::Continue), - }, + Ok(()) => Ok(None), + Err(ExecutionInterrupt::ControlFlow(control_flow_interrupt, _)) => { + Ok(Some(control_flow_interrupt)) + } + Err(error) => Err(error), } } } @@ -47,7 +42,7 @@ impl Interpret for CommandCodeInput { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.inner.interpret_into(interpreter, output) } } diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index 922dcbe8..9d72eaa6 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -25,7 +25,7 @@ macro_rules! define_field_inputs { } impl Parse for $inputs_type { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { $( let mut $required_field: Option<$required_type> = None; )* @@ -33,8 +33,7 @@ macro_rules! define_field_inputs { let mut $optional_field: Option<$optional_type> = None; )* - let content; - let brace = syn::braced!(content in input); + let (delim_span, content) = input.parse_group_matching(Delimiter::Brace)?; while !content.is_empty() { let ident = content.parse_any_ident()?; @@ -43,20 +42,20 @@ macro_rules! define_field_inputs { $( stringify!($required_field) => { if $required_field.is_some() { - return ident.err("duplicate field"); + return ident.parse_err("duplicate field"); } - $required_field = Some(content.parse()?); + $required_field = Some(content.parse_v2()?); } )* $( stringify!($optional_field) => { if $optional_field.is_some() { - return ident.err("duplicate field"); + return ident.parse_err("duplicate field"); } - $optional_field = Some(content.parse()?); + $optional_field = Some(content.parse_v2()?); } )* - _ => return ident.err("unexpected field"), + _ => return ident.parse_err("unexpected field"), } if !content.is_empty() { content.parse::()?; @@ -73,7 +72,7 @@ macro_rules! define_field_inputs { )* if !missing_fields.is_empty() { - return brace.span.err(format!( + return delim_span.join().parse_err(format!( "required fields are missing: {}", missing_fields.join(", ") )); diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 0d9b3075..56a19c2d 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -22,16 +22,16 @@ pub(crate) enum CommandStreamInput { } impl Parse for CommandStreamInput { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), - PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + PeekMatch::GroupedCommand(_) => Self::Command(input.parse_v2()?), + PeekMatch::FlattenedCommand(_) => Self::Command(input.parse_v2()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse_v2()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse_v2()?), + PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse_v2()?), + PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse_v2()?), _ => input.span() - .err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, + .parse_err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) } } @@ -53,14 +53,14 @@ impl Interpret for CommandStreamInput { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { match self { CommandStreamInput::Command(mut command) => { match command.output_kind() { CommandOutputKind::None | CommandOutputKind::Value | CommandOutputKind::Ident => { - command.err("The command does not output a stream") + command.execution_err("The command does not output a stream") } CommandOutputKind::FlattenedStream => parse_as_stream_input( command, @@ -121,7 +121,7 @@ fn parse_as_stream_input( interpreter: &mut Interpreter, error_message: impl FnOnce() -> String, output: &mut InterpretedStream, -) -> Result<()> { +) -> ExecutionResult<()> { let span = input.span_range(); input .interpret_to_new_stream(interpreter)? diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 7e4a1e52..0191bfa7 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -14,21 +14,23 @@ pub(crate) enum CommandValueInput { } impl Parse for CommandValueInput { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + PeekMatch::GroupedCommand(_) => Self::Command(input.parse_v2()?), + PeekMatch::FlattenedCommand(_) => Self::Command(input.parse_v2()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse_v2()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse_v2()?), + PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse_v2()?), PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { - return input.span().err("Destructurings are not supported here") + return input + .span() + .parse_err("Destructurings are not supported here") } PeekMatch::Group(_) | PeekMatch::Punct(_) | PeekMatch::Literal(_) | PeekMatch::Ident(_) - | PeekMatch::End => Self::Value(input.parse()?), + | PeekMatch::End => Self::Value(input.parse_v2()?), }) } } @@ -48,7 +50,7 @@ impl HasSpanRange for CommandValueInput { impl, I: Parse> InterpretValue for CommandValueInput { type InterpretedValue = I; - fn interpret(self, interpreter: &mut Interpreter) -> Result { + fn interpret(self, interpreter: &mut Interpreter) -> ExecutionResult { let descriptor = match self { CommandValueInput::Command(_) => "command output", CommandValueInput::GroupedVariable(_) => "grouped variable output", @@ -70,14 +72,16 @@ impl, I: Parse> InterpretValue for Comma unsafe { // RUST-ANALYZER SAFETY: We only use I with simple parse functions so far which don't care about // none-delimited groups - match interpreted_stream.syn_parse(I::parse) { - Ok(value) => Ok(value), - Err(err) => Err(err.concat(&format!( - "\nOccurred whilst parsing the {} to a {}.", - descriptor, - std::any::type_name::() - ))), - } + interpreted_stream + .syn_parse(I::parse) + .add_context_if_error_and_no_context(|| { + format!( + "Occurred whilst parsing the {} to a {}.", + descriptor, + std::any::type_name::() + ) + }) + .into_execution_result() } } } diff --git a/src/commands/concat_commands.rs b/src/interpretation/commands/concat_commands.rs similarity index 92% rename from src/commands/concat_commands.rs rename to src/interpretation/commands/concat_commands.rs index 4d9a26d0..6c1fb1f9 100644 --- a/src/commands/concat_commands.rs +++ b/src/interpretation/commands/concat_commands.rs @@ -8,7 +8,7 @@ fn concat_into_string( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> ExecutionResult { let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? @@ -21,7 +21,7 @@ fn concat_into_ident( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> ExecutionResult { let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? @@ -37,7 +37,7 @@ fn concat_into_literal( input: InterpretationStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> Result { +) -> ExecutionResult { let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? @@ -67,13 +67,16 @@ macro_rules! define_literal_concat_command { impl ValueCommandDefinition for $command { const COMMAND_NAME: &'static str = $command_name; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute( + self: Box, + interpreter: &mut Interpreter, + ) -> ExecutionResult { Ok($output_fn(self.arguments, interpreter, $conversion_fn)?.into()) } } @@ -96,13 +99,13 @@ macro_rules! define_ident_concat_command { impl IdentCommandDefinition for $command { const COMMAND_NAME: &'static str = $command_name; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { $output_fn(self.arguments, interpreter, $conversion_fn).into() } } diff --git a/src/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs similarity index 76% rename from src/commands/control_flow_commands.rs rename to src/interpretation/commands/control_flow_commands.rs index 1fc8e60b..3395a845 100644 --- a/src/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -15,11 +15,11 @@ impl CommandType for IfCommand { impl ControlFlowCommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { - let condition = input.parse()?; - let true_code = input.parse()?; + let condition = input.parse_v2()?; + let true_code = input.parse_v2()?; let mut else_ifs = Vec::new(); let mut else_code = None; while !input.is_empty() { @@ -27,11 +27,11 @@ impl ControlFlowCommandDefinition for IfCommand { if input.peek_ident_matching("elif") { input.parse_ident_matching("elif")?; input.parse::()?; - else_ifs.push((input.parse()?, input.parse()?)); + else_ifs.push((input.parse_v2()?, input.parse_v2()?)); } else { input.parse_ident_matching("else")?; input.parse::()?; - else_code = Some(input.parse()?); + else_code = Some(input.parse_v2()?); break; } } @@ -50,7 +50,7 @@ impl ControlFlowCommandDefinition for IfCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let evaluated_condition = self .condition .evaluate(interpreter)? @@ -93,12 +93,12 @@ impl CommandType for WhileCommand { impl ControlFlowCommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - condition: input.parse()?, - loop_code: input.parse()?, + condition: input.parse_v2()?, + loop_code: input.parse_v2()?, }) }, "Expected [!while! (condition) { code }]", @@ -109,7 +109,7 @@ impl ControlFlowCommandDefinition for WhileCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let mut iteration_counter = interpreter.start_iteration_counter(&self.condition); loop { iteration_counter.increment_and_check()?; @@ -130,9 +130,9 @@ impl ControlFlowCommandDefinition for WhileCommand { .clone() .interpret_loop_content_into(interpreter, output)? { - LoopCondition::None => {} - LoopCondition::Continue => continue, - LoopCondition::Break => break, + None => {} + Some(ControlFlowInterrupt::Continue) => continue, + Some(ControlFlowInterrupt::Break) => break, } } @@ -152,11 +152,11 @@ impl CommandType for LoopCommand { impl ControlFlowCommandDefinition for LoopCommand { const COMMAND_NAME: &'static str = "loop"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - loop_code: input.parse()?, + loop_code: input.parse_v2()?, }) }, "Expected [!loop! { ... }]", @@ -167,7 +167,7 @@ impl ControlFlowCommandDefinition for LoopCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let mut iteration_counter = interpreter.start_iteration_counter(&self.loop_code); loop { @@ -177,9 +177,9 @@ impl ControlFlowCommandDefinition for LoopCommand { .clone() .interpret_loop_content_into(interpreter, output)? { - LoopCondition::None => {} - LoopCondition::Continue => continue, - LoopCondition::Break => break, + None => {} + Some(ControlFlowInterrupt::Continue) => continue, + Some(ControlFlowInterrupt::Break) => break, } } Ok(()) @@ -202,14 +202,14 @@ impl CommandType for ForCommand { impl ControlFlowCommandDefinition for ForCommand { const COMMAND_NAME: &'static str = "for"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - parse_place: input.parse()?, - in_token: input.parse()?, - input: input.parse()?, - loop_code: input.parse()?, + parse_place: input.parse_v2()?, + in_token: input.parse_v2()?, + input: input.parse_v2()?, + loop_code: input.parse_v2()?, }) }, "Expected [!for! #x in [ ... ] { code }]", @@ -220,7 +220,7 @@ impl ControlFlowCommandDefinition for ForCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let stream = self.input.interpret_to_new_stream(interpreter)?; let mut iteration_counter = interpreter.start_iteration_counter(&self.in_token); @@ -234,9 +234,9 @@ impl ControlFlowCommandDefinition for ForCommand { .clone() .interpret_loop_content_into(interpreter, output)? { - LoopCondition::None => {} - LoopCondition::Continue => continue, - LoopCondition::Break => break, + None => {} + Some(ControlFlowInterrupt::Continue) => continue, + Some(ControlFlowInterrupt::Break) => break, } } @@ -256,15 +256,18 @@ impl CommandType for ContinueCommand { impl NoOutputCommandDefinition for ContinueCommand { const COMMAND_NAME: &'static str = "continue"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.assert_empty("The !continue! command takes no arguments")?; Ok(Self { span: arguments.full_span_range().span(), }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { - Err(interpreter.start_loop_action(self.span, LoopCondition::Continue)) + fn execute(self: Box, _: &mut Interpreter) -> ExecutionResult<()> { + ExecutionResult::Err(ExecutionInterrupt::ControlFlow( + ControlFlowInterrupt::Continue, + self.span, + )) } } @@ -280,14 +283,17 @@ impl CommandType for BreakCommand { impl NoOutputCommandDefinition for BreakCommand { const COMMAND_NAME: &'static str = "break"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.assert_empty("The !break! command takes no arguments")?; Ok(Self { span: arguments.full_span_range().span(), }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { - Err(interpreter.start_loop_action(self.span, LoopCondition::Break)) + fn execute(self: Box, _: &mut Interpreter) -> ExecutionResult<()> { + ExecutionResult::Err(ExecutionInterrupt::ControlFlow( + ControlFlowInterrupt::Break, + self.span, + )) } } diff --git a/src/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs similarity index 90% rename from src/commands/core_commands.rs rename to src/interpretation/commands/core_commands.rs index e8adbc10..52700687 100644 --- a/src/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -15,11 +15,11 @@ impl CommandType for SetCommand { impl NoOutputCommandDefinition for SetCommand { const COMMAND_NAME: &'static str = "set"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - variable: input.parse()?, + variable: input.parse_v2()?, equals: input.parse()?, arguments: input.parse_with(arguments.full_span_range())?, }) @@ -28,7 +28,7 @@ impl NoOutputCommandDefinition for SetCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; self.variable.set(interpreter, result_tokens)?; Ok(()) @@ -50,11 +50,11 @@ impl CommandType for ExtendCommand { impl NoOutputCommandDefinition for ExtendCommand { const COMMAND_NAME: &'static str = "extend"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - variable: input.parse()?, + variable: input.parse_v2()?, plus_equals: input.parse()?, arguments: input.parse_all_for_interpretation(arguments.full_span_range())?, }) @@ -63,7 +63,7 @@ impl NoOutputCommandDefinition for ExtendCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { let variable_data = self.variable.get_existing_for_mutation(interpreter)?; self.arguments.interpret_into( interpreter, @@ -85,7 +85,7 @@ impl CommandType for RawCommand { impl StreamCommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { token_stream: arguments.read_all_as_raw_token_stream(), }) @@ -95,7 +95,7 @@ impl StreamCommandDefinition for RawCommand { self: Box, _interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { output.extend_raw_tokens(self.token_stream); Ok(()) } @@ -111,13 +111,13 @@ impl CommandType for IgnoreCommand { impl NoOutputCommandDefinition for IgnoreCommand { const COMMAND_NAME: &'static str = "ignore"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { // Avoid a syn parse error by reading all the tokens let _ = arguments.read_all_as_raw_token_stream(); Ok(Self) } - fn execute(self: Box, _interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, _interpreter: &mut Interpreter) -> ExecutionResult<()> { Ok(()) } } @@ -134,13 +134,13 @@ impl CommandType for VoidCommand { impl NoOutputCommandDefinition for VoidCommand { const COMMAND_NAME: &'static str = "void"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { inner: arguments.parse_all_for_interpretation()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { let _ = self.inner.interpret_to_new_stream(interpreter)?; Ok(()) } @@ -167,13 +167,13 @@ define_field_inputs! { impl NoOutputCommandDefinition for SettingsCommand { const COMMAND_NAME: &'static str = "settings"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { inputs: arguments.fully_parse_as()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { if let Some(limit) = self.inputs.iteration_limit { let limit: usize = limit.interpret(interpreter)?.base10_parse()?; interpreter.set_iteration_limit(Some(limit)); @@ -211,12 +211,12 @@ define_field_inputs! { impl NoOutputCommandDefinition for ErrorCommand { const COMMAND_NAME: &'static str = "error"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { if input.peek(syn::token::Brace) { Ok(Self { - inputs: EitherErrorInput::Fields(input.parse()?), + inputs: EitherErrorInput::Fields(input.parse_v2()?), }) } else { Ok(Self { @@ -233,14 +233,14 @@ impl NoOutputCommandDefinition for ErrorCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { let fields = match self.inputs { EitherErrorInput::Fields(error_inputs) => error_inputs, EitherErrorInput::JustMessage(stream) => { let error_message = stream .interpret_to_new_stream(interpreter)? .concat_recursive(&ConcatBehaviour::standard()); - return Span::call_site().err(error_message); + return Span::call_site().execution_err(error_message); } }; @@ -287,7 +287,7 @@ impl NoOutputCommandDefinition for ErrorCommand { None => Span::call_site().span_range(), }; - error_span.err(message) + error_span.execution_err(message) } } @@ -303,13 +303,13 @@ impl CommandType for DebugCommand { impl ValueCommandDefinition for DebugCommand { const COMMAND_NAME: &'static str = "debug"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { inner: arguments.parse_all_for_interpretation()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { let span = self.inner.span(); let debug_string = self .inner diff --git a/src/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs similarity index 82% rename from src/commands/destructuring_commands.rs rename to src/interpretation/commands/destructuring_commands.rs index 58129a4b..796946b3 100644 --- a/src/commands/destructuring_commands.rs +++ b/src/interpretation/commands/destructuring_commands.rs @@ -15,12 +15,12 @@ impl CommandType for LetCommand { impl NoOutputCommandDefinition for LetCommand { const COMMAND_NAME: &'static str = "let"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - destructuring: input.parse()?, - equals: input.parse()?, + destructuring: input.parse_v2()?, + equals: input.parse_v2()?, arguments: input.parse_with(arguments.full_span_range())?, }) }, @@ -28,7 +28,7 @@ impl NoOutputCommandDefinition for LetCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; self.destructuring .handle_destructure_from_stream(result_tokens, interpreter) diff --git a/src/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs similarity index 88% rename from src/commands/expression_commands.rs rename to src/interpretation/commands/expression_commands.rs index 83d6b405..97ef1233 100644 --- a/src/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -12,18 +12,18 @@ impl CommandType for EvaluateCommand { impl ValueCommandDefinition for EvaluateCommand { const COMMAND_NAME: &'static str = "evaluate"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - expression: input.parse()?, + expression: input.parse_v2()?, }) }, "Expected [!evaluate! ...] containing a valid preinterpret expression", ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { let expression = self.expression.start_expression_builder(interpreter)?; Ok(expression.evaluate()?.into_token_tree()) } @@ -45,28 +45,28 @@ impl CommandType for AssignCommand { impl NoOutputCommandDefinition for AssignCommand { const COMMAND_NAME: &'static str = "assign"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - variable: input.parse()?, + variable: input.parse_v2()?, operator: { let operator: Punct = input.parse()?; match operator.as_char() { '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => return operator.err("Expected one of + - * / % & | or ^"), + _ => return operator.parse_err("Expected one of + - * / % & | or ^"), } operator }, equals: input.parse()?, - expression: input.parse()?, + expression: input.parse_v2()?, }) }, "Expected [!assign! #variable += ...] for + or some other operator supported in an expression", ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result<()> { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { let Self { variable, operator, @@ -100,13 +100,13 @@ impl CommandType for RangeCommand { impl StreamCommandDefinition for RangeCommand { const COMMAND_NAME: &'static str = "range"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { - left: input.parse()?, - range_limits: input.parse()?, - right: input.parse()?, + left: input.parse_v2()?, + range_limits: input.parse_v2()?, + right: input.parse_v2()?, }) }, "Expected a rust range expression such as [!range! 1..4]", @@ -117,7 +117,7 @@ impl StreamCommandDefinition for RangeCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let range_span_range = self.range_limits.span_range(); let range_span = self.range_limits.span(); @@ -178,7 +178,7 @@ enum RangeLimits { } impl Parse for RangeLimits { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { if input.peek(Token![..=]) { Ok(RangeLimits::Closed(input.parse()?)) } else { diff --git a/src/commands/mod.rs b/src/interpretation/commands/mod.rs similarity index 100% rename from src/commands/mod.rs rename to src/interpretation/commands/mod.rs diff --git a/src/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs similarity index 94% rename from src/commands/token_commands.rs rename to src/interpretation/commands/token_commands.rs index 79c25d1c..47f02fde 100644 --- a/src/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -12,13 +12,13 @@ impl CommandType for IsEmptyCommand { impl ValueCommandDefinition for IsEmptyCommand { const COMMAND_NAME: &'static str = "is_empty"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; Ok(Ident::new_bool(interpreted.is_empty(), output_span).into()) @@ -37,13 +37,13 @@ impl CommandType for LengthCommand { impl ValueCommandDefinition for LengthCommand { const COMMAND_NAME: &'static str = "length"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> Result { + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { let output_span = self.arguments.span_range().span(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; let length_literal = Literal::usize_unsuffixed(interpreted.len()).with_span(output_span); @@ -63,7 +63,7 @@ impl CommandType for GroupCommand { impl StreamCommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { arguments: arguments.parse_all_for_interpretation()?, }) @@ -73,7 +73,7 @@ impl StreamCommandDefinition for GroupCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { // The grouping happens automatically because a non-flattened // stream command is outputted in a group. self.arguments.interpret_into(interpreter, output) @@ -105,7 +105,7 @@ define_field_inputs! { impl StreamCommandDefinition for IntersperseCommand { const COMMAND_NAME: &'static str = "intersperse"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { inputs: arguments.fully_parse_as()?, }) @@ -115,7 +115,7 @@ impl StreamCommandDefinition for IntersperseCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let items = self .inputs .items @@ -174,7 +174,7 @@ impl SeparatorAppender { interpreter: &mut Interpreter, remaining: RemainingItemCount, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { match self.separator(remaining) { TrailingSeparator::Normal => self.separator.clone().interpret_into(interpreter, output), TrailingSeparator::Final => match self.final_separator.take() { @@ -244,7 +244,7 @@ define_field_inputs! { impl StreamCommandDefinition for SplitCommand { const COMMAND_NAME: &'static str = "split"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { inputs: arguments.fully_parse_as()?, }) @@ -254,7 +254,7 @@ impl StreamCommandDefinition for SplitCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let output_span = self.inputs.stream.span(); let stream = self.inputs.stream.interpret_to_new_stream(interpreter)?; let separator = self.inputs.separator.interpret_to_new_stream(interpreter)?; @@ -292,11 +292,11 @@ fn handle_split( drop_empty_start: bool, drop_empty_middle: bool, drop_empty_end: bool, -) -> Result<()> { +) -> ExecutionResult<()> { unsafe { // RUST-ANALYZER SAFETY: This is as safe as we can get. // Typically the separator won't contain none-delimited groups, so we're OK - input.syn_parse(move |input: ParseStream| -> Result<()> { + input.syn_parse(move |input| { let mut current_item = InterpretedStream::new(); let mut drop_empty_next = drop_empty_start; while !input.is_empty() { @@ -317,9 +317,8 @@ fn handle_split( output.push_new_group(current_item, Delimiter::None, output_span); } Ok(()) - })?; + }) } - Ok(()) } #[derive(Clone)] @@ -334,7 +333,7 @@ impl CommandType for CommaSplitCommand { impl StreamCommandDefinition for CommaSplitCommand { const COMMAND_NAME: &'static str = "comma_split"; - fn parse(arguments: CommandArguments) -> Result { + fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { input: arguments.parse_all_for_interpretation()?, }) @@ -344,7 +343,7 @@ impl StreamCommandDefinition for CommaSplitCommand { self: Box, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let output_span = self.input.span(); let stream = self.input.interpret_to_new_stream(interpreter)?; let separator = { diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index 5a0c47f9..89f229e8 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -5,9 +5,12 @@ pub(crate) trait Interpret: Sized { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()>; + ) -> ExecutionResult<()>; - fn interpret_to_new_stream(self, interpreter: &mut Interpreter) -> Result { + fn interpret_to_new_stream( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { let mut output = InterpretedStream::new(); self.interpret_into(interpreter, &mut output)?; Ok(output) @@ -17,7 +20,7 @@ pub(crate) trait Interpret: Sized { pub(crate) trait InterpretValue: Sized { type InterpretedValue; - fn interpret(self, interpreter: &mut Interpreter) -> Result; + fn interpret(self, interpreter: &mut Interpreter) -> ExecutionResult; } impl + HasSpanRange, I: ToTokens> Interpret for T { @@ -25,7 +28,7 @@ impl + HasSpanRange, I: ToTokens> Interp self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { output.extend_with_raw_tokens_from(self.interpret(interpreter)?); Ok(()) } @@ -34,7 +37,7 @@ impl + HasSpanRange, I: ToTokens> Interp impl InterpretValue for T { type InterpretedValue = Self; - fn interpret(self, _interpreter: &mut Interpreter) -> Result { + fn interpret(self, _interpreter: &mut Interpreter) -> ExecutionResult { Ok(self) } } diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 1e042157..3658fbcf 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -12,20 +12,22 @@ pub(crate) enum InterpretationItem { } impl Parse for InterpretationItem { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => InterpretationItem::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => InterpretationItem::Command(input.parse()?), - PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), - PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), + PeekMatch::GroupedCommand(_) => InterpretationItem::Command(input.parse_v2()?), + PeekMatch::FlattenedCommand(_) => InterpretationItem::Command(input.parse_v2()?), + PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse_v2()?), + PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse_v2()?), + PeekMatch::FlattenedVariable => { + InterpretationItem::FlattenedVariable(input.parse_v2()?) + } PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { - return input.span().err("Destructurings are not supported here") + return input.parse_err("Destructurings are not supported here") } PeekMatch::Punct(_) => InterpretationItem::Punct(input.parse_any_punct()?), PeekMatch::Ident(_) => InterpretationItem::Ident(input.parse_any_ident()?), PeekMatch::Literal(_) => InterpretationItem::Literal(input.parse()?), - PeekMatch::End => return input.span().err("Expected some item"), + PeekMatch::End => return input.parse_err("Expected some item"), }) } } @@ -127,7 +129,7 @@ impl Interpret for InterpretationItem { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { match self { InterpretationItem::Command(command_invocation) => { command_invocation.interpret_into(interpreter, output)?; diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 11ca6440..e77f8a15 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -7,26 +7,13 @@ pub(crate) struct InterpretationStream { span_range: SpanRange, } -impl InterpretationStream { - pub(crate) fn parse_from_token_stream( - token_stream: TokenStream, - span_range: SpanRange, - ) -> Result { - Self::create_parser(span_range).parse2(token_stream) - } - - fn create_parser(context: SpanRange) -> impl FnOnce(ParseStream) -> Result { - move |input: ParseStream| Self::parse_with_context(input, context) - } -} - impl ContextualParse for InterpretationStream { type Context = SpanRange; - fn parse_with_context(input: ParseStream, span_range: Self::Context) -> Result { + fn parse_with_context(input: ParseStream, span_range: Self::Context) -> ParseResult { let mut items = Vec::new(); while !input.is_empty() { - items.push(input.parse()?); + items.push(input.parse_v2()?); } Ok(Self { items, span_range }) } @@ -37,7 +24,7 @@ impl Interpret for InterpretationStream { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { for item in self.items { item.interpret_into(interpreter, output)?; } @@ -66,7 +53,7 @@ impl InterpretationGroup { } impl Parse for InterpretationGroup { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_delimiter()?; let content = content.parse_with(delim_span.span_range())?; Ok(Self { @@ -82,7 +69,7 @@ impl Interpret for InterpretationGroup { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { let inner = self.content.interpret_to_new_stream(interpreter)?; output.push_new_group(inner, self.source_delimiter, self.source_delim_span.join()); Ok(()) @@ -111,7 +98,7 @@ impl RawGroup { } impl Parse for RawGroup { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_delimiter()?; let content = content.parse()?; Ok(Self { @@ -123,7 +110,11 @@ impl Parse for RawGroup { } impl Interpret for RawGroup { - fn interpret_into(self, _: &mut Interpreter, output: &mut InterpretedStream) -> Result<()> { + fn interpret_into( + self, + _: &mut Interpreter, + output: &mut InterpretedStream, + ) -> ExecutionResult<()> { output.push_new_group( InterpretedStream::raw(self.content), self.source_delimeter, @@ -138,7 +129,7 @@ impl Express for RawGroup { self, _: &mut Interpreter, expression_stream: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { expression_stream.push_grouped( |inner| { inner.extend_raw_tokens(self.content); diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index aed21831..47874675 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -64,10 +64,10 @@ impl InterpretedStream { pub(crate) fn push_grouped( &mut self, - appender: impl FnOnce(&mut Self) -> Result<()>, + appender: impl FnOnce(&mut Self) -> ExecutionResult<()>, delimiter: Delimiter, span: Span, - ) -> Result<()> { + ) -> ExecutionResult<()> { let mut inner = Self::new(); appender(&mut inner)?; self.push_new_group(inner, delimiter, span); @@ -137,8 +137,11 @@ impl InterpretedStream { /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 /// /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. - pub(crate) unsafe fn syn_parse(self, parser: P) -> Result { - parser.parse2(self.into_token_stream()) + pub(crate) unsafe fn syn_parse>( + self, + parser: impl FnOnce(ParseStream) -> Result, + ) -> Result { + self.into_token_stream().parse_with(parser) } pub(crate) fn append_into(self, output: &mut InterpretedStream) { @@ -214,8 +217,8 @@ impl InterpretedStream { pub(crate) fn unwrap_singleton_group( self, check_group: impl FnOnce(Delimiter) -> bool, - create_error: impl FnOnce() -> Error, - ) -> Result { + create_error: impl FnOnce() -> SynError, + ) -> ParseResult { let mut item_vec = self.into_item_vec(); if item_vec.len() == 1 { match item_vec.pop().unwrap() { @@ -232,7 +235,7 @@ impl InterpretedStream { _ => {} } } - Err(create_error()) + Err(create_error().into()) } pub(crate) fn into_item_vec(self) -> Vec { diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index a15043d0..309a9317 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -8,15 +8,6 @@ use std::rc::Rc; pub(crate) struct Interpreter { config: InterpreterConfig, variable_data: HashMap, - loop_condition: LoopCondition, -} - -#[derive(Clone, Copy)] -#[must_use] -pub(crate) enum LoopCondition { - None, - Continue, - Break, } #[derive(Clone)] @@ -34,23 +25,30 @@ impl VariableData { pub(crate) fn get<'d>( &'d self, variable: &impl IsVariable, - ) -> Result> { + ) -> ExecutionResult> { self.value.try_borrow().map_err(|_| { - variable.error("The variable cannot be read if it is currently being modified") + variable + .error("The variable cannot be read if it is currently being modified") + .into() }) } pub(crate) fn get_mut<'d>( &'d self, variable: &impl IsVariable, - ) -> Result> { + ) -> ExecutionResult> { self.value.try_borrow_mut().map_err(|_| { - variable - .error("The variable cannot be modified if it is already currently being modified") + variable.execution_error( + "The variable cannot be modified if it is already currently being modified", + ) }) } - pub(crate) fn set(&self, variable: &impl IsVariable, content: InterpretedStream) -> Result<()> { + pub(crate) fn set( + &self, + variable: &impl IsVariable, + content: InterpretedStream, + ) -> ExecutionResult<()> { *self.get_mut(variable)? = content; Ok(()) } @@ -67,7 +65,6 @@ impl Interpreter { Self { config: Default::default(), variable_data: Default::default(), - loop_condition: LoopCondition::None, } } @@ -75,7 +72,7 @@ impl Interpreter { &mut self, variable: &impl IsVariable, tokens: InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { match self.variable_data.entry(variable.get_name()) { Entry::Occupied(mut entry) => { entry.get_mut().set(variable, tokens)?; @@ -90,11 +87,11 @@ impl Interpreter { pub(crate) fn get_existing_variable_data( &self, variable: &impl IsVariable, - make_error: impl FnOnce() -> Error, - ) -> Result<&VariableData> { + make_error: impl FnOnce() -> SynError, + ) -> ExecutionResult<&VariableData> { self.variable_data .get(&variable.get_name()) - .ok_or_else(make_error) + .ok_or_else(|| make_error().into()) } pub(crate) fn start_iteration_counter<'s, S: HasSpanRange>( @@ -111,22 +108,6 @@ impl Interpreter { pub(crate) fn set_iteration_limit(&mut self, limit: Option) { self.config.iteration_limit = limit; } - - /// Panics if called with [`LoopCondition::None`] - pub(crate) fn start_loop_action(&mut self, source: Span, action: LoopCondition) -> Error { - self.loop_condition = action; - match action { - LoopCondition::None => panic!("Not allowed"), - LoopCondition::Continue => { - source.error("The continue command is only allowed inside a loop") - } - LoopCondition::Break => source.error("The break command is only allowed inside a loop"), - } - } - - pub(crate) fn outstanding_loop_condition(&mut self) -> LoopCondition { - std::mem::replace(&mut self.loop_condition, LoopCondition::None) - } } pub(crate) struct IterationCounter<'a, S: HasSpanRange> { @@ -136,20 +117,20 @@ pub(crate) struct IterationCounter<'a, S: HasSpanRange> { } impl IterationCounter<'_, S> { - pub(crate) fn add_and_check(&mut self, count: usize) -> Result<()> { + pub(crate) fn add_and_check(&mut self, count: usize) -> ExecutionResult<()> { self.count = self.count.wrapping_add(count); self.check() } - pub(crate) fn increment_and_check(&mut self) -> Result<()> { + pub(crate) fn increment_and_check(&mut self) -> ExecutionResult<()> { self.count += 1; self.check() } - pub(crate) fn check(&self) -> Result<()> { + pub(crate) fn check(&self) -> ExecutionResult<()> { if let Some(limit) = self.iteration_limit { if self.count > limit { - return self.span_source.err(format!("Iteration limit of {} exceeded.\nIf needed, the limit can be reconfigured with [!settings! {{ iteration_limit: X }}]", limit)); + return self.span_source.execution_err(format!("Iteration limit of {} exceeded.\nIf needed, the limit can be reconfigured with [!settings! {{ iteration_limit: X }}]", limit)); } } Ok(()) diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index e6af51f9..4b972a92 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -4,6 +4,7 @@ mod command_code_input; mod command_field_inputs; mod command_stream_input; mod command_value_input; +mod commands; mod interpret_traits; mod interpretation_item; mod interpretation_stream; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 4cbbdff9..9ed58f40 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -11,7 +11,7 @@ pub(crate) struct GroupedVariable { } impl Parse for GroupedVariable { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { input.try_parse_or_message( |input| { Ok(Self { @@ -29,14 +29,14 @@ impl GroupedVariable { &self, interpreter: &mut Interpreter, value: InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { interpreter.set_variable(self, value) } pub(crate) fn get_existing_for_mutation( &self, interpreter: &Interpreter, - ) -> Result { + ) -> ExecutionResult { Ok(interpreter .get_existing_variable_data(self, || { self.error(format!("The variable {} wasn't already set", self)) @@ -48,7 +48,7 @@ impl GroupedVariable { &self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? .append_cloned_into(output); @@ -59,7 +59,7 @@ impl GroupedVariable { &self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { output.push_new_group( self.read_existing(interpreter)?.get(self)?.clone(), Delimiter::None, @@ -68,7 +68,7 @@ impl GroupedVariable { Ok(()) } - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i VariableData> { + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { interpreter.get_existing_variable_data( self, || self.error(format!( @@ -91,7 +91,7 @@ impl Interpret for &GroupedVariable { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.substitute_grouped_into(interpreter, output) } } @@ -101,7 +101,7 @@ impl Express for &GroupedVariable { self, interpreter: &mut Interpreter, expression_stream: &mut ExpressionBuilder, - ) -> Result<()> { + ) -> ExecutionResult<()> { expression_stream.push_grouped( |inner| self.substitute_ungrouped_contents_into(interpreter, inner), self.span(), @@ -136,7 +136,7 @@ pub(crate) struct FlattenedVariable { } impl Parse for FlattenedVariable { - fn parse(input: ParseStream) -> Result { + fn parse(input: ParseStream) -> ParseResult { input.try_parse_or_message( |input| { Ok(Self { @@ -155,14 +155,14 @@ impl FlattenedVariable { &self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? .append_cloned_into(output); Ok(()) } - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> Result<&'i VariableData> { + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { interpreter.get_existing_variable_data( self, || self.error(format!( @@ -189,18 +189,22 @@ impl Interpret for &FlattenedVariable { self, interpreter: &mut Interpreter, output: &mut InterpretedStream, - ) -> Result<()> { + ) -> ExecutionResult<()> { self.substitute_into(interpreter, output) } } impl Express for &FlattenedVariable { - fn add_to_expression(self, _: &mut Interpreter, _: &mut ExpressionBuilder) -> Result<()> { + fn add_to_expression( + self, + _: &mut Interpreter, + _: &mut ExpressionBuilder, + ) -> ExecutionResult<()> { // Just like with commands, we throw an error in the flattened case so // that we can determine in future the exact structure of the expression // at parse time. self.flatten - .err("Flattened variables cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression") + .execution_err("Flattened variables cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression") } } diff --git a/src/lib.rs b/src/lib.rs index e8e9b983..803a6f74 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -510,13 +510,12 @@ //! //! Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in this crate by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. //! -mod commands; mod destructuring; mod expressions; +mod extensions; mod internal_prelude; mod interpretation; -mod string_conversion; -mod traits; +mod misc; use internal_prelude::*; @@ -544,11 +543,19 @@ pub fn preinterpret(token_stream: proc_macro::TokenStream) -> proc_macro::TokenS .into() } -fn preinterpret_internal(input: TokenStream) -> Result { +fn preinterpret_internal(input: TokenStream) -> SynResult { let mut interpreter = Interpreter::new(); - let interpretation_stream = - InterpretationStream::parse_from_token_stream(input, Span::call_site().span_range())?; - let interpreted_stream = interpretation_stream.interpret_to_new_stream(&mut interpreter)?; + + let interpretation_stream = input + .parse_with(|input| { + InterpretationStream::parse_with_context(input, Span::call_site().span_range()) + }) + .convert_to_final_result()?; + + let interpreted_stream = interpretation_stream + .interpret_to_new_stream(&mut interpreter) + .convert_to_final_result()?; + unsafe { // RUST-ANALYZER-SAFETY: This might drop transparent groups in the output of // rust-analyzer. There's not much we can do here... diff --git a/src/misc/errors.rs b/src/misc/errors.rs new file mode 100644 index 00000000..a131f681 --- /dev/null +++ b/src/misc/errors.rs @@ -0,0 +1,107 @@ +use crate::internal_prelude::*; + +pub(crate) type ParseResult = core::result::Result; + +pub(crate) trait ParseResultExt { + /// This is not a `From` because it wants to be explicit + fn convert_to_final_result(self) -> syn::Result; + fn add_context_if_error_and_no_context(self, context: impl FnOnce() -> String) -> Self; + fn into_execution_result(self) -> ExecutionResult; +} + +impl ParseResultExt for ParseResult { + fn convert_to_final_result(self) -> syn::Result { + self.map_err(|error| error.convert_to_final_error()) + } + + fn add_context_if_error_and_no_context(self, context: impl FnOnce() -> String) -> Self { + self.map_err(|error| error.add_context_if_none(context())) + } + + fn into_execution_result(self) -> ExecutionResult { + self.map_err(|error| error.into()) + } +} + +#[derive(Debug)] +pub(crate) enum ParseError { + Standard(syn::Error), + Contextual(syn::Error, String), +} + +impl From for ParseError { + fn from(e: syn::Error) -> Self { + ParseError::Standard(e) + } +} + +impl ParseError { + /// This is not a `From` because it wants to be explicit + pub(crate) fn convert_to_final_error(self) -> syn::Error { + match self { + ParseError::Standard(e) => e, + ParseError::Contextual(e, message) => e.concat(&format!("\n{}", message)), + } + } + + pub(crate) fn add_context_if_none(self, context: impl std::fmt::Display) -> Self { + match self { + ParseError::Standard(e) => ParseError::Contextual(e, context.to_string()), + other => other, + } + } +} + +// Ideally this would be our own enum with Completed / Interrupted variants, +// but we want it to work with `?` and defining custom FromResidual is not +// possible on stable (at least according to our MSRV). +pub(crate) type ExecutionResult = core::result::Result; + +pub(crate) trait ExecutionResultExt { + /// This is not a `From` because it wants to be explicit + fn convert_to_final_result(self) -> syn::Result; +} + +impl ExecutionResultExt for ExecutionResult { + fn convert_to_final_result(self) -> syn::Result { + self.map_err(|error| error.convert_to_final_error()) + } +} + +pub(crate) enum ExecutionInterrupt { + Error(syn::Error), + DestructureError(ParseError), + ControlFlow(ControlFlowInterrupt, Span), +} + +pub(crate) enum ControlFlowInterrupt { + Break, + Continue, +} + +impl From for ExecutionInterrupt { + fn from(e: syn::Error) -> Self { + ExecutionInterrupt::Error(e) + } +} + +impl From for ExecutionInterrupt { + fn from(e: ParseError) -> Self { + ExecutionInterrupt::DestructureError(e) + } +} + +impl ExecutionInterrupt { + pub(crate) fn convert_to_final_error(self) -> syn::Error { + match self { + ExecutionInterrupt::Error(e) => e, + ExecutionInterrupt::DestructureError(e) => e.convert_to_final_error(), + ExecutionInterrupt::ControlFlow(ControlFlowInterrupt::Break, span) => { + syn::Error::new(span, "Break can only be used inside a loop") + } + ExecutionInterrupt::ControlFlow(ControlFlowInterrupt::Continue, span) => { + syn::Error::new(span, "Continue can only be used inside a loop") + } + } + } +} diff --git a/src/misc/mod.rs b/src/misc/mod.rs new file mode 100644 index 00000000..38882786 --- /dev/null +++ b/src/misc/mod.rs @@ -0,0 +1,7 @@ +mod errors; +mod parse_traits; +mod string_conversion; + +pub(crate) use errors::*; +pub(crate) use parse_traits::*; +pub(crate) use string_conversion::*; diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs new file mode 100644 index 00000000..bca1aa07 --- /dev/null +++ b/src/misc/parse_traits.rs @@ -0,0 +1,17 @@ +use crate::internal_prelude::*; + +pub(crate) trait ContextualParse: Sized { + type Context; + + fn parse_with_context(input: ParseStream, context: Self::Context) -> ParseResult; +} + +pub(crate) trait Parse: Sized { + fn parse(input: ParseStream) -> ParseResult; +} + +impl Parse for T { + fn parse(input: ParseStream) -> ParseResult { + Ok(T::parse(input)?) + } +} diff --git a/src/string_conversion.rs b/src/misc/string_conversion.rs similarity index 100% rename from src/string_conversion.rs rename to src/misc/string_conversion.rs diff --git a/src/traits.rs b/src/traits.rs deleted file mode 100644 index b7770918..00000000 --- a/src/traits.rs +++ /dev/null @@ -1,430 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) trait IdentExt: Sized { - fn new_bool(value: bool, span: Span) -> Self; - fn with_span(self, span: Span) -> Self; -} - -impl IdentExt for Ident { - fn new_bool(value: bool, span: Span) -> Self { - Ident::new(&value.to_string(), span) - } - - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -pub(crate) trait CursorExt: Sized { - /// Because syn doesn't parse ' as a punct (not aligned with the TokenTree abstraction) - fn any_punct(self) -> Option<(Punct, Self)>; - fn ident_matching(self, content: &str) -> Option<(Ident, Self)>; - fn punct_matching(self, char: char) -> Option<(Punct, Self)>; - fn literal_matching(self, content: &str) -> Option<(Literal, Self)>; - fn group_matching(self, expected_delimiter: Delimiter) -> Option<(DelimSpan, Self, Self)>; -} - -impl CursorExt for Cursor<'_> { - fn any_punct(self) -> Option<(Punct, Self)> { - match self.token_tree() { - Some((TokenTree::Punct(punct), next)) => Some((punct, next)), - _ => None, - } - } - - fn ident_matching(self, content: &str) -> Option<(Ident, Self)> { - match self.ident() { - Some((ident, next)) if ident == content => Some((ident, next)), - _ => None, - } - } - - fn punct_matching(self, char: char) -> Option<(Punct, Self)> { - // self.punct() is a little more efficient, but can't match ' - let matcher = if char == '\'' { - self.any_punct() - } else { - self.punct() - }; - match matcher { - Some((punct, next)) if punct.as_char() == char => Some((punct, next)), - _ => None, - } - } - - fn literal_matching(self, content: &str) -> Option<(Literal, Self)> { - match self.literal() { - Some((literal, next)) if literal.to_string() == content => Some((literal, next)), - _ => None, - } - } - - fn group_matching(self, expected_delimiter: Delimiter) -> Option<(DelimSpan, Self, Self)> { - match self.any_group() { - Some((inner_cursor, delimiter, delim_span, next_outer_cursor)) - if delimiter == expected_delimiter => - { - Some((delim_span, inner_cursor, next_outer_cursor)) - } - _ => None, - } - } -} - -pub(crate) trait LiteralExt: Sized { - #[allow(unused)] - fn content_if_string(&self) -> Option; - fn content_if_string_like(&self) -> Option; -} - -impl LiteralExt for Literal { - fn content_if_string(&self) -> Option { - match parse_str::(&self.to_string()).unwrap() { - Lit::Str(lit_str) => Some(lit_str.value()), - _ => None, - } - } - - fn content_if_string_like(&self) -> Option { - match parse_str::(&self.to_string()).unwrap() { - Lit::Str(lit_str) => Some(lit_str.value()), - Lit::Char(lit_char) => Some(lit_char.value().to_string()), - Lit::CStr(lit_cstr) => Some(lit_cstr.value().to_string_lossy().to_string()), - _ => None, - } - } -} - -pub(crate) trait TokenStreamExt: Sized { - #[allow(unused)] - fn push(&mut self, token: TokenTree); - fn flatten_transparent_groups(self) -> Self; -} - -impl TokenStreamExt for TokenStream { - fn push(&mut self, token: TokenTree) { - self.extend(iter::once(token)); - } - - fn flatten_transparent_groups(self) -> Self { - let mut output = TokenStream::new(); - for token in self { - match token { - TokenTree::Group(group) if group.delimiter() == Delimiter::None => { - output.extend(group.stream().flatten_transparent_groups()); - } - other => output.extend(iter::once(other)), - } - } - output - } -} - -pub(crate) trait TokenTreeExt: Sized { - fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; - fn into_singleton_group(self, delimiter: Delimiter) -> Self; -} - -impl TokenTreeExt for TokenTree { - fn group(inner_tokens: TokenStream, delimiter: Delimiter, span: Span) -> Self { - TokenTree::Group(Group::new(delimiter, inner_tokens).with_span(span)) - } - - fn into_singleton_group(self, delimiter: Delimiter) -> Self { - let span = self.span(); - Self::group(self.into_token_stream(), delimiter, span) - } -} - -pub(crate) trait ParserExt { - fn parse_with(&self, context: T::Context) -> Result; - fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result; - fn try_parse_or_message Result, M: std::fmt::Display>( - &self, - func: F, - message: M, - ) -> Result; - fn parse_any_ident(&self) -> Result; - fn parse_any_punct(&self) -> Result; - fn peek_ident_matching(&self, content: &str) -> bool; - fn parse_ident_matching(&self, content: &str) -> Result; - fn peek_punct_matching(&self, punct: char) -> bool; - fn parse_punct_matching(&self, content: char) -> Result; - fn peek_literal_matching(&self, content: &str) -> bool; - fn parse_literal_matching(&self, content: &str) -> Result; - fn peek_group_matching(&self, delimiter: Delimiter) -> bool; - fn parse_group_matching(&self, delimiter: Delimiter) -> Result<(DelimSpan, ParseBuffer)>; -} - -impl ParserExt for ParseBuffer<'_> { - fn parse_with(&self, context: T::Context) -> Result { - T::parse_with_context(self, context) - } - - fn parse_all_for_interpretation(&self, span_range: SpanRange) -> Result { - self.parse_with(span_range) - } - - fn try_parse_or_message Result, M: std::fmt::Display>( - &self, - parse: F, - message: M, - ) -> Result { - let error_span = self.span(); - parse(self).map_err(|_| error_span.error(message)) - } - - fn parse_any_ident(&self) -> Result { - Ident::parse_any(self) - } - - fn parse_any_punct(&self) -> Result { - // Annoyingly, ' behaves weirdly in syn, so we need to handle it - match self.parse::()? { - TokenTree::Punct(punct) => Ok(punct), - _ => self.span().err("expected punctuation"), - } - } - - fn peek_ident_matching(&self, content: &str) -> bool { - self.cursor().ident_matching(content).is_some() - } - - fn parse_ident_matching(&self, content: &str) -> Result { - self.step(|cursor| { - cursor - .ident_matching(content) - .ok_or_else(|| cursor.span().error(format!("expected {}", content))) - }) - } - - fn peek_punct_matching(&self, punct: char) -> bool { - self.cursor().punct_matching(punct).is_some() - } - - fn parse_punct_matching(&self, punct: char) -> Result { - self.step(|cursor| { - cursor - .punct_matching(punct) - .ok_or_else(|| cursor.span().error(format!("expected {}", punct))) - }) - } - - fn peek_literal_matching(&self, content: &str) -> bool { - self.cursor().literal_matching(content).is_some() - } - - fn parse_literal_matching(&self, content: &str) -> Result { - self.step(|cursor| { - cursor - .literal_matching(content) - .ok_or_else(|| cursor.span().error(format!("expected {}", content))) - }) - } - - fn peek_group_matching(&self, delimiter: Delimiter) -> bool { - self.cursor().group_matching(delimiter).is_some() - } - - fn parse_group_matching( - &self, - expected_delimiter: Delimiter, - ) -> Result<(DelimSpan, ParseBuffer)> { - let (delimiter, delim_span, inner_stream) = self.parse_any_delimiter()?; - if delimiter != expected_delimiter { - return delim_span.open().err(match expected_delimiter { - Delimiter::Parenthesis => "Expected (", - Delimiter::Brace => "Expected {", - Delimiter::Bracket => "Expected [", - Delimiter::None => "Expected start of transparent group", - }); - } - Ok((delim_span, inner_stream)) - } -} - -pub(crate) trait ContextualParse: Sized { - type Context; - - fn parse_with_context(input: ParseStream, context: Self::Context) -> Result; -} - -#[allow(unused)] -pub(crate) trait SynErrorExt: Sized { - fn concat(self, extra: &str) -> Self; -} - -impl SynErrorExt for syn::Error { - fn concat(self, extra: &str) -> Self { - let mut message = self.to_string(); - message.push_str(extra); - Self::new(self.span(), message) - } -} - -pub(crate) trait SpanErrorExt: Sized { - fn err(&self, message: impl std::fmt::Display) -> syn::Result { - Err(self.error(message)) - } - - fn error(&self, message: impl std::fmt::Display) -> syn::Error; -} - -impl SpanErrorExt for T { - fn error(&self, message: impl std::fmt::Display) -> syn::Error { - self.span_range().create_error(message) - } -} - -pub(crate) trait WithSpanExt { - fn with_span(self, span: Span) -> Self; -} - -impl WithSpanExt for Literal { - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -impl WithSpanExt for Punct { - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -impl WithSpanExt for Group { - fn with_span(mut self, span: Span) -> Self { - self.set_span(span); - self - } -} - -pub(crate) trait HasSpanRange { - fn span_range(&self) -> SpanRange; - - fn span(&self) -> Span { - self.span_range().span() - } -} - -/// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] -/// and falls back to the span of the first token when not available. -/// -/// Instead, [`syn::Error`] uses a trick involving a span range. This effectively -/// allows capturing this trick when we're not immediately creating an error. -/// -/// When [`proc_macro::Span::join`] is stabilised and [`syn::spanned`] works, -/// we can swap [`SpanRange`] contents for [`Span`] (or even remove it and [`HasSpanRange`]). -#[derive(Copy, Clone)] -pub(crate) struct SpanRange { - start: Span, - end: Span, -} - -impl SpanRange { - pub(crate) fn new_between(start: Span, end: Span) -> Self { - Self { start, end } - } - - fn create_error(&self, message: impl std::fmt::Display) -> syn::Error { - syn::Error::new_spanned(self, message) - } - - /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) - /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) - pub(crate) fn span(&self) -> Span { - ::span(self) - } - - pub(crate) fn start(&self) -> Span { - self.start - } - - #[allow(unused)] - pub(crate) fn end(&self) -> Span { - self.end - } -} - -// This is implemented so we can create an error from it using `Error::new_spanned(..)` -impl ToTokens for SpanRange { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.extend([ - TokenTree::Punct(Punct::new('<', Spacing::Alone).with_span(self.start)), - TokenTree::Punct(Punct::new('>', Spacing::Alone).with_span(self.end)), - ]); - } -} - -impl HasSpanRange for SpanRange { - fn span_range(&self) -> SpanRange { - *self - } -} - -impl HasSpanRange for Span { - fn span_range(&self) -> SpanRange { - SpanRange::new_between(*self, *self) - } -} - -impl HasSpanRange for TokenTree { - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} - -impl HasSpanRange for Group { - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} - -impl HasSpanRange for DelimSpan { - fn span_range(&self) -> SpanRange { - // We could use self.open() => self.close() here, but using - // self.join() is better as it can be round-tripped to a span - // as the whole span, rather than just the start or end. - self.join().span_range() - } -} - -impl HasSpanRange for T { - fn span_range(&self) -> SpanRange { - let mut iter = self.into_token_stream().into_iter(); - let start = iter.next().map_or_else(Span::call_site, |t| t.span()); - let end = iter.last().map_or(start, |t| t.span()); - SpanRange { start, end } - } -} - -/// This should only be used for syn built-ins or when there isn't a better -/// span range available -pub(crate) trait AutoSpanRange {} - -macro_rules! impl_auto_span_range { - ($($ty:ty),* $(,)?) => { - $( - impl AutoSpanRange for $ty {} - )* - }; -} - -impl_auto_span_range! { - TokenStream, - Ident, - Punct, - Literal, - syn::Expr, - syn::ExprBinary, - syn::ExprUnary, - syn::BinOp, - syn::UnOp, - syn::Type, - syn::TypePath, - syn::token::DotDot, - syn::token::In, -} diff --git a/tests/compilation_failures/complex/nested.stderr b/tests/compilation_failures/complex/nested.stderr index 05071229..c360b421 100644 --- a/tests/compilation_failures/complex/nested.stderr +++ b/tests/compilation_failures/complex/nested.stderr @@ -1,5 +1,5 @@ error: required fields are missing: message - Occurred whilst parsing [!error! ..] - Expected [!error! "Expected X, found: " #world] or [!error! { + Occurred whilst parsing [!error! ...] - Expected [!error! "Expected X, found: " #world] or [!error! { // The error message to display message: "...", // An optional [token stream], to determine where to show the error message diff --git a/tests/compilation_failures/control_flow/break_outside_a_loop.stderr b/tests/compilation_failures/control_flow/break_outside_a_loop.stderr index fb5f5429..a134bbda 100644 --- a/tests/compilation_failures/control_flow/break_outside_a_loop.stderr +++ b/tests/compilation_failures/control_flow/break_outside_a_loop.stderr @@ -1,4 +1,4 @@ -error: The break command is only allowed inside a loop +error: Break can only be used inside a loop --> tests/compilation_failures/control_flow/break_outside_a_loop.rs:4:23 | 4 | preinterpret!(1 + [!break!] 2); diff --git a/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr b/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr index 1512593e..c486fb6c 100644 --- a/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr +++ b/tests/compilation_failures/control_flow/continue_outside_a_loop.stderr @@ -1,4 +1,4 @@ -error: The continue command is only allowed inside a loop +error: Continue can only be used inside a loop --> tests/compilation_failures/control_flow/continue_outside_a_loop.rs:4:23 | 4 | preinterpret!(1 + [!continue!] 2); diff --git a/tests/compilation_failures/core/error_invalid_structure.stderr b/tests/compilation_failures/core/error_invalid_structure.stderr index 468b1187..6bb513cd 100644 --- a/tests/compilation_failures/core/error_invalid_structure.stderr +++ b/tests/compilation_failures/core/error_invalid_structure.stderr @@ -1,5 +1,5 @@ error: required fields are missing: message - Occurred whilst parsing [!error! ..] - Expected [!error! "Expected X, found: " #world] or [!error! { + Occurred whilst parsing [!error! ...] - Expected [!error! "Expected X, found: " #world] or [!error! { // The error message to display message: "...", // An optional [token stream], to determine where to show the error message diff --git a/tests/compilation_failures/core/extend_flattened_variable.stderr b/tests/compilation_failures/core/extend_flattened_variable.stderr index 11789f0c..f3d07f43 100644 --- a/tests/compilation_failures/core/extend_flattened_variable.stderr +++ b/tests/compilation_failures/core/extend_flattened_variable.stderr @@ -1,5 +1,5 @@ error: Expected #variable - Occurred whilst parsing [!extend! ..] - Expected [!extend! #variable += ..] + Occurred whilst parsing [!extend! ...] - Expected [!extend! #variable += ..] --> tests/compilation_failures/core/extend_flattened_variable.rs:6:19 | 6 | [!extend! #..variable += 2] diff --git a/tests/compilation_failures/core/set_flattened_variable.stderr b/tests/compilation_failures/core/set_flattened_variable.stderr index 3629546d..07e7cb05 100644 --- a/tests/compilation_failures/core/set_flattened_variable.stderr +++ b/tests/compilation_failures/core/set_flattened_variable.stderr @@ -1,5 +1,5 @@ error: Expected #variable - Occurred whilst parsing [!set! ..] - Expected [!set! #variable = ..] + Occurred whilst parsing [!set! ...] - Expected [!set! #variable = ..] --> tests/compilation_failures/core/set_flattened_variable.rs:5:16 | 5 | [!set! #..variable = 1] diff --git a/tests/compilation_failures/destructuring/double_flattened_variable.stderr b/tests/compilation_failures/destructuring/double_flattened_variable.stderr index 8f63bc32..ca3bf2e6 100644 --- a/tests/compilation_failures/destructuring/double_flattened_variable.stderr +++ b/tests/compilation_failures/destructuring/double_flattened_variable.stderr @@ -1,5 +1,5 @@ error: This cannot follow a flattened destructure match - Occurred whilst parsing [!let! ..] - Expected [!let! = ...] + Occurred whilst parsing [!let! ...] - Expected [!let! = ...] --> tests/compilation_failures/destructuring/double_flattened_variable.rs:4:31 | 4 | preinterpret!([!let! #..x #..y = Hello World]); diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr index 7ecf4b90..72ba03ca 100644 --- a/tests/compilation_failures/expressions/braces.stderr +++ b/tests/compilation_failures/expressions/braces.stderr @@ -1,5 +1,5 @@ error: Expected an expression - Occurred whilst parsing [!evaluate! ..] - Expected [!evaluate! ...] containing a valid preinterpret expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression --> tests/compilation_failures/expressions/braces.rs:5:21 | 5 | [!evaluate! {true}] diff --git a/tests/compilation_failures/expressions/inner_braces.stderr b/tests/compilation_failures/expressions/inner_braces.stderr index 98ba4703..537fcac9 100644 --- a/tests/compilation_failures/expressions/inner_braces.stderr +++ b/tests/compilation_failures/expressions/inner_braces.stderr @@ -1,5 +1,5 @@ error: Expected an expression - Occurred whilst parsing [!evaluate! ..] - Expected [!evaluate! ...] containing a valid preinterpret expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression --> tests/compilation_failures/expressions/inner_braces.rs:5:22 | 5 | [!evaluate! ({true})] From a4c542ea1960f531a47a2568eb582502d214f812 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 25 Jan 2025 00:21:18 +0000 Subject: [PATCH 059/126] refactor: Replace parse_v2 with parse --- src/destructuring/destructure_group.rs | 4 +- src/destructuring/destructure_item.rs | 6 +-- src/destructuring/destructure_variable.rs | 6 +-- src/destructuring/destructurer.rs | 2 +- src/destructuring/fields.rs | 10 ++-- src/expressions/expression_stream.rs | 18 +++---- src/extensions/parsing.rs | 27 +++++----- src/internal_prelude.rs | 5 +- src/interpretation/command_field_inputs.rs | 4 +- src/interpretation/command_stream_input.rs | 12 ++--- src/interpretation/command_value_input.rs | 12 ++--- .../commands/control_flow_commands.rs | 22 ++++---- src/interpretation/commands/core_commands.rs | 6 +-- .../commands/destructuring_commands.rs | 4 +- .../commands/expression_commands.rs | 12 ++--- src/interpretation/interpretation_item.rs | 12 ++--- src/interpretation/interpretation_stream.rs | 6 +-- src/misc/parse_traits.rs | 54 ++++++++++++++++++- 18 files changed, 138 insertions(+), 84 deletions(-) diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs index 21486902..3d7f4003 100644 --- a/src/destructuring/destructure_group.rs +++ b/src/destructuring/destructure_group.rs @@ -8,10 +8,10 @@ pub(crate) struct DestructureGroup { impl Parse for DestructureGroup { fn parse(input: ParseStream) -> ParseResult { - let (delimiter, _, content) = input.parse_any_delimiter()?; + let (delimiter, _, content) = input.parse_any_group()?; Ok(Self { delimiter, - inner: content.parse_v2()?, + inner: content.parse()?, }) } } diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs index 65d2e418..d20ee791 100644 --- a/src/destructuring/destructure_item.rs +++ b/src/destructuring/destructure_item.rs @@ -21,7 +21,7 @@ impl DestructureItem { PeekMatch::GroupedCommand(Some(command_kind)) if matches!(command_kind.output_kind(None), Ok(CommandOutputKind::None)) => { - Self::NoneOutputCommand(input.parse_v2()?) + Self::NoneOutputCommand(input.parse()?) } PeekMatch::GroupedCommand(_) => return input.parse_err( "Grouped commands returning a value are not supported in destructuring positions", @@ -35,8 +35,8 @@ impl DestructureItem { | PeekMatch::AppendVariableDestructuring => { Self::Variable(DestructureVariable::parse_until::(input)?) } - PeekMatch::Group(_) => Self::ExactGroup(input.parse_v2()?), - PeekMatch::Destructurer(_) => Self::Destructurer(input.parse_v2()?), + PeekMatch::Group(_) => Self::ExactGroup(input.parse()?), + PeekMatch::Destructurer(_) => Self::Destructurer(input.parse()?), PeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), PeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), PeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index 54fde891..cddc2f88 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -185,7 +185,7 @@ impl HandleDestructure for DestructureVariable { ) -> ExecutionResult<()> { match self { DestructureVariable::Grouped { .. } => { - let content = input.parse_v2::()?.into_interpreted(); + let content = input.parse::()?.into_interpreted(); interpreter.set_variable(self, content)?; } DestructureVariable::Flattened { until, .. } => { @@ -196,13 +196,13 @@ impl HandleDestructure for DestructureVariable { DestructureVariable::GroupedAppendGrouped { .. } => { let variable_data = self.get_variable_data(interpreter)?; input - .parse_v2::()? + .parse::()? .push_as_token_tree(variable_data.get_mut(self)?.deref_mut()); } DestructureVariable::GroupedAppendFlattened { .. } => { let variable_data = self.get_variable_data(interpreter)?; input - .parse_v2::()? + .parse::()? .flatten_into(variable_data.get_mut(self)?.deref_mut()); } DestructureVariable::FlattenedAppendGrouped { marker, until, .. } => { diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index 68db02f8..2107fc2d 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -45,7 +45,7 @@ impl<'a> DestructurerArguments<'a> { } pub(crate) fn fully_parse_no_error_override(&self) -> ParseResult { - self.parse_stream.parse_v2() + self.parse_stream.parse() } pub(crate) fn fully_parse_as(&self) -> ParseResult { diff --git a/src/destructuring/fields.rs b/src/destructuring/fields.rs index 2f5f2a48..7b94821f 100644 --- a/src/destructuring/fields.rs +++ b/src/destructuring/fields.rs @@ -42,7 +42,7 @@ impl FieldsParseDefinition { example: &str, explanation: Option<&str>, is_required: bool, - parse: impl Fn(syn::parse::ParseStream) -> ParseResult + 'static, + parse: impl Fn(ParseStream) -> ParseResult + 'static, set: impl Fn(&mut T, F) + 'static, ) -> Self { if self @@ -71,7 +71,7 @@ impl FieldsParseDefinition { pub(crate) fn create_syn_parser( self, error_span_range: SpanRange, - ) -> impl FnOnce(ParseStream) -> ParseResult { + ) -> impl FnOnce(SynParseStream) -> ParseResult { fn inner( input: ParseStream, new_builder: T, @@ -120,9 +120,9 @@ impl FieldsParseDefinition { Ok(builder) } - move |input: syn::parse::ParseStream| { + move |input: SynParseStream| { inner( - input, + input.into(), self.new_builder, &self.field_definitions, error_span_range, @@ -172,5 +172,5 @@ struct FieldParseDefinition { example: String, explanation: Option, #[allow(clippy::type_complexity)] - parse_and_set: Box ParseResult<()>>, + parse_and_set: Box ParseResult<()>>, } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index d194218b..ea97f0fe 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -38,14 +38,12 @@ impl Parse for ExpressionInput { // before code blocks or .. in [!range!] so we can break on those. // These aren't valid inside expressions we support anyway, so it's good enough for now. let item = match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => ExpressionItem::Command(input.parse_v2()?), - PeekMatch::FlattenedCommand(_) => ExpressionItem::Command(input.parse_v2()?), - PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse_v2()?), - PeekMatch::FlattenedVariable => { - ExpressionItem::FlattenedVariable(input.parse_v2()?) - } + PeekMatch::GroupedCommand(_) => ExpressionItem::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => ExpressionItem::Command(input.parse()?), + PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), PeekMatch::Group(Delimiter::Brace | Delimiter::Bracket) => break, - PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse_v2()?), + PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse()?), PeekMatch::Destructurer(_) | PeekMatch::AppendVariableDestructuring => { return input .span() @@ -54,7 +52,7 @@ impl Parse for ExpressionInput { PeekMatch::Punct(punct) if punct.as_char() == '.' => break, PeekMatch::Punct(_) => ExpressionItem::Punct(input.parse_any_punct()?), PeekMatch::Ident(_) => ExpressionItem::Ident(input.parse_any_ident()?), - PeekMatch::Literal(_) => ExpressionItem::Literal(input.parse_v2()?), + PeekMatch::Literal(_) => ExpressionItem::Literal(input.parse()?), PeekMatch::End => return input.span().parse_err("Expected an expression"), }; items.push(item); @@ -160,11 +158,11 @@ pub(crate) struct ExpressionGroup { impl Parse for ExpressionGroup { fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, content) = input.parse_any_delimiter()?; + let (delimiter, delim_span, content) = input.parse_any_group()?; Ok(Self { source_delimiter: delimiter, source_delim_span: delim_span, - content: content.parse_v2()?, + content: content.parse()?, }) } } diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index 2623aea2..f2f22af1 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -13,8 +13,8 @@ impl TokenStreamParseExt for TokenStream { parser: impl FnOnce(ParseStream) -> Result, ) -> Result { let mut result = None; - let parse_result = (|input: ParseStream| -> SynResult<()> { - result = Some(parser(input)); + let parse_result = (|input: SynParseStream| -> SynResult<()> { + result = Some(parser(input.into())); match &result { // Some fallback error to ensure that we don't go down the unexpected branch inside parse2 Some(Err(_)) => Err(SynError::new(Span::call_site(), "")), @@ -91,8 +91,7 @@ impl CursorExt for Cursor<'_> { } } -pub(crate) trait ParserExt { - fn parse_v2(&self) -> ParseResult; +pub(crate) trait ParserBufferExt { fn parse_with(&self, context: T::Context) -> ParseResult; fn parse_all_for_interpretation( &self, @@ -111,17 +110,14 @@ pub(crate) trait ParserExt { fn parse_punct_matching(&self, content: char) -> ParseResult; fn peek_literal_matching(&self, content: &str) -> bool; fn parse_literal_matching(&self, content: &str) -> ParseResult; + fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)>; fn peek_group_matching(&self, delimiter: Delimiter) -> bool; fn parse_group_matching(&self, delimiter: Delimiter) -> ParseResult<(DelimSpan, ParseBuffer)>; fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult; fn parse_error(&self, message: impl std::fmt::Display) -> ParseError; } -impl ParserExt for ParseBuffer<'_> { - fn parse_v2(&self) -> ParseResult { - T::parse(self) - } - +impl ParserBufferExt for ParseBuffer<'_> { fn parse_with(&self, context: T::Context) -> ParseResult { T::parse_with_context(self, context) } @@ -143,7 +139,7 @@ impl ParserExt for ParseBuffer<'_> { } fn parse_any_ident(&self) -> ParseResult { - Ok(Ident::parse_any(self)?) + Ok(self.call(Ident::parse_any)?) } fn parse_any_punct(&self) -> ParseResult { @@ -190,6 +186,12 @@ impl ParserExt for ParseBuffer<'_> { })?) } + fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)> { + use syn::parse::discouraged::AnyDelimiter; + let (delimiter, delim_span, parse_buffer) = self.parse_any_delimiter()?; + Ok((delimiter, delim_span, parse_buffer.into())) + } + fn peek_group_matching(&self, delimiter: Delimiter) -> bool { self.cursor().group_matching(delimiter).is_some() } @@ -198,7 +200,8 @@ impl ParserExt for ParseBuffer<'_> { &self, expected_delimiter: Delimiter, ) -> ParseResult<(DelimSpan, ParseBuffer)> { - let (delimiter, delim_span, inner_stream) = self.parse_any_delimiter()?; + use syn::parse::discouraged::AnyDelimiter; + let (delimiter, delim_span, inner) = self.parse_any_delimiter()?; if delimiter != expected_delimiter { return delim_span.open().parse_err(match expected_delimiter { Delimiter::Parenthesis => "Expected (", @@ -207,7 +210,7 @@ impl ParserExt for ParseBuffer<'_> { Delimiter::None => "Expected start of transparent group", }); } - Ok((delim_span, inner_stream)) + Ok((delim_span, inner.into())) } fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index d5494f3c..e7939c56 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -7,7 +7,10 @@ pub(crate) use quote::ToTokens; pub(crate) use std::{collections::HashMap, str::FromStr}; pub(crate) use syn::buffer::Cursor; pub(crate) use syn::ext::IdentExt as SynIdentExt; -pub(crate) use syn::parse::{discouraged::*, Parse as SynParse, ParseBuffer, ParseStream, Parser}; +pub(crate) use syn::parse::{ + discouraged::Speculative, Parse as SynParse, ParseBuffer as SynParseBuffer, + ParseStream as SynParseStream, Parser as SynParser, +}; pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Token, UnOp}; pub(crate) use syn::{Error as SynError, Result as SynResult}; diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index 9d72eaa6..bfa204aa 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -44,7 +44,7 @@ macro_rules! define_field_inputs { if $required_field.is_some() { return ident.parse_err("duplicate field"); } - $required_field = Some(content.parse_v2()?); + $required_field = Some(content.parse()?); } )* $( @@ -52,7 +52,7 @@ macro_rules! define_field_inputs { if $optional_field.is_some() { return ident.parse_err("duplicate field"); } - $optional_field = Some(content.parse_v2()?); + $optional_field = Some(content.parse()?); } )* _ => return ident.parse_err("unexpected field"), diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 56a19c2d..d896d541 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -24,12 +24,12 @@ pub(crate) enum CommandStreamInput { impl Parse for CommandStreamInput { fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => Self::Command(input.parse_v2()?), - PeekMatch::FlattenedCommand(_) => Self::Command(input.parse_v2()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse_v2()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse_v2()?), - PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse_v2()?), - PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse_v2()?), + PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), + PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), _ => input.span() .parse_err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 0191bfa7..9b125f61 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -16,11 +16,11 @@ pub(crate) enum CommandValueInput { impl Parse for CommandValueInput { fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => Self::Command(input.parse_v2()?), - PeekMatch::FlattenedCommand(_) => Self::Command(input.parse_v2()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse_v2()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse_v2()?), - PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse_v2()?), + PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), + PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { return input .span() @@ -30,7 +30,7 @@ impl Parse for CommandValueInput { | PeekMatch::Punct(_) | PeekMatch::Literal(_) | PeekMatch::Ident(_) - | PeekMatch::End => Self::Value(input.parse_v2()?), + | PeekMatch::End => Self::Value(input.parse()?), }) } } diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 3395a845..da9e33db 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -18,8 +18,8 @@ impl ControlFlowCommandDefinition for IfCommand { fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { - let condition = input.parse_v2()?; - let true_code = input.parse_v2()?; + let condition = input.parse()?; + let true_code = input.parse()?; let mut else_ifs = Vec::new(); let mut else_code = None; while !input.is_empty() { @@ -27,11 +27,11 @@ impl ControlFlowCommandDefinition for IfCommand { if input.peek_ident_matching("elif") { input.parse_ident_matching("elif")?; input.parse::()?; - else_ifs.push((input.parse_v2()?, input.parse_v2()?)); + else_ifs.push((input.parse()?, input.parse()?)); } else { input.parse_ident_matching("else")?; input.parse::()?; - else_code = Some(input.parse_v2()?); + else_code = Some(input.parse()?); break; } } @@ -97,8 +97,8 @@ impl ControlFlowCommandDefinition for WhileCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - condition: input.parse_v2()?, - loop_code: input.parse_v2()?, + condition: input.parse()?, + loop_code: input.parse()?, }) }, "Expected [!while! (condition) { code }]", @@ -156,7 +156,7 @@ impl ControlFlowCommandDefinition for LoopCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - loop_code: input.parse_v2()?, + loop_code: input.parse()?, }) }, "Expected [!loop! { ... }]", @@ -206,10 +206,10 @@ impl ControlFlowCommandDefinition for ForCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - parse_place: input.parse_v2()?, - in_token: input.parse_v2()?, - input: input.parse_v2()?, - loop_code: input.parse_v2()?, + parse_place: input.parse()?, + in_token: input.parse()?, + input: input.parse()?, + loop_code: input.parse()?, }) }, "Expected [!for! #x in [ ... ] { code }]", diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 52700687..39b6a54a 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -19,7 +19,7 @@ impl NoOutputCommandDefinition for SetCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - variable: input.parse_v2()?, + variable: input.parse()?, equals: input.parse()?, arguments: input.parse_with(arguments.full_span_range())?, }) @@ -54,7 +54,7 @@ impl NoOutputCommandDefinition for ExtendCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - variable: input.parse_v2()?, + variable: input.parse()?, plus_equals: input.parse()?, arguments: input.parse_all_for_interpretation(arguments.full_span_range())?, }) @@ -216,7 +216,7 @@ impl NoOutputCommandDefinition for ErrorCommand { |input| { if input.peek(syn::token::Brace) { Ok(Self { - inputs: EitherErrorInput::Fields(input.parse_v2()?), + inputs: EitherErrorInput::Fields(input.parse()?), }) } else { Ok(Self { diff --git a/src/interpretation/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs index 796946b3..267d7431 100644 --- a/src/interpretation/commands/destructuring_commands.rs +++ b/src/interpretation/commands/destructuring_commands.rs @@ -19,8 +19,8 @@ impl NoOutputCommandDefinition for LetCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - destructuring: input.parse_v2()?, - equals: input.parse_v2()?, + destructuring: input.parse()?, + equals: input.parse()?, arguments: input.parse_with(arguments.full_span_range())?, }) }, diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 97ef1233..8c7ffb4c 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -16,7 +16,7 @@ impl ValueCommandDefinition for EvaluateCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - expression: input.parse_v2()?, + expression: input.parse()?, }) }, "Expected [!evaluate! ...] containing a valid preinterpret expression", @@ -49,7 +49,7 @@ impl NoOutputCommandDefinition for AssignCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - variable: input.parse_v2()?, + variable: input.parse()?, operator: { let operator: Punct = input.parse()?; match operator.as_char() { @@ -59,7 +59,7 @@ impl NoOutputCommandDefinition for AssignCommand { operator }, equals: input.parse()?, - expression: input.parse_v2()?, + expression: input.parse()?, }) }, "Expected [!assign! #variable += ...] for + or some other operator supported in an expression", @@ -104,9 +104,9 @@ impl StreamCommandDefinition for RangeCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - left: input.parse_v2()?, - range_limits: input.parse_v2()?, - right: input.parse_v2()?, + left: input.parse()?, + range_limits: input.parse()?, + right: input.parse()?, }) }, "Expected a rust range expression such as [!range! 1..4]", diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 3658fbcf..04d78ba0 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -14,13 +14,11 @@ pub(crate) enum InterpretationItem { impl Parse for InterpretationItem { fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => InterpretationItem::Command(input.parse_v2()?), - PeekMatch::FlattenedCommand(_) => InterpretationItem::Command(input.parse_v2()?), - PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse_v2()?), - PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse_v2()?), - PeekMatch::FlattenedVariable => { - InterpretationItem::FlattenedVariable(input.parse_v2()?) - } + PeekMatch::GroupedCommand(_) => InterpretationItem::Command(input.parse()?), + PeekMatch::FlattenedCommand(_) => InterpretationItem::Command(input.parse()?), + PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), + PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), + PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { return input.parse_err("Destructurings are not supported here") } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index e77f8a15..dce8e81e 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -13,7 +13,7 @@ impl ContextualParse for InterpretationStream { fn parse_with_context(input: ParseStream, span_range: Self::Context) -> ParseResult { let mut items = Vec::new(); while !input.is_empty() { - items.push(input.parse_v2()?); + items.push(input.parse()?); } Ok(Self { items, span_range }) } @@ -54,7 +54,7 @@ impl InterpretationGroup { impl Parse for InterpretationGroup { fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, content) = input.parse_any_delimiter()?; + let (delimiter, delim_span, content) = input.parse_any_group()?; let content = content.parse_with(delim_span.span_range())?; Ok(Self { source_delimiter: delimiter, @@ -99,7 +99,7 @@ impl RawGroup { impl Parse for RawGroup { fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, content) = input.parse_any_delimiter()?; + let (delimiter, delim_span, content) = input.parse_any_group()?; let content = content.parse()?; Ok(Self { source_delimeter: delimiter, diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index bca1aa07..efd8a271 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -1,3 +1,5 @@ +use std::ops::Deref; + use crate::internal_prelude::*; pub(crate) trait ContextualParse: Sized { @@ -12,6 +14,56 @@ pub(crate) trait Parse: Sized { impl Parse for T { fn parse(input: ParseStream) -> ParseResult { - Ok(T::parse(input)?) + Ok(T::parse(&input.inner)?) + } +} + +pub(crate) type ParseStream<'a> = &'a ParseBuffer<'a>; + +// We create our own ParseBuffer mostly so we can overwrite +// parse to return ParseResult instead of syn::Result +#[repr(transparent)] +pub(crate) struct ParseBuffer<'a> { + inner: SynParseBuffer<'a>, +} + +impl<'a> From> for ParseBuffer<'a> { + fn from(inner: syn::parse::ParseBuffer<'a>) -> Self { + Self { inner } + } +} + +// This is From<&'a SynParseBuffer<'a>> for &'a ParseBuffer<'a> +impl<'a> From> for ParseStream<'a> { + fn from(syn_parse_stream: SynParseStream<'a>) -> Self { + unsafe { + // SAFETY: This is safe because [Syn]ParseStream<'a> = &'a [Syn]ParseBuffer<'a> + // And ParseBuffer<'a> is marked as #[repr(transparent)] so has identical layout to SynParseBuffer<'a> + // So this is a transmute between compound types with identical layouts which is safe. + core::mem::transmute::, ParseStream<'a>>(syn_parse_stream) + } + } +} + +impl<'a> ParseBuffer<'a> { + // Methods on SynParseBuffer are available courtesy of Deref below + // But the following methods are replaced, for ease of use: + + pub(crate) fn parse(&self) -> ParseResult { + T::parse(self) + } + + pub(crate) fn fork(&self) -> ParseBuffer<'a> { + ParseBuffer { + inner: self.inner.fork(), + } + } +} + +impl<'a> Deref for ParseBuffer<'a> { + type Target = SynParseBuffer<'a>; + + fn deref(&self) -> &Self::Target { + &self.inner } } From 3511a30cef79e4347149c02810a26e3ca8f17169 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 26 Jan 2025 01:14:40 +0000 Subject: [PATCH 060/126] feature: Add !zip! command --- CHANGELOG.md | 9 +- src/interpretation/command.rs | 1 + src/interpretation/command_stream_input.rs | 31 +++++ src/interpretation/command_value_input.rs | 73 ++++++++++- .../commands/control_flow_commands.rs | 2 +- src/interpretation/commands/core_commands.rs | 4 +- src/interpretation/commands/token_commands.rs | 120 ++++++++++++++++-- src/interpretation/interpret_traits.rs | 17 +-- src/interpretation/interpreted_stream.rs | 25 +++- .../tokens/zip_different_length_streams.rs | 7 + .../zip_different_length_streams.stderr | 5 + .../tokens/zip_no_streams.rs | 7 + .../tokens/zip_no_streams.stderr | 5 + tests/tokens.rs | 88 +++++++++++++ 14 files changed, 358 insertions(+), 36 deletions(-) create mode 100644 tests/compilation_failures/tokens/zip_different_length_streams.rs create mode 100644 tests/compilation_failures/tokens/zip_different_length_streams.stderr create mode 100644 tests/compilation_failures/tokens/zip_no_streams.rs create mode 100644 tests/compilation_failures/tokens/zip_no_streams.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index d5de79d1..6a9b6be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ * `[!assign! #x += ]` for `+` and other supported operators * `[!range! 0..5]` outputs `0 1 2 3 4` * Control flow commands: - * `[!if! { ... }]` and `[!if! { ... } !else! { ... }]` + * `[!if! { ... }]` and `[!if! { ... } !elif! { ... } !else! { ... }]` * `[!while! { ... }]` * `[!for! in [ ... ] { ... }]` * `[!loop! { ... }]` @@ -34,6 +34,7 @@ * `[!intersperse! { ... }]` which inserts separator tokens between each token tree in a stream. * `[!split! ...]` * `[!comma_split! ...]` + * `[!zip! (#countries #flags #capitals)]` which can be used to combine multiple streams together * Destructuring commands: * `[!let! = ...]` does destructuring/parsing (see next section). Note `[!let! #..x = ...]` is equivalent to `[!set! #x = ...]` @@ -75,10 +76,7 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* Add our own ParseBuffer / ParseStream types so we can e.g. override `parse` * Add compile error tests for all the standard destructuring errors -* `[!zip! ([Hello Goodbye] [World Friend])]` => `[(Hello World), (Goodbye Friend)]` and/or `[!zip! { streams: (#countries #flags #capitals), error_on_length_mismatch?: true }]` with `InterpretValue>>` - * e.g. `[!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)]` * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` * Change `(!content! ...)` to unwrap none groups, to be more permissive @@ -92,7 +90,8 @@ Destructuring performs parsing of a token stream. It supports: * `(!optional! ...)` * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` - * `(!match! ...)` (with `#..x` as a catch-all) + * `(!match! ...)` (with `#..x` as a catch-all) but without arms... + so I guess it's more of an `(!any! ...)` * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much * `[!match! ...]` command - similar to the match destructurer, but a command... * Check all `#[allow(unused)]` and remove any which aren't needed diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 7b3c8c40..335be051 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -455,6 +455,7 @@ define_command_kind! { IntersperseCommand, SplitCommand, CommaSplitCommand, + ZipCommand, } #[derive(Clone)] diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index d896d541..6ad2dbe9 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -132,3 +132,34 @@ fn parse_as_stream_input( .append_into(output); Ok(()) } + +impl InterpretValue for CommandStreamInput { + type InterpretedValue = InterpretedCommandStream; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + Ok(InterpretedCommandStream { + stream: self.interpret_to_new_stream(interpreter)?, + }) + } +} + +pub(crate) struct InterpretedCommandStream { + pub(crate) stream: InterpretedStream, +} + +impl Parse for InterpretedCommandStream { + fn parse(input: ParseStream) -> ParseResult { + // We assume we're parsing an already interpreted raw stream here, so we replicate + // parse_as_stream_input + let (delimiter, delim_span, inner) = input.parse_any_group()?; + match delimiter { + Delimiter::Bracket | Delimiter::None => Ok(Self { + stream: InterpretedStream::raw(inner.parse()?), + }), + _ => delim_span.parse_err("Expected a [ ... ] or transparent group"), + } + } +} diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 9b125f61..8062538b 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -50,7 +50,7 @@ impl HasSpanRange for CommandValueInput { impl, I: Parse> InterpretValue for CommandValueInput { type InterpretedValue = I; - fn interpret(self, interpreter: &mut Interpreter) -> ExecutionResult { + fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { let descriptor = match self { CommandValueInput::Command(_) => "command output", CommandValueInput::GroupedVariable(_) => "grouped variable output", @@ -67,7 +67,7 @@ impl, I: Parse> InterpretValue for Comma variable.interpret_to_new_stream(interpreter)? } CommandValueInput::Code(code) => code.interpret_to_new_stream(interpreter)?, - CommandValueInput::Value(value) => return value.interpret(interpreter), + CommandValueInput::Value(value) => return value.interpret_to_value(interpreter), }; unsafe { // RUST-ANALYZER SAFETY: We only use I with simple parse functions so far which don't care about @@ -85,3 +85,72 @@ impl, I: Parse> InterpretValue for Comma } } } + +#[derive(Clone)] +pub(crate) struct Grouped { + pub(crate) delimiter: Delimiter, + pub(crate) delim_span: DelimSpan, + pub(crate) inner: T, +} + +impl Parse for Grouped { + fn parse(input: ParseStream) -> ParseResult { + let (delimiter, delim_span, inner) = input.parse_any_group()?; + Ok(Self { + delimiter, + delim_span, + inner: inner.parse()?, + }) + } +} + +impl InterpretValue for Grouped +where + T: InterpretValue, +{ + type InterpretedValue = Grouped; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + Ok(Grouped { + delimiter: self.delimiter, + delim_span: self.delim_span, + inner: self.inner.interpret_to_value(interpreter)?, + }) + } +} + +#[derive(Clone)] +pub(crate) struct Repeated { + pub(crate) inner: Vec, +} + +impl Parse for Repeated { + fn parse(input: ParseStream) -> ParseResult { + let mut inner = vec![]; + while !input.is_empty() { + inner.push(input.parse::()?); + } + Ok(Self { inner }) + } +} + +impl InterpretValue for Repeated +where + T: InterpretValue, +{ + type InterpretedValue = Repeated; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut interpreted = Vec::with_capacity(self.inner.len()); + for item in self.inner.into_iter() { + interpreted.push(item.interpret_to_value(interpreter)?); + } + Ok(Repeated { inner: interpreted }) + } +} diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index da9e33db..832adfd4 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -225,7 +225,7 @@ impl ControlFlowCommandDefinition for ForCommand { let mut iteration_counter = interpreter.start_iteration_counter(&self.in_token); - for token in stream.into_item_vec() { + for token in stream { iteration_counter.increment_and_check()?; self.parse_place .handle_destructure_from_stream(token.into(), interpreter)?; diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 39b6a54a..0c1edc69 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -175,7 +175,7 @@ impl NoOutputCommandDefinition for SettingsCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { if let Some(limit) = self.inputs.iteration_limit { - let limit: usize = limit.interpret(interpreter)?.base10_parse()?; + let limit: usize = limit.interpret_to_value(interpreter)?.base10_parse()?; interpreter.set_iteration_limit(Some(limit)); } Ok(()) @@ -244,7 +244,7 @@ impl NoOutputCommandDefinition for ErrorCommand { } }; - let message = fields.message.interpret(interpreter)?.value(); + let message = fields.message.interpret_to_value(interpreter)?.value(); let error_span = match fields.spans { Some(spans) => { diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 47f02fde..fc3aaae5 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -116,13 +116,9 @@ impl StreamCommandDefinition for IntersperseCommand { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> ExecutionResult<()> { - let items = self - .inputs - .items - .interpret_to_new_stream(interpreter)? - .into_item_vec(); + let items = self.inputs.items.interpret_to_new_stream(interpreter)?; let add_trailing = match self.inputs.add_trailing { - Some(add_trailing) => add_trailing.interpret(interpreter)?.value(), + Some(add_trailing) => add_trailing.interpret_to_value(interpreter)?.value(), None => false, }; @@ -139,7 +135,7 @@ impl StreamCommandDefinition for IntersperseCommand { let mut items = items.into_iter().peekable(); let mut this_item = items.next().unwrap(); // Safe to unwrap as non-empty loop { - output.push_segment_item(this_item); + output.push_interpreted_item(this_item); let next_item = items.next(); match next_item { Some(next_item) => { @@ -260,15 +256,15 @@ impl StreamCommandDefinition for SplitCommand { let separator = self.inputs.separator.interpret_to_new_stream(interpreter)?; let drop_empty_start = match self.inputs.drop_empty_start { - Some(value) => value.interpret(interpreter)?.value(), + Some(value) => value.interpret_to_value(interpreter)?.value(), None => false, }; let drop_empty_middle = match self.inputs.drop_empty_middle { - Some(value) => value.interpret(interpreter)?.value(), + Some(value) => value.interpret_to_value(interpreter)?.value(), None => false, }; let drop_empty_end = match self.inputs.drop_empty_end { - Some(value) => value.interpret(interpreter)?.value(), + Some(value) => value.interpret_to_value(interpreter)?.value(), None => true, }; @@ -357,3 +353,107 @@ impl StreamCommandDefinition for CommaSplitCommand { handle_split(stream, output, output_span, separator, false, false, true) } } + +#[derive(Clone)] +pub(crate) struct ZipCommand { + inputs: EitherZipInput, +} + +type Streams = CommandValueInput>>; + +#[derive(Clone)] +enum EitherZipInput { + Fields(ZipInputs), + JustStream(Streams), +} + +define_field_inputs! { + ZipInputs { + required: { + streams: Streams = r#"([Hello Goodbye] [World Friend])"# ("A group of one or more streams to zip together. The outer brackets are used for the group."), + }, + optional: { + error_on_length_mismatch: CommandValueInput = "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), + } + } +} + +impl CommandType for ZipCommand { + type OutputKind = OutputKindStream; +} + +impl StreamCommandDefinition for ZipCommand { + const COMMAND_NAME: &'static str = "zip"; + + fn parse(arguments: CommandArguments) -> ParseResult { + arguments.fully_parse_or_error( + |input| { + if input.peek(syn::token::Brace) { + Ok(Self { + inputs: EitherZipInput::Fields(input.parse()?), + }) + } else { + Ok(Self { + inputs: EitherZipInput::JustStream(input.parse()?), + }) + } + }, + format!( + "Expected [!zip! (#a #b #c)] or [!zip! {}]", + ZipInputs::fields_description() + ), + ) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut InterpretedStream, + ) -> ExecutionResult<()> { + let (grouped_streams, error_on_length_mismatch) = match self.inputs { + EitherZipInput::Fields(inputs) => (inputs.streams, inputs.error_on_length_mismatch), + EitherZipInput::JustStream(streams) => (streams, None), + }; + let grouped_streams = grouped_streams.interpret_to_value(interpreter)?; + let error_on_length_mismatch = match error_on_length_mismatch { + Some(value) => value.interpret_to_value(interpreter)?.value(), + None => true, + }; + let Grouped { + delimiter, + delim_span, + inner: Repeated { inner: streams }, + } = grouped_streams; + if streams.is_empty() { + return delim_span + .join() + .execution_err("At least one stream is required to zip"); + } + let stream_lengths = streams + .iter() + .map(|stream| stream.stream.len()) + .collect::>(); + let min_stream_length = *stream_lengths.iter().min().unwrap(); + if error_on_length_mismatch { + let max_stream_length = *stream_lengths.iter().max().unwrap(); + if min_stream_length != max_stream_length { + return delim_span.join().execution_err(format!( + "Streams have different lengths and zip's error_on_length_mismatch is true. The lengths vary from {} to {}", + min_stream_length, max_stream_length + )); + } + } + let mut iters: Vec<_> = streams + .into_iter() + .map(|stream| stream.stream.into_iter()) + .collect(); + for _ in 0..min_stream_length { + let mut inner = InterpretedStream::new(); + for iter in iters.iter_mut() { + inner.push_interpreted_item(iter.next().unwrap()); + } + output.push_new_group(inner, delimiter, delim_span.span()); + } + Ok(()) + } +} diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index 89f229e8..b02693da 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -20,24 +20,19 @@ pub(crate) trait Interpret: Sized { pub(crate) trait InterpretValue: Sized { type InterpretedValue; - fn interpret(self, interpreter: &mut Interpreter) -> ExecutionResult; -} - -impl + HasSpanRange, I: ToTokens> Interpret for T { - fn interpret_into( + fn interpret_to_value( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, - ) -> ExecutionResult<()> { - output.extend_with_raw_tokens_from(self.interpret(interpreter)?); - Ok(()) - } + ) -> ExecutionResult; } impl InterpretValue for T { type InterpretedValue = Self; - fn interpret(self, _interpreter: &mut Interpreter) -> ExecutionResult { + fn interpret_to_value( + self, + _interpreter: &mut Interpreter, + ) -> ExecutionResult { Ok(self) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 47874675..e69dd85b 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -31,7 +31,7 @@ pub(crate) enum InterpretedTokenTree { impl From for InterpretedStream { fn from(value: InterpretedTokenTree) -> Self { let mut new = Self::new(); - new.push_segment_item(value); + new.push_interpreted_item(value); new } } @@ -92,7 +92,7 @@ impl InterpretedStream { self.extend_raw_tokens(tokens.into_token_stream()) } - pub(crate) fn push_segment_item(&mut self, segment_item: InterpretedTokenTree) { + pub(crate) fn push_interpreted_item(&mut self, segment_item: InterpretedTokenTree) { match segment_item { InterpretedTokenTree::TokenTree(token_tree) => { self.push_raw_token_tree(token_tree); @@ -344,6 +344,15 @@ impl InterpretedStream { } } +impl IntoIterator for InterpretedStream { + type IntoIter = std::vec::IntoIter; + type Item = InterpretedTokenTree; + + fn into_iter(self) -> Self::IntoIter { + self.into_item_vec().into_iter() + } +} + pub(crate) struct ConcatBehaviour<'a> { pub(crate) between_token_trees: Option<&'a str>, pub(crate) output_transparent_group_as_command: bool, @@ -398,9 +407,15 @@ impl ConcatBehaviour<'_> { output.push(')'); } Delimiter::Brace => { - output.push('{'); - inner(output); - output.push('}'); + if is_empty { + output.push('{'); + inner(output); + output.push('}'); + } else { + output.push_str("{ "); + inner(output); + output.push_str(" }"); + } } Delimiter::Bracket => { output.push('['); diff --git a/tests/compilation_failures/tokens/zip_different_length_streams.rs b/tests/compilation_failures/tokens/zip_different_length_streams.rs new file mode 100644 index 00000000..f6589845 --- /dev/null +++ b/tests/compilation_failures/tokens/zip_different_length_streams.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!zip! ([A B C] [1 2 3 4])] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/zip_different_length_streams.stderr b/tests/compilation_failures/tokens/zip_different_length_streams.stderr new file mode 100644 index 00000000..0572cc5a --- /dev/null +++ b/tests/compilation_failures/tokens/zip_different_length_streams.stderr @@ -0,0 +1,5 @@ +error: Streams have different lengths and zip's error_on_length_mismatch is true. The lengths vary from 3 to 4 + --> tests/compilation_failures/tokens/zip_different_length_streams.rs:5:16 + | +5 | [!zip! ([A B C] [1 2 3 4])] + | ^^^^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/tokens/zip_no_streams.rs b/tests/compilation_failures/tokens/zip_no_streams.rs new file mode 100644 index 00000000..0dd8fe05 --- /dev/null +++ b/tests/compilation_failures/tokens/zip_no_streams.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + preinterpret! { + [!zip! ()] + } +} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/zip_no_streams.stderr b/tests/compilation_failures/tokens/zip_no_streams.stderr new file mode 100644 index 00000000..788a385d --- /dev/null +++ b/tests/compilation_failures/tokens/zip_no_streams.stderr @@ -0,0 +1,5 @@ +error: At least one stream is required to zip + --> tests/compilation_failures/tokens/zip_no_streams.rs:5:16 + | +5 | [!zip! ()] + | ^^ diff --git a/tests/tokens.rs b/tests/tokens.rs index ac5866b4..297d93b6 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -390,3 +390,91 @@ fn test_comma_split() { "[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]" ); } + +#[test] +fn test_zip() { + assert_preinterpret_eq!( + { [!debug! [!..zip! ([Hello Goodbye] [World Friend])]] }, + "(Hello World) (Goodbye Friend)" + ); + assert_preinterpret_eq!( + { + [!set! #countries = France Germany Italy] + [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] + [!set! #capitals = Paris Berlin Rome] + [!debug! [!zip! { + streams: [#countries #flags #capitals], + }]] + }, + r#"[!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]"#, + ); + assert_preinterpret_eq!( + { + [!set! #longer = A B C D] + [!set! #shorter = 1 2 3] + [!set! #combined = #longer #shorter] + [!debug! [!..zip! { + streams: #combined, + error_on_length_mismatch: false, + }]] + }, + r#"[!group! A 1] [!group! B 2] [!group! C 3]"#, + ); + assert_preinterpret_eq!( + { + [!set! #letters = A B C] + [!set! #numbers = 1 2 3] + [!set! #combined = #letters #numbers] + [!debug! [!..zip! { + streams: { + { #..combined } + }, + }]] + }, + r#"{ A 1 } { B 2 } { C 3 }"#, + ); + assert_preinterpret_eq!( + { + [!set! #letters = A B C] + [!set! #numbers = 1 2 3] + [!set! #combined = { #letters #numbers }] + [!debug! [!..zip! { + streams: #..combined, + }]] + }, + r#"{ A 1 } { B 2 } { C 3 }"#, + ); + assert_preinterpret_eq!( + { + [!set! #letters = A B C] + [!set! #numbers = 1 2 3] + [!set! #combined = [#letters #numbers]] + [!debug! [!..zip! #..combined]] + }, + r#"[A 1] [B 2] [C 3]"#, + ); +} + +#[test] +fn test_zip_with_for() { + assert_preinterpret_eq!( + { + [!set! #countries = France Germany Italy] + [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] + [!set! #capitals = Paris Berlin Rome] + [!set! #facts = [!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)] { + [!string! "=> The capital of " #country " is " #capital " and its flag is " #flag] + }]] + + [!string! "The facts are:\n" [!intersperse! { + items: #facts, + separator: ["\n"], + }] "\n"] + }, + r#"The facts are: +=> The capital of France is Paris and its flag is 🇫🇷 +=> The capital of Germany is Berlin and its flag is 🇩🇪 +=> The capital of Italy is Rome and its flag is 🇮🇹 +"#, + ); +} From aedc01a51bf0b9b342387568a5da03315607c26a Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 26 Jan 2025 13:42:01 +0000 Subject: [PATCH 061/126] tweak: Minor error message improvement when parsing groups --- CHANGELOG.md | 6 +- src/destructuring/destructure_group.rs | 2 +- src/destructuring/destructure_raw.rs | 2 +- src/destructuring/destructure_variable.rs | 2 +- src/destructuring/destructurer.rs | 2 +- src/destructuring/destructurers.rs | 2 +- src/destructuring/fields.rs | 2 +- src/extensions/parsing.rs | 68 +++++++++++++++---- src/interpretation/command.rs | 2 +- src/interpretation/command_code_input.rs | 2 +- src/interpretation/command_field_inputs.rs | 2 +- src/interpretation/command_stream_input.rs | 22 +++--- ...tersperse_stream_input_braces_issue.stderr | 2 +- ...rsperse_stream_input_variable_issue.stderr | 2 +- 14 files changed, 79 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9b6be1..b39af1ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ Destructuring performs parsing of a token stream. It supports: ### To come +* The `[!assign! ...]` operator is optional, if not present, it functions as `[!set! #x = [!evaluate! ...]]` * Add compile error tests for all the standard destructuring errors * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` @@ -90,10 +91,9 @@ Destructuring performs parsing of a token stream. It supports: * `(!optional! ...)` * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` - * `(!match! ...)` (with `#..x` as a catch-all) but without arms... - so I guess it's more of an `(!any! ...)` + * `[!match! ...]` command + * `(!any! ...)` (with `#..x` as a catch-all) like the [!match!] command but without arms... * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much -* `[!match! ...]` command - similar to the match destructurer, but a command... * Check all `#[allow(unused)]` and remove any which aren't needed * Rework expression parsing, in order to: * Fix comments in the expression files diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs index 3d7f4003..2ad59121 100644 --- a/src/destructuring/destructure_group.rs +++ b/src/destructuring/destructure_group.rs @@ -22,7 +22,7 @@ impl HandleDestructure for DestructureGroup { input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { - let (_, inner) = input.parse_group_matching(self.delimiter)?; + let (_, inner) = input.parse_specific_group(self.delimiter)?; self.inner.handle_destructure(&inner, interpreter) } } diff --git a/src/destructuring/destructure_raw.rs b/src/destructuring/destructure_raw.rs index 72878cdc..c9cf0199 100644 --- a/src/destructuring/destructure_raw.rs +++ b/src/destructuring/destructure_raw.rs @@ -92,7 +92,7 @@ impl RawDestructureGroup { } pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { - let (_, inner) = input.parse_group_matching(self.delimiter)?; + let (_, inner) = input.parse_specific_group(self.delimiter)?; self.inner.handle_destructure(&inner) } } diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index cddc2f88..90b01683 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -322,7 +322,7 @@ impl ParseUntil { ParseUntil::End => output.extend_raw_tokens(input.parse::()?), ParseUntil::Group(delimiter) => { while !input.is_empty() { - if input.peek_group_matching(*delimiter) { + if input.peek_specific_group(*delimiter) { return Ok(()); } output.push_raw_token_tree(input.parse()?); diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index 2107fc2d..a1e22401 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -87,7 +87,7 @@ pub(crate) struct Destructurer { impl Parse for Destructurer { fn parse(input: ParseStream) -> ParseResult { - let (delim_span, content) = input.parse_group_matching(Delimiter::Parenthesis)?; + let (delim_span, content) = input.parse_specific_group(Delimiter::Parenthesis)?; content.parse::()?; let destructurer_name = content.parse_any_ident()?; let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index ed610798..19544a57 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -168,7 +168,7 @@ impl DestructurerDefinition for GroupDestructurer { input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { - let (_, inner) = input.parse_group_matching(Delimiter::None)?; + let (_, inner) = input.parse_specific_group(Delimiter::None)?; self.inner.handle_destructure(&inner, interpreter) } } diff --git a/src/destructuring/fields.rs b/src/destructuring/fields.rs index 7b94821f..d4d7ce06 100644 --- a/src/destructuring/fields.rs +++ b/src/destructuring/fields.rs @@ -79,7 +79,7 @@ impl FieldsParseDefinition { error_span_range: SpanRange, ) -> ParseResult { let mut builder = new_builder; - let (_, content) = input.parse_group_matching(Delimiter::Brace)?; + let (_, content) = input.parse_specific_group(Delimiter::Brace)?; let mut required_field_names: BTreeSet<_> = field_definitions .0 diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index f2f22af1..baa02e9c 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -111,8 +111,13 @@ pub(crate) trait ParserBufferExt { fn peek_literal_matching(&self, content: &str) -> bool; fn parse_literal_matching(&self, content: &str) -> ParseResult; fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)>; - fn peek_group_matching(&self, delimiter: Delimiter) -> bool; - fn parse_group_matching(&self, delimiter: Delimiter) -> ParseResult<(DelimSpan, ParseBuffer)>; + fn peek_specific_group(&self, delimiter: Delimiter) -> bool; + fn parse_group_matching( + &self, + matching: impl FnOnce(Delimiter) -> bool, + expected_message: impl FnOnce() -> String, + ) -> ParseResult<(DelimSpan, ParseBuffer)>; + fn parse_specific_group(&self, delimiter: Delimiter) -> ParseResult<(DelimSpan, ParseBuffer)>; fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult; fn parse_error(&self, message: impl std::fmt::Display) -> ParseError; } @@ -192,25 +197,34 @@ impl ParserBufferExt for ParseBuffer<'_> { Ok((delimiter, delim_span, parse_buffer.into())) } - fn peek_group_matching(&self, delimiter: Delimiter) -> bool { + fn peek_specific_group(&self, delimiter: Delimiter) -> bool { self.cursor().group_matching(delimiter).is_some() } fn parse_group_matching( &self, - expected_delimiter: Delimiter, + matching: impl FnOnce(Delimiter) -> bool, + expected_message: impl FnOnce() -> String, ) -> ParseResult<(DelimSpan, ParseBuffer)> { use syn::parse::discouraged::AnyDelimiter; - let (delimiter, delim_span, inner) = self.parse_any_delimiter()?; - if delimiter != expected_delimiter { - return delim_span.open().parse_err(match expected_delimiter { - Delimiter::Parenthesis => "Expected (", - Delimiter::Brace => "Expected {", - Delimiter::Bracket => "Expected [", - Delimiter::None => "Expected start of transparent group", - }); - } - Ok((delim_span, inner.into())) + let error_span = match self.parse_any_delimiter() { + Ok((delimiter, delim_span, inner)) if matching(delimiter) => { + return Ok((delim_span, inner.into())); + } + Ok((_, delim_span, _)) => delim_span.open(), + Err(error) => error.span(), + }; + error_span.parse_err(expected_message()) + } + + fn parse_specific_group( + &self, + expected_delimiter: Delimiter, + ) -> ParseResult<(DelimSpan, ParseBuffer)> { + self.parse_group_matching( + |delimiter| delimiter == expected_delimiter, + || format!("Expected {}", expected_delimiter.description_of_open()), + ) } fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { @@ -221,3 +235,29 @@ impl ParserBufferExt for ParseBuffer<'_> { self.span().parse_error(message) } } + +pub(crate) trait DelimiterExt { + fn description_of_open(&self) -> &'static str; + #[allow(unused)] + fn description_of_group(&self) -> &'static str; +} + +impl DelimiterExt for Delimiter { + fn description_of_open(&self) -> &'static str { + match self { + Delimiter::Parenthesis => "(", + Delimiter::Brace => "{", + Delimiter::Bracket => "[", + Delimiter::None => "start of transparent group, from a grouped #variable substitution or stream-based command such as [!group! ...]", + } + } + + fn description_of_group(&self) -> &'static str { + match self { + Delimiter::Parenthesis => "(...)", + Delimiter::Brace => "{ ... }", + Delimiter::Bracket => "[...]", + Delimiter::None => "transparent group, from a grouped #variable substitution or stream-based command such as [!group! ...]", + } + } +} diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 335be051..132d888d 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -467,7 +467,7 @@ pub(crate) struct Command { impl Parse for Command { fn parse(input: ParseStream) -> ParseResult { - let (delim_span, content) = input.parse_group_matching(Delimiter::Bracket)?; + let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; content.parse::()?; let flattening = if content.peek(Token![.]) { Some(content.parse::()?) diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index d9ac32d1..883364dd 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -9,7 +9,7 @@ pub(crate) struct CommandCodeInput { impl Parse for CommandCodeInput { fn parse(input: ParseStream) -> ParseResult { - let (delim_span, content) = input.parse_group_matching(Delimiter::Brace)?; + let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; let inner = content.parse_with(delim_span.join().span_range())?; Ok(Self { delim_span, inner }) } diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index bfa204aa..86f576a2 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -33,7 +33,7 @@ macro_rules! define_field_inputs { let mut $optional_field: Option<$optional_type> = None; )* - let (delim_span, content) = input.parse_group_matching(Delimiter::Brace)?; + let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; while !content.is_empty() { let ident = content.parse_any_ident()?; diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 6ad2dbe9..e90e1cd2 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -66,7 +66,7 @@ impl Interpret for CommandStreamInput { command, interpreter, || { - "Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.".to_string() + "Expected output of flattened command to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to remove the .., to use the command output as-is.".to_string() }, output, ), @@ -81,7 +81,7 @@ impl Interpret for CommandStreamInput { command, interpreter, || { - "Expected output of control flow command to contain a single [ ... ] or transparent group.".to_string() + "Expected output of control flow command to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...].".to_string() }, output, ), @@ -92,7 +92,7 @@ impl Interpret for CommandStreamInput { interpreter, || { format!( - "Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.", + "Expected variable to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to use {} instead, to use the content of the variable as the stream.", variable.display_grouped_variable_token(), ) }, @@ -105,7 +105,7 @@ impl Interpret for CommandStreamInput { code, interpreter, || { - "Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string() + "Expected the { ... } block to output a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string() }, output, ), @@ -154,12 +154,12 @@ impl Parse for InterpretedCommandStream { fn parse(input: ParseStream) -> ParseResult { // We assume we're parsing an already interpreted raw stream here, so we replicate // parse_as_stream_input - let (delimiter, delim_span, inner) = input.parse_any_group()?; - match delimiter { - Delimiter::Bracket | Delimiter::None => Ok(Self { - stream: InterpretedStream::raw(inner.parse()?), - }), - _ => delim_span.parse_err("Expected a [ ... ] or transparent group"), - } + let (_, inner) = input.parse_group_matching( + |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None), + || "Expected [...] or a transparent group from a #variable or stream-output command such as [!group! ...]".to_string(), + )?; + Ok(Self { + stream: InterpretedStream::raw(inner.parse()?), + }) } } diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr index 10070cbb..4c40c9aa 100644 --- a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr +++ b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr @@ -1,4 +1,4 @@ -error: Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream. +error: Expected the { ... } block to output a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream. --> tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs:6:20 | 6 | items: { 1 2 3 }, diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr index 95fb83a0..930db528 100644 --- a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr +++ b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr @@ -1,4 +1,4 @@ -error: Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use #x instead, to use the content of the variable as the stream. +error: Expected variable to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to use #x instead, to use the content of the variable as the stream. --> tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs:7:20 | 7 | items: #..x, From 738c3cc6fdfc478b8464672bd9073b0a91c6a795 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 26 Jan 2025 16:11:09 +0000 Subject: [PATCH 062/126] feature: Assign's operator is now optional --- .../commands/expression_commands.rs | 24 ++++++++++++------- tests/expressions.rs | 13 +++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 8c7ffb4c..d070938b 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -32,7 +32,7 @@ impl ValueCommandDefinition for EvaluateCommand { #[derive(Clone)] pub(crate) struct AssignCommand { variable: GroupedVariable, - operator: Punct, + operator: Option, #[allow(unused)] equals: Token![=], expression: ExpressionInput, @@ -51,18 +51,22 @@ impl NoOutputCommandDefinition for AssignCommand { Ok(Self { variable: input.parse()?, operator: { - let operator: Punct = input.parse()?; - match operator.as_char() { - '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => return operator.parse_err("Expected one of + - * / % & | or ^"), + if input.peek(Token![=]) { + None + } else { + let operator: Punct = input.parse()?; + match operator.as_char() { + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} + _ => return operator.parse_err("Expected one of + - * / % & | or ^"), + } + Some(operator) } - operator }, equals: input.parse()?, expression: input.parse()?, }) }, - "Expected [!assign! #variable += ...] for + or some other operator supported in an expression", + "Expected [!assign! #variable = ] or [!assign! #variable X= ] for X one of + - * / % & | or ^", ) } @@ -75,8 +79,10 @@ impl NoOutputCommandDefinition for AssignCommand { } = *self; let mut builder = ExpressionBuilder::new(); - variable.add_to_expression(interpreter, &mut builder)?; - builder.push_punct(operator); + if let Some(operator) = operator { + variable.add_to_expression(interpreter, &mut builder)?; + builder.push_punct(operator); + } builder.extend_with_evaluation_output(expression.evaluate(interpreter)?); let output = builder.evaluate()?.into_token_tree(); diff --git a/tests/expressions.rs b/tests/expressions.rs index ae360279..d51c5275 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -76,9 +76,16 @@ fn test_basic_evaluate_works() { fn assign_works() { assert_preinterpret_eq!( { - [!set! #x = 8 + 2] // 10 - [!assign! #x /= 1 + 1] // 5 - [!assign! #x += 2 + #x] // 12 + [!assign! #x = 5 + 5] + [!debug! #..x] + }, + "10" + ); + assert_preinterpret_eq!( + { + [!set! #x = 8 + 2] // 8 + 2 (not evaluated) + [!assign! #x /= 1 + 1] // ((8 + 2) / (1 + 1)) => 5 + [!assign! #x += 2 + #x] // ((10) + 2) => 12 #x }, 12 From 69c9271400e6afaea2fad4010a0e6dd1c612fc89 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 28 Jan 2025 18:26:01 +0000 Subject: [PATCH 063/126] tests: Add more tests and improve long expression perf --- CHANGELOG.md | 17 +++++++++--- src/expressions/evaluation_tree.rs | 14 ++++++---- src/expressions/operations.rs | 17 +++++++----- src/extensions/errors_and_spans.rs | 27 +++++++++++++------ src/interpretation/commands/core_commands.rs | 2 +- .../destructure_with_flattened_command.rs | 5 ++++ .../destructure_with_flattened_command.stderr | 6 +++++ ...structure_with_ident_flattened_variable.rs | 5 ++++ ...cture_with_ident_flattened_variable.stderr | 6 +++++ ...ructure_with_literal_flattened_variable.rs | 5 ++++ ...ure_with_literal_flattened_variable.stderr | 6 +++++ .../destructure_with_outputting_command.rs | 5 ++++ ...destructure_with_outputting_command.stderr | 6 +++++ ...structure_with_punct_flattened_variable.rs | 5 ++++ ...cture_with_punct_flattened_variable.stderr | 6 +++++ .../destructuring/invalid_content_too_long.rs | 5 ++++ .../invalid_content_too_long.stderr | 5 ++++ .../invalid_content_too_short.rs | 5 ++++ .../invalid_content_too_short.stderr | 7 +++++ .../invalid_content_wrong_group.rs | 5 ++++ .../invalid_content_wrong_group.stderr | 5 ++++ .../invalid_content_wrong_group_2.rs | 5 ++++ .../invalid_content_wrong_group_2.stderr | 5 ++++ .../invalid_content_wrong_ident.rs | 5 ++++ .../invalid_content_wrong_ident.stderr | 5 ++++ .../invalid_content_wrong_punct.rs | 5 ++++ .../invalid_content_wrong_punct.stderr | 5 ++++ .../invalid_group_content_too_long.rs | 5 ++++ .../invalid_group_content_too_long.stderr | 5 ++++ .../invalid_group_content_too_short.rs | 5 ++++ .../invalid_group_content_too_short.stderr | 5 ++++ tests/expressions.rs | 20 ++++++++++++++ 32 files changed, 209 insertions(+), 25 deletions(-) create mode 100644 tests/compilation_failures/destructuring/destructure_with_flattened_command.rs create mode 100644 tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr create mode 100644 tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs create mode 100644 tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr create mode 100644 tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs create mode 100644 tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr create mode 100644 tests/compilation_failures/destructuring/destructure_with_outputting_command.rs create mode 100644 tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr create mode 100644 tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs create mode 100644 tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_content_too_long.rs create mode 100644 tests/compilation_failures/destructuring/invalid_content_too_long.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_content_too_short.rs create mode 100644 tests/compilation_failures/destructuring/invalid_content_too_short.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_group.rs create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs create mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_group_content_too_long.rs create mode 100644 tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr create mode 100644 tests/compilation_failures/destructuring/invalid_group_content_too_short.rs create mode 100644 tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index b39af1ff..fec87d1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,8 +76,9 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* The `[!assign! ...]` operator is optional, if not present, it functions as `[!set! #x = [!evaluate! ...]]` -* Add compile error tests for all the standard destructuring errors +* Expression refactoring: + * Get rid of source span, and instead provide it when getting an output + * Also remove ToTokens etc * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` * Change `(!content! ...)` to unwrap none groups, to be more permissive @@ -99,12 +100,20 @@ Destructuring performs parsing of a token stream. It supports: * Fix comments in the expression files * Enable lazy && and || * Enable support for code blocks { .. } in expressions, and remove hacks where expression parsing stops at {} or . + * Remove stack overflow possibilities when parsing a long nested expression * Pushed to 0.4: * Fork of syn to: * Fix issues in Rust Analyzer - * Improve performance - * Permit `[!parse_while! [!PARSE! ...] from #x { ... }]` + * Improve performance (?) + * Permit `[!parse_while! (!stream! ...) from #x { ... }]` * Fix `any_punct()` to ignore none groups + * Groups can either be: + * Raw Groups + * Or created groups, where we store `DelimSpan` for re-parsing and accessing the open/close delimiters (this will let us improve `invalid_content_wrong_group`) + * Better error messages + * See e.g. invalid_content_too_short where ideally the error message would be on the last token in the stream. Perhaps End gets a span from the previous error? + * See e.g. invalid_content_too_long where `unexpected token` is quite vague. + Maybe we can't sensibly do better though... * Further syn parsings (e.g. item, fields, etc) * Work on book * Input paradigms: diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index 64b64302..57718e1d 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -79,6 +79,7 @@ impl<'a> EvaluationTreeBuilder<'a> { ); } Expr::Group(expr) => { + // We handle these as a no-op operation so that they get the span from the group self.add_unary_operation( placement, UnaryOperation::for_group_expression(expr)?, @@ -89,6 +90,7 @@ impl<'a> EvaluationTreeBuilder<'a> { self.add_literal(placement, EvaluationValue::for_literal_expression(expr)?); } Expr::Paren(expr) => { + // We handle these as a no-op operation so that they get the span from the paren self.add_unary_operation( placement, UnaryOperation::for_paren_expression(expr)?, @@ -103,9 +105,11 @@ impl<'a> EvaluationTreeBuilder<'a> { ); } other_expression => { - return other_expression.execution_err( - "This expression is not supported in preinterpret expressions", - ); + return other_expression + .span_range_from_iterating_over_all_tokens() + .execution_err( + "This expression is not supported in preinterpret expressions", + ); } } } @@ -116,14 +120,14 @@ impl<'a> EvaluationTreeBuilder<'a> { fn add_binary_operation( &mut self, - placement: ResultPlacement, + result_placement: ResultPlacement, operation: BinaryOperation, lhs: &'a Expr, rhs: &'a Expr, ) { let parent_node_stack_index = self.evaluation_stack.len(); self.evaluation_stack.push(EvaluationNode { - result_placement: placement, + result_placement, content: EvaluationNodeContent::Operator(EvaluationOperator::Binary { operation, left_input: None, diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index f712eb72..0d3a658b 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -78,9 +78,11 @@ impl UnaryOperation { let ident = match type_path.path.get_ident() { Some(ident) => ident, None => { - return type_path.execution_err( - "This type is not supported in preinterpret cast expressions", - ) + return type_path + .span_range_from_iterating_over_all_tokens() + .execution_err( + "This type is not supported in preinterpret cast expressions", + ) } }; match ident.to_string().as_str() { @@ -108,13 +110,14 @@ impl UnaryOperation { } } other => other + .span_range_from_iterating_over_all_tokens() .execution_err("This type is not supported in preinterpret cast expressions"), } } Ok(Self { - span_for_output: expr.expr.span_range(), - operator_span: expr.as_token.span.span_range(), + span_for_output: expr.as_token.span_range(), + operator_span: expr.as_token.span_range(), operator: UnaryOperator::Cast(extract_type(&expr.ty)?), }) } @@ -146,7 +149,7 @@ impl UnaryOperation { } }; Ok(Self { - span_for_output: expr.span_range(), + span_for_output: expr.op.span_range(), operator_span: expr.op.span_range(), operator, }) @@ -249,7 +252,7 @@ impl BinaryOperation { } }; Ok(Self { - span_for_output: expr.span_range(), + span_for_output: expr.op.span_range(), operator_span: expr.op.span_range(), operator, }) diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index 549ecb57..e5854011 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -133,8 +133,13 @@ impl HasSpanRange for DelimSpan { } } -impl HasSpanRange for T { - fn span_range(&self) -> SpanRange { +pub(crate) trait SlowSpanRange { + /// This name is purposefully very long to discourage use, as it can cause nasty performance issues + fn span_range_from_iterating_over_all_tokens(&self) -> SpanRange; +} + +impl SlowSpanRange for T { + fn span_range_from_iterating_over_all_tokens(&self) -> SpanRange { let mut iter = self.into_token_stream().into_iter(); let start = iter.next().map_or_else(Span::call_site, |t| t.span()); let end = iter.last().map_or(start, |t| t.span()); @@ -154,18 +159,24 @@ macro_rules! impl_auto_span_range { }; } +impl HasSpanRange for T { + fn span_range(&self) -> SpanRange { + // AutoSpanRange should only be used for tokens with a small number of tokens + SlowSpanRange::span_range_from_iterating_over_all_tokens(&self) + } +} + +// This should only be used for types with a bounded number of tokens +// otherwise, span_range_from_iterating_over_all_tokens() can be used +// directly with a longer name to make the performance hit clearer, so +// it's only used in error cases. impl_auto_span_range! { - TokenStream, Ident, Punct, Literal, - syn::Expr, - syn::ExprBinary, - syn::ExprUnary, syn::BinOp, syn::UnOp, - syn::Type, - syn::TypePath, + syn::token::As, syn::token::DotDot, syn::token::In, } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 0c1edc69..e31e0304 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -281,7 +281,7 @@ impl NoOutputCommandDefinition for ErrorCommand { if error_span_stream.is_empty() { Span::call_site().span_range() } else { - error_span_stream.span_range() + error_span_stream.span_range_from_iterating_over_all_tokens() } } None => Span::call_site().span_range(), diff --git a/tests/compilation_failures/destructuring/destructure_with_flattened_command.rs b/tests/compilation_failures/destructuring/destructure_with_flattened_command.rs new file mode 100644 index 00000000..72d60b09 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_flattened_command.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! [!..group! Output] = [!group! Output]]); +} diff --git a/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr b/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr new file mode 100644 index 00000000..a3e716e7 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr @@ -0,0 +1,6 @@ +error: Flattened commands are not supported in destructuring positions + Occurred whilst parsing [!let! ...] - Expected [!let! = ...] + --> tests/compilation_failures/destructuring/destructure_with_flattened_command.rs:4:26 + | +4 | preinterpret!([!let! [!..group! Output] = [!group! Output]]); + | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs b/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs new file mode 100644 index 00000000..487613f4 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! (!ident! #..x) = Hello]); +} diff --git a/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr b/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr new file mode 100644 index 00000000..a1968590 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr @@ -0,0 +1,6 @@ +error: A flattened input variable is not supported here + Occurred whilst parsing (!ident! ...) - Expected (!ident! #x) or (!ident #>>x) or (!ident!) + --> tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs:4:35 + | +4 | preinterpret!([!let! (!ident! #..x) = Hello]); + | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs b/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs new file mode 100644 index 00000000..daf6bf81 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! (!literal! #..x) = "Hello"]); +} diff --git a/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr b/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr new file mode 100644 index 00000000..eb9e2f7d --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr @@ -0,0 +1,6 @@ +error: A flattened input variable is not supported here + Occurred whilst parsing (!literal! ...) - Expected (!literal! #x) or (!literal! #>>x) or (!literal!) + --> tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs:4:37 + | +4 | preinterpret!([!let! (!literal! #..x) = "Hello"]); + | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_outputting_command.rs b/tests/compilation_failures/destructuring/destructure_with_outputting_command.rs new file mode 100644 index 00000000..41748784 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_outputting_command.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! [!group! Output] = [!group! Output]]); +} diff --git a/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr b/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr new file mode 100644 index 00000000..8ed41c10 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr @@ -0,0 +1,6 @@ +error: Grouped commands returning a value are not supported in destructuring positions + Occurred whilst parsing [!let! ...] - Expected [!let! = ...] + --> tests/compilation_failures/destructuring/destructure_with_outputting_command.rs:4:26 + | +4 | preinterpret!([!let! [!group! Output] = [!group! Output]]); + | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs b/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs new file mode 100644 index 00000000..0c69d7ac --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! (!punct! #..x) = @]); +} diff --git a/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr b/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr new file mode 100644 index 00000000..9c38ca23 --- /dev/null +++ b/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr @@ -0,0 +1,6 @@ +error: A flattened input variable is not supported here + Occurred whilst parsing (!punct! ...) - Expected (!punct! #x) or (!punct! #>>x) or (!punct!) + --> tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs:4:35 + | +4 | preinterpret!([!let! (!punct! #..x) = @]); + | ^ diff --git a/tests/compilation_failures/destructuring/invalid_content_too_long.rs b/tests/compilation_failures/destructuring/invalid_content_too_long.rs new file mode 100644 index 00000000..20ac159b --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_too_long.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! Hello World = Hello World!!!]); +} diff --git a/tests/compilation_failures/destructuring/invalid_content_too_long.stderr b/tests/compilation_failures/destructuring/invalid_content_too_long.stderr new file mode 100644 index 00000000..f83502fe --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_too_long.stderr @@ -0,0 +1,5 @@ +error: unexpected token + --> tests/compilation_failures/destructuring/invalid_content_too_long.rs:4:51 + | +4 | preinterpret!([!let! Hello World = Hello World!!!]); + | ^ diff --git a/tests/compilation_failures/destructuring/invalid_content_too_short.rs b/tests/compilation_failures/destructuring/invalid_content_too_short.rs new file mode 100644 index 00000000..081ed084 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_too_short.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! Hello World = Hello]); +} diff --git a/tests/compilation_failures/destructuring/invalid_content_too_short.stderr b/tests/compilation_failures/destructuring/invalid_content_too_short.stderr new file mode 100644 index 00000000..f7193556 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_too_short.stderr @@ -0,0 +1,7 @@ +error: expected World + --> tests/compilation_failures/destructuring/invalid_content_too_short.rs:4:5 + | +4 | preinterpret!([!let! Hello World = Hello]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `preinterpret` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group.rs b/tests/compilation_failures/destructuring/invalid_content_wrong_group.rs new file mode 100644 index 00000000..28819476 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_group.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! (#..x) = [Hello World]]); +} diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr b/tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr new file mode 100644 index 00000000..9db00a4f --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr @@ -0,0 +1,5 @@ +error: Expected ( + --> tests/compilation_failures/destructuring/invalid_content_wrong_group.rs:4:35 + | +4 | preinterpret!([!let! (#..x) = [Hello World]]); + | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs b/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs new file mode 100644 index 00000000..025d39fe --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! (!group! #..x) = [Hello World]]); +} diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr b/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr new file mode 100644 index 00000000..04570fa9 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr @@ -0,0 +1,5 @@ +error: Expected start of transparent group, from a grouped #variable substitution or stream-based command such as [!group! ...] + --> tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs:4:43 + | +4 | preinterpret!([!let! (!group! #..x) = [Hello World]]); + | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs b/tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs new file mode 100644 index 00000000..959e900f --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! Hello World = Hello Earth]); +} diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr b/tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr new file mode 100644 index 00000000..0f9c6db8 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr @@ -0,0 +1,5 @@ +error: expected World + --> tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs:4:46 + | +4 | preinterpret!([!let! Hello World = Hello Earth]); + | ^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs b/tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs new file mode 100644 index 00000000..ab6bf1d4 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! Hello _ World = Hello World]); +} diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr b/tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr new file mode 100644 index 00000000..7fa785ab --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr @@ -0,0 +1,5 @@ +error: expected _ + --> tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs:4:48 + | +4 | preinterpret!([!let! Hello _ World = Hello World]); + | ^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_long.rs b/tests/compilation_failures/destructuring/invalid_group_content_too_long.rs new file mode 100644 index 00000000..b3553ecc --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_group_content_too_long.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! Group: (Hello World) = Group: (Hello World!!!)]); +} diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr b/tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr new file mode 100644 index 00000000..14ea0a33 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr @@ -0,0 +1,5 @@ +error: unexpected token, expected `)` + --> tests/compilation_failures/destructuring/invalid_group_content_too_long.rs:4:68 + | +4 | preinterpret!([!let! Group: (Hello World) = Group: (Hello World!!!)]); + | ^ diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_short.rs b/tests/compilation_failures/destructuring/invalid_group_content_too_short.rs new file mode 100644 index 00000000..22f8edce --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_group_content_too_short.rs @@ -0,0 +1,5 @@ +use preinterpret::*; + +fn main() { + preinterpret!([!let! Group: (Hello World!!) = Group: (Hello World)]); +} diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr b/tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr new file mode 100644 index 00000000..4db82988 --- /dev/null +++ b/tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr @@ -0,0 +1,5 @@ +error: expected ! + --> tests/compilation_failures/destructuring/invalid_group_content_too_short.rs:4:58 + | +4 | preinterpret!([!let! Group: (Hello World!!) = Group: (Hello World)]); + | ^^^^^^^^^^^^^ diff --git a/tests/expressions.rs b/tests/expressions.rs index d51c5275..69654e92 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -72,6 +72,26 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! "Zoo" > "Aardvark"], true); } +#[test] +fn test_very_long_expression_works() { + // Any larger than this gets hit by a stack overflow - which is in the syn library, + // during parsing the Expr, rather than in preinterpret. + // When we implement expression parsing in preinterpret, we can potentially flatten + // the parsing loop and get better performance. + assert_preinterpret_eq!({ + [!set! #x = 0 + [!for! #i in [!range! 0..1000] { + [!for! #j in [!range! 0..25] { + + 1 + }] + }] + ] + [!evaluate! #x] + }, + 25000 + ); +} + #[test] fn assign_works() { assert_preinterpret_eq!( From 886784165f595f904a631d3d7764e67afaa3b2e1 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 28 Jan 2025 23:39:47 +0000 Subject: [PATCH 064/126] tweak: Simplify expression span handling --- CHANGELOG.md | 3 - src/expressions/boolean.rs | 44 +-- src/expressions/character.rs | 35 +-- src/expressions/evaluation_tree.rs | 279 ++++-------------- src/expressions/expression_stream.rs | 9 +- src/expressions/float.rs | 60 ++-- src/expressions/integer.rs | 77 +++-- src/expressions/operations.rs | 91 +++--- src/expressions/string.rs | 44 ++- src/expressions/value.rs | 267 ++++++++++++++--- src/extensions/errors_and_spans.rs | 1 + src/interpretation/command.rs | 2 +- src/interpretation/command_arguments.rs | 20 +- .../commands/control_flow_commands.rs | 13 +- src/interpretation/commands/core_commands.rs | 7 +- .../commands/destructuring_commands.rs | 2 +- .../commands/expression_commands.rs | 15 +- 17 files changed, 481 insertions(+), 488 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec87d1d..56efe385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,9 +76,6 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* Expression refactoring: - * Get rid of source span, and instead provide it when getting an output - * Also remove ToTokens etc * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` * Change `(!content! ...)` to unwrap none groups, to be more permissive diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index cdfa300e..2b3fdb08 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -1,22 +1,16 @@ use super::*; pub(crate) struct EvaluationBoolean { - pub(super) source_span: SpanRange, pub(super) value: bool, + /// The span of the source code that generated this boolean value. + /// It may not have a value if generated from a complex expression. + pub(super) source_span: Option, } impl EvaluationBoolean { - pub(super) fn new(value: bool, source_span: SpanRange) -> Self { - Self { value, source_span } - } - - pub(crate) fn value(&self) -> bool { - self.value - } - pub(super) fn for_litbool(lit: &syn::LitBool) -> Self { Self { - source_span: lit.span().span_range(), + source_span: Some(lit.span()), value: lit.value, } } @@ -24,7 +18,7 @@ impl EvaluationBoolean { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let input = self.value; match operation.operator { UnaryOperator::Neg => operation.unsupported_for_value_type_err("boolean"), @@ -58,7 +52,7 @@ impl EvaluationBoolean { self, _right: EvaluationInteger, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { operation.unsupported_for_value_type_err("boolean") @@ -70,7 +64,7 @@ impl EvaluationBoolean { self, rhs: Self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation.paired_operator() { @@ -93,24 +87,16 @@ impl EvaluationBoolean { } } - pub(super) fn to_ident(&self) -> Ident { - Ident::new_bool(self.value, self.source_span.start()) - } - - #[allow(unused)] - pub(super) fn to_lit_bool(&self) -> LitBool { - LitBool::new(self.value, self.source_span.span()) - } -} - -impl ToEvaluationOutput for bool { - fn to_output(self, span: SpanRange) -> EvaluationOutput { - EvaluationValue::Boolean(EvaluationBoolean::new(self, span)).into() + pub(super) fn to_ident(&self, fallback_span: Span) -> Ident { + Ident::new_bool(self.value, self.source_span.unwrap_or(fallback_span)) } } -impl quote::ToTokens for EvaluationBoolean { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_ident().to_tokens(tokens) +impl ToEvaluationValue for bool { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::Boolean(EvaluationBoolean { + value: self, + source_span, + }) } } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 7de7ba2c..ef7403c0 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -2,25 +2,23 @@ use super::*; pub(crate) struct EvaluationChar { pub(super) value: char, - pub(super) source_span: SpanRange, + /// The span of the source code that generated this boolean value. + /// It may not have a value if generated from a complex expression. + pub(super) source_span: Option, } impl EvaluationChar { - pub(super) fn new(value: char, source_span: SpanRange) -> Self { - Self { value, source_span } - } - pub(super) fn for_litchar(lit: &syn::LitChar) -> Self { Self { value: lit.value(), - source_span: lit.span().span_range(), + source_span: Some(lit.span()), } } pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let char = self.value; match operation.operator { UnaryOperator::NoOp => operation.output(char), @@ -55,7 +53,7 @@ impl EvaluationChar { self, _right: EvaluationInteger, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { operation.unsupported_for_value_type_err("char") } @@ -63,7 +61,7 @@ impl EvaluationChar { self, rhs: Self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation.paired_operator() { @@ -86,19 +84,16 @@ impl EvaluationChar { } } - pub(super) fn to_literal(&self) -> Literal { - Literal::character(self.value).with_span(self.source_span.start()) - } -} - -impl ToEvaluationOutput for char { - fn to_output(self, span: SpanRange) -> EvaluationOutput { - EvaluationValue::Char(EvaluationChar::new(self, span)).into() + pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { + Literal::character(self.value).with_span(self.source_span.unwrap_or(fallback_span)) } } -impl quote::ToTokens for EvaluationChar { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_literal().to_tokens(tokens) +impl ToEvaluationValue for char { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::Char(EvaluationChar { + value: self, + source_span, + }) } } diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs index 57718e1d..ad4aea0a 100644 --- a/src/expressions/evaluation_tree.rs +++ b/src/expressions/evaluation_tree.rs @@ -4,11 +4,15 @@ pub(super) struct EvaluationTree { /// We store the tree as a normalized stack of nodes to make it easier to evaluate /// without risking hitting stack overflow issues for deeply unbalanced trees evaluation_stack: Vec, + fallback_output_span: Span, } impl EvaluationTree { - pub(super) fn build_from(expression: &Expr) -> ExecutionResult { - EvaluationTreeBuilder::new(expression).build() + pub(super) fn build_from( + fallback_output_span: Span, + expression: &Expr, + ) -> ExecutionResult { + EvaluationTreeBuilder::new(expression).build(fallback_output_span) } pub(super) fn evaluate(mut self) -> ExecutionResult { @@ -17,7 +21,12 @@ impl EvaluationTree { .expect("The builder should ensure that the stack is non-empty and has a final element of a RootResult which results in a return below."); let result = content.evaluate()?; match result_placement { - ResultPlacement::RootResult => return Ok(result), + ResultPlacement::RootResult => { + return Ok(EvaluationOutput { + value: result, + fallback_output_span: self.fallback_output_span, + }) + } ResultPlacement::UnaryOperationInput { parent_node_stack_index, } => { @@ -60,7 +69,7 @@ impl<'a> EvaluationTreeBuilder<'a> { /// Attempts to construct a preinterpret expression tree from a syn [Expr]. /// It tries to align with the [rustc expression] building approach. /// [rustc expression]: https://doc.rust-lang.org/reference/expressions.html - fn build(mut self) -> ExecutionResult { + fn build(mut self, fallback_output_span: Span) -> ExecutionResult { while let Some((expression, placement)) = self.work_stack.pop() { match expression { Expr::Binary(expr) => { @@ -115,6 +124,7 @@ impl<'a> EvaluationTreeBuilder<'a> { } Ok(EvaluationTree { evaluation_stack: self.evaluation_stack, + fallback_output_span, }) } @@ -174,7 +184,7 @@ impl<'a> EvaluationTreeBuilder<'a> { fn add_literal(&mut self, placement: ResultPlacement, literal: EvaluationValue) { self.evaluation_stack.push(EvaluationNode { result_placement: placement, - content: EvaluationNodeContent::Literal(literal), + content: EvaluationNodeContent::Value(literal), }); } } @@ -185,19 +195,19 @@ struct EvaluationNode { } enum EvaluationNodeContent { - Literal(EvaluationValue), + Value(EvaluationValue), Operator(EvaluationOperator), } impl EvaluationNodeContent { - fn evaluate(self) -> ExecutionResult { + fn evaluate(self) -> ExecutionResult { match self { - Self::Literal(literal) => Ok(EvaluationOutput::Value(literal)), + Self::Value(value) => Ok(value), Self::Operator(operator) => operator.evaluate(), } } - fn set_unary_input(&mut self, input: EvaluationOutput) { + fn set_unary_input(&mut self, input: EvaluationValue) { match self { Self::Operator(EvaluationOperator::Unary { input: existing_input, @@ -207,7 +217,7 @@ impl EvaluationNodeContent { } } - fn set_binary_left_input(&mut self, input: EvaluationOutput) { + fn set_binary_left_input(&mut self, input: EvaluationValue) { match self { Self::Operator(EvaluationOperator::Binary { left_input: existing_input, @@ -217,7 +227,7 @@ impl EvaluationNodeContent { } } - fn set_binary_right_input(&mut self, input: EvaluationOutput) { + fn set_binary_right_input(&mut self, input: EvaluationValue) { match self { Self::Operator(EvaluationOperator::Binary { right_input: existing_input, @@ -228,239 +238,66 @@ impl EvaluationNodeContent { } } -pub(crate) enum EvaluationOutput { - Value(EvaluationValue), -} - -pub(super) trait ToEvaluationOutput: Sized { - fn to_output(self, span: SpanRange) -> EvaluationOutput; +pub(crate) struct EvaluationOutput { + value: EvaluationValue, + fallback_output_span: Span, } impl EvaluationOutput { - pub(super) fn expect_value_pair( - self, - operator: PairedBinaryOperator, - right: EvaluationOutput, - operator_span: SpanRange, - ) -> ExecutionResult { - let left_lit = self.into_value(); - let right_lit = right.into_value(); - Ok(match (left_lit, right_lit) { - (EvaluationValue::Integer(left), EvaluationValue::Integer(right)) => { - let integer_pair = match (left.value, right.value) { - (EvaluationIntegerValue::Untyped(untyped_lhs), rhs) => match rhs { - EvaluationIntegerValue::Untyped(untyped_rhs) => { - EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) - } - EvaluationIntegerValue::U8(rhs) => { - EvaluationIntegerValuePair::U8(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::U16(rhs) => { - EvaluationIntegerValuePair::U16(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::U32(rhs) => { - EvaluationIntegerValuePair::U32(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::U64(rhs) => { - EvaluationIntegerValuePair::U64(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::U128(rhs) => { - EvaluationIntegerValuePair::U128(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::Usize(rhs) => { - EvaluationIntegerValuePair::Usize(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::I8(rhs) => { - EvaluationIntegerValuePair::I8(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::I16(rhs) => { - EvaluationIntegerValuePair::I16(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::I32(rhs) => { - EvaluationIntegerValuePair::I32(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::I64(rhs) => { - EvaluationIntegerValuePair::I64(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::I128(rhs) => { - EvaluationIntegerValuePair::I128(untyped_lhs.parse_as()?, rhs) - } - EvaluationIntegerValue::Isize(rhs) => { - EvaluationIntegerValuePair::Isize(untyped_lhs.parse_as()?, rhs) - } - }, - (lhs, EvaluationIntegerValue::Untyped(untyped_rhs)) => match lhs { - EvaluationIntegerValue::Untyped(untyped_lhs) => { - EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) - } - EvaluationIntegerValue::U8(lhs) => { - EvaluationIntegerValuePair::U8(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::U16(lhs) => { - EvaluationIntegerValuePair::U16(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::U32(lhs) => { - EvaluationIntegerValuePair::U32(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::U64(lhs) => { - EvaluationIntegerValuePair::U64(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::U128(lhs) => { - EvaluationIntegerValuePair::U128(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::Usize(lhs) => { - EvaluationIntegerValuePair::Usize(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::I8(lhs) => { - EvaluationIntegerValuePair::I8(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::I16(lhs) => { - EvaluationIntegerValuePair::I16(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::I32(lhs) => { - EvaluationIntegerValuePair::I32(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::I64(lhs) => { - EvaluationIntegerValuePair::I64(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::I128(lhs) => { - EvaluationIntegerValuePair::I128(lhs, untyped_rhs.parse_as()?) - } - EvaluationIntegerValue::Isize(lhs) => { - EvaluationIntegerValuePair::Isize(lhs, untyped_rhs.parse_as()?) - } - }, - (EvaluationIntegerValue::U8(lhs), EvaluationIntegerValue::U8(rhs)) => { - EvaluationIntegerValuePair::U8(lhs, rhs) - } - (EvaluationIntegerValue::U16(lhs), EvaluationIntegerValue::U16(rhs)) => { - EvaluationIntegerValuePair::U16(lhs, rhs) - } - (EvaluationIntegerValue::U32(lhs), EvaluationIntegerValue::U32(rhs)) => { - EvaluationIntegerValuePair::U32(lhs, rhs) - } - (EvaluationIntegerValue::U64(lhs), EvaluationIntegerValue::U64(rhs)) => { - EvaluationIntegerValuePair::U64(lhs, rhs) - } - (EvaluationIntegerValue::U128(lhs), EvaluationIntegerValue::U128(rhs)) => { - EvaluationIntegerValuePair::U128(lhs, rhs) - } - (EvaluationIntegerValue::Usize(lhs), EvaluationIntegerValue::Usize(rhs)) => { - EvaluationIntegerValuePair::Usize(lhs, rhs) - } - (EvaluationIntegerValue::I8(lhs), EvaluationIntegerValue::I8(rhs)) => { - EvaluationIntegerValuePair::I8(lhs, rhs) - } - (EvaluationIntegerValue::I16(lhs), EvaluationIntegerValue::I16(rhs)) => { - EvaluationIntegerValuePair::I16(lhs, rhs) - } - (EvaluationIntegerValue::I32(lhs), EvaluationIntegerValue::I32(rhs)) => { - EvaluationIntegerValuePair::I32(lhs, rhs) - } - (EvaluationIntegerValue::I64(lhs), EvaluationIntegerValue::I64(rhs)) => { - EvaluationIntegerValuePair::I64(lhs, rhs) - } - (EvaluationIntegerValue::I128(lhs), EvaluationIntegerValue::I128(rhs)) => { - EvaluationIntegerValuePair::I128(lhs, rhs) - } - (EvaluationIntegerValue::Isize(lhs), EvaluationIntegerValue::Isize(rhs)) => { - EvaluationIntegerValuePair::Isize(lhs, rhs) - } - (left_value, right_value) => { - return operator_span.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); - } - }; - EvaluationLiteralPair::Integer(integer_pair) - } - (EvaluationValue::Boolean(left), EvaluationValue::Boolean(right)) => { - EvaluationLiteralPair::BooleanPair(left, right) - } - (EvaluationValue::Float(left), EvaluationValue::Float(right)) => { - let float_pair = match (left.value, right.value) { - (EvaluationFloatValue::Untyped(untyped_lhs), rhs) => match rhs { - EvaluationFloatValue::Untyped(untyped_rhs) => { - EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) - } - EvaluationFloatValue::F32(rhs) => { - EvaluationFloatValuePair::F32(untyped_lhs.parse_as()?, rhs) - } - EvaluationFloatValue::F64(rhs) => { - EvaluationFloatValuePair::F64(untyped_lhs.parse_as()?, rhs) - } - }, - (lhs, EvaluationFloatValue::Untyped(untyped_rhs)) => match lhs { - EvaluationFloatValue::Untyped(untyped_lhs) => { - EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) - } - EvaluationFloatValue::F32(lhs) => { - EvaluationFloatValuePair::F32(lhs, untyped_rhs.parse_as()?) - } - EvaluationFloatValue::F64(lhs) => { - EvaluationFloatValuePair::F64(lhs, untyped_rhs.parse_as()?) - } - }, - (EvaluationFloatValue::F32(lhs), EvaluationFloatValue::F32(rhs)) => { - EvaluationFloatValuePair::F32(lhs, rhs) - } - (EvaluationFloatValue::F64(lhs), EvaluationFloatValue::F64(rhs)) => { - EvaluationFloatValuePair::F64(lhs, rhs) - } - (left_value, right_value) => { - return operator_span.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); - } - }; - EvaluationLiteralPair::Float(float_pair) - } - (EvaluationValue::String(left), EvaluationValue::String(right)) => { - EvaluationLiteralPair::StringPair(left, right) - } - (EvaluationValue::Char(left), EvaluationValue::Char(right)) => { - EvaluationLiteralPair::CharPair(left, right) - } - (left, right) => { - return operator_span.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); - } - }) + #[allow(unused)] + pub(super) fn into_value(self) -> EvaluationValue { + self.value } + #[allow(unused)] pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { - match self.into_value() { - EvaluationValue::Integer(value) => Ok(value), - other => other.source_span().execution_err(error_message), + let error_span = self.span(); + match self.value.into_integer() { + Some(integer) => Ok(integer), + None => error_span.execution_err(error_message), } } - pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { - match self.into_value() { - EvaluationValue::Boolean(value) => Ok(value), - other => other.source_span().execution_err(error_message), + pub(crate) fn try_into_i128(self, error_message: &str) -> ExecutionResult { + let error_span = self.span(); + match self + .value + .into_integer() + .and_then(|integer| integer.try_into_i128()) + { + Some(integer) => Ok(integer), + None => error_span.execution_err(error_message), } } - pub(super) fn into_value(self) -> EvaluationValue { - match self { - Self::Value(value) => value, + pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { + let error_span = self.span(); + match self.value.into_bool() { + Some(boolean) => Ok(boolean.value), + None => error_span.execution_err(error_message), } } - pub(crate) fn into_token_tree(self) -> TokenTree { - match self { - Self::Value(value) => value.into_token_tree(), - } + pub(crate) fn to_token_tree(&self) -> TokenTree { + self.value.to_token_tree(self.fallback_output_span) } } -impl From for EvaluationOutput { - fn from(literal: EvaluationValue) -> Self { - Self::Value(literal) +impl HasSpanRange for EvaluationOutput { + fn span(&self) -> Span { + self.value + .source_span() + .unwrap_or(self.fallback_output_span) + } + + fn span_range(&self) -> SpanRange { + self.span().span_range() } } impl quote::ToTokens for EvaluationOutput { fn to_tokens(&self, tokens: &mut TokenStream) { - match self { - EvaluationOutput::Value(evaluation_literal) => evaluation_literal.to_tokens(tokens), - } + self.to_token_tree().to_tokens(tokens); } } diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs index ea97f0fe..8cf03822 100644 --- a/src/expressions/expression_stream.rs +++ b/src/expressions/expression_stream.rs @@ -91,7 +91,8 @@ impl ExpressionInput { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - self.start_expression_builder(interpreter)?.evaluate() + let span = self.span(); + self.start_expression_builder(interpreter)?.evaluate(span) } } @@ -227,7 +228,7 @@ impl ExpressionBuilder { pub(crate) fn extend_with_evaluation_output(&mut self, value: EvaluationOutput) { self.interpreted_stream - .extend_with_raw_tokens_from(value.into_token_tree()); + .extend_with_raw_tokens_from(value.to_token_tree()); } pub(crate) fn push_expression_group( @@ -240,7 +241,7 @@ impl ExpressionBuilder { .push_new_group(contents.interpreted_stream, delimiter, span); } - pub(crate) fn evaluate(self) -> ExecutionResult { + pub(crate) fn evaluate(self, fallback_output_span: Span) -> ExecutionResult { // Parsing into a rust expression is overkill here. // // In future we could choose to implement a subset of the grammar which we actually can use/need. @@ -258,6 +259,6 @@ impl ExpressionBuilder { self.interpreted_stream.syn_parse(::parse)? }; - EvaluationTree::build_from(&expression)?.evaluate() + EvaluationTree::build_from(fallback_output_span, &expression)?.evaluate() } } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 197fff89..3ed2c7ae 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -2,26 +2,24 @@ use super::*; use crate::internal_prelude::*; pub(crate) struct EvaluationFloat { - pub(super) source_span: SpanRange, pub(super) value: EvaluationFloatValue, + /// The span of the source code that generated this boolean value. + /// It may not have a value if generated from a complex expression. + pub(super) source_span: Option, } impl EvaluationFloat { - pub(super) fn new(value: EvaluationFloatValue, source_span: SpanRange) -> Self { - Self { value, source_span } - } - pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ExecutionResult { Ok(Self { - source_span: lit.span().span_range(), value: EvaluationFloatValue::for_litfloat(lit)?, + source_span: Some(lit.span()), }) } pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => input.handle_unary_operation(&operation), EvaluationFloatValue::F32(input) => input.handle_unary_operation(&operation), @@ -33,7 +31,7 @@ impl EvaluationFloat { self, right: EvaluationInteger, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => { input.handle_integer_binary_operation(right, &operation) @@ -47,16 +45,10 @@ impl EvaluationFloat { } } - pub(super) fn to_literal(&self) -> Literal { + pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { self.value .to_unspanned_literal() - .with_span(self.source_span.start()) - } -} - -impl quote::ToTokens for EvaluationFloat { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_literal().to_tokens(tokens) + .with_span(self.source_span.unwrap_or(fallback_span)) } } @@ -70,7 +62,7 @@ impl EvaluationFloatValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::F32(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -142,7 +134,7 @@ impl UntypedFloat { pub(super) fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let input = self.parse_fallback()?; match operation.operator { UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), @@ -180,7 +172,7 @@ impl UntypedFloat { self, _rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { operation.unsupported_for_value_type_err("untyped float") @@ -192,7 +184,7 @@ impl UntypedFloat { self, rhs: Self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; match operation.paired_operator() { @@ -253,13 +245,12 @@ impl UntypedFloat { } } -impl ToEvaluationOutput for UntypedFloat { - fn to_output(self, span_range: SpanRange) -> EvaluationOutput { - EvaluationValue::Float(EvaluationFloat::new( - EvaluationFloatValue::Untyped(self), - span_range, - )) - .into() +impl ToEvaluationValue for UntypedFloat { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::Float(EvaluationFloat { + value: EvaluationFloatValue::Untyped(self), + source_span, + }) } } @@ -267,14 +258,17 @@ macro_rules! impl_float_operations { ( $($float_enum_variant:ident($float_type:ident)),* $(,)? ) => {$( - impl ToEvaluationOutput for $float_type { - fn to_output(self, span_range: SpanRange) -> EvaluationOutput { - EvaluationValue::Float(EvaluationFloat::new(EvaluationFloatValue::$float_enum_variant(self), span_range)).into() + impl ToEvaluationValue for $float_type { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::Float(EvaluationFloat { + value: EvaluationFloatValue::$float_enum_variant(self), + source_span + }) } } impl HandleUnaryOperation for $float_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { UnaryOperator::Neg => operation.output(-self), UnaryOperator::Not => operation.unsupported_for_value_type_err(stringify!($float_type)), @@ -303,7 +297,7 @@ macro_rules! impl_float_operations { } impl HandleBinaryOperation for $float_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { // Unlike integer arithmetic, float arithmetic does not overflow // and instead falls back to NaN or infinity. In future we could // allow trapping on these codes, but for now this is good enough @@ -336,7 +330,7 @@ macro_rules! impl_float_operations { self, _rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { operation.unsupported_for_value_type_err(stringify!($float_type)) diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 9c91ffae..42bc2e7f 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -1,24 +1,23 @@ use super::*; +#[derive(Clone)] pub(crate) struct EvaluationInteger { - pub(super) source_span: SpanRange, pub(super) value: EvaluationIntegerValue, + /// The span of the source code that generated this boolean value. + /// It may not have a value if generated from a complex expression. + pub(super) source_span: Option, } impl EvaluationInteger { - pub(super) fn new(value: EvaluationIntegerValue, source_span: SpanRange) -> Self { - Self { value, source_span } - } - pub(super) fn for_litint(lit: &syn::LitInt) -> ExecutionResult { Ok(Self { - source_span: lit.span().span_range(), value: EvaluationIntegerValue::for_litint(lit)?, + source_span: Some(lit.span()), }) } - pub(crate) fn try_into_i128(self) -> ExecutionResult { - let option_of_fallback = match self.value { + pub(crate) fn try_into_i128(self) -> Option { + match self.value { EvaluationIntegerValue::Untyped(x) => x.parse_fallback().ok(), EvaluationIntegerValue::U8(x) => Some(x.into()), EvaluationIntegerValue::U16(x) => Some(x.into()), @@ -32,19 +31,13 @@ impl EvaluationInteger { EvaluationIntegerValue::I64(x) => Some(x.into()), EvaluationIntegerValue::I128(x) => Some(x), EvaluationIntegerValue::Isize(x) => x.try_into().ok(), - }; - match option_of_fallback { - Some(value) => Ok(value), - None => self - .source_span - .execution_err("The integer does not fit in a i128".to_string()), } } pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), EvaluationIntegerValue::U8(input) => input.handle_unary_operation(&operation), @@ -66,7 +59,7 @@ impl EvaluationInteger { self, right: EvaluationInteger, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => { input.handle_integer_binary_operation(right, &operation) @@ -110,16 +103,10 @@ impl EvaluationInteger { } } - pub(super) fn to_literal(&self) -> Literal { + pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { self.value .to_unspanned_literal() - .with_span(self.source_span.start()) - } -} - -impl quote::ToTokens for EvaluationInteger { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_literal().to_tokens(tokens) + .with_span(self.source_span.unwrap_or(fallback_span)) } } @@ -143,7 +130,7 @@ impl EvaluationIntegerValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::U8(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -179,6 +166,7 @@ pub(super) enum IntegerKind { Usize, } +#[derive(Clone)] pub(super) enum EvaluationIntegerValue { Untyped(UntypedInteger), U8(u8), @@ -256,6 +244,7 @@ impl EvaluationIntegerValue { } } +#[derive(Clone)] pub(super) struct UntypedInteger( /// The span of the literal is ignored, and will be set when converted to an output. syn::LitInt, @@ -275,7 +264,7 @@ impl UntypedInteger { pub(super) fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let input = self.parse_fallback()?; match operation.operator { UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), @@ -313,7 +302,7 @@ impl UntypedInteger { self, rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft => match rhs.value { @@ -357,7 +346,7 @@ impl UntypedInteger { self, rhs: Self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; let overflow_error = || { @@ -437,13 +426,12 @@ impl UntypedInteger { } } -impl ToEvaluationOutput for UntypedInteger { - fn to_output(self, span_range: SpanRange) -> EvaluationOutput { - EvaluationValue::Integer(EvaluationInteger::new( - EvaluationIntegerValue::Untyped(self), - span_range, - )) - .into() +impl ToEvaluationValue for UntypedInteger { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::Integer(EvaluationInteger { + value: EvaluationIntegerValue::Untyped(self), + source_span, + }) } } @@ -452,14 +440,17 @@ macro_rules! impl_int_operations_except_unary { ( $($integer_enum_variant:ident($integer_type:ident)),* $(,)? ) => {$( - impl ToEvaluationOutput for $integer_type { - fn to_output(self, span_range: SpanRange) -> EvaluationOutput { - EvaluationValue::Integer(EvaluationInteger::new(EvaluationIntegerValue::$integer_enum_variant(self), span_range)).into() + impl ToEvaluationValue for $integer_type { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::Integer(EvaluationInteger { + value: EvaluationIntegerValue::$integer_enum_variant(self), + source_span, + }) } } impl HandleBinaryOperation for $integer_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { let lhs = self; let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.operator.symbol(), rhs); match operation.paired_operator() { @@ -486,7 +477,7 @@ macro_rules! impl_int_operations_except_unary { self, rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self; match operation.integer_operator() { IntegerBinaryOperator::ShiftLeft => { @@ -532,7 +523,7 @@ macro_rules! impl_int_operations_except_unary { macro_rules! impl_unsigned_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self), UnaryOperator::Neg @@ -568,7 +559,7 @@ macro_rules! impl_unsigned_unary_operations { macro_rules! impl_signed_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self), UnaryOperator::Neg => operation.output(-self), @@ -604,7 +595,7 @@ impl HandleUnaryOperation for u8 { fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self), UnaryOperator::Neg | UnaryOperator::Not => { diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 0d3a658b..bfc3c027 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -3,17 +3,17 @@ use super::*; pub(super) enum EvaluationOperator { Unary { operation: UnaryOperation, - input: Option, + input: Option, }, Binary { operation: BinaryOperation, - left_input: Option, - right_input: Option, + left_input: Option, + right_input: Option, }, } impl EvaluationOperator { - pub(super) fn evaluate(self) -> ExecutionResult { + pub(super) fn evaluate(self) -> ExecutionResult { const OPERATOR_INPUT_EXPECT_STR: &str = "Handling children on the stack ordering should ensure the parent input is always set when the parent is evaluated"; match self { @@ -34,8 +34,8 @@ impl EvaluationOperator { } pub(super) struct UnaryOperation { - pub(super) span_for_output: SpanRange, - pub(super) operator_span: SpanRange, + pub(super) source_span: Option, + pub(super) operator_span: Span, pub(super) operator: UnaryOperator, } @@ -47,7 +47,7 @@ impl UnaryOperation { pub(super) fn unsupported_for_value_type_err( &self, value_type: &'static str, - ) -> ExecutionResult { + ) -> ExecutionResult { Err(self.error(&format!( "The {} operator is not supported for {} values", self.operator.symbol(), @@ -55,15 +55,15 @@ impl UnaryOperation { ))) } - pub(super) fn err(&self, error_message: &'static str) -> ExecutionResult { + pub(super) fn err(&self, error_message: &'static str) -> ExecutionResult { Err(self.error(error_message)) } pub(super) fn output( &self, - output_value: impl ToEvaluationOutput, - ) -> ExecutionResult { - Ok(output_value.to_output(self.span_for_output)) + output_value: impl ToEvaluationValue, + ) -> ExecutionResult { + Ok(output_value.to_value(self.source_span)) } pub(super) fn for_cast_expression(expr: &syn::ExprCast) -> ExecutionResult { @@ -116,24 +116,26 @@ impl UnaryOperation { } Ok(Self { - span_for_output: expr.as_token.span_range(), - operator_span: expr.as_token.span_range(), + source_span: None, + operator_span: expr.as_token.span(), operator: UnaryOperator::Cast(extract_type(&expr.ty)?), }) } pub(super) fn for_group_expression(expr: &syn::ExprGroup) -> ExecutionResult { + let span = expr.group_token.span; Ok(Self { - span_for_output: expr.group_token.span.span_range(), - operator_span: expr.group_token.span.span_range(), + source_span: Some(span), + operator_span: span, operator: UnaryOperator::NoOp, }) } pub(super) fn for_paren_expression(expr: &syn::ExprParen) -> ExecutionResult { + let span = expr.paren_token.span.join(); Ok(Self { - span_for_output: expr.paren_token.span.span_range(), - operator_span: expr.paren_token.span.span_range(), + source_span: Some(span), + operator_span: span, operator: UnaryOperator::NoOp, }) } @@ -144,19 +146,19 @@ impl UnaryOperation { UnOp::Not(_) => UnaryOperator::Not, other_unary_op => { return other_unary_op.execution_err( - "This unary operator is not supported in preinterpret expressions", + "This unary operator is not supported in a preinterpret expression", ); } }; Ok(Self { - span_for_output: expr.op.span_range(), - operator_span: expr.op.span_range(), + source_span: None, + operator_span: expr.op.span(), operator, }) } - pub(super) fn evaluate(self, input: EvaluationOutput) -> ExecutionResult { - input.into_value().handle_unary_operation(self) + pub(super) fn evaluate(self, input: EvaluationValue) -> ExecutionResult { + input.handle_unary_operation(self) } } @@ -180,15 +182,14 @@ impl UnaryOperator { } pub(super) trait HandleUnaryOperation: Sized { - fn handle_unary_operation( - self, - operation: &UnaryOperation, - ) -> ExecutionResult; + fn handle_unary_operation(self, operation: &UnaryOperation) + -> ExecutionResult; } pub(super) struct BinaryOperation { - pub(super) span_for_output: SpanRange, - pub(super) operator_span: SpanRange, + /// Only present if there is a single span for the source tokens + pub(super) source_span: Option, + pub(super) operator_span: Span, pub(super) operator: BinaryOperator, } @@ -200,7 +201,7 @@ impl BinaryOperation { pub(super) fn unsupported_for_value_type_err( &self, value_type: &'static str, - ) -> ExecutionResult { + ) -> ExecutionResult { Err(self.error(&format!( "The {} operator is not supported for {} values", self.operator.symbol(), @@ -210,16 +211,16 @@ impl BinaryOperation { pub(super) fn output( &self, - output_value: impl ToEvaluationOutput, - ) -> ExecutionResult { - Ok(output_value.to_output(self.span_for_output)) + output_value: impl ToEvaluationValue, + ) -> ExecutionResult { + Ok(output_value.to_value(self.source_span)) } pub(super) fn output_if_some( &self, - output_value: Option, + output_value: Option, error_message: impl FnOnce() -> String, - ) -> ExecutionResult { + ) -> ExecutionResult { match output_value { Some(output_value) => self.output(output_value), None => self.operator_span.execution_err(error_message()), @@ -252,26 +253,28 @@ impl BinaryOperation { } }; Ok(Self { - span_for_output: expr.op.span_range(), - operator_span: expr.op.span_range(), + source_span: None, + operator_span: expr.op.span(), operator, }) } fn evaluate( self, - left: EvaluationOutput, - right: EvaluationOutput, - ) -> ExecutionResult { + left: EvaluationValue, + right: EvaluationValue, + ) -> ExecutionResult { match self.operator { BinaryOperator::Paired(operator) => { let value_pair = left.expect_value_pair(operator, right, self.operator_span)?; value_pair.handle_paired_binary_operation(self) } BinaryOperator::Integer(_) => { - let right = right.expect_integer("The shift amount must be an integer")?; - left.into_value() - .handle_integer_binary_operation(right, self) + let right = right.into_integer().ok_or_else(|| { + self.operator_span + .execution_error("The shift amount must be an integer") + })?; + left.handle_integer_binary_operation(right, self) } } } @@ -369,11 +372,11 @@ pub(super) trait HandleBinaryOperation: Sized { self, rhs: Self, operation: &BinaryOperation, - ) -> ExecutionResult; + ) -> ExecutionResult; fn handle_integer_binary_operation( self, rhs: EvaluationInteger, operation: &BinaryOperation, - ) -> ExecutionResult; + ) -> ExecutionResult; } diff --git a/src/expressions/string.rs b/src/expressions/string.rs index b708ef3b..7dd45c92 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -2,25 +2,23 @@ use super::*; pub(crate) struct EvaluationString { pub(super) value: String, - pub(super) source_span: SpanRange, + /// The span of the source code that generated this boolean value. + /// It may not have a value if generated from a complex expression. + pub(super) source_span: Option, } impl EvaluationString { - pub(super) fn new(value: String, source_span: SpanRange) -> Self { - Self { value, source_span } - } - pub(super) fn for_litstr(lit: &syn::LitStr) -> Self { Self { value: lit.value(), - source_span: lit.span().span_range(), + source_span: Some(lit.span()), } } pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation.operator { UnaryOperator::NoOp => operation.output(self.value), UnaryOperator::Neg | UnaryOperator::Not | UnaryOperator::Cast(_) => { @@ -33,7 +31,7 @@ impl EvaluationString { self, _right: EvaluationInteger, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { operation.unsupported_for_value_type_err("string") } @@ -41,7 +39,7 @@ impl EvaluationString { self, rhs: Self, operation: &BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation.paired_operator() { @@ -64,25 +62,25 @@ impl EvaluationString { } } - pub(super) fn to_literal(&self) -> Literal { - Literal::string(&self.value).with_span(self.source_span.start()) - } -} - -impl ToEvaluationOutput for String { - fn to_output(self, span: SpanRange) -> EvaluationOutput { - EvaluationValue::String(EvaluationString::new(self, span)).into() + pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { + Literal::string(&self.value).with_span(self.source_span.unwrap_or(fallback_span)) } } -impl ToEvaluationOutput for &str { - fn to_output(self, span: SpanRange) -> EvaluationOutput { - EvaluationValue::String(EvaluationString::new(self.to_string(), span)).into() +impl ToEvaluationValue for String { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::String(EvaluationString { + value: self, + source_span, + }) } } -impl quote::ToTokens for EvaluationString { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_literal().to_tokens(tokens) +impl ToEvaluationValue for &str { + fn to_value(self, source_span: Option) -> EvaluationValue { + EvaluationValue::String(EvaluationString { + value: self.to_string(), + source_span, + }) } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 7dc78925..12a88344 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -8,17 +8,11 @@ pub(crate) enum EvaluationValue { Char(EvaluationChar), } -impl EvaluationValue { - pub(super) fn into_token_tree(self) -> TokenTree { - match self { - Self::Integer(int) => int.to_literal().into(), - Self::Float(float) => float.to_literal().into(), - Self::Boolean(bool) => bool.to_ident().into(), - Self::String(string) => string.to_literal().into(), - Self::Char(char) => char.to_literal().into(), - } - } +pub(super) trait ToEvaluationValue: Sized { + fn to_value(self, source_span: Option) -> EvaluationValue; +} +impl EvaluationValue { pub(super) fn for_literal_expression(expr: &ExprLit) -> ExecutionResult { // https://docs.rs/syn/latest/syn/enum.Lit.html Ok(match &expr.lit { @@ -35,6 +29,215 @@ impl EvaluationValue { }) } + pub(super) fn expect_value_pair( + self, + operator: PairedBinaryOperator, + right: Self, + operator_span: Span, + ) -> ExecutionResult { + Ok(match (self, right) { + (EvaluationValue::Integer(left), EvaluationValue::Integer(right)) => { + let integer_pair = match (left.value, right.value) { + (EvaluationIntegerValue::Untyped(untyped_lhs), rhs) => match rhs { + EvaluationIntegerValue::Untyped(untyped_rhs) => { + EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationIntegerValue::U8(rhs) => { + EvaluationIntegerValuePair::U8(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U16(rhs) => { + EvaluationIntegerValuePair::U16(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U32(rhs) => { + EvaluationIntegerValuePair::U32(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U64(rhs) => { + EvaluationIntegerValuePair::U64(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::U128(rhs) => { + EvaluationIntegerValuePair::U128(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::Usize(rhs) => { + EvaluationIntegerValuePair::Usize(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I8(rhs) => { + EvaluationIntegerValuePair::I8(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I16(rhs) => { + EvaluationIntegerValuePair::I16(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I32(rhs) => { + EvaluationIntegerValuePair::I32(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I64(rhs) => { + EvaluationIntegerValuePair::I64(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::I128(rhs) => { + EvaluationIntegerValuePair::I128(untyped_lhs.parse_as()?, rhs) + } + EvaluationIntegerValue::Isize(rhs) => { + EvaluationIntegerValuePair::Isize(untyped_lhs.parse_as()?, rhs) + } + }, + (lhs, EvaluationIntegerValue::Untyped(untyped_rhs)) => match lhs { + EvaluationIntegerValue::Untyped(untyped_lhs) => { + EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationIntegerValue::U8(lhs) => { + EvaluationIntegerValuePair::U8(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U16(lhs) => { + EvaluationIntegerValuePair::U16(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U32(lhs) => { + EvaluationIntegerValuePair::U32(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U64(lhs) => { + EvaluationIntegerValuePair::U64(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::U128(lhs) => { + EvaluationIntegerValuePair::U128(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::Usize(lhs) => { + EvaluationIntegerValuePair::Usize(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I8(lhs) => { + EvaluationIntegerValuePair::I8(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I16(lhs) => { + EvaluationIntegerValuePair::I16(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I32(lhs) => { + EvaluationIntegerValuePair::I32(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I64(lhs) => { + EvaluationIntegerValuePair::I64(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::I128(lhs) => { + EvaluationIntegerValuePair::I128(lhs, untyped_rhs.parse_as()?) + } + EvaluationIntegerValue::Isize(lhs) => { + EvaluationIntegerValuePair::Isize(lhs, untyped_rhs.parse_as()?) + } + }, + (EvaluationIntegerValue::U8(lhs), EvaluationIntegerValue::U8(rhs)) => { + EvaluationIntegerValuePair::U8(lhs, rhs) + } + (EvaluationIntegerValue::U16(lhs), EvaluationIntegerValue::U16(rhs)) => { + EvaluationIntegerValuePair::U16(lhs, rhs) + } + (EvaluationIntegerValue::U32(lhs), EvaluationIntegerValue::U32(rhs)) => { + EvaluationIntegerValuePair::U32(lhs, rhs) + } + (EvaluationIntegerValue::U64(lhs), EvaluationIntegerValue::U64(rhs)) => { + EvaluationIntegerValuePair::U64(lhs, rhs) + } + (EvaluationIntegerValue::U128(lhs), EvaluationIntegerValue::U128(rhs)) => { + EvaluationIntegerValuePair::U128(lhs, rhs) + } + (EvaluationIntegerValue::Usize(lhs), EvaluationIntegerValue::Usize(rhs)) => { + EvaluationIntegerValuePair::Usize(lhs, rhs) + } + (EvaluationIntegerValue::I8(lhs), EvaluationIntegerValue::I8(rhs)) => { + EvaluationIntegerValuePair::I8(lhs, rhs) + } + (EvaluationIntegerValue::I16(lhs), EvaluationIntegerValue::I16(rhs)) => { + EvaluationIntegerValuePair::I16(lhs, rhs) + } + (EvaluationIntegerValue::I32(lhs), EvaluationIntegerValue::I32(rhs)) => { + EvaluationIntegerValuePair::I32(lhs, rhs) + } + (EvaluationIntegerValue::I64(lhs), EvaluationIntegerValue::I64(rhs)) => { + EvaluationIntegerValuePair::I64(lhs, rhs) + } + (EvaluationIntegerValue::I128(lhs), EvaluationIntegerValue::I128(rhs)) => { + EvaluationIntegerValuePair::I128(lhs, rhs) + } + (EvaluationIntegerValue::Isize(lhs), EvaluationIntegerValue::Isize(rhs)) => { + EvaluationIntegerValuePair::Isize(lhs, rhs) + } + (left_value, right_value) => { + return operator_span.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + } + }; + EvaluationLiteralPair::Integer(integer_pair) + } + (EvaluationValue::Boolean(left), EvaluationValue::Boolean(right)) => { + EvaluationLiteralPair::BooleanPair(left, right) + } + (EvaluationValue::Float(left), EvaluationValue::Float(right)) => { + let float_pair = match (left.value, right.value) { + (EvaluationFloatValue::Untyped(untyped_lhs), rhs) => match rhs { + EvaluationFloatValue::Untyped(untyped_rhs) => { + EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationFloatValue::F32(rhs) => { + EvaluationFloatValuePair::F32(untyped_lhs.parse_as()?, rhs) + } + EvaluationFloatValue::F64(rhs) => { + EvaluationFloatValuePair::F64(untyped_lhs.parse_as()?, rhs) + } + }, + (lhs, EvaluationFloatValue::Untyped(untyped_rhs)) => match lhs { + EvaluationFloatValue::Untyped(untyped_lhs) => { + EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) + } + EvaluationFloatValue::F32(lhs) => { + EvaluationFloatValuePair::F32(lhs, untyped_rhs.parse_as()?) + } + EvaluationFloatValue::F64(lhs) => { + EvaluationFloatValuePair::F64(lhs, untyped_rhs.parse_as()?) + } + }, + (EvaluationFloatValue::F32(lhs), EvaluationFloatValue::F32(rhs)) => { + EvaluationFloatValuePair::F32(lhs, rhs) + } + (EvaluationFloatValue::F64(lhs), EvaluationFloatValue::F64(rhs)) => { + EvaluationFloatValuePair::F64(lhs, rhs) + } + (left_value, right_value) => { + return operator_span.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + } + }; + EvaluationLiteralPair::Float(float_pair) + } + (EvaluationValue::String(left), EvaluationValue::String(right)) => { + EvaluationLiteralPair::StringPair(left, right) + } + (EvaluationValue::Char(left), EvaluationValue::Char(right)) => { + EvaluationLiteralPair::CharPair(left, right) + } + (left, right) => { + return operator_span.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); + } + }) + } + + pub(crate) fn into_integer(self) -> Option { + match self { + EvaluationValue::Integer(value) => Some(value), + _ => None, + } + } + + pub(crate) fn into_bool(self) -> Option { + match self { + EvaluationValue::Boolean(value) => Some(value), + _ => None, + } + } + + /// The span is used if there isn't already a span available + pub(super) fn to_token_tree(&self, fallback_output_span: Span) -> TokenTree { + match self { + Self::Integer(int) => int.to_literal(fallback_output_span).into(), + Self::Float(float) => float.to_literal(fallback_output_span).into(), + Self::Boolean(bool) => bool.to_ident(fallback_output_span).into(), + Self::String(string) => string.to_literal(fallback_output_span).into(), + Self::Char(char) => char.to_literal(fallback_output_span).into(), + } + } + pub(super) fn describe_type(&self) -> &'static str { match self { Self::Integer(int) => int.value.describe_type(), @@ -45,10 +248,20 @@ impl EvaluationValue { } } + pub(super) fn source_span(&self) -> Option { + match self { + Self::Integer(int) => int.source_span, + Self::Float(float) => float.source_span, + Self::Boolean(bool) => bool.source_span, + Self::String(str) => str.source_span, + Self::Char(char) => char.source_span, + } + } + pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { EvaluationValue::Integer(value) => value.handle_unary_operation(operation), EvaluationValue::Float(value) => value.handle_unary_operation(operation), @@ -62,7 +275,7 @@ impl EvaluationValue { self, right: EvaluationInteger, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { EvaluationValue::Integer(value) => { value.handle_integer_binary_operation(right, operation) @@ -79,34 +292,6 @@ impl EvaluationValue { EvaluationValue::Char(value) => value.handle_integer_binary_operation(right, operation), } } - - pub(super) fn source_span(&self) -> SpanRange { - match self { - EvaluationValue::Integer(value) => value.source_span, - EvaluationValue::Float(value) => value.source_span, - EvaluationValue::Boolean(value) => value.source_span, - EvaluationValue::String(value) => value.source_span, - EvaluationValue::Char(value) => value.source_span, - } - } -} - -impl HasSpanRange for EvaluationValue { - fn span_range(&self) -> SpanRange { - self.source_span() - } -} - -impl quote::ToTokens for EvaluationValue { - fn to_tokens(&self, tokens: &mut TokenStream) { - match self { - Self::Integer(value) => value.to_tokens(tokens), - Self::Float(value) => value.to_tokens(tokens), - Self::Boolean(value) => value.to_tokens(tokens), - Self::String(value) => value.to_tokens(tokens), - Self::Char(value) => value.to_tokens(tokens), - } - } } #[derive(Copy, Clone)] @@ -129,7 +314,7 @@ impl EvaluationLiteralPair { pub(super) fn handle_paired_binary_operation( self, operation: BinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { Self::Integer(pair) => pair.handle_paired_binary_operation(&operation), Self::Float(pair) => pair.handle_paired_binary_operation(&operation), diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index e5854011..94aef270 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -80,6 +80,7 @@ impl SpanRange { ::span(self) } + #[allow(unused)] pub(crate) fn start(&self) -> Span { self.start } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 132d888d..480d463b 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -492,7 +492,7 @@ impl Parse for Command { let invocation = command_kind.parse_invocation(CommandArguments::new( &content, command_name, - delim_span.span_range(), + delim_span.join(), ))?; Ok(Self { invocation, diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index 191e710f..e6042c2e 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -4,25 +4,29 @@ use crate::internal_prelude::*; pub(crate) struct CommandArguments<'a> { parse_stream: ParseStream<'a>, command_name: Ident, - /// The span range of the original stream, before tokens were consumed - full_span_range: SpanRange, + /// The span of the [ ... ] which contained the command + command_span: Span, } impl<'a> CommandArguments<'a> { pub(crate) fn new( parse_stream: ParseStream<'a>, command_name: Ident, - span_range: SpanRange, + command_span: Span, ) -> Self { Self { parse_stream, command_name, - full_span_range: span_range, + command_span, } } - pub(crate) fn full_span_range(&self) -> SpanRange { - self.full_span_range + pub(crate) fn command_span(&self) -> Span { + self.command_span + } + + pub(crate) fn command_span_range(&self) -> SpanRange { + self.command_span.span_range() } /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message @@ -30,7 +34,7 @@ impl<'a> CommandArguments<'a> { if self.parse_stream.is_empty() { Ok(()) } else { - self.full_span_range.parse_err(error_message) + self.command_span.parse_err(error_message) } } @@ -65,7 +69,7 @@ impl<'a> CommandArguments<'a> { pub(crate) fn parse_all_for_interpretation(&self) -> ParseResult { self.parse_stream - .parse_all_for_interpretation(self.full_span_range) + .parse_all_for_interpretation(self.command_span.span_range()) } pub(crate) fn read_all_as_raw_token_stream(&self) -> TokenStream { diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 832adfd4..baf7e208 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -54,8 +54,7 @@ impl ControlFlowCommandDefinition for IfCommand { let evaluated_condition = self .condition .evaluate(interpreter)? - .expect_bool("An if condition must evaluate to a boolean")? - .value(); + .expect_bool("An if condition must evaluate to a boolean")?; if evaluated_condition { return self.true_code.interpret_into(interpreter, output); @@ -64,8 +63,7 @@ impl ControlFlowCommandDefinition for IfCommand { for (condition, code) in self.else_ifs { let evaluated_condition = condition .evaluate(interpreter)? - .expect_bool("An else if condition must evaluate to a boolean")? - .value(); + .expect_bool("An else if condition must evaluate to a boolean")?; if evaluated_condition { return code.interpret_into(interpreter, output); @@ -118,8 +116,7 @@ impl ControlFlowCommandDefinition for WhileCommand { .condition .clone() .evaluate(interpreter)? - .expect_bool("An if condition must evaluate to a boolean")? - .value(); + .expect_bool("An if condition must evaluate to a boolean")?; if !evaluated_condition { break; @@ -259,7 +256,7 @@ impl NoOutputCommandDefinition for ContinueCommand { fn parse(arguments: CommandArguments) -> ParseResult { arguments.assert_empty("The !continue! command takes no arguments")?; Ok(Self { - span: arguments.full_span_range().span(), + span: arguments.command_span(), }) } @@ -286,7 +283,7 @@ impl NoOutputCommandDefinition for BreakCommand { fn parse(arguments: CommandArguments) -> ParseResult { arguments.assert_empty("The !break! command takes no arguments")?; Ok(Self { - span: arguments.full_span_range().span(), + span: arguments.command_span(), }) } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index e31e0304..16390ae8 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -21,7 +21,7 @@ impl NoOutputCommandDefinition for SetCommand { Ok(Self { variable: input.parse()?, equals: input.parse()?, - arguments: input.parse_with(arguments.full_span_range())?, + arguments: input.parse_with(arguments.command_span_range())?, }) }, "Expected [!set! #variable = ..]", @@ -56,7 +56,8 @@ impl NoOutputCommandDefinition for ExtendCommand { Ok(Self { variable: input.parse()?, plus_equals: input.parse()?, - arguments: input.parse_all_for_interpretation(arguments.full_span_range())?, + arguments: input + .parse_all_for_interpretation(arguments.command_span_range())?, }) }, "Expected [!extend! #variable += ..]", @@ -221,7 +222,7 @@ impl NoOutputCommandDefinition for ErrorCommand { } else { Ok(Self { inputs: EitherErrorInput::JustMessage( - input.parse_with(arguments.full_span_range())?, + input.parse_with(arguments.command_span_range())?, ), }) } diff --git a/src/interpretation/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs index 267d7431..4cf92526 100644 --- a/src/interpretation/commands/destructuring_commands.rs +++ b/src/interpretation/commands/destructuring_commands.rs @@ -21,7 +21,7 @@ impl NoOutputCommandDefinition for LetCommand { Ok(Self { destructuring: input.parse()?, equals: input.parse()?, - arguments: input.parse_with(arguments.full_span_range())?, + arguments: input.parse_with(arguments.command_span_range())?, }) }, "Expected [!let! = ...]", diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index d070938b..82c7e3a6 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -3,6 +3,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct EvaluateCommand { expression: ExpressionInput, + command_span: Span, } impl CommandType for EvaluateCommand { @@ -17,6 +18,7 @@ impl ValueCommandDefinition for EvaluateCommand { |input| { Ok(Self { expression: input.parse()?, + command_span: arguments.command_span(), }) }, "Expected [!evaluate! ...] containing a valid preinterpret expression", @@ -25,7 +27,7 @@ impl ValueCommandDefinition for EvaluateCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { let expression = self.expression.start_expression_builder(interpreter)?; - Ok(expression.evaluate()?.into_token_tree()) + Ok(expression.evaluate(self.command_span)?.to_token_tree()) } } @@ -36,6 +38,7 @@ pub(crate) struct AssignCommand { #[allow(unused)] equals: Token![=], expression: ExpressionInput, + command_span: Span, } impl CommandType for AssignCommand { @@ -64,6 +67,7 @@ impl NoOutputCommandDefinition for AssignCommand { }, equals: input.parse()?, expression: input.parse()?, + command_span: arguments.command_span(), }) }, "Expected [!assign! #variable = ] or [!assign! #variable X= ] for X one of + - * / % & | or ^", @@ -76,6 +80,7 @@ impl NoOutputCommandDefinition for AssignCommand { operator, equals: _, expression, + command_span, } = *self; let mut builder = ExpressionBuilder::new(); @@ -85,7 +90,7 @@ impl NoOutputCommandDefinition for AssignCommand { } builder.extend_with_evaluation_output(expression.evaluate(interpreter)?); - let output = builder.evaluate()?.into_token_tree(); + let output = builder.evaluate(command_span)?.to_token_tree(); variable.set(interpreter, output.into())?; Ok(()) @@ -130,13 +135,11 @@ impl StreamCommandDefinition for RangeCommand { let left = self .left .evaluate(interpreter)? - .expect_integer("The left side of the range must be an integer")? - .try_into_i128()?; + .try_into_i128("The left side of the range must be an i128-compatible integer")?; let right = self .right .evaluate(interpreter)? - .expect_integer("The right side of the range must be an integer")? - .try_into_i128()?; + .try_into_i128("The right side of the range must be an i128-compatible integer")?; if left > right { return Ok(()); From e602db8cfe561685930bf85fcba1f0d2ce2bdb94 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 28 Jan 2025 23:53:58 +0000 Subject: [PATCH 065/126] tweak: Try to fix tests on nightly --- .github/workflows/ci.yml | 2 ++ tests/control_flow.rs | 1 - tests/core.rs | 1 - tests/destructuring.rs | 5 ++++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bf38cc4..3f0760d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: - name: Enable type layout randomization run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV if: matrix.rust == 'nightly' + - name: Add TEST_RUST_MODE env variable + run: echo TEST_RUST_MODE=${{matrix.rust}} >> $GITHUB_ENV - run: cargo test - uses: actions/upload-artifact@v4 if: matrix.rust == 'nightly' && always() diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 26977503..e30ab4b2 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -12,7 +12,6 @@ macro_rules! assert_preinterpret_eq { #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_control_flow_compilation_failures() { let t = trybuild::TestCases::new(); - // In particular, the "error" command is tested here. t.compile_fail("tests/compilation_failures/control_flow/*.rs"); } diff --git a/tests/core.rs b/tests/core.rs index bd46e285..24cb8ca3 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -10,7 +10,6 @@ macro_rules! assert_preinterpret_eq { #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_core_compilation_failures() { let t = trybuild::TestCases::new(); - // In particular, the "error" command is tested here. t.compile_fail("tests/compilation_failures/core/*.rs"); } diff --git a/tests/destructuring.rs b/tests/destructuring.rs index 3bf9e6f3..71f41f8a 100644 --- a/tests/destructuring.rs +++ b/tests/destructuring.rs @@ -9,8 +9,11 @@ macro_rules! assert_preinterpret_eq { #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_destructuring_compilation_failures() { + if option_env!("TEST_RUST_MODE") == Some("nightly") { + // Some of the outputs are different on nightly, so don't test these + return; + } let t = trybuild::TestCases::new(); - // In particular, the "error" command is tested here. t.compile_fail("tests/compilation_failures/destructuring/*.rs"); } From cde9caa40d011ff4c95b4492d0361e9f2e527cba Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 30 Jan 2025 03:05:20 +0000 Subject: [PATCH 066/126] refactor: Improve expression parsing --- CHANGELOG.md | 14 +- src/expressions/boolean.rs | 11 +- src/expressions/character.rs | 9 +- src/expressions/evaluation.rs | 201 +++++++ src/expressions/evaluation_tree.rs | 309 ----------- src/expressions/expression.rs | 73 +++ src/expressions/expression_parsing.rs | 504 ++++++++++++++++++ src/expressions/expression_stream.rs | 264 --------- src/expressions/float.rs | 33 +- src/expressions/integer.rs | 46 +- src/expressions/mod.rs | 10 +- src/expressions/operations.rs | 213 +++----- src/expressions/string.rs | 7 +- src/expressions/value.rs | 7 +- src/extensions/errors_and_spans.rs | 11 + src/extensions/parsing.rs | 73 +++ src/internal_prelude.rs | 2 +- src/interpretation/command.rs | 117 +--- src/interpretation/command_arguments.rs | 3 +- .../commands/control_flow_commands.rs | 6 +- .../commands/expression_commands.rs | 42 +- src/interpretation/interpretation_stream.rs | 16 - src/interpretation/interpreted_stream.rs | 4 - src/interpretation/variable.rs | 27 - src/misc/mod.rs | 18 + .../expressions/braces.stderr | 6 - .../expressions/{braces.rs => brackets.rs} | 2 +- .../expressions/brackets.stderr | 6 + .../fix_me_negative_max_int_fails.stderr | 1 + .../flattened_commands_in_expressions.stderr | 7 +- .../flattened_variables_in_expressions.stderr | 7 +- ...variable_with_incomplete_expression.stderr | 8 +- .../expressions/inner_braces.stderr | 6 - .../{inner_braces.rs => inner_brackets.rs} | 2 +- .../expressions/inner_brackets.stderr | 6 + .../no_output_commands_in_expressions.stderr | 7 +- tests/expressions.rs | 18 +- 37 files changed, 1123 insertions(+), 973 deletions(-) create mode 100644 src/expressions/evaluation.rs delete mode 100644 src/expressions/evaluation_tree.rs create mode 100644 src/expressions/expression.rs create mode 100644 src/expressions/expression_parsing.rs delete mode 100644 src/expressions/expression_stream.rs delete mode 100644 tests/compilation_failures/expressions/braces.stderr rename tests/compilation_failures/expressions/{braces.rs => brackets.rs} (71%) create mode 100644 tests/compilation_failures/expressions/brackets.stderr delete mode 100644 tests/compilation_failures/expressions/inner_braces.stderr rename tests/compilation_failures/expressions/{inner_braces.rs => inner_brackets.rs} (69%) create mode 100644 tests/compilation_failures/expressions/inner_brackets.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 56efe385..952e6897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,13 +76,16 @@ Destructuring performs parsing of a token stream. It supports: ### To come +* Complete expression rework: + * Enable lazy && and || + * Enable support for code blocks { .. } in expressions + * Flatten `UnaryOperation` and `UnaryOperator` + * Flatten `BinaryOperation` and `BinaryOperator` + * Search / resolve TODOs * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` * Change `(!content! ...)` to unwrap none groups, to be more permissive * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` -* Add more tests - * e.g. for various expressions - * e.g. for long sums * Destructurers * `(!fields! ...)` and `(!subfields! ...)` * Add ability to fork (copy on write?) / revert the interpreter state and can then add: @@ -93,11 +96,6 @@ Destructuring performs parsing of a token stream. It supports: * `(!any! ...)` (with `#..x` as a catch-all) like the [!match!] command but without arms... * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much * Check all `#[allow(unused)]` and remove any which aren't needed -* Rework expression parsing, in order to: - * Fix comments in the expression files - * Enable lazy && and || - * Enable support for code blocks { .. } in expressions, and remove hacks where expression parsing stops at {} or . - * Remove stack overflow possibilities when parsing a long nested expression * Pushed to 0.4: * Fork of syn to: * Fix issues in Rust Analyzer diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 2b3fdb08..af9ddea3 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -1,5 +1,6 @@ use super::*; +#[derive(Clone)] pub(crate) struct EvaluationBoolean { pub(super) value: bool, /// The span of the source code that generated this boolean value. @@ -8,7 +9,7 @@ pub(crate) struct EvaluationBoolean { } impl EvaluationBoolean { - pub(super) fn for_litbool(lit: &syn::LitBool) -> Self { + pub(super) fn for_litbool(lit: syn::LitBool) -> Self { Self { source_span: Some(lit.span()), value: lit.value, @@ -21,10 +22,10 @@ impl EvaluationBoolean { ) -> ExecutionResult { let input = self.value; match operation.operator { - UnaryOperator::Neg => operation.unsupported_for_value_type_err("boolean"), - UnaryOperator::Not => operation.output(!input), - UnaryOperator::NoOp => operation.output(input), - UnaryOperator::Cast(target) => match target { + UnaryOperator::Neg { .. } => operation.unsupported_for_value_type_err("boolean"), + UnaryOperator::Not { .. } => operation.output(!input), + UnaryOperator::GroupedNoOp { .. } => operation.output(input), + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index ef7403c0..ccc01a01 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -1,5 +1,6 @@ use super::*; +#[derive(Clone)] pub(crate) struct EvaluationChar { pub(super) value: char, /// The span of the source code that generated this boolean value. @@ -8,7 +9,7 @@ pub(crate) struct EvaluationChar { } impl EvaluationChar { - pub(super) fn for_litchar(lit: &syn::LitChar) -> Self { + pub(super) fn for_litchar(lit: syn::LitChar) -> Self { Self { value: lit.value(), source_span: Some(lit.span()), @@ -21,11 +22,11 @@ impl EvaluationChar { ) -> ExecutionResult { let char = self.value; match operation.operator { - UnaryOperator::NoOp => operation.output(char), - UnaryOperator::Neg | UnaryOperator::Not => { + UnaryOperator::GroupedNoOp { .. } => operation.output(char), + UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } => { operation.unsupported_for_value_type_err("char") } - UnaryOperator::Cast(target) => match target { + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(char as FallbackInteger)) } diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs new file mode 100644 index 00000000..c8e7c0b6 --- /dev/null +++ b/src/expressions/evaluation.rs @@ -0,0 +1,201 @@ +use super::*; + +pub(super) struct ExpressionEvaluator<'a> { + nodes: &'a [ExpressionNode], + operation_stack: Vec, +} + +impl<'a> ExpressionEvaluator<'a> { + pub(super) fn new(nodes: &'a [ExpressionNode]) -> Self { + Self { + nodes, + operation_stack: Vec::new(), + } + } + + pub(super) fn evaluate( + mut self, + root: ExpressionNodeId, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut next = self.begin_node_evaluation(root, interpreter)?; + + loop { + match next { + NextAction::HandleValue(evaluation_value) => { + let top_of_stack = match self.operation_stack.pop() { + Some(top) => top, + None => return Ok(evaluation_value), + }; + next = self.continue_node_evaluation(top_of_stack, evaluation_value)?; + } + NextAction::EnterNode(next_node) => { + next = self.begin_node_evaluation(next_node, interpreter)?; + } + } + } + } + + fn begin_node_evaluation( + &mut self, + node_id: ExpressionNodeId, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + Ok(match &self.nodes[node_id.0] { + ExpressionNode::Leaf(leaf) => { + let interpreted = match leaf { + ExpressionLeaf::Command(command) => { + command.clone().interpret_to_new_stream(interpreter)? + } + ExpressionLeaf::GroupedVariable(grouped_variable) => { + grouped_variable.interpret_to_new_stream(interpreter)? + } + ExpressionLeaf::CodeBlock(code_block) => { + code_block.clone().interpret_to_new_stream(interpreter)? + } + ExpressionLeaf::Value(value) => { + return Ok(NextAction::HandleValue(value.clone())) + } + }; + let parsed_expression = unsafe { + // RUST-ANALYZER SAFETY: This isn't very safe, as it could have a none-delimited group in it + interpreted.syn_parse(Expression::parse)? + }; + NextAction::HandleValue(parsed_expression.evaluate_to_value(interpreter)?) + } + ExpressionNode::UnaryOperation { operation, input } => { + self.operation_stack + .push(EvaluationStackFrame::UnaryOperation { + operation: operation.clone(), + }); + NextAction::EnterNode(*input) + } + ExpressionNode::BinaryOperation { + operation, + left_input, + right_input, + } => { + self.operation_stack + .push(EvaluationStackFrame::BinaryOperation { + operation: operation.clone(), + state: BinaryPath::OnLeftBranch { + right: *right_input, + }, + }); + NextAction::EnterNode(*left_input) + } + }) + } + + fn continue_node_evaluation( + &mut self, + top_of_stack: EvaluationStackFrame, + evaluation_value: EvaluationValue, + ) -> ExecutionResult { + Ok(match top_of_stack { + EvaluationStackFrame::UnaryOperation { operation } => { + let result = operation.evaluate(evaluation_value)?; + NextAction::HandleValue(result) + } + EvaluationStackFrame::BinaryOperation { operation, state } => match state { + BinaryPath::OnLeftBranch { right } => { + self.operation_stack + .push(EvaluationStackFrame::BinaryOperation { + operation, + state: BinaryPath::OnRightBranch { + left: evaluation_value, + }, + }); + NextAction::EnterNode(right) + } + BinaryPath::OnRightBranch { left } => { + let result = operation.evaluate(left, evaluation_value)?; + NextAction::HandleValue(result) + } + }, + }) + } +} + +enum NextAction { + HandleValue(EvaluationValue), + EnterNode(ExpressionNodeId), +} + +enum EvaluationStackFrame { + UnaryOperation { + operation: UnaryOperation, + }, + BinaryOperation { + operation: BinaryOperation, + state: BinaryPath, + }, +} + +enum BinaryPath { + OnLeftBranch { right: ExpressionNodeId }, + OnRightBranch { left: EvaluationValue }, +} + +pub(crate) struct EvaluationOutput { + pub(super) value: EvaluationValue, + pub(super) fallback_output_span: Span, +} + +impl EvaluationOutput { + #[allow(unused)] + pub(super) fn into_value(self) -> EvaluationValue { + self.value + } + + #[allow(unused)] + pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { + let error_span = self.span(); + match self.value.into_integer() { + Some(integer) => Ok(integer), + None => error_span.execution_err(error_message), + } + } + + pub(crate) fn try_into_i128(self, error_message: &str) -> ExecutionResult { + let error_span = self.span(); + match self + .value + .into_integer() + .and_then(|integer| integer.try_into_i128()) + { + Some(integer) => Ok(integer), + None => error_span.execution_err(error_message), + } + } + + pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { + let error_span = self.span(); + match self.value.into_bool() { + Some(boolean) => Ok(boolean.value), + None => error_span.execution_err(error_message), + } + } + + pub(crate) fn to_token_tree(&self) -> TokenTree { + self.value.to_token_tree(self.fallback_output_span) + } +} + +impl HasSpanRange for EvaluationOutput { + fn span(&self) -> Span { + self.value + .source_span() + .unwrap_or(self.fallback_output_span) + } + + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + +impl quote::ToTokens for EvaluationOutput { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.to_token_tree().to_tokens(tokens); + } +} diff --git a/src/expressions/evaluation_tree.rs b/src/expressions/evaluation_tree.rs deleted file mode 100644 index ad4aea0a..00000000 --- a/src/expressions/evaluation_tree.rs +++ /dev/null @@ -1,309 +0,0 @@ -use super::*; - -pub(super) struct EvaluationTree { - /// We store the tree as a normalized stack of nodes to make it easier to evaluate - /// without risking hitting stack overflow issues for deeply unbalanced trees - evaluation_stack: Vec, - fallback_output_span: Span, -} - -impl EvaluationTree { - pub(super) fn build_from( - fallback_output_span: Span, - expression: &Expr, - ) -> ExecutionResult { - EvaluationTreeBuilder::new(expression).build(fallback_output_span) - } - - pub(super) fn evaluate(mut self) -> ExecutionResult { - loop { - let EvaluationNode { result_placement, content } = self.evaluation_stack.pop() - .expect("The builder should ensure that the stack is non-empty and has a final element of a RootResult which results in a return below."); - let result = content.evaluate()?; - match result_placement { - ResultPlacement::RootResult => { - return Ok(EvaluationOutput { - value: result, - fallback_output_span: self.fallback_output_span, - }) - } - ResultPlacement::UnaryOperationInput { - parent_node_stack_index, - } => { - self.evaluation_stack[parent_node_stack_index] - .content - .set_unary_input(result); - } - ResultPlacement::BinaryOperationLeftChild { - parent_node_stack_index, - } => { - self.evaluation_stack[parent_node_stack_index] - .content - .set_binary_left_input(result); - } - ResultPlacement::BinaryOperationRightChild { - parent_node_stack_index, - } => { - self.evaluation_stack[parent_node_stack_index] - .content - .set_binary_right_input(result); - } - } - } - } -} - -struct EvaluationTreeBuilder<'a> { - work_stack: Vec<(&'a Expr, ResultPlacement)>, - evaluation_stack: Vec, -} - -impl<'a> EvaluationTreeBuilder<'a> { - fn new(expression: &'a Expr) -> Self { - Self { - work_stack: vec![(expression, ResultPlacement::RootResult)], - evaluation_stack: Vec::new(), - } - } - - /// Attempts to construct a preinterpret expression tree from a syn [Expr]. - /// It tries to align with the [rustc expression] building approach. - /// [rustc expression]: https://doc.rust-lang.org/reference/expressions.html - fn build(mut self, fallback_output_span: Span) -> ExecutionResult { - while let Some((expression, placement)) = self.work_stack.pop() { - match expression { - Expr::Binary(expr) => { - self.add_binary_operation( - placement, - BinaryOperation::for_binary_expression(expr)?, - &expr.left, - &expr.right, - ); - } - Expr::Cast(expr) => { - self.add_unary_operation( - placement, - UnaryOperation::for_cast_expression(expr)?, - &expr.expr, - ); - } - Expr::Group(expr) => { - // We handle these as a no-op operation so that they get the span from the group - self.add_unary_operation( - placement, - UnaryOperation::for_group_expression(expr)?, - &expr.expr, - ); - } - Expr::Lit(expr) => { - self.add_literal(placement, EvaluationValue::for_literal_expression(expr)?); - } - Expr::Paren(expr) => { - // We handle these as a no-op operation so that they get the span from the paren - self.add_unary_operation( - placement, - UnaryOperation::for_paren_expression(expr)?, - &expr.expr, - ); - } - Expr::Unary(expr) => { - self.add_unary_operation( - placement, - UnaryOperation::for_unary_expression(expr)?, - &expr.expr, - ); - } - other_expression => { - return other_expression - .span_range_from_iterating_over_all_tokens() - .execution_err( - "This expression is not supported in preinterpret expressions", - ); - } - } - } - Ok(EvaluationTree { - evaluation_stack: self.evaluation_stack, - fallback_output_span, - }) - } - - fn add_binary_operation( - &mut self, - result_placement: ResultPlacement, - operation: BinaryOperation, - lhs: &'a Expr, - rhs: &'a Expr, - ) { - let parent_node_stack_index = self.evaluation_stack.len(); - self.evaluation_stack.push(EvaluationNode { - result_placement, - content: EvaluationNodeContent::Operator(EvaluationOperator::Binary { - operation, - left_input: None, - right_input: None, - }), - }); - // Note - we put the lhs towards the end of the stack so it's evaluated first - self.work_stack.push(( - rhs, - ResultPlacement::BinaryOperationRightChild { - parent_node_stack_index, - }, - )); - self.work_stack.push(( - lhs, - ResultPlacement::BinaryOperationLeftChild { - parent_node_stack_index, - }, - )); - } - - fn add_unary_operation( - &mut self, - placement: ResultPlacement, - operation: UnaryOperation, - input: &'a Expr, - ) { - let parent_node_stack_index = self.evaluation_stack.len(); - self.evaluation_stack.push(EvaluationNode { - result_placement: placement, - content: EvaluationNodeContent::Operator(EvaluationOperator::Unary { - operation, - input: None, - }), - }); - self.work_stack.push(( - input, - ResultPlacement::UnaryOperationInput { - parent_node_stack_index, - }, - )); - } - - fn add_literal(&mut self, placement: ResultPlacement, literal: EvaluationValue) { - self.evaluation_stack.push(EvaluationNode { - result_placement: placement, - content: EvaluationNodeContent::Value(literal), - }); - } -} - -struct EvaluationNode { - result_placement: ResultPlacement, - content: EvaluationNodeContent, -} - -enum EvaluationNodeContent { - Value(EvaluationValue), - Operator(EvaluationOperator), -} - -impl EvaluationNodeContent { - fn evaluate(self) -> ExecutionResult { - match self { - Self::Value(value) => Ok(value), - Self::Operator(operator) => operator.evaluate(), - } - } - - fn set_unary_input(&mut self, input: EvaluationValue) { - match self { - Self::Operator(EvaluationOperator::Unary { - input: existing_input, - .. - }) => *existing_input = Some(input), - _ => panic!("Attempted to set unary input on a non-unary operator"), - } - } - - fn set_binary_left_input(&mut self, input: EvaluationValue) { - match self { - Self::Operator(EvaluationOperator::Binary { - left_input: existing_input, - .. - }) => *existing_input = Some(input), - _ => panic!("Attempted to set binary left input on a non-binary operator"), - } - } - - fn set_binary_right_input(&mut self, input: EvaluationValue) { - match self { - Self::Operator(EvaluationOperator::Binary { - right_input: existing_input, - .. - }) => *existing_input = Some(input), - _ => panic!("Attempted to set binary right input on a non-binary operator"), - } - } -} - -pub(crate) struct EvaluationOutput { - value: EvaluationValue, - fallback_output_span: Span, -} - -impl EvaluationOutput { - #[allow(unused)] - pub(super) fn into_value(self) -> EvaluationValue { - self.value - } - - #[allow(unused)] - pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { - let error_span = self.span(); - match self.value.into_integer() { - Some(integer) => Ok(integer), - None => error_span.execution_err(error_message), - } - } - - pub(crate) fn try_into_i128(self, error_message: &str) -> ExecutionResult { - let error_span = self.span(); - match self - .value - .into_integer() - .and_then(|integer| integer.try_into_i128()) - { - Some(integer) => Ok(integer), - None => error_span.execution_err(error_message), - } - } - - pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { - let error_span = self.span(); - match self.value.into_bool() { - Some(boolean) => Ok(boolean.value), - None => error_span.execution_err(error_message), - } - } - - pub(crate) fn to_token_tree(&self) -> TokenTree { - self.value.to_token_tree(self.fallback_output_span) - } -} - -impl HasSpanRange for EvaluationOutput { - fn span(&self) -> Span { - self.value - .source_span() - .unwrap_or(self.fallback_output_span) - } - - fn span_range(&self) -> SpanRange { - self.span().span_range() - } -} - -impl quote::ToTokens for EvaluationOutput { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_token_tree().to_tokens(tokens); - } -} - -enum ResultPlacement { - RootResult, - UnaryOperationInput { parent_node_stack_index: usize }, - BinaryOperationLeftChild { parent_node_stack_index: usize }, - BinaryOperationRightChild { parent_node_stack_index: usize }, -} diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs new file mode 100644 index 00000000..e582843d --- /dev/null +++ b/src/expressions/expression.rs @@ -0,0 +1,73 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct Expression { + pub(super) root: ExpressionNodeId, + pub(super) span_range: SpanRange, + pub(super) nodes: std::rc::Rc<[ExpressionNode]>, +} + +impl HasSpanRange for Expression { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl Parse for Expression { + fn parse(input: ParseStream) -> ParseResult { + ExpressionParser::new(input).parse() + } +} + +impl Expression { + pub(crate) fn evaluate( + &self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + Ok(EvaluationOutput { + value: self.evaluate_to_value(interpreter)?, + fallback_output_span: self.span_range.span(), + }) + } + + pub(crate) fn evaluate_with_span( + &self, + interpreter: &mut Interpreter, + fallback_output_span: Span, + ) -> ExecutionResult { + Ok(EvaluationOutput { + value: self.evaluate_to_value(interpreter)?, + fallback_output_span, + }) + } + + pub(crate) fn evaluate_to_value( + &self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + ExpressionEvaluator::new(&self.nodes).evaluate(self.root, interpreter) + } +} + +#[derive(Clone, Copy)] +pub(super) struct ExpressionNodeId(pub(super) usize); + +pub(super) enum ExpressionNode { + Leaf(ExpressionLeaf), + UnaryOperation { + operation: UnaryOperation, + input: ExpressionNodeId, + }, + BinaryOperation { + operation: BinaryOperation, + left_input: ExpressionNodeId, + right_input: ExpressionNodeId, + }, +} + +pub(super) enum ExpressionLeaf { + Command(Command), + GroupedVariable(GroupedVariable), + CodeBlock(CommandCodeInput), + Value(EvaluationValue), +} diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs new file mode 100644 index 00000000..0d20d9a8 --- /dev/null +++ b/src/expressions/expression_parsing.rs @@ -0,0 +1,504 @@ +use super::*; + +pub(super) struct ExpressionParser<'a> { + streams: ParseStreamStack<'a>, + nodes: ExpressionNodes, + expression_stack: Vec, + span_range: SpanRange, +} + +impl<'a> ExpressionParser<'a> { + pub(super) fn new(input: ParseStream<'a>) -> Self { + Self { + streams: ParseStreamStack::new(input), + nodes: ExpressionNodes::new(), + expression_stack: Vec::with_capacity(10), + span_range: SpanRange::new_single(input.span()), + } + } + + pub(super) fn parse(mut self) -> ParseResult { + let mut work_item = self.push_stack_frame(ExpressionStackFrame::Root); + loop { + work_item = match work_item { + WorkItem::RequireUnaryAtom => { + let unary_atom = self.parse_unary_atom()?; + self.extend_with_unary_atom(unary_atom)? + } + WorkItem::TryParseAndApplyExtension { node } => { + let extension = self.parse_extension()?; + self.attempt_extension(node, extension)? + } + WorkItem::TryApplyAlreadyParsedExtension { node, extension } => { + self.attempt_extension(node, extension)? + } + WorkItem::Finished { root } => { + return Ok(self.nodes.complete(root, self.span_range)); + } + } + } + } + + fn extend_with_unary_atom(&mut self, unary_atom: UnaryAtom) -> ParseResult { + Ok(match unary_atom { + UnaryAtom::Command(command) => { + self.span_range.set_end(command.span()); + self.add_leaf(ExpressionLeaf::Command(command)) + } + UnaryAtom::GroupedVariable(variable) => { + self.span_range.set_end(variable.span()); + self.add_leaf(ExpressionLeaf::GroupedVariable(variable)) + } + UnaryAtom::CodeBlock(code_block) => { + self.span_range.set_end(code_block.span()); + self.add_leaf(ExpressionLeaf::CodeBlock(code_block)) + } + UnaryAtom::Value(value) => { + if let Some(span) = value.source_span() { + self.span_range.set_end(span); + } + self.add_leaf(ExpressionLeaf::Value(value)) + } + UnaryAtom::Group(delim_span) => { + self.span_range.set_end(delim_span.close()); + self.push_stack_frame(ExpressionStackFrame::Group { delim_span }) + } + UnaryAtom::UnaryOperation(operation) => { + self.span_range.set_end(operation.operator.span()); + self.push_stack_frame(ExpressionStackFrame::IncompletePrefixOperation { operation }) + } + }) + } + + fn attempt_extension( + &mut self, + node: ExpressionNodeId, + extension: NodeExtension, + ) -> ParseResult { + let parent_precedence = self.parent_precedence(); + let extension_precendence = extension.precedence(); + if extension_precendence > parent_precedence { + // If the extension has a higher precedence than the parent, then the left_node gets pulled into + // becoming part of the new operation + Ok(match extension { + NodeExtension::PostfixOperation(operation) => WorkItem::TryParseAndApplyExtension { + node: self.nodes.add_node(ExpressionNode::UnaryOperation { + operation, + input: node, + }), + }, + NodeExtension::BinaryOperation(operation) => { + self.push_stack_frame(ExpressionStackFrame::IncompleteBinaryOperation { + lhs: node, + operation, + }) + } + NodeExtension::NoneMatched => { + panic!("Not possible, as this has minimum precedence") + } + }) + } else { + // Otherwise, the node gets combined into the parent stack frame, and the extension + // is attempted to be applied against the next parent stack frame + let parent_stack_frame = self.pop_stack_frame(); + Ok(match parent_stack_frame { + ExpressionStackFrame::Root => { + assert!(matches!(extension, NodeExtension::NoneMatched)); + WorkItem::Finished { root: node } + } + ExpressionStackFrame::Group { delim_span } => { + assert!(matches!(extension, NodeExtension::NoneMatched)); + self.streams.exit_group(); + let operation = UnaryOperation { + operator: UnaryOperator::GroupedNoOp { + span: delim_span.join(), + }, + }; + WorkItem::TryParseAndApplyExtension { + node: self.nodes.add_node(ExpressionNode::UnaryOperation { + operation, + input: node, + }), + } + } + ExpressionStackFrame::IncompletePrefixOperation { operation } => { + WorkItem::TryApplyAlreadyParsedExtension { + node: self.nodes.add_node(ExpressionNode::UnaryOperation { + operation, + input: node, + }), + extension, + } + } + ExpressionStackFrame::IncompleteBinaryOperation { lhs, operation } => { + WorkItem::TryApplyAlreadyParsedExtension { + node: self.nodes.add_node(ExpressionNode::BinaryOperation { + operation, + left_input: lhs, + right_input: node, + }), + extension, + } + } + }) + } + } + + fn parse_extension(&mut self) -> ParseResult { + Ok(match self.streams.peek_grammar() { + PeekMatch::Punct(punct) => match punct.as_char() { + '.' => NodeExtension::NoneMatched, + _ => { + let operation = BinaryOperation::for_binary_operator(self.streams.parse()?)?; + NodeExtension::BinaryOperation(operation) + } + }, + PeekMatch::Ident(ident) if ident == "as" => { + let cast_operation = UnaryOperation::for_cast_operation(self.streams.parse()?, { + let target_type = self.streams.parse::()?; + self.span_range.set_end(target_type.span()); + target_type + })?; + NodeExtension::PostfixOperation(cast_operation) + } + _ => NodeExtension::NoneMatched, + }) + } + + fn parse_unary_atom(&mut self) -> ParseResult { + Ok(match self.streams.peek_grammar() { + PeekMatch::GroupedCommand(Some(command_kind)) => { + match command_kind.grouped_output_kind().expression_support() { + Ok(()) => UnaryAtom::Command(self.streams.parse()?), + Err(error_message) => return self.streams.parse_err(error_message), + } + } + PeekMatch::GroupedCommand(None) => return self.streams.parse_err("Invalid command"), + PeekMatch::FlattenedCommand(_) => return self.streams.parse_err(CommandOutputKind::FlattenedStream.expression_support().unwrap_err()), + PeekMatch::GroupedVariable => UnaryAtom::GroupedVariable(self.streams.parse()?), + PeekMatch::FlattenedVariable => return self.streams.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), + PeekMatch::AppendVariableDestructuring => return self.streams.parse_err("Append variable operations are not supported in an expression"), + PeekMatch::Destructurer(_) => return self.streams.parse_err("Destructurings are not supported in an expression"), + PeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { + let (_, delim_span) = self.streams.parse_and_enter_group()?; + UnaryAtom::Group(delim_span) + }, + PeekMatch::Group(Delimiter::Brace) => UnaryAtom::CodeBlock(self.streams.parse()?), + PeekMatch::Group(Delimiter::Bracket) => return self.streams.parse_err("Square brackets [ .. ] are not supported in an expression"), + PeekMatch::Punct(_) => { + let unary_operation = UnaryOperation::for_unary_operator(self.streams.parse()?)?; + UnaryAtom::UnaryOperation(unary_operation) + }, + PeekMatch::Ident(_) => { + UnaryAtom::Value(EvaluationValue::Boolean(EvaluationBoolean::for_litbool(self.streams.parse()?))) + }, + PeekMatch::Literal(_) => { + UnaryAtom::Value(EvaluationValue::for_literal(self.streams.parse()?)?) + }, + PeekMatch::End => return self.streams.parse_err("The expression ended in an incomplete state"), + }) + } + + fn add_leaf(&mut self, leaf: ExpressionLeaf) -> WorkItem { + let node = self.nodes.add_node(ExpressionNode::Leaf(leaf)); + WorkItem::TryParseAndApplyExtension { node } + } + + fn push_stack_frame(&mut self, frame: ExpressionStackFrame) -> WorkItem { + self.expression_stack.push(frame); + WorkItem::RequireUnaryAtom + } + + /// Panics if called after the Root has been popped + fn pop_stack_frame(&mut self) -> ExpressionStackFrame { + self.expression_stack + .pop() + .expect("There should always be at least a root") + } + + /// Panics if called after the Root has been popped + fn parent_precedence(&self) -> OperatorPrecendence { + self.expression_stack + .last() + .map(|s| s.precedence()) + .unwrap() + } +} + +pub(super) struct ExpressionNodes { + nodes: Vec, +} + +impl ExpressionNodes { + pub(super) fn new() -> Self { + Self { nodes: Vec::new() } + } + + pub(super) fn add_node(&mut self, node: ExpressionNode) -> ExpressionNodeId { + let node_id = ExpressionNodeId(self.nodes.len()); + self.nodes.push(node); + node_id + } + + pub(super) fn complete(self, root: ExpressionNodeId, span_range: SpanRange) -> Expression { + Expression { + root, + span_range, + nodes: self.nodes.into(), + } + } +} + +//=============================================================================================== +// The OperatorPrecendence is an amended copy of the Precedence enum from Syn. +// +// Syn is dual-licensed under MIT and Apache, and a subset of it is reproduced from version 2.0.96 +// of syn, and then further edited as a derivative work as part of preinterpret, which is released +// under the same licenses. +// +// LICENSE-MIT: https://github.com/dtolnay/syn/blob/2.0.96/LICENSE-MIT +// LICENSE-APACHE: https://github.com/dtolnay/syn/blob/2.0.96/LICENSE-APACHE +//=============================================================================================== + +/// This is an amended copy of the `Precedence` enum from Syn. +/// +/// Reference: https://doc.rust-lang.org/reference/expressions.html#expression-precedence +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[allow(unused)] +pub(crate) enum OperatorPrecendence { + // return, break, closures + Jump, + /// = += -= *= /= %= &= |= ^= <<= >>= + Assign, + // .. ..= + Range, + // || + Or, + // && + And, + // let + Let, + /// == != < > <= >= + Compare, + // | + BitOr, + // ^ + BitXor, + // & + BitAnd, + // << >> + Shift, + // + - + Sum, + // * / % + Product, + // as + Cast, + // unary - * ! & &mut + Prefix, + // paths, loops, function calls, array indexing, field expressions, method calls + Unambiguous, +} + +impl OperatorPrecendence { + pub(crate) const MIN: Self = OperatorPrecendence::Jump; + + pub(crate) fn of_unary_operator(op: &UnaryOperator) -> Self { + match op { + UnaryOperator::GroupedNoOp { .. } => Self::Unambiguous, + UnaryOperator::Cast { .. } => Self::Cast, + UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } => Self::Prefix, + } + } + + pub(crate) fn of_binary_operator(op: &BinaryOperator) -> Self { + match op { + BinaryOperator::Integer(op) => Self::of_integer_binary_operator(op), + BinaryOperator::Paired(op) => Self::of_paired_binary_operator(op), + } + } + + fn of_integer_binary_operator(op: &IntegerBinaryOperator) -> Self { + match op { + IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + OperatorPrecendence::Shift + } + } + } + + fn of_paired_binary_operator(op: &PairedBinaryOperator) -> Self { + match op { + PairedBinaryOperator::Addition | PairedBinaryOperator::Subtraction => Self::Sum, + PairedBinaryOperator::Multiplication + | PairedBinaryOperator::Division + | PairedBinaryOperator::Remainder => Self::Product, + PairedBinaryOperator::LogicalAnd => Self::And, + PairedBinaryOperator::LogicalOr => Self::Or, + PairedBinaryOperator::BitXor => Self::BitXor, + PairedBinaryOperator::BitAnd => Self::BitAnd, + PairedBinaryOperator::BitOr => Self::BitOr, + PairedBinaryOperator::Equal + | PairedBinaryOperator::LessThan + | PairedBinaryOperator::LessThanOrEqual + | PairedBinaryOperator::NotEqual + | PairedBinaryOperator::GreaterThanOrEqual + | PairedBinaryOperator::GreaterThan => Self::Compare, + } + } +} + +/// Use of a stack avoids recursion, which hits limits with long/deep expressions. +/// +/// ## Expression Types (in decreasing precedence) +/// * Leaf Expression: Command, Variable, Value (literal, true/false) +/// * Prefix Expression: prefix-based unary operators +/// * Postfix Expression: postfix-based unary operators such as casting +/// * Binary Expression: binary operators +/// +/// ## Worked Algorithm Sketch +/// Let ParseStream P be: +/// a b cdx e fg h i j k +/// 1 + -(1) + (2 + 4) * 3 +/// +/// The algorithm proceeds as follows: +/// ```text +/// => Start +/// ===> PushParseBuffer([P]) +/// ===> WorkStack: [Root] +/// => Root detects leaf a:1 +/// ===> PushEvalNode(A: Leaf(a:1), NewParentPrecedence: Root => >MIN) +/// ===> WorkStack: [Root, TryExtend(A, >MIN)] +/// => TryExtend detects binop b:+ +/// ===> WorkStack: [Root, BinOp(A, b:+)] +/// => BinOp detects unop c:- +/// ===> WorkStack: [Root, BinOp(A, b:+), PrefixOp(c:-)] +/// => PrefixOp detects group d:([D]) +/// ===> PushParseBuffer([D]) +/// ===> WorkStack: [Root, BinOp(A, b:+), PrefixOp(c:-), Group(d:Paren), EmptyExpression] +/// => EmptyExpression detects leaf x:1 +/// ===> PushEvalNode(X: Leaf(x:1), NewParentPrecedence: Group => >MIN) +/// ===> WorkStack: [Root, BinOp(A, +), PrefixOp(c:-), Group(d:Paren), TryExtend(X, >MIN)] +/// => TryExtend detects no valid extension, cascade X with Group: +/// ===> PopWorkStack: It's a group, so PopParseBuffer, PushEvalNode(D: UnOp(d:(), X), NewParentPrecedence: PrefixOp => >PREFIX) +/// ===> WorkStack: [Root, BinOp(A, b:+), PrefixOp(c:-), TryExtend(D, >PREFIX)] +/// => TryExtend detects no valid extension (it's impossible), cascade D with PrefixOp: +/// ===> PopWorkStack: PushEvalNode(C: UnOp(c:-, D), NewParentPrecedence: BinOp => >SUM) +/// ===> WorkStack: [Root, BinOp(A, b:+), TryExtend(C, >SUM)] +/// => TryExtend detects no valid extension, so cascade C with BinOp: +/// ===> PopWorkStack: PushEvalNode(B: BinOp(A, b:+, C), NewParentPrecedence: EMPTY => >MIN) +/// ===> WorkStack: [Root, TryExtend(B, >MIN)] +/// => TryExtend detects binop e:+ +/// ===> WorkStack: [Root, BinOp(B, e:+)] +/// => BinOp detects group f:([F]) +/// ===> PushParseBuffer([F]) +/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), EmptyExpression] +/// => EmptyExpression detects leaf g:2 +/// ===> PushEvalNode(G: Leaf(g:2), NewParentPrecedence: Group => >MIN) +/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), TryExtend(G, >MIN)] +/// => TryExtend detects binop h:+ +/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), BinOp(G, h:+)] +/// => BinOp detects leaf i:2 +/// ===> PushEvalNode(I: Leaf(i:2), NewParentPrecedence: BinOp => >SUM) +/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), BinOp(G, h:+), TryExtend(I, >SUM)] +/// => TryExtend detects no valid extension, so cascade I with BinOp: +/// ===> PopWorkStack: PushEvalNode(H: BinOp(G, h:+, I), NewParentPrecedence: Group => >MIN) +/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), TryExtend(H, >MIN)] +/// => TryExtend detects no valid extension, so cascade H with Group: +/// ===> PopWorkStack: It's a group, so PopParseBuffer, PushEvalNode(F: UnOp(f:Paren, H), NewParentPrecedence: BinOp => >SUM) +/// ===> WorkStack: [Root, BinOp(B, e:+), TryExtend(F, >SUM)] +/// => TryExtend detects binop j:* +/// ===> WorkStack: [Root, BinOp(B, e:+), BinOp(F, j:*)] +/// => BinOp detects leaf k:3 +/// ===> PushEvalNode(K: Leaf(k:3), NewParentPrecedence: BinOp => >PRODUCT) +/// ===> WorkStack: [Root, BinOp(B, e:+), BinOp(F, j:*), TryExtend(K, >PRODUCT)] +/// => TryExtend detects no valid extension, so cascade K with BinOp: +/// ===> PopWorkStack: PushEvalNode(J: BinOp(F, j:*, K), NewParentPrecedence: BinOp => >SUM) +/// ===> WorkStack: [Root, BinOp(B, e:+), TryExtend(J, >SUM)] +/// => TryExtend detects no valid extension, so cascade J with BinOp: +/// ===> PopWorkStack: PushEvalNode(E: BinOp(B, e:*, J), NewParentPrecedence: Root => >MIN) +/// ===> WorkStack: [Root, TryExtend(E, >MIN)] +/// => TryExtend detects no valid extension, so cascade E with Root: +/// ===> DONE Root = E +/// ``` +/// +/// TODO SPECIAL CASES: +/// * Some operators can't follow others, e.g. comparison operators can't be chained +enum ExpressionStackFrame { + /// A marker for the root of the expression + Root, + /// A marker for the parenthesized or transparent group. + /// * When the group is opened, we add its inside to the parse stream stack + /// * When the group is closed, we pop it from the parse stream stack + Group { delim_span: DelimSpan }, + /// An incomplete unary prefix operation + /// NB: unary postfix operations such as `as` casting go straight to ExtendableNode + IncompletePrefixOperation { operation: UnaryOperation }, + /// An incomplete binary operation + IncompleteBinaryOperation { + lhs: ExpressionNodeId, + operation: BinaryOperation, + }, +} + +impl ExpressionStackFrame { + fn precedence(&self) -> OperatorPrecendence { + match self { + ExpressionStackFrame::Root => OperatorPrecendence::MIN, + ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::IncompletePrefixOperation { operation, .. } => { + OperatorPrecendence::of_unary_operator(&operation.operator) + } + ExpressionStackFrame::IncompleteBinaryOperation { operation, .. } => { + OperatorPrecendence::of_binary_operator(&operation.operator) + } + } + } +} + +enum WorkItem { + /// We require reading a UnaryAtom, such as a command, variable, literal, or unary operation + RequireUnaryAtom, + /// We have a partial expression, which can possibly be extended to the right. + /// => If on the right, there is an operation of higher precedence than its parent, + /// it gets pulled into being part of the right expression + /// => Otherwise it will be combined into its parent + TryParseAndApplyExtension { + node: ExpressionNodeId, + }, + TryApplyAlreadyParsedExtension { + node: ExpressionNodeId, + extension: NodeExtension, + }, + Finished { + root: ExpressionNodeId, + }, +} + +enum UnaryAtom { + Command(Command), + GroupedVariable(GroupedVariable), + CodeBlock(CommandCodeInput), + Value(EvaluationValue), + Group(DelimSpan), + UnaryOperation(UnaryOperation), +} + +enum NodeExtension { + PostfixOperation(UnaryOperation), + BinaryOperation(BinaryOperation), + NoneMatched, +} + +impl NodeExtension { + fn precedence(&self) -> OperatorPrecendence { + match self { + NodeExtension::PostfixOperation(op) => { + OperatorPrecendence::of_unary_operator(&op.operator) + } + NodeExtension::BinaryOperation(op) => { + OperatorPrecendence::of_binary_operator(&op.operator) + } + NodeExtension::NoneMatched => OperatorPrecendence::MIN, + } + } +} diff --git a/src/expressions/expression_stream.rs b/src/expressions/expression_stream.rs deleted file mode 100644 index 8cf03822..00000000 --- a/src/expressions/expression_stream.rs +++ /dev/null @@ -1,264 +0,0 @@ -use super::*; - -pub(crate) trait Express: Sized { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()>; - - fn start_expression_builder( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let mut output = ExpressionBuilder::new(); - self.add_to_expression(interpreter, &mut output)?; - Ok(output) - } -} - -/// This abstraction is a bit ropey... -/// -/// Ideally we'd properly handle building expressions into an expression tree, -/// but that requires duplicating some portion of the syn parser for the rust expression tree... -/// -/// Instead, to be lazy for now, we interpret the stream at intepretation time to substitute -/// in variables and commands, and then parse the resulting expression with syn. -#[derive(Clone)] -pub(crate) struct ExpressionInput { - items: Vec, -} - -impl Parse for ExpressionInput { - fn parse(input: ParseStream) -> ParseResult { - let mut items = Vec::new(); - while !input.is_empty() { - // Until we create a proper ExpressionInput parser which builds up a syntax tree - // then we need to have a way to stop parsing... currently ExpressionInput comes - // before code blocks or .. in [!range!] so we can break on those. - // These aren't valid inside expressions we support anyway, so it's good enough for now. - let item = match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => ExpressionItem::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => ExpressionItem::Command(input.parse()?), - PeekMatch::GroupedVariable => ExpressionItem::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => ExpressionItem::FlattenedVariable(input.parse()?), - PeekMatch::Group(Delimiter::Brace | Delimiter::Bracket) => break, - PeekMatch::Group(_) => ExpressionItem::ExpressionGroup(input.parse()?), - PeekMatch::Destructurer(_) | PeekMatch::AppendVariableDestructuring => { - return input - .span() - .parse_err("Destructuring is not supported in an expression"); - } - PeekMatch::Punct(punct) if punct.as_char() == '.' => break, - PeekMatch::Punct(_) => ExpressionItem::Punct(input.parse_any_punct()?), - PeekMatch::Ident(_) => ExpressionItem::Ident(input.parse_any_ident()?), - PeekMatch::Literal(_) => ExpressionItem::Literal(input.parse()?), - PeekMatch::End => return input.span().parse_err("Expected an expression"), - }; - items.push(item); - } - if items.is_empty() { - return input.span().parse_err("Expected an expression"); - } - Ok(Self { items }) - } -} - -impl HasSpanRange for ExpressionInput { - fn span_range(&self) -> SpanRange { - SpanRange::new_between( - self.items.first().unwrap().span(), - self.items.last().unwrap().span(), - ) - } -} - -impl Express for ExpressionInput { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - for item in self.items { - item.add_to_expression(interpreter, builder)?; - } - Ok(()) - } -} - -impl ExpressionInput { - pub(crate) fn evaluate( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let span = self.span(); - self.start_expression_builder(interpreter)?.evaluate(span) - } -} - -#[derive(Clone)] -pub(crate) enum ExpressionItem { - Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), - ExpressionGroup(ExpressionGroup), - Punct(Punct), - Ident(Ident), - Literal(Literal), -} - -impl HasSpanRange for ExpressionItem { - fn span_range(&self) -> SpanRange { - match self { - ExpressionItem::Command(command) => command.span_range(), - ExpressionItem::GroupedVariable(grouped_variable) => grouped_variable.span_range(), - ExpressionItem::FlattenedVariable(flattened_variable) => { - flattened_variable.span_range() - } - ExpressionItem::ExpressionGroup(expression_group) => expression_group.span_range(), - ExpressionItem::Punct(punct) => punct.span_range(), - ExpressionItem::Ident(ident) => ident.span_range(), - ExpressionItem::Literal(literal) => literal.span_range(), - } - } -} - -impl Express for ExpressionItem { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - match self { - ExpressionItem::Command(command_invocation) => { - command_invocation.add_to_expression(interpreter, builder)?; - } - ExpressionItem::FlattenedVariable(variable) => { - variable.add_to_expression(interpreter, builder)?; - } - ExpressionItem::GroupedVariable(variable) => { - variable.add_to_expression(interpreter, builder)?; - } - ExpressionItem::ExpressionGroup(group) => { - group.add_to_expression(interpreter, builder)?; - } - ExpressionItem::Punct(punct) => builder.push_punct(punct), - ExpressionItem::Ident(ident) => builder.push_ident(ident), - ExpressionItem::Literal(literal) => builder.push_literal(literal), - } - Ok(()) - } -} - -#[derive(Clone)] -pub(crate) struct ExpressionGroup { - source_delimiter: Delimiter, - source_delim_span: DelimSpan, - content: ExpressionInput, -} - -impl Parse for ExpressionGroup { - fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, content) = input.parse_any_group()?; - Ok(Self { - source_delimiter: delimiter, - source_delim_span: delim_span, - content: content.parse()?, - }) - } -} - -impl Express for ExpressionGroup { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - builder.push_expression_group( - self.content.start_expression_builder(interpreter)?, - self.source_delimiter, - self.source_delim_span.join(), - ); - Ok(()) - } -} - -impl HasSpanRange for ExpressionGroup { - fn span_range(&self) -> SpanRange { - self.source_delim_span.span_range() - } -} - -#[derive(Clone)] -pub(crate) struct ExpressionBuilder { - interpreted_stream: InterpretedStream, -} - -impl ExpressionBuilder { - pub(crate) fn new() -> Self { - Self { - interpreted_stream: InterpretedStream::new(), - } - } - - pub(crate) fn push_literal(&mut self, literal: Literal) { - self.interpreted_stream.push_literal(literal); - } - - /// Only true and false make sense, but allow all here and catch others at evaluation time - pub(crate) fn push_ident(&mut self, ident: Ident) { - self.interpreted_stream.push_ident(ident); - } - - pub(crate) fn push_punct(&mut self, punct: Punct) { - self.interpreted_stream.push_punct(punct); - } - - pub(crate) fn push_grouped( - &mut self, - appender: impl FnOnce(&mut InterpretedStream) -> ExecutionResult<()>, - span: Span, - ) -> ExecutionResult<()> { - // Currently using Expr::Parse, it ignores transparent groups, which is a little too permissive. - // Instead, we use parentheses to ensure that the group has to be a valid expression itself, without being flattened. - // This also works around the SAFETY issue in syn_parse below - self.interpreted_stream - .push_grouped(appender, Delimiter::Parenthesis, span) - } - - pub(crate) fn extend_with_evaluation_output(&mut self, value: EvaluationOutput) { - self.interpreted_stream - .extend_with_raw_tokens_from(value.to_token_tree()); - } - - pub(crate) fn push_expression_group( - &mut self, - contents: Self, - delimiter: Delimiter, - span: Span, - ) { - self.interpreted_stream - .push_new_group(contents.interpreted_stream, delimiter, span); - } - - pub(crate) fn evaluate(self, fallback_output_span: Span) -> ExecutionResult { - // Parsing into a rust expression is overkill here. - // - // In future we could choose to implement a subset of the grammar which we actually can use/need. - // - // That said, it's useful for now for two reasons: - // * Aligning with rust syntax - // * Saving implementation work, particularly given that syn is likely pulled into most code bases anyway - // - // Because of the kind of expressions we're parsing (i.e. no {} allowed), - // we can get by with parsing it as `Expr::parse` rather than with - // `Expr::parse_without_eager_brace` or `Expr::parse_with_earlier_boundary_rule`. - let expression = unsafe { - // RUST-ANALYZER SAFETY: We wrap commands and variables in `()` instead of none-delimited groups in expressions, - // so it doesn't matter that we can drop none-delimited groups - self.interpreted_stream.syn_parse(::parse)? - }; - - EvaluationTree::build_from(fallback_output_span, &expression)?.evaluate() - } -} diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 3ed2c7ae..8fb8cbd2 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -1,6 +1,7 @@ use super::*; use crate::internal_prelude::*; +#[derive(Clone)] pub(crate) struct EvaluationFloat { pub(super) value: EvaluationFloatValue, /// The span of the source code that generated this boolean value. @@ -9,10 +10,11 @@ pub(crate) struct EvaluationFloat { } impl EvaluationFloat { - pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ExecutionResult { + pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { + let source_span = Some(lit.span()); Ok(Self { value: EvaluationFloatValue::for_litfloat(lit)?, - source_span: Some(lit.span()), + source_span, }) } @@ -71,6 +73,7 @@ impl EvaluationFloatValuePair { } } +#[derive(Clone)] pub(super) enum EvaluationFloatValue { Untyped(UntypedFloat), F32(f32), @@ -78,13 +81,13 @@ pub(super) enum EvaluationFloatValue { } impl EvaluationFloatValue { - pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ExecutionResult { + pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { Ok(match lit.suffix() { "" => Self::Untyped(UntypedFloat::new_from_lit_float(lit)), "f32" => Self::F32(lit.base10_parse()?), "f64" => Self::F64(lit.base10_parse()?), suffix => { - return lit.span().execution_err(format!( + return lit.span().parse_err(format!( "The literal suffix {suffix} is not supported in preinterpret expressions" )); } @@ -115,6 +118,7 @@ pub(super) enum FloatKind { F64, } +#[derive(Clone)] pub(super) struct UntypedFloat( /// The span of the literal is ignored, and will be set when converted to an output. LitFloat, @@ -122,9 +126,8 @@ pub(super) struct UntypedFloat( pub(super) type FallbackFloat = f64; impl UntypedFloat { - pub(super) fn new_from_lit_float(lit_float: &LitFloat) -> Self { - // LitFloat doesn't support Clone, so we have to do this - Self::new_from_literal(lit_float.token()) + pub(super) fn new_from_lit_float(lit_float: LitFloat) -> Self { + Self(lit_float) } pub(super) fn new_from_literal(literal: Literal) -> Self { @@ -137,10 +140,10 @@ impl UntypedFloat { ) -> ExecutionResult { let input = self.parse_fallback()?; match operation.operator { - UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), - UnaryOperator::Not => operation.unsupported_for_value_type_err("untyped float"), - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Cast(target) => match target { + UnaryOperator::Neg { .. } => operation.output(Self::from_fallback(-input)), + UnaryOperator::Not { .. } => operation.unsupported_for_value_type_err("untyped float"), + UnaryOperator::GroupedNoOp { .. } => operation.output(self), + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } @@ -270,10 +273,10 @@ macro_rules! impl_float_operations { impl HandleUnaryOperation for $float_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { - UnaryOperator::Neg => operation.output(-self), - UnaryOperator::Not => operation.unsupported_for_value_type_err(stringify!($float_type)), - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Cast(target) => match target { + UnaryOperator::Neg { .. } => operation.output(-self), + UnaryOperator::Not { .. } => operation.unsupported_for_value_type_err(stringify!($float_type)), + UnaryOperator::GroupedNoOp { .. } => operation.output(self), + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 42bc2e7f..8ccd8dc9 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -9,10 +9,11 @@ pub(crate) struct EvaluationInteger { } impl EvaluationInteger { - pub(super) fn for_litint(lit: &syn::LitInt) -> ExecutionResult { + pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { + let source_span = Some(lit.span()); Ok(Self { value: EvaluationIntegerValue::for_litint(lit)?, - source_span: Some(lit.span()), + source_span, }) } @@ -184,7 +185,7 @@ pub(super) enum EvaluationIntegerValue { } impl EvaluationIntegerValue { - pub(super) fn for_litint(lit: &syn::LitInt) -> ExecutionResult { + pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { Ok(match lit.suffix() { "" => Self::Untyped(UntypedInteger::new_from_lit_int(lit)), "u8" => Self::U8(lit.base10_parse()?), @@ -200,7 +201,7 @@ impl EvaluationIntegerValue { "i128" => Self::I128(lit.base10_parse()?), "isize" => Self::Isize(lit.base10_parse()?), suffix => { - return lit.span().execution_err(format!( + return lit.span().parse_err(format!( "The literal suffix {suffix} is not supported in preinterpret expressions" )); } @@ -252,9 +253,8 @@ pub(super) struct UntypedInteger( pub(super) type FallbackInteger = i128; impl UntypedInteger { - pub(super) fn new_from_lit_int(lit_int: &LitInt) -> Self { - // LitInt doesn't support Clone, so we have to do this - Self::new_from_literal(lit_int.token()) + pub(super) fn new_from_lit_int(lit_int: LitInt) -> Self { + Self(lit_int) } pub(super) fn new_from_literal(literal: Literal) -> Self { @@ -267,10 +267,12 @@ impl UntypedInteger { ) -> ExecutionResult { let input = self.parse_fallback()?; match operation.operator { - UnaryOperator::Neg => operation.output(Self::from_fallback(-input)), - UnaryOperator::Not => operation.unsupported_for_value_type_err("untyped integer"), - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Cast(target) => match target { + UnaryOperator::Neg { .. } => operation.output(Self::from_fallback(-input)), + UnaryOperator::Not { .. } => { + operation.unsupported_for_value_type_err("untyped integer") + } + UnaryOperator::GroupedNoOp { .. } => operation.output(self), + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } @@ -525,12 +527,12 @@ macro_rules! impl_unsigned_unary_operations { impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Neg - | UnaryOperator::Not => { + UnaryOperator::GroupedNoOp { .. } => operation.output(self), + UnaryOperator::Neg { .. } + | UnaryOperator::Not { .. } => { operation.unsupported_for_value_type_err(stringify!($integer_type)) }, - UnaryOperator::Cast(target) => match target { + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), @@ -561,12 +563,12 @@ macro_rules! impl_signed_unary_operations { impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation.operator { - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Neg => operation.output(-self), - UnaryOperator::Not => { + UnaryOperator::GroupedNoOp { .. } => operation.output(self), + UnaryOperator::Neg { .. } => operation.output(-self), + UnaryOperator::Not { .. } => { operation.unsupported_for_value_type_err(stringify!($integer_type)) }, - UnaryOperator::Cast(target) => match target { + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), @@ -597,11 +599,11 @@ impl HandleUnaryOperation for u8 { operation: &UnaryOperation, ) -> ExecutionResult { match operation.operator { - UnaryOperator::NoOp => operation.output(self), - UnaryOperator::Neg | UnaryOperator::Not => { + UnaryOperator::GroupedNoOp { .. } => operation.output(self), + UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } => { operation.unsupported_for_value_type_err("u8") } - UnaryOperator::Cast(target) => match target { + UnaryOperator::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(self as FallbackInteger)) } diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index ea2810ff..f35e8f14 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -1,7 +1,8 @@ mod boolean; mod character; -mod evaluation_tree; -mod expression_stream; +mod evaluation; +mod expression; +mod expression_parsing; mod float; mod integer; mod operations; @@ -12,11 +13,12 @@ mod value; use crate::internal_prelude::*; use boolean::*; use character::*; -use evaluation_tree::*; +use evaluation::*; +use expression_parsing::*; use float::*; use integer::*; use operations::*; use string::*; use value::*; -pub(crate) use expression_stream::*; +pub(crate) use expression::*; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index bfc3c027..2b1e8105 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -1,47 +1,13 @@ use super::*; -pub(super) enum EvaluationOperator { - Unary { - operation: UnaryOperation, - input: Option, - }, - Binary { - operation: BinaryOperation, - left_input: Option, - right_input: Option, - }, -} - -impl EvaluationOperator { - pub(super) fn evaluate(self) -> ExecutionResult { - const OPERATOR_INPUT_EXPECT_STR: &str = "Handling children on the stack ordering should ensure the parent input is always set when the parent is evaluated"; - - match self { - Self::Unary { - operation: operator, - input, - } => operator.evaluate(input.expect(OPERATOR_INPUT_EXPECT_STR)), - Self::Binary { - operation: operator, - left_input, - right_input, - } => operator.evaluate( - left_input.expect(OPERATOR_INPUT_EXPECT_STR), - right_input.expect(OPERATOR_INPUT_EXPECT_STR), - ), - } - } -} - +#[derive(Clone)] pub(super) struct UnaryOperation { - pub(super) source_span: Option, - pub(super) operator_span: Span, pub(super) operator: UnaryOperator, } impl UnaryOperation { fn error(&self, error_message: &str) -> ExecutionInterrupt { - self.operator_span.execution_error(error_message) + self.operator.execution_error(error_message) } pub(super) fn unsupported_for_value_type_err( @@ -63,98 +29,53 @@ impl UnaryOperation { &self, output_value: impl ToEvaluationValue, ) -> ExecutionResult { - Ok(output_value.to_value(self.source_span)) + Ok(output_value.to_value(self.operator.source_span_for_output())) } - pub(super) fn for_cast_expression(expr: &syn::ExprCast) -> ExecutionResult { - fn extract_type(ty: &syn::Type) -> ExecutionResult { - match ty { - syn::Type::Group(group) => extract_type(&group.elem), - syn::Type::Path(type_path) - if type_path.qself.is_none() - && type_path.path.leading_colon.is_none() - && type_path.path.segments.len() == 1 => - { - let ident = match type_path.path.get_ident() { - Some(ident) => ident, - None => { - return type_path - .span_range_from_iterating_over_all_tokens() - .execution_err( - "This type is not supported in preinterpret cast expressions", - ) - } - }; - match ident.to_string().as_str() { - "int" | "integer" => Ok(CastTarget::Integer(IntegerKind::Untyped)), - "u8" => Ok(CastTarget::Integer(IntegerKind::U8)), - "u16" => Ok(CastTarget::Integer(IntegerKind::U16)), - "u32" => Ok(CastTarget::Integer(IntegerKind::U32)), - "u64" => Ok(CastTarget::Integer(IntegerKind::U64)), - "u128" => Ok(CastTarget::Integer(IntegerKind::U128)), - "usize" => Ok(CastTarget::Integer(IntegerKind::Usize)), - "i8" => Ok(CastTarget::Integer(IntegerKind::I8)), - "i16" => Ok(CastTarget::Integer(IntegerKind::I16)), - "i32" => Ok(CastTarget::Integer(IntegerKind::I32)), - "i64" => Ok(CastTarget::Integer(IntegerKind::I64)), - "i128" => Ok(CastTarget::Integer(IntegerKind::I128)), - "isize" => Ok(CastTarget::Integer(IntegerKind::Isize)), - "float" => Ok(CastTarget::Float(FloatKind::Untyped)), - "f32" => Ok(CastTarget::Float(FloatKind::F32)), - "f64" => Ok(CastTarget::Float(FloatKind::F64)), - "bool" => Ok(CastTarget::Boolean), - "char" => Ok(CastTarget::Char), - _ => ident.execution_err( - "This type is not supported in preinterpret cast expressions", - ), - } - } - other => other - .span_range_from_iterating_over_all_tokens() - .execution_err("This type is not supported in preinterpret cast expressions"), + pub(super) fn for_cast_operation( + as_token: Token![as], + target_type: Ident, + ) -> ParseResult { + let target = match target_type.to_string().as_str() { + "int" | "integer" => CastTarget::Integer(IntegerKind::Untyped), + "u8" => CastTarget::Integer(IntegerKind::U8), + "u16" => CastTarget::Integer(IntegerKind::U16), + "u32" => CastTarget::Integer(IntegerKind::U32), + "u64" => CastTarget::Integer(IntegerKind::U64), + "u128" => CastTarget::Integer(IntegerKind::U128), + "usize" => CastTarget::Integer(IntegerKind::Usize), + "i8" => CastTarget::Integer(IntegerKind::I8), + "i16" => CastTarget::Integer(IntegerKind::I16), + "i32" => CastTarget::Integer(IntegerKind::I32), + "i64" => CastTarget::Integer(IntegerKind::I64), + "i128" => CastTarget::Integer(IntegerKind::I128), + "isize" => CastTarget::Integer(IntegerKind::Isize), + "float" => CastTarget::Float(FloatKind::Untyped), + "f32" => CastTarget::Float(FloatKind::F32), + "f64" => CastTarget::Float(FloatKind::F64), + "bool" => CastTarget::Boolean, + "char" => CastTarget::Char, + _ => { + return target_type + .parse_err("This type is not supported in preinterpret cast expressions") } - } - - Ok(Self { - source_span: None, - operator_span: expr.as_token.span(), - operator: UnaryOperator::Cast(extract_type(&expr.ty)?), - }) - } - - pub(super) fn for_group_expression(expr: &syn::ExprGroup) -> ExecutionResult { - let span = expr.group_token.span; - Ok(Self { - source_span: Some(span), - operator_span: span, - operator: UnaryOperator::NoOp, - }) - } - - pub(super) fn for_paren_expression(expr: &syn::ExprParen) -> ExecutionResult { - let span = expr.paren_token.span.join(); + }; Ok(Self { - source_span: Some(span), - operator_span: span, - operator: UnaryOperator::NoOp, + operator: UnaryOperator::Cast { as_token, target }, }) } - pub(super) fn for_unary_expression(expr: &syn::ExprUnary) -> ExecutionResult { - let operator = match &expr.op { - UnOp::Neg(_) => UnaryOperator::Neg, - UnOp::Not(_) => UnaryOperator::Not, + pub(super) fn for_unary_operator(operator: syn::UnOp) -> ParseResult { + let operator = match operator { + UnOp::Neg(token) => UnaryOperator::Neg { token }, + UnOp::Not(token) => UnaryOperator::Not { token }, other_unary_op => { - return other_unary_op.execution_err( + return other_unary_op.parse_err( "This unary operator is not supported in a preinterpret expression", ); } }; - Ok(Self { - source_span: None, - operator_span: expr.op.span(), - operator, - }) + Ok(Self { operator }) } pub(super) fn evaluate(self, input: EvaluationValue) -> ExecutionResult { @@ -164,28 +85,62 @@ impl UnaryOperation { #[derive(Copy, Clone)] pub(super) enum UnaryOperator { - Neg, - Not, - NoOp, - Cast(CastTarget), + Neg { + token: Token![-], + }, + Not { + token: Token![!], + }, + GroupedNoOp { + span: Span, + }, + Cast { + as_token: Token![as], + target: CastTarget, + }, } impl UnaryOperator { + pub(crate) fn source_span_for_output(&self) -> Option { + match self { + UnaryOperator::Neg { .. } => None, + UnaryOperator::Not { .. } => None, + UnaryOperator::GroupedNoOp { span } => Some(*span), + UnaryOperator::Cast { .. } => None, + } + } + pub(crate) fn symbol(&self) -> &'static str { match self { - UnaryOperator::Neg => "-", - UnaryOperator::Not => "!", - UnaryOperator::NoOp => "", - UnaryOperator::Cast(_) => "as", + UnaryOperator::Neg { .. } => "-", + UnaryOperator::Not { .. } => "!", + UnaryOperator::GroupedNoOp { .. } => "", + UnaryOperator::Cast { .. } => "as", } } } +impl HasSpanRange for UnaryOperator { + fn span(&self) -> Span { + match self { + UnaryOperator::Neg { token } => token.span, + UnaryOperator::Not { token } => token.span, + UnaryOperator::GroupedNoOp { span } => *span, + UnaryOperator::Cast { as_token, .. } => as_token.span, + } + } + + fn span_range(&self) -> SpanRange { + self.span().span_range() + } +} + pub(super) trait HandleUnaryOperation: Sized { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult; } +#[derive(Clone)] pub(super) struct BinaryOperation { /// Only present if there is a single span for the source tokens pub(super) source_span: Option, @@ -227,8 +182,8 @@ impl BinaryOperation { } } - pub(super) fn for_binary_expression(expr: &syn::ExprBinary) -> ExecutionResult { - let operator = match &expr.op { + pub(super) fn for_binary_operator(syn_operator: syn::BinOp) -> ParseResult { + let operator = match syn_operator { syn::BinOp::Add(_) => BinaryOperator::Paired(PairedBinaryOperator::Addition), syn::BinOp::Sub(_) => BinaryOperator::Paired(PairedBinaryOperator::Subtraction), syn::BinOp::Mul(_) => BinaryOperator::Paired(PairedBinaryOperator::Multiplication), @@ -249,17 +204,17 @@ impl BinaryOperation { syn::BinOp::Gt(_) => BinaryOperator::Paired(PairedBinaryOperator::GreaterThan), other_binary_operation => { return other_binary_operation - .execution_err("This operation is not supported in preinterpret expressions") + .parse_err("This operation is not supported in preinterpret expressions") } }; Ok(Self { source_span: None, - operator_span: expr.op.span(), + operator_span: syn_operator.span(), operator, }) } - fn evaluate( + pub(super) fn evaluate( self, left: EvaluationValue, right: EvaluationValue, diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 7dd45c92..9241a60a 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -1,5 +1,6 @@ use super::*; +#[derive(Clone)] pub(crate) struct EvaluationString { pub(super) value: String, /// The span of the source code that generated this boolean value. @@ -8,7 +9,7 @@ pub(crate) struct EvaluationString { } impl EvaluationString { - pub(super) fn for_litstr(lit: &syn::LitStr) -> Self { + pub(super) fn for_litstr(lit: syn::LitStr) -> Self { Self { value: lit.value(), source_span: Some(lit.span()), @@ -20,8 +21,8 @@ impl EvaluationString { operation: UnaryOperation, ) -> ExecutionResult { match operation.operator { - UnaryOperator::NoOp => operation.output(self.value), - UnaryOperator::Neg | UnaryOperator::Not | UnaryOperator::Cast(_) => { + UnaryOperator::GroupedNoOp { .. } => operation.output(self.value), + UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } | UnaryOperator::Cast { .. } => { operation.unsupported_for_value_type_err("string") } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 12a88344..70eca4f1 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -1,5 +1,6 @@ use super::*; +#[derive(Clone)] pub(crate) enum EvaluationValue { Integer(EvaluationInteger), Float(EvaluationFloat), @@ -13,9 +14,9 @@ pub(super) trait ToEvaluationValue: Sized { } impl EvaluationValue { - pub(super) fn for_literal_expression(expr: &ExprLit) -> ExecutionResult { + pub(super) fn for_literal(lit: syn::Lit) -> ParseResult { // https://docs.rs/syn/latest/syn/enum.Lit.html - Ok(match &expr.lit { + Ok(match lit { Lit::Int(lit) => Self::Integer(EvaluationInteger::for_litint(lit)?), Lit::Float(lit) => Self::Float(EvaluationFloat::for_litfloat(lit)?), Lit::Bool(lit) => Self::Boolean(EvaluationBoolean::for_litbool(lit)), @@ -24,7 +25,7 @@ impl EvaluationValue { other_literal => { return other_literal .span() - .execution_err("This literal is not supported in preinterpret expressions"); + .parse_err("This literal is not supported in preinterpret expressions"); } }) } diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index 94aef270..fc399cf7 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -66,6 +66,13 @@ pub(crate) struct SpanRange { } impl SpanRange { + pub(crate) fn new_single(span: Span) -> Self { + Self { + start: span, + end: span, + } + } + pub(crate) fn new_between(start: Span, end: Span) -> Self { Self { start, end } } @@ -80,6 +87,10 @@ impl SpanRange { ::span(self) } + pub(crate) fn set_end(&mut self, end: Span) { + self.end = end; + } + #[allow(unused)] pub(crate) fn start(&self) -> Span { self.start diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index baa02e9c..1f3f5a6b 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -261,3 +261,76 @@ impl DelimiterExt for Delimiter { } } } + +/// Allows storing a stack of parse buffers for certain parse strategies which require +/// handling multiple groups in parallel. +pub(crate) struct ParseStreamStack<'a> { + base: ParseStream<'a>, + group_stack: Vec>, +} + +impl<'a> ParseStreamStack<'a> { + pub(crate) fn new(base: ParseStream<'a>) -> Self { + Self { + base, + group_stack: Vec::new(), + } + } + + fn current(&self) -> ParseStream<'_> { + self.group_stack.last().unwrap_or(self.base) + } + + pub(crate) fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { + self.current().parse_err(message) + } + + pub(crate) fn peek_grammar(&mut self) -> PeekMatch { + detect_preinterpret_grammar(self.current().cursor()) + } + + pub(crate) fn parse(&mut self) -> ParseResult { + self.current().parse() + } + + pub(crate) fn parse_and_enter_group(&mut self) -> ParseResult<(Delimiter, DelimSpan)> { + let (delimiter, delim_span, inner) = self.current().parse_any_group()?; + let inner = unsafe { + // SAFETY: This is safe because the lifetime is there for two reasons: + // (A) Prevent mixing up different buffers from e.g. different groups, + // (B) Ensure the buffers are dropped in the correct order so that the unexpected drop glue triggers + // in the correct order. + // + // This invariant is maintained by this `ParseStreamStack` struct: + // (A) Is enforced by the fact we're parsing the group from the top parse buffer current(). + // (B) Is enforced by a combination of: + // ==> exit_group() ensures the parse buffers are dropped in the correct order + // ==> If a user forgets to do it (or e.g. an error path or panic causes exit_group not to be called) + // Then the drop glue ensures the groups are dropped in the correct order. + std::mem::transmute::, ParseBuffer<'a>>(inner) + }; + self.group_stack.push(inner); + Ok((delimiter, delim_span)) + } + + /// Should be paired with `parse_and_enter_group`. + /// + /// If the group is not finished, the next attempt to read from the parent will trigger an error, + /// in accordance with the drop glue on `ParseBuffer`. + /// + /// ### Panics + /// Panics if there is no group available. + pub(crate) fn exit_group(&mut self) { + self.group_stack + .pop() + .expect("finish_group must be paired with push_group"); + } +} + +impl Drop for ParseStreamStack<'_> { + fn drop(&mut self) { + while !self.group_stack.is_empty() { + self.exit_group(); + } + } +} diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index e7939c56..25373c1c 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -11,7 +11,7 @@ pub(crate) use syn::parse::{ discouraged::Speculative, Parse as SynParse, ParseBuffer as SynParseBuffer, ParseStream as SynParseStream, Parser as SynParser, }; -pub(crate) use syn::{parse_str, Expr, ExprLit, Lit, LitBool, LitFloat, LitInt, Token, UnOp}; +pub(crate) use syn::{parse_str, Lit, LitBool, LitFloat, LitInt, Token, UnOp}; pub(crate) use syn::{Error as SynError, Result as SynResult}; pub(crate) use crate::destructuring::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 480d463b..ff9d9ed0 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -13,6 +13,18 @@ pub(crate) enum CommandOutputKind { ControlFlowCodeStream, } +impl CommandOutputKind { + pub(crate) fn expression_support(&self) -> Result<(), &'static str> { + match self { + CommandOutputKind::Value | CommandOutputKind::GroupedStream => Ok(()), + CommandOutputKind::None => Err("A command which returns nothing cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), + CommandOutputKind::Ident => Err("A command which returns idents cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), + CommandOutputKind::FlattenedStream => Err("A command which returns a flattened stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), + CommandOutputKind::ControlFlowCodeStream => Err("A control flow command which returns a code stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), + } + } +} + pub(crate) trait CommandType { type OutputKind: OutputKind; } @@ -34,12 +46,6 @@ trait CommandInvocation { context: ExecutionContext, output: &mut InterpretedStream, ) -> ExecutionResult<()>; - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()>; } trait ClonableCommandInvocation: CommandInvocation { @@ -66,12 +72,6 @@ trait CommandInvocationAs { context: ExecutionContext, output: &mut InterpretedStream, ) -> ExecutionResult<()>; - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()>; } impl> CommandInvocation for C { @@ -82,16 +82,6 @@ impl> CommandInvocation for ) -> ExecutionResult<()> { >::execute_into(self, context, output) } - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - >::execute_into_expression( - self, context, builder, - ) - } } //=============== @@ -129,16 +119,6 @@ impl CommandInvocationAs for C { self.execute(context.interpreter)?; Ok(()) } - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - _: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - context.delim_span - .join() - .execution_err("Commands with no output cannot be used directly in expressions.\nConsider wrapping it inside a command such as [!group! ..] which returns an expression") - } } //================ @@ -175,19 +155,6 @@ impl CommandInvocationAs for C { output.push_raw_token_tree(self.execute(context.interpreter)?); Ok(()) } - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - match self.execute(context.interpreter)? { - TokenTree::Literal(literal) => builder.push_literal(literal), - TokenTree::Ident(ident) => builder.push_ident(ident), - _ => panic!("Value Output Commands should only output literals or idents"), - } - Ok(()) - } } //================ @@ -224,15 +191,6 @@ impl CommandInvocationAs for C { output.push_ident(self.execute(context.interpreter)?); Ok(()) } - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - builder.push_ident(self.execute(context.interpreter)?); - Ok(()) - } } //================= @@ -279,22 +237,6 @@ impl CommandInvocationAs for C { _ => unreachable!(), } } - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - if let CommandOutputKind::FlattenedStream = context.output_kind { - return context.delim_span - .join() - .execution_err("Flattened commands cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"); - } - builder.push_grouped( - |output| self.execute(context.interpreter, output), - context.delim_span.join(), - ) - } } //====================== @@ -333,17 +275,6 @@ impl CommandInvocationAs ) -> ExecutionResult<()> { self.execute(context.interpreter, output) } - - fn execute_into_expression( - self: Box, - context: ExecutionContext, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - builder.push_grouped( - |output| self.execute(context.interpreter, output), - context.delim_span.join(), - ) - } } //========================= @@ -373,6 +304,11 @@ macro_rules! define_command_kind { }) } + pub(crate) fn grouped_output_kind(&self) -> CommandOutputKind { + // Guaranteed to be Ok if no flattening is provided + self.output_kind(None).unwrap() + } + pub(crate) fn output_kind(&self, flattening: Option) -> ParseResult { match self { $( @@ -533,22 +469,3 @@ impl Interpret for Command { self.invocation.execute_into(context, output) } } - -impl Express for Command { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - builder: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - let context = ExecutionContext { - interpreter, - output_kind: self.output_kind, - delim_span: self.source_group_span, - }; - // This is set up so that we can determine the exact expression - // structure at parse time, and in future refactor to parsing - // the expression ourselves, and then executing an expression AST - // rather than going via a syn::expression - self.invocation.execute_into_expression(context, builder) - } -} diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index e6042c2e..ccca64e2 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -34,7 +34,8 @@ impl<'a> CommandArguments<'a> { if self.parse_stream.is_empty() { Ok(()) } else { - self.command_span.parse_err(error_message) + self.parse_stream + .parse_err(format!("Unexpected extra tokens. {}", error_message)) } } diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index baf7e208..b07e0077 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -2,9 +2,9 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IfCommand { - condition: ExpressionInput, + condition: Expression, true_code: CommandCodeInput, - else_ifs: Vec<(ExpressionInput, CommandCodeInput)>, + else_ifs: Vec<(Expression, CommandCodeInput)>, else_code: Option, } @@ -80,7 +80,7 @@ impl ControlFlowCommandDefinition for IfCommand { #[derive(Clone)] pub(crate) struct WhileCommand { - condition: ExpressionInput, + condition: Expression, loop_code: CommandCodeInput, } diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 82c7e3a6..6135dc69 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct EvaluateCommand { - expression: ExpressionInput, + expression: Expression, command_span: Span, } @@ -26,8 +26,10 @@ impl ValueCommandDefinition for EvaluateCommand { } fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { - let expression = self.expression.start_expression_builder(interpreter)?; - Ok(expression.evaluate(self.command_span)?.to_token_tree()) + Ok(self + .expression + .evaluate_with_span(interpreter, self.command_span)? + .to_token_tree()) } } @@ -37,7 +39,7 @@ pub(crate) struct AssignCommand { operator: Option, #[allow(unused)] equals: Token![=], - expression: ExpressionInput, + expression: Expression, command_span: Span, } @@ -83,14 +85,28 @@ impl NoOutputCommandDefinition for AssignCommand { command_span, } = *self; - let mut builder = ExpressionBuilder::new(); - if let Some(operator) = operator { - variable.add_to_expression(interpreter, &mut builder)?; - builder.push_punct(operator); - } - builder.extend_with_evaluation_output(expression.evaluate(interpreter)?); + let expression = if let Some(operator) = operator { + let mut calculation = TokenStream::new(); + unsafe { + // RUST-ANALYZER SAFETY: Hopefully it won't contain a none-delimited group + variable + .interpret_to_new_stream(interpreter)? + .syn_parse(Expression::parse)? + .evaluate(interpreter)? + .to_tokens(&mut calculation); + }; + operator.to_tokens(&mut calculation); + expression + .evaluate(interpreter)? + .to_tokens(&mut calculation); + calculation.parse_with(Expression::parse)? + } else { + expression + }; - let output = builder.evaluate(command_span)?.to_token_tree(); + let output = expression + .evaluate_with_span(interpreter, command_span)? + .to_token_tree(); variable.set(interpreter, output.into())?; Ok(()) @@ -99,9 +115,9 @@ impl NoOutputCommandDefinition for AssignCommand { #[derive(Clone)] pub(crate) struct RangeCommand { - left: ExpressionInput, + left: Expression, range_limits: RangeLimits, - right: ExpressionInput, + right: Expression, } impl CommandType for RangeCommand { diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index dce8e81e..656ddd6d 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -124,22 +124,6 @@ impl Interpret for RawGroup { } } -impl Express for RawGroup { - fn add_to_expression( - self, - _: &mut Interpreter, - expression_stream: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - expression_stream.push_grouped( - |inner| { - inner.extend_raw_tokens(self.content); - Ok(()) - }, - self.source_delim_span.join(), - ) - } -} - impl HasSpanRange for RawGroup { fn span_range(&self) -> SpanRange { self.source_delim_span.span_range() diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index e69dd85b..5780a6f2 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -88,10 +88,6 @@ impl InterpretedStream { self.token_length += 1; } - pub(crate) fn extend_with_raw_tokens_from(&mut self, tokens: impl ToTokens) { - self.extend_raw_tokens(tokens.into_token_stream()) - } - pub(crate) fn push_interpreted_item(&mut self, segment_item: InterpretedTokenTree) { match segment_item { InterpretedTokenTree::TokenTree(token_tree) => { diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 9ed58f40..43a7f8e1 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -96,19 +96,6 @@ impl Interpret for &GroupedVariable { } } -impl Express for &GroupedVariable { - fn add_to_expression( - self, - interpreter: &mut Interpreter, - expression_stream: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - expression_stream.push_grouped( - |inner| self.substitute_ungrouped_contents_into(interpreter, inner), - self.span(), - ) - } -} - impl HasSpanRange for GroupedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) @@ -194,20 +181,6 @@ impl Interpret for &FlattenedVariable { } } -impl Express for &FlattenedVariable { - fn add_to_expression( - self, - _: &mut Interpreter, - _: &mut ExpressionBuilder, - ) -> ExecutionResult<()> { - // Just like with commands, we throw an error in the flattened case so - // that we can determine in future the exact structure of the expression - // at parse time. - self.flatten - .execution_err("Flattened variables cannot be used directly in expressions.\nConsider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression") - } -} - impl HasSpanRange for FlattenedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) diff --git a/src/misc/mod.rs b/src/misc/mod.rs index 38882786..475a8180 100644 --- a/src/misc/mod.rs +++ b/src/misc/mod.rs @@ -5,3 +5,21 @@ mod string_conversion; pub(crate) use errors::*; pub(crate) use parse_traits::*; pub(crate) use string_conversion::*; + +#[allow(unused)] +pub(crate) fn print_if_slow( + inner: impl FnOnce() -> T, + slow_threshold: std::time::Duration, + print_message: impl FnOnce(&T, std::time::Duration) -> String, +) -> T { + use std::time::*; + let before = SystemTime::now(); + let output = inner(); + let after = SystemTime::now(); + + let elapsed = after.duration_since(before).unwrap(); + if elapsed >= slow_threshold { + println!("{}", print_message(&output, elapsed)); + } + output +} diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr deleted file mode 100644 index 72ba03ca..00000000 --- a/tests/compilation_failures/expressions/braces.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Expected an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/braces.rs:5:21 - | -5 | [!evaluate! {true}] - | ^ diff --git a/tests/compilation_failures/expressions/braces.rs b/tests/compilation_failures/expressions/brackets.rs similarity index 71% rename from tests/compilation_failures/expressions/braces.rs rename to tests/compilation_failures/expressions/brackets.rs index 21f141fd..a85e809a 100644 --- a/tests/compilation_failures/expressions/braces.rs +++ b/tests/compilation_failures/expressions/brackets.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! {true}] + [!evaluate! [true]] }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/brackets.stderr b/tests/compilation_failures/expressions/brackets.stderr new file mode 100644 index 00000000..72795d65 --- /dev/null +++ b/tests/compilation_failures/expressions/brackets.stderr @@ -0,0 +1,6 @@ +error: Square brackets [ .. ] are not supported in an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/brackets.rs:5:21 + | +5 | [!evaluate! [true]] + | ^ diff --git a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr index 03455a9d..d442be70 100644 --- a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr +++ b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr @@ -1,4 +1,5 @@ error: number too large to fit in target type + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression --> tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs:8:22 | 8 | [!evaluate! -128i8] diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr index 05d5138b..9d64994e 100644 --- a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr @@ -1,6 +1,7 @@ -error: Flattened commands cannot be used directly in expressions. - Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression +error: A command which returns a flattened stream cannot be used directly in expressions. + Consider wrapping it inside a { } block which returns an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:25 | 5 | [!evaluate! 5 + [!..range! 1..2]] - | ^^^^^^^^^^^^^^^^ + | ^ diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr index 42767567..70cced4f 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr @@ -1,6 +1,5 @@ -error: Flattened variables cannot be used directly in expressions. - Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression - --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:6:24 +error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:6:23 | 6 | [!evaluate! 5 #..partial_sum] - | ^^ + | ^ diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr index 68aa6002..5e2a1cec 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr @@ -1,5 +1,5 @@ -error: expected an expression - --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:21 +error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:6:23 | -5 | [!set! #x = + 1] - | ^ +6 | [!evaluate! 1 #x] + | ^ diff --git a/tests/compilation_failures/expressions/inner_braces.stderr b/tests/compilation_failures/expressions/inner_braces.stderr deleted file mode 100644 index 537fcac9..00000000 --- a/tests/compilation_failures/expressions/inner_braces.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Expected an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/inner_braces.rs:5:22 - | -5 | [!evaluate! ({true})] - | ^ diff --git a/tests/compilation_failures/expressions/inner_braces.rs b/tests/compilation_failures/expressions/inner_brackets.rs similarity index 69% rename from tests/compilation_failures/expressions/inner_braces.rs rename to tests/compilation_failures/expressions/inner_brackets.rs index 464c4653..0607a729 100644 --- a/tests/compilation_failures/expressions/inner_braces.rs +++ b/tests/compilation_failures/expressions/inner_brackets.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! ({true})] + [!evaluate! ([true])] }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/inner_brackets.stderr b/tests/compilation_failures/expressions/inner_brackets.stderr new file mode 100644 index 00000000..947d2f82 --- /dev/null +++ b/tests/compilation_failures/expressions/inner_brackets.stderr @@ -0,0 +1,6 @@ +error: Square brackets [ .. ] are not supported in an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/inner_brackets.rs:5:22 + | +5 | [!evaluate! ([true])] + | ^ diff --git a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr index 800497eb..c906afbb 100644 --- a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr +++ b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr @@ -1,6 +1,7 @@ -error: Commands with no output cannot be used directly in expressions. - Consider wrapping it inside a command such as [!group! ..] which returns an expression +error: A command which returns nothing cannot be used directly in expressions. + Consider wrapping it inside a { } block which returns an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression --> tests/compilation_failures/expressions/no_output_commands_in_expressions.rs:5:25 | 5 | [!evaluate! 5 + [!set! #x = 2] 2] - | ^^^^^^^^^^^^^^ + | ^ diff --git a/tests/expressions.rs b/tests/expressions.rs index 69654e92..df1118d7 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -74,21 +74,11 @@ fn test_basic_evaluate_works() { #[test] fn test_very_long_expression_works() { - // Any larger than this gets hit by a stack overflow - which is in the syn library, - // during parsing the Expr, rather than in preinterpret. - // When we implement expression parsing in preinterpret, we can potentially flatten - // the parsing loop and get better performance. - assert_preinterpret_eq!({ - [!set! #x = 0 - [!for! #i in [!range! 0..1000] { - [!for! #j in [!range! 0..25] { - + 1 - }] - }] - ] - [!evaluate! #x] + assert_preinterpret_eq!( + { + [!settings! { iteration_limit: 100000 }][!evaluate! [!group! 0 [!for! #i in [!range! 0..100000] { + 1 }]]] }, - 25000 + 100000 ); } From dea5beab27d7b5c2cefea743d088a91f22dafba6 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 30 Jan 2025 22:37:16 +0000 Subject: [PATCH 067/126] fix: || and && short-circuit --- CHANGELOG.md | 13 ++- src/destructuring/destructurer.rs | 14 +-- src/destructuring/destructurers.rs | 5 +- src/expressions/evaluation.rs | 26 ++--- src/expressions/expression.rs | 2 +- src/expressions/expression_parsing.rs | 2 +- src/expressions/operations.rs | 47 +++++++- src/extensions/errors_and_spans.rs | 110 ++++++++++++------ src/extensions/parsing.rs | 12 +- src/interpretation/command.rs | 6 +- src/interpretation/command_arguments.rs | 6 +- src/interpretation/command_code_input.rs | 8 +- src/interpretation/commands/core_commands.rs | 7 +- .../commands/destructuring_commands.rs | 2 +- .../commands/expression_commands.rs | 8 +- src/interpretation/commands/token_commands.rs | 8 +- src/interpretation/interpretation_stream.rs | 28 ++--- src/interpretation/variable.rs | 2 +- src/lib.rs | 4 +- ...ix_me_comparison_operators_are_not_lazy.rs | 11 -- ...e_comparison_operators_are_not_lazy.stderr | 13 --- tests/expressions.rs | 36 +++++- 22 files changed, 222 insertions(+), 148 deletions(-) delete mode 100644 tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs delete mode 100644 tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 952e6897..a03326ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,11 +44,15 @@ Expressions can be evaluated with `[!evaluate! ...]` and are also used in the `i Currently supported are: * Integer, Float, Bool, String and Char literals -* The operators: `+ - * / % & | ^ || &&` +* The operators: `+ - * / % & | ^` +* The lazy boolean operators: `|| &&` * The comparison operators: `== != < > <= >=` * The shift operators: `>> <<` * Casting with `as` including to untyped integers/floats with `as int` and `as float` * () and none-delimited groups for precedence +* Embedded `#x` grouped variables, whose contents are parsed as an expression + and evaluated. +* `{ ... }` for creating sub-expressions, which are parsed from the resultant token stream. Expressions behave intuitively as you'd expect from writing regular rust code, except they happen at compile time. @@ -77,10 +81,12 @@ Destructuring performs parsing of a token stream. It supports: ### To come * Complete expression rework: - * Enable lazy && and || - * Enable support for code blocks { .. } in expressions + * Add more tests for operator precedence (e.g. the worked example) * Flatten `UnaryOperation` and `UnaryOperator` * Flatten `BinaryOperation` and `BinaryOperator` + * Disallow expressions/commands in re-evaluated `{ .. }` blocks and command outputs + * Revise the comment in expression_parsing.rs + * Create `ParseInterpreted` and `ParseSource` via `ParseStream`, `SourceParseStream` and `InterpretedParseStream`, and add grammar peaking to `SourceParseStream` only. Add `[!reinterpret! ...]` command for an `eval` style command. * Search / resolve TODOs * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` @@ -95,6 +101,7 @@ Destructuring performs parsing of a token stream. It supports: * `[!match! ...]` command * `(!any! ...)` (with `#..x` as a catch-all) like the [!match!] command but without arms... * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much +* TODO check * Check all `#[allow(unused)]` and remove any which aren't needed * Pushed to 0.4: * Fork of syn to: diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index a1e22401..d077bf7d 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -14,7 +14,7 @@ pub(crate) trait DestructurerDefinition: Clone { pub(crate) struct DestructurerArguments<'a> { parse_stream: ParseStream<'a>, destructurer_name: Ident, - full_span_range: SpanRange, + full_span: Span, } #[allow(unused)] @@ -22,17 +22,17 @@ impl<'a> DestructurerArguments<'a> { pub(crate) fn new( parse_stream: ParseStream<'a>, destructurer_name: Ident, - span_range: SpanRange, + full_span: Span, ) -> Self { Self { parse_stream, destructurer_name, - full_span_range: span_range, + full_span, } } - pub(crate) fn full_span_range(&self) -> SpanRange { - self.full_span_range + pub(crate) fn full_span(&self) -> Span { + self.full_span } /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message @@ -40,7 +40,7 @@ impl<'a> DestructurerArguments<'a> { if self.parse_stream.is_empty() { Ok(()) } else { - self.full_span_range.parse_err(error_message) + self.full_span.parse_err(error_message) } } @@ -104,7 +104,7 @@ impl Parse for Destructurer { let instance = destructurer_kind.parse_instance(DestructurerArguments::new( &content, destructurer_name, - delim_span.join().span_range(), + delim_span.join(), ))?; Ok(Self { instance, diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index 19544a57..8c58b717 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -205,10 +205,7 @@ impl DestructurerDefinition for ContentDestructurer { arguments.fully_parse_or_error( |input| { Ok(Self { - stream: InterpretationStream::parse_with_context( - input, - arguments.full_span_range(), - )?, + stream: InterpretationStream::parse_with_context(input, arguments.full_span())?, }) }, "Expected (!content! ... interpretable input ...)", diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index c8e7c0b6..6dc8fd59 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -99,14 +99,18 @@ impl<'a> ExpressionEvaluator<'a> { } EvaluationStackFrame::BinaryOperation { operation, state } => match state { BinaryPath::OnLeftBranch { right } => { - self.operation_stack - .push(EvaluationStackFrame::BinaryOperation { - operation, - state: BinaryPath::OnRightBranch { - left: evaluation_value, - }, - }); - NextAction::EnterNode(right) + if let Some(result) = operation.lazy_evaluate(&evaluation_value)? { + NextAction::HandleValue(result) + } else { + self.operation_stack + .push(EvaluationStackFrame::BinaryOperation { + operation, + state: BinaryPath::OnRightBranch { + left: evaluation_value, + }, + }); + NextAction::EnterNode(right) + } } BinaryPath::OnRightBranch { left } => { let result = operation.evaluate(left, evaluation_value)?; @@ -182,16 +186,12 @@ impl EvaluationOutput { } } -impl HasSpanRange for EvaluationOutput { +impl HasSpan for EvaluationOutput { fn span(&self) -> Span { self.value .source_span() .unwrap_or(self.fallback_output_span) } - - fn span_range(&self) -> SpanRange { - self.span().span_range() - } } impl quote::ToTokens for EvaluationOutput { diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index e582843d..c6a4a43c 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -26,7 +26,7 @@ impl Expression { ) -> ExecutionResult { Ok(EvaluationOutput { value: self.evaluate_to_value(interpreter)?, - fallback_output_span: self.span_range.span(), + fallback_output_span: self.span_range.join_into_span_else_start(), }) } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 0d20d9a8..649afed2 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -46,7 +46,7 @@ impl<'a> ExpressionParser<'a> { self.add_leaf(ExpressionLeaf::Command(command)) } UnaryAtom::GroupedVariable(variable) => { - self.span_range.set_end(variable.span()); + self.span_range.set_end(variable.span_range().end()); self.add_leaf(ExpressionLeaf::GroupedVariable(variable)) } UnaryAtom::CodeBlock(code_block) => { diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 2b1e8105..19189b97 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -120,7 +120,7 @@ impl UnaryOperator { } } -impl HasSpanRange for UnaryOperator { +impl HasSpan for UnaryOperator { fn span(&self) -> Span { match self { UnaryOperator::Neg { token } => token.span, @@ -129,10 +129,6 @@ impl HasSpanRange for UnaryOperator { UnaryOperator::Cast { as_token, .. } => as_token.span, } } - - fn span_range(&self) -> SpanRange { - self.span().span_range() - } } pub(super) trait HandleUnaryOperation: Sized { @@ -209,11 +205,44 @@ impl BinaryOperation { }; Ok(Self { source_span: None, - operator_span: syn_operator.span(), + operator_span: syn_operator.span_range().join_into_span_else_start(), operator, }) } + pub(super) fn lazy_evaluate( + &self, + left: &EvaluationValue, + ) -> ExecutionResult> { + match self.operator { + BinaryOperator::Paired(PairedBinaryOperator::LogicalAnd) => { + match left.clone().into_bool() { + Some(bool) => { + if !bool.value { + Ok(Some(EvaluationValue::Boolean(bool))) + } else { + Ok(None) + } + } + None => self.execution_err("The left operand was not a boolean"), + } + } + BinaryOperator::Paired(PairedBinaryOperator::LogicalOr) => { + match left.clone().into_bool() { + Some(bool) => { + if bool.value { + Ok(Some(EvaluationValue::Boolean(bool))) + } else { + Ok(None) + } + } + None => self.execution_err("The left operand was not a boolean"), + } + } + _ => Ok(None), + } + } + pub(super) fn evaluate( self, left: EvaluationValue, @@ -249,6 +278,12 @@ impl BinaryOperation { } } +impl HasSpan for BinaryOperation { + fn span(&self) -> Span { + self.operator_span + } +} + #[derive(Copy, Clone)] pub(super) enum BinaryOperator { Paired(PairedBinaryOperator), diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index fc399cf7..a634209c 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -43,16 +43,31 @@ impl SpanErrorExt for T { } } +/// This is intended to be implemented only for types which have a cheap span. +/// It is cheaper than [`syn::spanned`], which requires streaming the whole type, +/// and can be very slow for e.g. large expressions. +pub(crate) trait HasSpan { + fn span(&self) -> Span; +} + +/// This is intended to be implemented only for types which have a cheap SpanRange. +/// It is cheaper than [`syn::spanned`], which requires streaming the whole type, +/// and can be very slow for e.g. large expressions. +/// +/// See also [`SlowSpanRange`] for the equivalent of [`syn::spanned`]. pub(crate) trait HasSpanRange { fn span_range(&self) -> SpanRange; +} - fn span(&self) -> Span { - self.span_range().span() +impl HasSpanRange for T { + fn span_range(&self) -> SpanRange { + SpanRange::new_single(self.span()) } } -/// [`syn::spanned`] has the limitation that it uses [`proc_macro::Span::join`] -/// and falls back to the span of the first token when not available. +/// [`syn::spanned`] is potentially unexpectedly expensive, and has the +/// limitation that it uses [`proc_macro::Span::join`] and falls back to the +/// span of the first token when not available. /// /// Instead, [`syn::Error`] uses a trick involving a span range. This effectively /// allows capturing this trick when we're not immediately creating an error. @@ -83,7 +98,7 @@ impl SpanRange { /// * On nightly, this gives a span covering the full range (the same result as `Span::join` would) /// * On stable, this gives the span of the first token of the group (because [`proc_macro::Span::join`] is not supported) - pub(crate) fn span(&self) -> Span { + pub(crate) fn join_into_span_else_start(&self) -> Span { ::span(self) } @@ -118,30 +133,57 @@ impl HasSpanRange for SpanRange { } } -impl HasSpanRange for Span { - fn span_range(&self) -> SpanRange { - SpanRange::new_between(*self, *self) +impl HasSpan for Span { + fn span(&self) -> Span { + *self } } -impl HasSpanRange for TokenTree { - fn span_range(&self) -> SpanRange { - self.span().span_range() +impl HasSpan for TokenTree { + fn span(&self) -> Span { + self.span() } } -impl HasSpanRange for Group { - fn span_range(&self) -> SpanRange { - self.span().span_range() +impl HasSpan for Group { + fn span(&self) -> Span { + self.span() } } -impl HasSpanRange for DelimSpan { - fn span_range(&self) -> SpanRange { - // We could use self.open() => self.close() here, but using - // self.join() is better as it can be round-tripped to a span - // as the whole span, rather than just the start or end. - self.join().span_range() +impl HasSpan for DelimSpan { + fn span(&self) -> Span { + self.join() + } +} + +impl HasSpan for Ident { + fn span(&self) -> Span { + self.span() + } +} + +impl HasSpan for Punct { + fn span(&self) -> Span { + self.span() + } +} + +impl HasSpan for Literal { + fn span(&self) -> Span { + self.span() + } +} + +impl HasSpan for Token![as] { + fn span(&self) -> Span { + self.span + } +} + +impl HasSpan for Token![in] { + fn span(&self) -> Span { + self.span } } @@ -159,36 +201,28 @@ impl SlowSpanRange for T { } } -/// This should only be used for syn built-ins or when there isn't a better -/// span range available -pub(crate) trait AutoSpanRange {} - +// This should only be used for types implementing ToTokens, which have +// a small, bounded number of tokens, with sensible performance. macro_rules! impl_auto_span_range { ($($ty:ty),* $(,)?) => { $( - impl AutoSpanRange for $ty {} + impl HasSpanRange for $ty { + fn span_range(&self) -> SpanRange { + SlowSpanRange::span_range_from_iterating_over_all_tokens(self) + } + } )* }; } -impl HasSpanRange for T { - fn span_range(&self) -> SpanRange { - // AutoSpanRange should only be used for tokens with a small number of tokens - SlowSpanRange::span_range_from_iterating_over_all_tokens(&self) - } -} - // This should only be used for types with a bounded number of tokens -// otherwise, span_range_from_iterating_over_all_tokens() can be used +// greater than one. +// If exactly one, implement HasSpan. +// Otherwise, span_range_from_iterating_over_all_tokens() can be used // directly with a longer name to make the performance hit clearer, so // it's only used in error cases. impl_auto_span_range! { - Ident, - Punct, - Literal, syn::BinOp, syn::UnOp, - syn::token::As, syn::token::DotDot, - syn::token::In, } diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index 1f3f5a6b..d667f543 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -93,10 +93,7 @@ impl CursorExt for Cursor<'_> { pub(crate) trait ParserBufferExt { fn parse_with(&self, context: T::Context) -> ParseResult; - fn parse_all_for_interpretation( - &self, - span_range: SpanRange, - ) -> ParseResult; + fn parse_all_for_interpretation(&self, span: Span) -> ParseResult; fn try_parse_or_message ParseResult, M: std::fmt::Display>( &self, func: F, @@ -127,11 +124,8 @@ impl ParserBufferExt for ParseBuffer<'_> { T::parse_with_context(self, context) } - fn parse_all_for_interpretation( - &self, - span_range: SpanRange, - ) -> ParseResult { - self.parse_with(span_range) + fn parse_all_for_interpretation(&self, span: Span) -> ParseResult { + self.parse_with(span) } fn try_parse_or_message ParseResult, M: std::fmt::Display>( diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index ff9d9ed0..a45dae27 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -449,9 +449,9 @@ impl Command { } } -impl HasSpanRange for Command { - fn span_range(&self) -> SpanRange { - self.source_group_span.span_range() +impl HasSpan for Command { + fn span(&self) -> Span { + self.source_group_span.join() } } diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index ccca64e2..2d14b9e6 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -25,10 +25,6 @@ impl<'a> CommandArguments<'a> { self.command_span } - pub(crate) fn command_span_range(&self) -> SpanRange { - self.command_span.span_range() - } - /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> ParseResult<()> { if self.parse_stream.is_empty() { @@ -70,7 +66,7 @@ impl<'a> CommandArguments<'a> { pub(crate) fn parse_all_for_interpretation(&self) -> ParseResult { self.parse_stream - .parse_all_for_interpretation(self.command_span.span_range()) + .parse_all_for_interpretation(self.command_span) } pub(crate) fn read_all_as_raw_token_stream(&self) -> TokenStream { diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index 883364dd..18da676e 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -10,14 +10,14 @@ pub(crate) struct CommandCodeInput { impl Parse for CommandCodeInput { fn parse(input: ParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; - let inner = content.parse_with(delim_span.join().span_range())?; + let inner = content.parse_with(delim_span.join())?; Ok(Self { delim_span, inner }) } } -impl HasSpanRange for CommandCodeInput { - fn span_range(&self) -> SpanRange { - self.delim_span.span_range() +impl HasSpan for CommandCodeInput { + fn span(&self) -> Span { + self.delim_span.join() } } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 16390ae8..731f06d4 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -21,7 +21,7 @@ impl NoOutputCommandDefinition for SetCommand { Ok(Self { variable: input.parse()?, equals: input.parse()?, - arguments: input.parse_with(arguments.command_span_range())?, + arguments: input.parse_with(arguments.command_span())?, }) }, "Expected [!set! #variable = ..]", @@ -56,8 +56,7 @@ impl NoOutputCommandDefinition for ExtendCommand { Ok(Self { variable: input.parse()?, plus_equals: input.parse()?, - arguments: input - .parse_all_for_interpretation(arguments.command_span_range())?, + arguments: input.parse_all_for_interpretation(arguments.command_span())?, }) }, "Expected [!extend! #variable += ..]", @@ -222,7 +221,7 @@ impl NoOutputCommandDefinition for ErrorCommand { } else { Ok(Self { inputs: EitherErrorInput::JustMessage( - input.parse_with(arguments.command_span_range())?, + input.parse_with(arguments.command_span())?, ), }) } diff --git a/src/interpretation/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs index 4cf92526..0ef941f3 100644 --- a/src/interpretation/commands/destructuring_commands.rs +++ b/src/interpretation/commands/destructuring_commands.rs @@ -21,7 +21,7 @@ impl NoOutputCommandDefinition for LetCommand { Ok(Self { destructuring: input.parse()?, equals: input.parse()?, - arguments: input.parse_with(arguments.command_span_range())?, + arguments: input.parse_with(arguments.command_span())?, }) }, "Expected [!let! = ...]", diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 6135dc69..55651384 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -146,7 +146,7 @@ impl StreamCommandDefinition for RangeCommand { output: &mut InterpretedStream, ) -> ExecutionResult<()> { let range_span_range = self.range_limits.span_range(); - let range_span = self.range_limits.span(); + let range_span = range_span_range.join_into_span_else_start(); let left = self .left @@ -221,7 +221,11 @@ impl ToTokens for RangeLimits { } } -impl AutoSpanRange for RangeLimits {} +impl HasSpanRange for RangeLimits { + fn span_range(&self) -> SpanRange { + self.span_range_from_iterating_over_all_tokens() + } +} impl RangeLimits { fn length_of_range(&self, left: i128, right: i128) -> Option { diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index fc3aaae5..f82e954d 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -19,7 +19,7 @@ impl ValueCommandDefinition for IsEmptyCommand { } fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { - let output_span = self.arguments.span_range().span(); + let output_span = self.arguments.span_range().join_into_span_else_start(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; Ok(Ident::new_bool(interpreted.is_empty(), output_span).into()) } @@ -44,7 +44,7 @@ impl ValueCommandDefinition for LengthCommand { } fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { - let output_span = self.arguments.span_range().span(); + let output_span = self.arguments.span_range().join_into_span_else_start(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; let length_literal = Literal::usize_unsuffixed(interpreted.len()).with_span(output_span); Ok(length_literal.into()) @@ -251,7 +251,7 @@ impl StreamCommandDefinition for SplitCommand { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> ExecutionResult<()> { - let output_span = self.inputs.stream.span(); + let output_span = self.inputs.stream.span_range().join_into_span_else_start(); let stream = self.inputs.stream.interpret_to_new_stream(interpreter)?; let separator = self.inputs.separator.interpret_to_new_stream(interpreter)?; @@ -340,7 +340,7 @@ impl StreamCommandDefinition for CommaSplitCommand { interpreter: &mut Interpreter, output: &mut InterpretedStream, ) -> ExecutionResult<()> { - let output_span = self.input.span(); + let output_span = self.input.span_range().join_into_span_else_start(); let stream = self.input.interpret_to_new_stream(interpreter)?; let separator = { let mut stream = RawDestructureStream::empty(); diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 656ddd6d..00864ead 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -4,18 +4,18 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct InterpretationStream { items: Vec, - span_range: SpanRange, + span: Span, } impl ContextualParse for InterpretationStream { - type Context = SpanRange; + type Context = Span; - fn parse_with_context(input: ParseStream, span_range: Self::Context) -> ParseResult { + fn parse_with_context(input: ParseStream, span: Self::Context) -> ParseResult { let mut items = Vec::new(); while !input.is_empty() { items.push(input.parse()?); } - Ok(Self { items, span_range }) + Ok(Self { items, span }) } } @@ -32,9 +32,9 @@ impl Interpret for InterpretationStream { } } -impl HasSpanRange for InterpretationStream { - fn span_range(&self) -> SpanRange { - self.span_range +impl HasSpan for InterpretationStream { + fn span(&self) -> Span { + self.span } } @@ -55,7 +55,7 @@ impl InterpretationGroup { impl Parse for InterpretationGroup { fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_group()?; - let content = content.parse_with(delim_span.span_range())?; + let content = content.parse_with(delim_span.join())?; Ok(Self { source_delimiter: delimiter, source_delim_span: delim_span, @@ -76,9 +76,9 @@ impl Interpret for InterpretationGroup { } } -impl HasSpanRange for InterpretationGroup { - fn span_range(&self) -> SpanRange { - self.source_delim_span.span_range() +impl HasSpan for InterpretationGroup { + fn span(&self) -> Span { + self.source_delim_span.join() } } @@ -124,8 +124,8 @@ impl Interpret for RawGroup { } } -impl HasSpanRange for RawGroup { - fn span_range(&self) -> SpanRange { - self.source_delim_span.span_range() +impl HasSpan for RawGroup { + fn span(&self) -> Span { + self.source_delim_span.join() } } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 43a7f8e1..5e2efe3a 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -63,7 +63,7 @@ impl GroupedVariable { output.push_new_group( self.read_existing(interpreter)?.get(self)?.clone(), Delimiter::None, - self.span(), + self.span_range().join_into_span_else_start(), ); Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 803a6f74..0dc4b6e1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -547,9 +547,7 @@ fn preinterpret_internal(input: TokenStream) -> SynResult { let mut interpreter = Interpreter::new(); let interpretation_stream = input - .parse_with(|input| { - InterpretationStream::parse_with_context(input, Span::call_site().span_range()) - }) + .parse_with(|input| InterpretationStream::parse_with_context(input, Span::call_site())) .convert_to_final_result()?; let interpreted_stream = interpretation_stream diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs deleted file mode 100644 index cf9e1327..00000000 --- a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs +++ /dev/null @@ -1,11 +0,0 @@ -use preinterpret::*; - -fn main() { - preinterpret!{ - [!set! #is_eager = false] - let _ = [!evaluate! false && [!group! [!set! #is_eager = true] true]]; - [!if! #is_eager { - [!error! { message: "The && expression is not evaluated lazily" }] - }] - } -} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr b/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr deleted file mode 100644 index 688810a9..00000000 --- a/tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.stderr +++ /dev/null @@ -1,13 +0,0 @@ -error: The && expression is not evaluated lazily - --> tests/compilation_failures/expressions/fix_me_comparison_operators_are_not_lazy.rs:4:5 - | -4 | / preinterpret!{ -5 | | [!set! #is_eager = false] -6 | | let _ = [!evaluate! false && [!group! [!set! #is_eager = true] true]]; -7 | | [!if! #is_eager { -8 | | [!error! { message: "The && expression is not evaluated lazily" }] -9 | | }] -10 | | } - | |_____^ - | - = note: this error originates in the macro `preinterpret` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/expressions.rs b/tests/expressions.rs index df1118d7..effcd00e 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -56,6 +56,14 @@ fn test_basic_evaluate_works() { }, 7 ); + assert_preinterpret_eq!( + { + [!set! #partial_sum = + 2] + // A { ... } block is evaluated as an expression after interpretation + [!evaluate! { 1 #..partial_sum }] + }, + 3 + ); assert_preinterpret_eq!( { [!evaluate! 1 + [!range! 1..2]] @@ -76,12 +84,38 @@ fn test_basic_evaluate_works() { fn test_very_long_expression_works() { assert_preinterpret_eq!( { - [!settings! { iteration_limit: 100000 }][!evaluate! [!group! 0 [!for! #i in [!range! 0..100000] { + 1 }]]] + [!settings! { + iteration_limit: 100000, + }][!evaluate! { + 0 [!for! #i in [!range! 0..100000] { + 1 }] + }] }, 100000 ); } +#[test] +fn boolean_operators_short_circuit() { + // && short-circuits if first operand is false + assert_preinterpret_eq!( + { + [!set! #is_lazy = true] + [!void! [!evaluate! false && { [!set! #is_lazy = false] true }]] + #is_lazy + }, + true + ); + // || short-circuits if first operand is true + assert_preinterpret_eq!( + { + [!set! #is_lazy = true] + [!void! [!evaluate! true || { [!set! #is_lazy = false] true }]] + #is_lazy + }, + true + ); +} + #[test] fn assign_works() { assert_preinterpret_eq!( From 95718697deae54555e802ab5af7842e5c95ef70f Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 30 Jan 2025 22:44:48 +0000 Subject: [PATCH 068/126] fix: Fix MSRV --- src/expressions/expression_parsing.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 649afed2..764f930b 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -265,7 +265,7 @@ impl ExpressionNodes { /// Reference: https://doc.rust-lang.org/reference/expressions.html#expression-precedence #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] #[allow(unused)] -pub(crate) enum OperatorPrecendence { +enum OperatorPrecendence { // return, break, closures Jump, /// = += -= *= /= %= &= |= ^= <<= >>= @@ -301,9 +301,9 @@ pub(crate) enum OperatorPrecendence { } impl OperatorPrecendence { - pub(crate) const MIN: Self = OperatorPrecendence::Jump; + const MIN: Self = OperatorPrecendence::Jump; - pub(crate) fn of_unary_operator(op: &UnaryOperator) -> Self { + fn of_unary_operator(op: &UnaryOperator) -> Self { match op { UnaryOperator::GroupedNoOp { .. } => Self::Unambiguous, UnaryOperator::Cast { .. } => Self::Cast, @@ -311,7 +311,7 @@ impl OperatorPrecendence { } } - pub(crate) fn of_binary_operator(op: &BinaryOperator) -> Self { + fn of_binary_operator(op: &BinaryOperator) -> Self { match op { BinaryOperator::Integer(op) => Self::of_integer_binary_operator(op), BinaryOperator::Paired(op) => Self::of_paired_binary_operator(op), From 17b32bd819dd09c40148f1dc77568271a43b2cab Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 30 Jan 2025 23:13:52 +0000 Subject: [PATCH 069/126] fix: !debug! now respects punct spacing --- src/interpretation/interpreted_stream.rs | 59 +++++++++++++++--------- tests/core.rs | 25 ++++++++++ 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 5780a6f2..a514c0ad 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -275,17 +275,17 @@ impl InterpretedStream { fn concat_recursive_interpreted_stream( behaviour: &ConcatBehaviour, output: &mut String, + prefix_spacing: Spacing, stream: InterpretedStream, ) { - let mut n = 0; + let mut spacing = prefix_spacing; for segment in stream.segments { - match segment { + spacing = match segment { InterpretedSegment::TokenVec(vec) => { - n += vec.len(); - concat_recursive_token_stream(behaviour, output, vec); + concat_recursive_token_stream(behaviour, output, spacing, vec) } InterpretedSegment::InterpretedGroup(delimiter, _, interpreted_stream) => { - behaviour.before_nth_token_tree(output, n); + behaviour.before_token_tree(output, spacing); behaviour.wrap_delimiters( output, delimiter, @@ -294,11 +294,12 @@ impl InterpretedStream { concat_recursive_interpreted_stream( behaviour, output, + Spacing::Joint, interpreted_stream, ); }, ); - n += 1; + Spacing::Alone } } } @@ -307,13 +308,16 @@ impl InterpretedStream { fn concat_recursive_token_stream( behaviour: &ConcatBehaviour, output: &mut String, + prefix_spacing: Spacing, token_stream: impl IntoIterator, - ) { - for (n, token_tree) in token_stream.into_iter().enumerate() { - behaviour.before_nth_token_tree(output, n); - match token_tree { + ) -> Spacing { + let mut spacing = prefix_spacing; + for token_tree in token_stream.into_iter() { + behaviour.before_token_tree(output, spacing); + spacing = match token_tree { TokenTree::Literal(literal) => { behaviour.handle_literal(output, literal); + Spacing::Alone } TokenTree::Group(group) => { let inner = group.stream(); @@ -322,20 +326,31 @@ impl InterpretedStream { group.delimiter(), inner.is_empty(), |output| { - concat_recursive_token_stream(behaviour, output, inner); + concat_recursive_token_stream( + behaviour, + output, + Spacing::Joint, + inner, + ); }, ); + Spacing::Alone } TokenTree::Punct(punct) => { output.push(punct.as_char()); + punct.spacing() + } + TokenTree::Ident(ident) => { + output.push_str(&ident.to_string()); + Spacing::Alone } - TokenTree::Ident(ident) => output.push_str(&ident.to_string()), } } + spacing } let mut output = String::new(); - concat_recursive_interpreted_stream(behaviour, &mut output, self); + concat_recursive_interpreted_stream(behaviour, &mut output, Spacing::Joint, self); output } } @@ -349,16 +364,16 @@ impl IntoIterator for InterpretedStream { } } -pub(crate) struct ConcatBehaviour<'a> { - pub(crate) between_token_trees: Option<&'a str>, +pub(crate) struct ConcatBehaviour { + pub(crate) add_space_between_token_trees: bool, pub(crate) output_transparent_group_as_command: bool, pub(crate) unwrap_contents_of_string_like_literals: bool, } -impl ConcatBehaviour<'_> { +impl ConcatBehaviour { pub(crate) fn standard() -> Self { Self { - between_token_trees: None, + add_space_between_token_trees: false, output_transparent_group_as_command: false, unwrap_contents_of_string_like_literals: true, } @@ -366,17 +381,15 @@ impl ConcatBehaviour<'_> { pub(crate) fn debug() -> Self { Self { - between_token_trees: Some(" "), + add_space_between_token_trees: true, output_transparent_group_as_command: true, unwrap_contents_of_string_like_literals: false, } } - fn before_nth_token_tree(&self, output: &mut String, n: usize) { - if let Some(between) = self.between_token_trees { - if n > 0 { - output.push_str(between); - } + fn before_token_tree(&self, output: &mut String, spacing: Spacing) { + if self.add_space_between_token_trees && spacing == Spacing::Alone { + output.push(' '); } } diff --git a/tests/core.rs b/tests/core.rs index 24cb8ca3..fdc1c6a9 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -80,3 +80,28 @@ fn test_void() { #x }, true); } + +#[test] +fn test_debug() { + // It keeps the semantic punctuation spacing intact + // (e.g. it keeps 'a and >> together) + assert_preinterpret_eq!( + [!debug! impl<'a, T> MyStruct<'a, T> { + pub fn new() -> Self { + !($crate::Test::CONSTANT >> 5 > 1) + } + }], + "impl < 'a , T > MyStruct < 'a , T > { pub fn new () -> Self { ! ($ crate :: Test :: CONSTANT >> 5 > 1) } }" + ); + // It shows transparent groups + // NOTE: The output code can't be used directly as preinterpret input + // because it doesn't stick [!raw! ...] around things which could be confused + // for the preinterpret grammar. Perhaps it could/should in future. + assert_preinterpret_eq!( + { + [!set! #x = Hello (World)] + [!debug! #x [!..raw! #test] "and" [!raw! ##] #..x] + }, + r###"[!group! Hello (World)] # test "and" [!group! ##] Hello (World)"### + ); +} From 80f1883e9a7504ec8b288598038e6fc466efdd1a Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 31 Jan 2025 23:48:18 +0000 Subject: [PATCH 070/126] refactor: Merge peeking of Commands --- CHANGELOG.md | 1 + src/destructuring/destructure_item.rs | 14 ++-- src/destructuring/destructure_variable.rs | 3 +- src/expressions/expression_parsing.rs | 7 +- src/interpretation/command.rs | 82 +++++++++++-------- src/interpretation/command_stream_input.rs | 3 +- src/interpretation/command_value_input.rs | 3 +- src/interpretation/interpretation_item.rs | 17 ++-- .../destructure_with_flattened_command.stderr | 2 +- ...destructure_with_outputting_command.stderr | 2 +- 10 files changed, 70 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a03326ab..9d67327f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ Destructuring performs parsing of a token stream. It supports: * Revise the comment in expression_parsing.rs * Create `ParseInterpreted` and `ParseSource` via `ParseStream`, `SourceParseStream` and `InterpretedParseStream`, and add grammar peaking to `SourceParseStream` only. Add `[!reinterpret! ...]` command for an `eval` style command. * Search / resolve TODOs +* Support `[!set! #x]` to be `[!set! #x =]`. * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` * Change `(!content! ...)` to unwrap none groups, to be more permissive diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs index d20ee791..1cc52cd8 100644 --- a/src/destructuring/destructure_item.rs +++ b/src/destructuring/destructure_item.rs @@ -18,17 +18,13 @@ impl DestructureItem { /// parsing `Hello` into `x`. pub(crate) fn parse_until(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(Some(command_kind)) - if matches!(command_kind.output_kind(None), Ok(CommandOutputKind::None)) => - { + PeekMatch::Command(Some(CommandOutputKind::None)) => { Self::NoneOutputCommand(input.parse()?) } - PeekMatch::GroupedCommand(_) => return input.parse_err( - "Grouped commands returning a value are not supported in destructuring positions", - ), - PeekMatch::FlattenedCommand(_) => { - return input - .parse_err("Flattened commands are not supported in destructuring positions") + PeekMatch::Command(_) => { + return input.parse_err( + "Commands which return something are not supported in destructuring positions", + ) } PeekMatch::GroupedVariable | PeekMatch::FlattenedVariable diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index 90b01683..eb552556 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -295,8 +295,7 @@ impl ParseUntil { return Ok(ParseUntil::End); } Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) - | PeekMatch::FlattenedCommand(_) + PeekMatch::Command(_) | PeekMatch::GroupedVariable | PeekMatch::FlattenedVariable | PeekMatch::Destructurer(_) diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 764f930b..897a590e 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -167,14 +167,13 @@ impl<'a> ExpressionParser<'a> { fn parse_unary_atom(&mut self) -> ParseResult { Ok(match self.streams.peek_grammar() { - PeekMatch::GroupedCommand(Some(command_kind)) => { - match command_kind.grouped_output_kind().expression_support() { + PeekMatch::Command(Some(output_kind)) => { + match output_kind.expression_support() { Ok(()) => UnaryAtom::Command(self.streams.parse()?), Err(error_message) => return self.streams.parse_err(error_message), } } - PeekMatch::GroupedCommand(None) => return self.streams.parse_err("Invalid command"), - PeekMatch::FlattenedCommand(_) => return self.streams.parse_err(CommandOutputKind::FlattenedStream.expression_support().unwrap_err()), + PeekMatch::Command(None) => return self.streams.parse_err("Invalid command"), PeekMatch::GroupedVariable => UnaryAtom::GroupedVariable(self.streams.parse()?), PeekMatch::FlattenedVariable => return self.streams.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), PeekMatch::AppendVariableDestructuring => return self.streams.parse_err("Append variable operations are not supported in an expression"), diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index a45dae27..128317dc 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -31,7 +31,8 @@ pub(crate) trait CommandType { pub(crate) trait OutputKind { type Output; - fn resolve(flattening: Option) -> ParseResult; + fn resolve_standard() -> CommandOutputKind; + fn resolve_flattened(error_span_range: SpanRange) -> ParseResult; } struct ExecutionContext<'a> { @@ -92,13 +93,12 @@ pub(crate) struct OutputKindNone; impl OutputKind for OutputKindNone { type Output = (); - fn resolve(flattening: Option) -> ParseResult { - match flattening { - Some(dots) => { - dots.parse_err("This command has no output, so cannot be flattened with ..") - } - None => Ok(CommandOutputKind::None), - } + fn resolve_standard() -> CommandOutputKind { + CommandOutputKind::None + } + + fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { + error_span_range.parse_err("This command has no output, so cannot be flattened with ..") } } @@ -129,12 +129,13 @@ pub(crate) struct OutputKindValue; impl OutputKind for OutputKindValue { type Output = TokenTree; - fn resolve(flattening: Option) -> ParseResult { - match flattening { - Some(dots) => dots - .parse_err("This command outputs a single value, so cannot be flattened with .."), - None => Ok(CommandOutputKind::Value), - } + fn resolve_standard() -> CommandOutputKind { + CommandOutputKind::Value + } + + fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { + error_span_range + .parse_err("This command outputs a single value, so cannot be flattened with ..") } } @@ -165,12 +166,13 @@ pub(crate) struct OutputKindIdent; impl OutputKind for OutputKindIdent { type Output = Ident; - fn resolve(flattening: Option) -> ParseResult { - match flattening { - Some(dots) => dots - .parse_err("This command outputs a single ident, so cannot be flattened with .."), - None => Ok(CommandOutputKind::Ident), - } + fn resolve_standard() -> CommandOutputKind { + CommandOutputKind::Ident + } + + fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { + error_span_range + .parse_err("This command outputs a single ident, so cannot be flattened with ..") } } @@ -201,11 +203,12 @@ pub(crate) struct OutputKindStream; impl OutputKind for OutputKindStream { type Output = InterpretedStream; - fn resolve(flattening: Option) -> ParseResult { - match flattening { - Some(_) => Ok(CommandOutputKind::FlattenedStream), - None => Ok(CommandOutputKind::GroupedStream), - } + fn resolve_standard() -> CommandOutputKind { + CommandOutputKind::GroupedStream + } + + fn resolve_flattened(_: SpanRange) -> ParseResult { + Ok(CommandOutputKind::FlattenedStream) } } @@ -247,11 +250,12 @@ pub(crate) struct OutputKindControlFlow; impl OutputKind for OutputKindControlFlow { type Output = (); - fn resolve(flattening: Option) -> ParseResult { - match flattening { - Some(dots) => dots.parse_err("This command is control flow, so is always flattened and cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command"), - None => Ok(CommandOutputKind::ControlFlowCodeStream), - } + fn resolve_standard() -> CommandOutputKind { + CommandOutputKind::ControlFlowCodeStream + } + + fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { + error_span_range.parse_err("This command is control flow, so is always flattened and cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command") } } @@ -304,15 +308,18 @@ macro_rules! define_command_kind { }) } - pub(crate) fn grouped_output_kind(&self) -> CommandOutputKind { - // Guaranteed to be Ok if no flattening is provided - self.output_kind(None).unwrap() + pub(crate) fn standard_output_kind(&self) -> CommandOutputKind { + match self { + $( + Self::$command => <$command as CommandType>::OutputKind::resolve_standard(), + )* + } } - pub(crate) fn output_kind(&self, flattening: Option) -> ParseResult { + pub(crate) fn flattened_output_kind(&self, error_span_range: SpanRange) -> ParseResult { match self { $( - Self::$command => <$command as CommandType>::OutputKind::resolve(flattening), + Self::$command => <$command as CommandType>::OutputKind::resolve_flattened(error_span_range), )* } } @@ -413,7 +420,10 @@ impl Parse for Command { let command_name = content.parse_any_ident()?; let (command_kind, output_kind) = match CommandKind::for_ident(&command_name) { Some(command_kind) => { - let output_kind = command_kind.output_kind(flattening)?; + let output_kind = match flattening { + Some(flattening) => command_kind.flattened_output_kind(flattening.span_range())?, + None => command_kind.standard_output_kind(), + }; (command_kind, output_kind) } None => command_name.span().err( diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index e90e1cd2..50a9048f 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -24,8 +24,7 @@ pub(crate) enum CommandStreamInput { impl Parse for CommandStreamInput { fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), + PeekMatch::Command(_) => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 8062538b..b9b87f4a 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -16,8 +16,7 @@ pub(crate) enum CommandValueInput { impl Parse for CommandValueInput { fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => Self::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => Self::Command(input.parse()?), + PeekMatch::Command(_) => Self::Command(input.parse()?), PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 04d78ba0..c53e3da7 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -14,8 +14,7 @@ pub(crate) enum InterpretationItem { impl Parse for InterpretationItem { fn parse(input: ParseStream) -> ParseResult { Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::GroupedCommand(_) => InterpretationItem::Command(input.parse()?), - PeekMatch::FlattenedCommand(_) => InterpretationItem::Command(input.parse()?), + PeekMatch::Command(_) => InterpretationItem::Command(input.parse()?), PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), @@ -32,8 +31,7 @@ impl Parse for InterpretationItem { #[allow(unused)] pub(crate) enum PeekMatch { - GroupedCommand(Option), - FlattenedCommand(Option), + Command(Option), GroupedVariable, FlattenedVariable, AppendVariableDestructuring, @@ -53,14 +51,19 @@ pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMa if let Some((_, next)) = next.punct_matching('!') { if let Some((ident, next)) = next.ident() { if next.punct_matching('!').is_some() { - return PeekMatch::GroupedCommand(CommandKind::for_ident(&ident)); + let output_kind = + CommandKind::for_ident(&ident).map(|kind| kind.standard_output_kind()); + return PeekMatch::Command(output_kind); } } - if let Some((_, next)) = next.punct_matching('.') { + if let Some((first, next)) = next.punct_matching('.') { if let Some((_, next)) = next.punct_matching('.') { if let Some((ident, next)) = next.ident() { if next.punct_matching('!').is_some() { - return PeekMatch::FlattenedCommand(CommandKind::for_ident(&ident)); + let output_kind = CommandKind::for_ident(&ident).and_then(|kind| { + kind.flattened_output_kind(first.span_range()).ok() + }); + return PeekMatch::Command(output_kind); } } } diff --git a/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr b/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr index a3e716e7..e518d650 100644 --- a/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr +++ b/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr @@ -1,4 +1,4 @@ -error: Flattened commands are not supported in destructuring positions +error: Commands which return something are not supported in destructuring positions Occurred whilst parsing [!let! ...] - Expected [!let! = ...] --> tests/compilation_failures/destructuring/destructure_with_flattened_command.rs:4:26 | diff --git a/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr b/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr index 8ed41c10..935d3c07 100644 --- a/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr +++ b/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr @@ -1,4 +1,4 @@ -error: Grouped commands returning a value are not supported in destructuring positions +error: Commands which return something are not supported in destructuring positions Occurred whilst parsing [!let! ...] - Expected [!let! = ...] --> tests/compilation_failures/destructuring/destructure_with_outputting_command.rs:4:26 | From 93ac5923431ab84926fb865f762294dbaebece87 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 1 Feb 2025 01:56:54 +0000 Subject: [PATCH 071/126] refactor: Simplify BinaryOperation and UnaryOperation --- CHANGELOG.md | 23 +- src/expressions/boolean.rs | 59 ++- src/expressions/character.rs | 48 +- src/expressions/expression_parsing.rs | 227 ++++----- src/expressions/float.rs | 125 ++--- src/expressions/integer.rs | 171 +++---- src/expressions/operations.rs | 465 ++++++++++-------- src/expressions/string.rs | 50 +- src/expressions/value.rs | 23 +- src/extensions/errors_and_spans.rs | 57 ++- src/extensions/parsing.rs | 19 + src/internal_prelude.rs | 2 +- .../expressions/invalid_binary_operator.rs | 11 + .../invalid_binary_operator.stderr | 5 + .../expressions/invalid_unary_operator.rs | 7 + .../expressions/invalid_unary_operator.stderr | 6 + tests/expressions.rs | 16 + 17 files changed, 728 insertions(+), 586 deletions(-) create mode 100644 tests/compilation_failures/expressions/invalid_binary_operator.rs create mode 100644 tests/compilation_failures/expressions/invalid_binary_operator.stderr create mode 100644 tests/compilation_failures/expressions/invalid_unary_operator.rs create mode 100644 tests/compilation_failures/expressions/invalid_unary_operator.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d67327f..14e2c4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,13 +81,8 @@ Destructuring performs parsing of a token stream. It supports: ### To come * Complete expression rework: - * Add more tests for operator precedence (e.g. the worked example) - * Flatten `UnaryOperation` and `UnaryOperator` - * Flatten `BinaryOperation` and `BinaryOperator` * Disallow expressions/commands in re-evaluated `{ .. }` blocks and command outputs - * Revise the comment in expression_parsing.rs * Create `ParseInterpreted` and `ParseSource` via `ParseStream`, `SourceParseStream` and `InterpretedParseStream`, and add grammar peaking to `SourceParseStream` only. Add `[!reinterpret! ...]` command for an `eval` style command. - * Search / resolve TODOs * Support `[!set! #x]` to be `[!set! #x =]`. * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` @@ -104,6 +99,15 @@ Destructuring performs parsing of a token stream. It supports: * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed +* Work on book + * Input paradigms: + * Streams + * StreamInput / ValueInput / CodeInput + * Including documenting expressions + * There are three main kinds of commands: + * Those taking a stream as-is + * Those taking some { fields } + * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` etc * Pushed to 0.4: * Fork of syn to: * Fix issues in Rust Analyzer @@ -118,15 +122,6 @@ Destructuring performs parsing of a token stream. It supports: * See e.g. invalid_content_too_long where `unexpected token` is quite vague. Maybe we can't sensibly do better though... * Further syn parsings (e.g. item, fields, etc) -* Work on book - * Input paradigms: - * Streams - * StreamInput / ValueInput / CodeInput - * Including documenting expressions - * There are three main kinds of commands: - * Those taking a stream as-is - * Those taking some { fields } - * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` etc # Major Version 0.2 diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index af9ddea3..46823d98 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -21,11 +21,11 @@ impl EvaluationBoolean { operation: UnaryOperation, ) -> ExecutionResult { let input = self.value; - match operation.operator { - UnaryOperator::Neg { .. } => operation.unsupported_for_value_type_err("boolean"), - UnaryOperator::Not { .. } => operation.output(!input), - UnaryOperator::GroupedNoOp { .. } => operation.output(input), - UnaryOperator::Cast { target, .. } => match target { + match operation { + UnaryOperation::Neg { .. } => operation.unsupported_for_value_type_err("boolean"), + UnaryOperation::Not { .. } => operation.output(!input), + UnaryOperation::GroupedNoOp { .. } => operation.output(input), + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } @@ -42,7 +42,7 @@ impl EvaluationBoolean { CastTarget::Integer(IntegerKind::U128) => operation.output(input as u128), CastTarget::Integer(IntegerKind::Usize) => operation.output(input as usize), CastTarget::Float(_) | CastTarget::Char => { - operation.err("This cast is not supported") + operation.execution_err("This cast is not supported") } CastTarget::Boolean => operation.output(self.value), }, @@ -52,10 +52,11 @@ impl EvaluationBoolean { pub(super) fn handle_integer_binary_operation( self, _right: EvaluationInteger, - operation: BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { - match operation.integer_operator() { - IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + match operation { + IntegerBinaryOperation::ShiftLeft { .. } + | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err("boolean") } } @@ -64,27 +65,31 @@ impl EvaluationBoolean { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - match operation.paired_operator() { - PairedBinaryOperator::Addition - | PairedBinaryOperator::Subtraction - | PairedBinaryOperator::Multiplication - | PairedBinaryOperator::Division => operation.unsupported_for_value_type_err("boolean"), - PairedBinaryOperator::LogicalAnd => operation.output(lhs && rhs), - PairedBinaryOperator::LogicalOr => operation.output(lhs || rhs), - PairedBinaryOperator::Remainder => operation.unsupported_for_value_type_err("boolean"), - PairedBinaryOperator::BitXor => operation.output(lhs ^ rhs), - PairedBinaryOperator::BitAnd => operation.output(lhs & rhs), - PairedBinaryOperator::BitOr => operation.output(lhs | rhs), - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(!lhs & rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs & !rhs), + match operation { + PairedBinaryOperation::Addition { .. } + | PairedBinaryOperation::Subtraction { .. } + | PairedBinaryOperation::Multiplication { .. } + | PairedBinaryOperation::Division { .. } => { + operation.unsupported_for_value_type_err("boolean") + } + PairedBinaryOperation::LogicalAnd { .. } => operation.output(lhs && rhs), + PairedBinaryOperation::LogicalOr { .. } => operation.output(lhs || rhs), + PairedBinaryOperation::Remainder { .. } => { + operation.unsupported_for_value_type_err("boolean") + } + PairedBinaryOperation::BitXor { .. } => operation.output(lhs ^ rhs), + PairedBinaryOperation::BitAnd { .. } => operation.output(lhs & rhs), + PairedBinaryOperation::BitOr { .. } => operation.output(lhs | rhs), + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(!lhs & rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs & !rhs), } } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index ccc01a01..53a7ef3b 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -21,12 +21,12 @@ impl EvaluationChar { operation: UnaryOperation, ) -> ExecutionResult { let char = self.value; - match operation.operator { - UnaryOperator::GroupedNoOp { .. } => operation.output(char), - UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } => { + match operation { + UnaryOperation::GroupedNoOp { .. } => operation.output(char), + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { operation.unsupported_for_value_type_err("char") } - UnaryOperator::Cast { target, .. } => match target { + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(char as FallbackInteger)) } @@ -53,7 +53,7 @@ impl EvaluationChar { pub(super) fn handle_integer_binary_operation( self, _right: EvaluationInteger, - operation: BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { operation.unsupported_for_value_type_err("char") } @@ -61,27 +61,29 @@ impl EvaluationChar { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - match operation.paired_operator() { - PairedBinaryOperator::Addition - | PairedBinaryOperator::Subtraction - | PairedBinaryOperator::Multiplication - | PairedBinaryOperator::Division - | PairedBinaryOperator::LogicalAnd - | PairedBinaryOperator::LogicalOr - | PairedBinaryOperator::Remainder - | PairedBinaryOperator::BitXor - | PairedBinaryOperator::BitAnd - | PairedBinaryOperator::BitOr => operation.unsupported_for_value_type_err("char"), - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + match operation { + PairedBinaryOperation::Addition { .. } + | PairedBinaryOperation::Subtraction { .. } + | PairedBinaryOperation::Multiplication { .. } + | PairedBinaryOperation::Division { .. } + | PairedBinaryOperation::LogicalAnd { .. } + | PairedBinaryOperation::LogicalOr { .. } + | PairedBinaryOperation::Remainder { .. } + | PairedBinaryOperation::BitXor { .. } + | PairedBinaryOperation::BitAnd { .. } + | PairedBinaryOperation::BitOr { .. } => { + operation.unsupported_for_value_type_err("char") + } + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), } } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 897a590e..fc5bb605 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -64,7 +64,7 @@ impl<'a> ExpressionParser<'a> { self.push_stack_frame(ExpressionStackFrame::Group { delim_span }) } UnaryAtom::UnaryOperation(operation) => { - self.span_range.set_end(operation.operator.span()); + self.span_range.set_end(operation.span()); self.push_stack_frame(ExpressionStackFrame::IncompletePrefixOperation { operation }) } }) @@ -109,14 +109,11 @@ impl<'a> ExpressionParser<'a> { ExpressionStackFrame::Group { delim_span } => { assert!(matches!(extension, NodeExtension::NoneMatched)); self.streams.exit_group(); - let operation = UnaryOperation { - operator: UnaryOperator::GroupedNoOp { - span: delim_span.join(), - }, - }; WorkItem::TryParseAndApplyExtension { node: self.nodes.add_node(ExpressionNode::UnaryOperation { - operation, + operation: UnaryOperation::GroupedNoOp { + span: delim_span.join(), + }, input: node, }), } @@ -146,12 +143,9 @@ impl<'a> ExpressionParser<'a> { fn parse_extension(&mut self) -> ParseResult { Ok(match self.streams.peek_grammar() { - PeekMatch::Punct(punct) => match punct.as_char() { - '.' => NodeExtension::NoneMatched, - _ => { - let operation = BinaryOperation::for_binary_operator(self.streams.parse()?)?; - NodeExtension::BinaryOperation(operation) - } + PeekMatch::Punct(_) => match self.streams.try_parse_or_revert::() { + Ok(operation) => NodeExtension::BinaryOperation(operation), + Err(_) => NodeExtension::NoneMatched, }, PeekMatch::Ident(ident) if ident == "as" => { let cast_operation = UnaryOperation::for_cast_operation(self.streams.parse()?, { @@ -185,8 +179,7 @@ impl<'a> ExpressionParser<'a> { PeekMatch::Group(Delimiter::Brace) => UnaryAtom::CodeBlock(self.streams.parse()?), PeekMatch::Group(Delimiter::Bracket) => return self.streams.parse_err("Square brackets [ .. ] are not supported in an expression"), PeekMatch::Punct(_) => { - let unary_operation = UnaryOperation::for_unary_operator(self.streams.parse()?)?; - UnaryAtom::UnaryOperation(unary_operation) + UnaryAtom::UnaryOperation(self.streams.parse_with(UnaryOperation::parse_from_prefix_punct)?) }, PeekMatch::Ident(_) => { UnaryAtom::Value(EvaluationValue::Boolean(EvaluationBoolean::for_litbool(self.streams.parse()?))) @@ -302,59 +295,55 @@ enum OperatorPrecendence { impl OperatorPrecendence { const MIN: Self = OperatorPrecendence::Jump; - fn of_unary_operator(op: &UnaryOperator) -> Self { + fn of_unary_operation(op: &UnaryOperation) -> Self { match op { - UnaryOperator::GroupedNoOp { .. } => Self::Unambiguous, - UnaryOperator::Cast { .. } => Self::Cast, - UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } => Self::Prefix, + UnaryOperation::GroupedNoOp { .. } => Self::Unambiguous, + UnaryOperation::Cast { .. } => Self::Cast, + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => Self::Prefix, } } - fn of_binary_operator(op: &BinaryOperator) -> Self { + fn of_binary_operation(op: &BinaryOperation) -> Self { match op { - BinaryOperator::Integer(op) => Self::of_integer_binary_operator(op), - BinaryOperator::Paired(op) => Self::of_paired_binary_operator(op), + BinaryOperation::Integer(op) => Self::of_integer_binary_operator(op), + BinaryOperation::Paired(op) => Self::of_paired_binary_operator(op), } } - fn of_integer_binary_operator(op: &IntegerBinaryOperator) -> Self { + fn of_integer_binary_operator(op: &IntegerBinaryOperation) -> Self { match op { - IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { - OperatorPrecendence::Shift - } + IntegerBinaryOperation::ShiftLeft { .. } + | IntegerBinaryOperation::ShiftRight { .. } => OperatorPrecendence::Shift, } } - fn of_paired_binary_operator(op: &PairedBinaryOperator) -> Self { + fn of_paired_binary_operator(op: &PairedBinaryOperation) -> Self { match op { - PairedBinaryOperator::Addition | PairedBinaryOperator::Subtraction => Self::Sum, - PairedBinaryOperator::Multiplication - | PairedBinaryOperator::Division - | PairedBinaryOperator::Remainder => Self::Product, - PairedBinaryOperator::LogicalAnd => Self::And, - PairedBinaryOperator::LogicalOr => Self::Or, - PairedBinaryOperator::BitXor => Self::BitXor, - PairedBinaryOperator::BitAnd => Self::BitAnd, - PairedBinaryOperator::BitOr => Self::BitOr, - PairedBinaryOperator::Equal - | PairedBinaryOperator::LessThan - | PairedBinaryOperator::LessThanOrEqual - | PairedBinaryOperator::NotEqual - | PairedBinaryOperator::GreaterThanOrEqual - | PairedBinaryOperator::GreaterThan => Self::Compare, + PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } => { + Self::Sum + } + PairedBinaryOperation::Multiplication { .. } + | PairedBinaryOperation::Division { .. } + | PairedBinaryOperation::Remainder { .. } => Self::Product, + PairedBinaryOperation::LogicalAnd { .. } => Self::And, + PairedBinaryOperation::LogicalOr { .. } => Self::Or, + PairedBinaryOperation::BitXor { .. } => Self::BitXor, + PairedBinaryOperation::BitAnd { .. } => Self::BitAnd, + PairedBinaryOperation::BitOr { .. } => Self::BitOr, + PairedBinaryOperation::Equal { .. } + | PairedBinaryOperation::LessThan { .. } + | PairedBinaryOperation::LessThanOrEqual { .. } + | PairedBinaryOperation::NotEqual { .. } + | PairedBinaryOperation::GreaterThanOrEqual { .. } + | PairedBinaryOperation::GreaterThan { .. } => Self::Compare, } } } /// Use of a stack avoids recursion, which hits limits with long/deep expressions. /// -/// ## Expression Types (in decreasing precedence) -/// * Leaf Expression: Command, Variable, Value (literal, true/false) -/// * Prefix Expression: prefix-based unary operators -/// * Postfix Expression: postfix-based unary operators such as casting -/// * Binary Expression: binary operators -/// /// ## Worked Algorithm Sketch +/// /// Let ParseStream P be: /// a b cdx e fg h i j k /// 1 + -(1) + (2 + 4) * 3 @@ -362,66 +351,84 @@ impl OperatorPrecendence { /// The algorithm proceeds as follows: /// ```text /// => Start -/// ===> PushParseBuffer([P]) -/// ===> WorkStack: [Root] -/// => Root detects leaf a:1 -/// ===> PushEvalNode(A: Leaf(a:1), NewParentPrecedence: Root => >MIN) -/// ===> WorkStack: [Root, TryExtend(A, >MIN)] -/// => TryExtend detects binop b:+ -/// ===> WorkStack: [Root, BinOp(A, b:+)] -/// => BinOp detects unop c:- -/// ===> WorkStack: [Root, BinOp(A, b:+), PrefixOp(c:-)] -/// => PrefixOp detects group d:([D]) +/// ===> Set parse stack to have a root parsebuffer of [P] +/// ===> Stack: [Root] +/// ===> WorkItem::RequireUnaryAtom +/// => Read leaf a:1 +/// ===> PushEvalNode(A: Leaf(a:1)) +/// ===> Stack: [Root] +/// ===> WorkItem::TryParseAndApplyExtension(A) with precedence >MIN from parent=Root +/// => Read binop b:+ +/// ===> Stack: [Root, BinOp(A, b:+)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read unop c:- +/// ===> Stack: [Root, BinOp(A, b:+), PrefixOp(c:-)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read group d:([D]) /// ===> PushParseBuffer([D]) -/// ===> WorkStack: [Root, BinOp(A, b:+), PrefixOp(c:-), Group(d:Paren), EmptyExpression] -/// => EmptyExpression detects leaf x:1 +/// ===> Stack: [Root, BinOp(A, b:+), PrefixOp(c:-), Group(d:Paren)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read leaf x:1 /// ===> PushEvalNode(X: Leaf(x:1), NewParentPrecedence: Group => >MIN) -/// ===> WorkStack: [Root, BinOp(A, +), PrefixOp(c:-), Group(d:Paren), TryExtend(X, >MIN)] -/// => TryExtend detects no valid extension, cascade X with Group: -/// ===> PopWorkStack: It's a group, so PopParseBuffer, PushEvalNode(D: UnOp(d:(), X), NewParentPrecedence: PrefixOp => >PREFIX) -/// ===> WorkStack: [Root, BinOp(A, b:+), PrefixOp(c:-), TryExtend(D, >PREFIX)] -/// => TryExtend detects no valid extension (it's impossible), cascade D with PrefixOp: -/// ===> PopWorkStack: PushEvalNode(C: UnOp(c:-, D), NewParentPrecedence: BinOp => >SUM) -/// ===> WorkStack: [Root, BinOp(A, b:+), TryExtend(C, >SUM)] -/// => TryExtend detects no valid extension, so cascade C with BinOp: -/// ===> PopWorkStack: PushEvalNode(B: BinOp(A, b:+, C), NewParentPrecedence: EMPTY => >MIN) -/// ===> WorkStack: [Root, TryExtend(B, >MIN)] -/// => TryExtend detects binop e:+ -/// ===> WorkStack: [Root, BinOp(B, e:+)] -/// => BinOp detects group f:([F]) +/// ===> Stack: [Root, BinOp(A, +), PrefixOp(c:-), Group(d:Paren)] +/// ===> WorkItem::TryParseAndApplyExtension(X) with precedence >MIN from parent=Group +/// => Extension of None is not valid, cascade X with Group: +/// ===> PopStack: It's a group, so PopParseBuffer, PushEvalNode(D: UnOp(d:(), X)) +/// ===> Stack: [Root, BinOp(A, b:+), PrefixOp(c:-), TryExtend(D, >PREFIX)] +/// ===> WorkItem::TryParseAndApplyExtension(X) with precedence >PREFIX from parent=PrefixOp(c:-) +/// => Extension of + is not valid, cascade D with PrefixOp: +/// ===> PopStack: PushEvalNode(C: UnOp(c:-, D)) +/// ===> Stack: [Root, BinOp(A, b:+)] +/// ===> WorkItem::TryApplyAlreadyParsedExtension(C, +) with precedence >SUM from parent=BinOp(A, b:+) +/// => Extension of + is not valid, so cascade C with BinOp: +/// ===> PopStack: PushEvalNode(B: BinOp(A, b:+, C)) +/// ===> Stack: [Root] +/// ===> WorkItem::TryApplyAlreadyParsedExtension(B, +) with precedence >MIN from parent=Root +/// => Read binop e:+ +/// ===> Stack: [Root, BinOp(B, e:+)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read group f:([F]) /// ===> PushParseBuffer([F]) -/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), EmptyExpression] -/// => EmptyExpression detects leaf g:2 -/// ===> PushEvalNode(G: Leaf(g:2), NewParentPrecedence: Group => >MIN) -/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), TryExtend(G, >MIN)] -/// => TryExtend detects binop h:+ -/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), BinOp(G, h:+)] -/// => BinOp detects leaf i:2 -/// ===> PushEvalNode(I: Leaf(i:2), NewParentPrecedence: BinOp => >SUM) -/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), BinOp(G, h:+), TryExtend(I, >SUM)] -/// => TryExtend detects no valid extension, so cascade I with BinOp: -/// ===> PopWorkStack: PushEvalNode(H: BinOp(G, h:+, I), NewParentPrecedence: Group => >MIN) -/// ===> WorkStack: [Root, BinOp(B, e:+), Group(f:Paren), TryExtend(H, >MIN)] -/// => TryExtend detects no valid extension, so cascade H with Group: -/// ===> PopWorkStack: It's a group, so PopParseBuffer, PushEvalNode(F: UnOp(f:Paren, H), NewParentPrecedence: BinOp => >SUM) -/// ===> WorkStack: [Root, BinOp(B, e:+), TryExtend(F, >SUM)] -/// => TryExtend detects binop j:* -/// ===> WorkStack: [Root, BinOp(B, e:+), BinOp(F, j:*)] -/// => BinOp detects leaf k:3 -/// ===> PushEvalNode(K: Leaf(k:3), NewParentPrecedence: BinOp => >PRODUCT) -/// ===> WorkStack: [Root, BinOp(B, e:+), BinOp(F, j:*), TryExtend(K, >PRODUCT)] -/// => TryExtend detects no valid extension, so cascade K with BinOp: -/// ===> PopWorkStack: PushEvalNode(J: BinOp(F, j:*, K), NewParentPrecedence: BinOp => >SUM) -/// ===> WorkStack: [Root, BinOp(B, e:+), TryExtend(J, >SUM)] -/// => TryExtend detects no valid extension, so cascade J with BinOp: -/// ===> PopWorkStack: PushEvalNode(E: BinOp(B, e:*, J), NewParentPrecedence: Root => >MIN) -/// ===> WorkStack: [Root, TryExtend(E, >MIN)] -/// => TryExtend detects no valid extension, so cascade E with Root: -/// ===> DONE Root = E +/// ===> Stack: [Root, BinOp(B, e:+), Group(f:Paren)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read leaf g:2 +/// ===> PushEvalNode(G: Leaf(g:2)) +/// ===> Stack: [Root, BinOp(B, e:+), Group(f:Paren)] +/// ===> WorkItem::TryParseAndApplyExtension(G) with precedence >MIN from parent=Group +/// => Read binop h:+ +/// ===> Stack: [Root, BinOp(B, e:+), Group(f:Paren), BinOp(G, h:+)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read leaf i:2 +/// ===> PushEvalNode(I: Leaf(i:2)) +/// ===> Stack: [Root, BinOp(B, e:+), Group(f:Paren), BinOp(G, h:+)] +/// ===> WorkItem::TryParseAndApplyExtension(I) with precedence >SUM from parent=BinOp(G, h:+) +/// => Extension of None is not valid, so cascade I with BinOp: +/// ===> PopStack: PushEvalNode(H: BinOp(G, h:+, I)) +/// ===> Stack: [Root, BinOp(B, e:+), Group(f:Paren)] +/// ===> WorkItem::TryApplyAlreadyParsedExtension(H, None) with precedence >MIN from parent=Group +/// => Extension of None is not valid, so cascade H with Group: +/// ===> PopStack: It's a group, so PopParseBuffer, PushEvalNode(F: UnOp(f:Paren, H)) +/// ===> Stack: [Root, BinOp(B, e:+)] +/// ===> WorkItem::TryParseAndApplyExtension(F) with precedence >SUM from parent=BinOp(B, e:+) +/// => Read binop j:* +/// ===> Stack: [Root, BinOp(B, e:+), BinOp(F, j:*)] +/// ===> WorkItem::RequireUnaryAtom +/// => Read leaf k:3 +/// ===> PushEvalNode(K: Leaf(k:3)) +/// ===> Stack: [Root, BinOp(B, e:+), BinOp(F, j:*)] +/// ===> WorkItem::TryParseAndApplyExtension(K) with precedence >PRODUCT from parent=BinOp(F, j:*) +/// => Extension of None is not valid, so cascade K with BinOp: +/// ===> PopStack: PushEvalNode(J: BinOp(F, j:*, K)) +/// ===> Stack: [Root, BinOp(B, e:+)] +/// ===> WorkItem::TryApplyAlreadyParsedExtension(J, None) with precedence >SUM from parent=BinOp(B, e:+) +/// => Extension of None is not valid, so cascade J with BinOp: +/// ===> PopStack: PushEvalNode(E: BinOp(B, e:*, J)) +/// ===> Stack: [Root] +/// ===> WorkItem::TryApplyAlreadyParsedExtension(E, None) with precedence >MIN from parent=Root +/// => Extension of None is not valid, so cascade E with Root: +/// ===> Stack: [] +/// ===> WorkItem::Finished(E) /// ``` -/// -/// TODO SPECIAL CASES: -/// * Some operators can't follow others, e.g. comparison operators can't be chained enum ExpressionStackFrame { /// A marker for the root of the expression Root, @@ -445,10 +452,10 @@ impl ExpressionStackFrame { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompletePrefixOperation { operation, .. } => { - OperatorPrecendence::of_unary_operator(&operation.operator) + OperatorPrecendence::of_unary_operation(operation) } ExpressionStackFrame::IncompleteBinaryOperation { operation, .. } => { - OperatorPrecendence::of_binary_operator(&operation.operator) + OperatorPrecendence::of_binary_operation(operation) } } } @@ -491,12 +498,8 @@ enum NodeExtension { impl NodeExtension { fn precedence(&self) -> OperatorPrecendence { match self { - NodeExtension::PostfixOperation(op) => { - OperatorPrecendence::of_unary_operator(&op.operator) - } - NodeExtension::BinaryOperation(op) => { - OperatorPrecendence::of_binary_operator(&op.operator) - } + NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), + NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), NodeExtension::NoneMatched => OperatorPrecendence::MIN, } } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 8fb8cbd2..acc328be 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -32,17 +32,17 @@ impl EvaluationFloat { pub(super) fn handle_integer_binary_operation( self, right: EvaluationInteger, - operation: BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationFloatValue::F32(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationFloatValue::F64(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } } } @@ -63,7 +63,7 @@ pub(super) enum EvaluationFloatValuePair { impl EvaluationFloatValuePair { pub(super) fn handle_paired_binary_operation( self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -139,11 +139,11 @@ impl UntypedFloat { operation: &UnaryOperation, ) -> ExecutionResult { let input = self.parse_fallback()?; - match operation.operator { - UnaryOperator::Neg { .. } => operation.output(Self::from_fallback(-input)), - UnaryOperator::Not { .. } => operation.unsupported_for_value_type_err("untyped float"), - UnaryOperator::GroupedNoOp { .. } => operation.output(self), - UnaryOperator::Cast { target, .. } => match target { + match operation { + UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), + UnaryOperation::Not { .. } => operation.unsupported_for_value_type_err("untyped float"), + UnaryOperation::GroupedNoOp { .. } => operation.output(self), + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } @@ -165,7 +165,7 @@ impl UntypedFloat { CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input), CastTarget::Boolean | CastTarget::Char => { - operation.err("This cast is not supported") + operation.execution_err("This cast is not supported") } }, } @@ -174,10 +174,11 @@ impl UntypedFloat { pub(super) fn handle_integer_binary_operation( self, _rhs: EvaluationInteger, - operation: &BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { - match operation.integer_operator() { - IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + match operation { + IntegerBinaryOperation::ShiftLeft { .. } + | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err("untyped float") } } @@ -186,32 +187,40 @@ impl UntypedFloat { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; - match operation.paired_operator() { - PairedBinaryOperator::Addition => operation.output(Self::from_fallback(lhs + rhs)), - PairedBinaryOperator::Subtraction => operation.output(Self::from_fallback(lhs - rhs)), - PairedBinaryOperator::Multiplication => { + match operation { + PairedBinaryOperation::Addition { .. } => { + operation.output(Self::from_fallback(lhs + rhs)) + } + PairedBinaryOperation::Subtraction { .. } => { + operation.output(Self::from_fallback(lhs - rhs)) + } + PairedBinaryOperation::Multiplication { .. } => { operation.output(Self::from_fallback(lhs * rhs)) } - PairedBinaryOperator::Division => operation.output(Self::from_fallback(lhs / rhs)), - PairedBinaryOperator::LogicalAnd | PairedBinaryOperator::LogicalOr => { + PairedBinaryOperation::Division { .. } => { + operation.output(Self::from_fallback(lhs / rhs)) + } + PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { operation.unsupported_for_value_type_err(stringify!($float_type)) } - PairedBinaryOperator::Remainder => operation.output(Self::from_fallback(lhs % rhs)), - PairedBinaryOperator::BitXor - | PairedBinaryOperator::BitAnd - | PairedBinaryOperator::BitOr => { + PairedBinaryOperation::Remainder { .. } => { + operation.output(Self::from_fallback(lhs % rhs)) + } + PairedBinaryOperation::BitXor { .. } + | PairedBinaryOperation::BitAnd { .. } + | PairedBinaryOperation::BitOr { .. } => { operation.unsupported_for_value_type_err("untyped float") } - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), } } @@ -272,11 +281,11 @@ macro_rules! impl_float_operations { impl HandleUnaryOperation for $float_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - match operation.operator { - UnaryOperator::Neg { .. } => operation.output(-self), - UnaryOperator::Not { .. } => operation.unsupported_for_value_type_err(stringify!($float_type)), - UnaryOperator::GroupedNoOp { .. } => operation.output(self), - UnaryOperator::Cast { target, .. } => match target { + match operation { + UnaryOperation::Neg { .. } => operation.output(-self), + UnaryOperation::Not { .. } => operation.unsupported_for_value_type_err(stringify!($float_type)), + UnaryOperation::GroupedNoOp { .. } => operation.output(self), + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), @@ -293,49 +302,49 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), - CastTarget::Boolean | CastTarget::Char => operation.err("This cast is not supported"), + CastTarget::Boolean | CastTarget::Char => operation.execution_err("This cast is not supported"), } } } } impl HandleBinaryOperation for $float_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { // Unlike integer arithmetic, float arithmetic does not overflow // and instead falls back to NaN or infinity. In future we could // allow trapping on these codes, but for now this is good enough let lhs = self; - match operation.paired_operator() { - PairedBinaryOperator::Addition => operation.output(lhs + rhs), - PairedBinaryOperator::Subtraction => operation.output(lhs - rhs), - PairedBinaryOperator::Multiplication => operation.output(lhs * rhs), - PairedBinaryOperator::Division => operation.output(lhs / rhs), - PairedBinaryOperator::LogicalAnd - | PairedBinaryOperator::LogicalOr => { + match operation { + PairedBinaryOperation::Addition { .. } => operation.output(lhs + rhs), + PairedBinaryOperation::Subtraction { .. } => operation.output(lhs - rhs), + PairedBinaryOperation::Multiplication { .. } => operation.output(lhs * rhs), + PairedBinaryOperation::Division { .. } => operation.output(lhs / rhs), + PairedBinaryOperation::LogicalAnd { .. } + | PairedBinaryOperation::LogicalOr { .. } => { operation.unsupported_for_value_type_err(stringify!($float_type)) } - PairedBinaryOperator::Remainder => operation.output(lhs % rhs), - PairedBinaryOperator::BitXor - | PairedBinaryOperator::BitAnd - | PairedBinaryOperator::BitOr => { + PairedBinaryOperation::Remainder { .. } => operation.output(lhs % rhs), + PairedBinaryOperation::BitXor { .. } + | PairedBinaryOperation::BitAnd { .. } + | PairedBinaryOperation::BitOr { .. } => { operation.unsupported_for_value_type_err(stringify!($float_type)) } - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), } } fn handle_integer_binary_operation( self, _rhs: EvaluationInteger, - operation: &BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { - match operation.integer_operator() { - IntegerBinaryOperator::ShiftLeft | IntegerBinaryOperator::ShiftRight => { + match operation { + IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err(stringify!($float_type)) }, } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 8ccd8dc9..d5c481c1 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -59,47 +59,47 @@ impl EvaluationInteger { pub(super) fn handle_integer_binary_operation( self, right: EvaluationInteger, - operation: BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::U8(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::U16(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::U32(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::U64(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::U128(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::Usize(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::I8(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::I16(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::I32(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::I64(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::I128(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } EvaluationIntegerValue::Isize(input) => { - input.handle_integer_binary_operation(right, &operation) + input.handle_integer_binary_operation(right, operation) } } } @@ -130,7 +130,7 @@ pub(super) enum EvaluationIntegerValuePair { impl EvaluationIntegerValuePair { pub(super) fn handle_paired_binary_operation( self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -266,13 +266,13 @@ impl UntypedInteger { operation: &UnaryOperation, ) -> ExecutionResult { let input = self.parse_fallback()?; - match operation.operator { - UnaryOperator::Neg { .. } => operation.output(Self::from_fallback(-input)), - UnaryOperator::Not { .. } => { + match operation { + UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), + UnaryOperation::Not { .. } => { operation.unsupported_for_value_type_err("untyped integer") } - UnaryOperator::GroupedNoOp { .. } => operation.output(self), - UnaryOperator::Cast { target, .. } => match target { + UnaryOperation::GroupedNoOp { .. } => operation.output(self), + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) } @@ -294,7 +294,7 @@ impl UntypedInteger { CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input as f64), CastTarget::Boolean | CastTarget::Char => { - operation.err("This cast is not supported") + operation.execution_err("This cast is not supported") } }, } @@ -303,11 +303,11 @@ impl UntypedInteger { pub(super) fn handle_integer_binary_operation( self, rhs: EvaluationInteger, - operation: &BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { let lhs = self.parse_fallback()?; - match operation.integer_operator() { - IntegerBinaryOperator::ShiftLeft => match rhs.value { + match operation { + IntegerBinaryOperation::ShiftLeft { .. } => match rhs.value { EvaluationIntegerValue::Untyped(rhs) => { operation.output(lhs << rhs.parse_fallback()?) } @@ -324,7 +324,7 @@ impl UntypedInteger { EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), }, - IntegerBinaryOperator::ShiftRight => match rhs.value { + IntegerBinaryOperation::ShiftRight { .. } => match rhs.value { EvaluationIntegerValue::Untyped(rhs) => { operation.output(lhs >> rhs.parse_fallback()?) } @@ -347,7 +347,7 @@ impl UntypedInteger { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; @@ -355,43 +355,47 @@ impl UntypedInteger { format!( "The untyped integer operation {:?} {} {:?} overflowed in i128 space", lhs, - operation.operator.symbol(), + operation.symbol(), rhs ) }; - match operation.paired_operator() { - PairedBinaryOperator::Addition => operation.output_if_some( + match operation { + PairedBinaryOperation::Addition { .. } => operation.output_if_some( lhs.checked_add(rhs).map(Self::from_fallback), overflow_error, ), - PairedBinaryOperator::Subtraction => operation.output_if_some( + PairedBinaryOperation::Subtraction { .. } => operation.output_if_some( lhs.checked_sub(rhs).map(Self::from_fallback), overflow_error, ), - PairedBinaryOperator::Multiplication => operation.output_if_some( + PairedBinaryOperation::Multiplication { .. } => operation.output_if_some( lhs.checked_mul(rhs).map(Self::from_fallback), overflow_error, ), - PairedBinaryOperator::Division => operation.output_if_some( + PairedBinaryOperation::Division { .. } => operation.output_if_some( lhs.checked_div(rhs).map(Self::from_fallback), overflow_error, ), - PairedBinaryOperator::LogicalAnd | PairedBinaryOperator::LogicalOr => { + PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { operation.unsupported_for_value_type_err("untyped integer") } - PairedBinaryOperator::Remainder => operation.output_if_some( + PairedBinaryOperation::Remainder { .. } => operation.output_if_some( lhs.checked_rem(rhs).map(Self::from_fallback), overflow_error, ), - PairedBinaryOperator::BitXor => operation.output(Self::from_fallback(lhs ^ rhs)), - PairedBinaryOperator::BitAnd => operation.output(Self::from_fallback(lhs & rhs)), - PairedBinaryOperator::BitOr => operation.output(Self::from_fallback(lhs | rhs)), - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + PairedBinaryOperation::BitXor { .. } => { + operation.output(Self::from_fallback(lhs ^ rhs)) + } + PairedBinaryOperation::BitAnd { .. } => { + operation.output(Self::from_fallback(lhs & rhs)) + } + PairedBinaryOperation::BitOr { .. } => operation.output(Self::from_fallback(lhs | rhs)), + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), } } @@ -452,37 +456,37 @@ macro_rules! impl_int_operations_except_unary { } impl HandleBinaryOperation for $integer_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &BinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { let lhs = self; - let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.operator.symbol(), rhs); - match operation.paired_operator() { - PairedBinaryOperator::Addition => operation.output_if_some(lhs.checked_add(rhs), overflow_error), - PairedBinaryOperator::Subtraction => operation.output_if_some(lhs.checked_sub(rhs), overflow_error), - PairedBinaryOperator::Multiplication => operation.output_if_some(lhs.checked_mul(rhs), overflow_error), - PairedBinaryOperator::Division => operation.output_if_some(lhs.checked_div(rhs), overflow_error), - PairedBinaryOperator::LogicalAnd - | PairedBinaryOperator::LogicalOr => operation.unsupported_for_value_type_err(stringify!($integer_type)), - PairedBinaryOperator::Remainder => operation.output_if_some(lhs.checked_rem(rhs), overflow_error), - PairedBinaryOperator::BitXor => operation.output(lhs ^ rhs), - PairedBinaryOperator::BitAnd => operation.output(lhs & rhs), - PairedBinaryOperator::BitOr => operation.output(lhs | rhs), - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.symbol(), rhs); + match operation { + PairedBinaryOperation::Addition { .. } => operation.output_if_some(lhs.checked_add(rhs), overflow_error), + PairedBinaryOperation::Subtraction { .. } => operation.output_if_some(lhs.checked_sub(rhs), overflow_error), + PairedBinaryOperation::Multiplication { .. } => operation.output_if_some(lhs.checked_mul(rhs), overflow_error), + PairedBinaryOperation::Division { .. } => operation.output_if_some(lhs.checked_div(rhs), overflow_error), + PairedBinaryOperation::LogicalAnd { .. } + | PairedBinaryOperation::LogicalOr { .. } => operation.unsupported_for_value_type_err(stringify!($integer_type)), + PairedBinaryOperation::Remainder { .. } => operation.output_if_some(lhs.checked_rem(rhs), overflow_error), + PairedBinaryOperation::BitXor { .. } => operation.output(lhs ^ rhs), + PairedBinaryOperation::BitAnd { .. } => operation.output(lhs & rhs), + PairedBinaryOperation::BitOr { .. } => operation.output(lhs | rhs), + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), } } fn handle_integer_binary_operation( self, rhs: EvaluationInteger, - operation: &BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { let lhs = self; - match operation.integer_operator() { - IntegerBinaryOperator::ShiftLeft => { + match operation { + IntegerBinaryOperation::ShiftLeft { .. } => { match rhs.value { EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), @@ -499,7 +503,7 @@ macro_rules! impl_int_operations_except_unary { EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), } }, - IntegerBinaryOperator::ShiftRight => { + IntegerBinaryOperation::ShiftRight { .. } => { match rhs.value { EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs >> rhs.parse_fallback()?), EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), @@ -526,13 +530,13 @@ macro_rules! impl_unsigned_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - match operation.operator { - UnaryOperator::GroupedNoOp { .. } => operation.output(self), - UnaryOperator::Neg { .. } - | UnaryOperator::Not { .. } => { + match operation { + UnaryOperation::GroupedNoOp { .. } => operation.output(self), + UnaryOperation::Neg { .. } + | UnaryOperation::Not { .. } => { operation.unsupported_for_value_type_err(stringify!($integer_type)) }, - UnaryOperator::Cast { target, .. } => match target { + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), @@ -549,8 +553,7 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), - // Technically u8 => char is supported, but we can add it later - CastTarget::Boolean | CastTarget::Char => operation.err("This cast is not supported"), + CastTarget::Boolean | CastTarget::Char => operation.execution_err("This cast is not supported"), } } } @@ -562,13 +565,13 @@ macro_rules! impl_signed_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - match operation.operator { - UnaryOperator::GroupedNoOp { .. } => operation.output(self), - UnaryOperator::Neg { .. } => operation.output(-self), - UnaryOperator::Not { .. } => { + match operation { + UnaryOperation::GroupedNoOp { .. } => operation.output(self), + UnaryOperation::Neg { .. } => operation.output(-self), + UnaryOperation::Not { .. } => { operation.unsupported_for_value_type_err(stringify!($integer_type)) }, - UnaryOperator::Cast { target, .. } => match target { + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), CastTarget::Integer(IntegerKind::I16) => operation.output(self as i16), @@ -585,7 +588,7 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), - CastTarget::Boolean | CastTarget::Char => operation.err("This cast is not supported"), + CastTarget::Boolean | CastTarget::Char => operation.execution_err("This cast is not supported"), } } } @@ -598,12 +601,12 @@ impl HandleUnaryOperation for u8 { self, operation: &UnaryOperation, ) -> ExecutionResult { - match operation.operator { - UnaryOperator::GroupedNoOp { .. } => operation.output(self), - UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } => { + match operation { + UnaryOperation::GroupedNoOp { .. } => operation.output(self), + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { operation.unsupported_for_value_type_err("u8") } - UnaryOperator::Cast { target, .. } => match target { + UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(self as FallbackInteger)) } @@ -625,7 +628,7 @@ impl HandleUnaryOperation for u8 { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Char => operation.output(self as char), - CastTarget::Boolean => operation.err("This cast is not supported"), + CastTarget::Boolean => operation.execution_err("This cast is not supported"), }, } } diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 19189b97..1bbe2517 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -1,35 +1,66 @@ use super::*; -#[derive(Clone)] -pub(super) struct UnaryOperation { - pub(super) operator: UnaryOperator, -} +pub(super) trait Operation: HasSpanRange { + fn output(&self, output_value: impl ToEvaluationValue) -> ExecutionResult { + Ok(output_value.to_value(self.source_span_for_output())) + } -impl UnaryOperation { - fn error(&self, error_message: &str) -> ExecutionInterrupt { - self.operator.execution_error(error_message) + fn output_if_some( + &self, + output_value: Option, + error_message: impl FnOnce() -> String, + ) -> ExecutionResult { + match output_value { + Some(output_value) => self.output(output_value), + None => self.execution_err(error_message()), + } } - pub(super) fn unsupported_for_value_type_err( + fn unsupported_for_value_type_err( &self, value_type: &'static str, ) -> ExecutionResult { - Err(self.error(&format!( + Err(self.execution_error(format!( "The {} operator is not supported for {} values", - self.operator.symbol(), + self.symbol(), value_type, ))) } - pub(super) fn err(&self, error_message: &'static str) -> ExecutionResult { - Err(self.error(error_message)) - } + fn source_span_for_output(&self) -> Option; + fn symbol(&self) -> &'static str; +} - pub(super) fn output( - &self, - output_value: impl ToEvaluationValue, - ) -> ExecutionResult { - Ok(output_value.to_value(self.operator.source_span_for_output())) +#[derive(Clone)] +pub(super) enum UnaryOperation { + Neg { + token: Token![-], + }, + Not { + token: Token![!], + }, + GroupedNoOp { + span: Span, + }, + Cast { + as_token: Token![as], + target: CastTarget, + }, +} + +impl UnaryOperation { + pub(super) fn parse_from_prefix_punct(input: ParseStream) -> ParseResult { + if input.peek(Token![-]) { + Ok(Self::Neg { + token: input.parse()?, + }) + } else if input.peek(Token![!]) { + Ok(Self::Not { + token: input.parse()?, + }) + } else { + input.parse_err("Expected ! or -") + } } pub(super) fn for_cast_operation( @@ -60,22 +91,7 @@ impl UnaryOperation { .parse_err("This type is not supported in preinterpret cast expressions") } }; - Ok(Self { - operator: UnaryOperator::Cast { as_token, target }, - }) - } - - pub(super) fn for_unary_operator(operator: syn::UnOp) -> ParseResult { - let operator = match operator { - UnOp::Neg(token) => UnaryOperator::Neg { token }, - UnOp::Not(token) => UnaryOperator::Not { token }, - other_unary_op => { - return other_unary_op.parse_err( - "This unary operator is not supported in a preinterpret expression", - ); - } - }; - Ok(Self { operator }) + Ok(Self::Cast { as_token, target }) } pub(super) fn evaluate(self, input: EvaluationValue) -> ExecutionResult { @@ -83,50 +99,33 @@ impl UnaryOperation { } } -#[derive(Copy, Clone)] -pub(super) enum UnaryOperator { - Neg { - token: Token![-], - }, - Not { - token: Token![!], - }, - GroupedNoOp { - span: Span, - }, - Cast { - as_token: Token![as], - target: CastTarget, - }, -} - -impl UnaryOperator { - pub(crate) fn source_span_for_output(&self) -> Option { +impl Operation for UnaryOperation { + fn source_span_for_output(&self) -> Option { match self { - UnaryOperator::Neg { .. } => None, - UnaryOperator::Not { .. } => None, - UnaryOperator::GroupedNoOp { span } => Some(*span), - UnaryOperator::Cast { .. } => None, + UnaryOperation::Neg { .. } => None, + UnaryOperation::Not { .. } => None, + UnaryOperation::GroupedNoOp { span } => Some(*span), + UnaryOperation::Cast { .. } => None, } } - pub(crate) fn symbol(&self) -> &'static str { + fn symbol(&self) -> &'static str { match self { - UnaryOperator::Neg { .. } => "-", - UnaryOperator::Not { .. } => "!", - UnaryOperator::GroupedNoOp { .. } => "", - UnaryOperator::Cast { .. } => "as", + UnaryOperation::Neg { .. } => "-", + UnaryOperation::Not { .. } => "!", + UnaryOperation::GroupedNoOp { .. } => "", + UnaryOperation::Cast { .. } => "as", } } } -impl HasSpan for UnaryOperator { +impl HasSpan for UnaryOperation { fn span(&self) -> Span { match self { - UnaryOperator::Neg { token } => token.span, - UnaryOperator::Not { token } => token.span, - UnaryOperator::GroupedNoOp { span } => *span, - UnaryOperator::Cast { as_token, .. } => as_token.span, + UnaryOperation::Neg { token } => token.span, + UnaryOperation::Not { token } => token.span, + UnaryOperation::GroupedNoOp { span } => *span, + UnaryOperation::Cast { as_token, .. } => as_token.span, } } } @@ -137,85 +136,93 @@ pub(super) trait HandleUnaryOperation: Sized { } #[derive(Clone)] -pub(super) struct BinaryOperation { - /// Only present if there is a single span for the source tokens - pub(super) source_span: Option, - pub(super) operator_span: Span, - pub(super) operator: BinaryOperator, +pub(super) enum BinaryOperation { + Paired(PairedBinaryOperation), + Integer(IntegerBinaryOperation), } -impl BinaryOperation { - fn error(&self, error_message: &str) -> ExecutionInterrupt { - self.operator_span.execution_error(error_message) - } - - pub(super) fn unsupported_for_value_type_err( - &self, - value_type: &'static str, - ) -> ExecutionResult { - Err(self.error(&format!( - "The {} operator is not supported for {} values", - self.operator.symbol(), - value_type, - ))) - } - - pub(super) fn output( - &self, - output_value: impl ToEvaluationValue, - ) -> ExecutionResult { - Ok(output_value.to_value(self.source_span)) - } - - pub(super) fn output_if_some( - &self, - output_value: Option, - error_message: impl FnOnce() -> String, - ) -> ExecutionResult { - match output_value { - Some(output_value) => self.output(output_value), - None => self.operator_span.execution_err(error_message()), +impl Parse for BinaryOperation { + fn parse(input: ParseStream) -> ParseResult { + // In line with Syn's BinOp, we use peek instead of lookahead + // ...I assume for slightly increased performance + // ...Or becuase 30 alternative options in the error message is too many + if input.peek(Token![+]) { + Ok(Self::Paired(PairedBinaryOperation::Addition( + input.parse()?, + ))) + } else if input.peek(Token![-]) { + Ok(Self::Paired(PairedBinaryOperation::Subtraction( + input.parse()?, + ))) + } else if input.peek(Token![*]) { + Ok(Self::Paired(PairedBinaryOperation::Multiplication( + input.parse()?, + ))) + } else if input.peek(Token![/]) { + Ok(Self::Paired(PairedBinaryOperation::Division( + input.parse()?, + ))) + } else if input.peek(Token![%]) { + Ok(Self::Paired(PairedBinaryOperation::Remainder( + input.parse()?, + ))) + } else if input.peek(Token![&&]) { + Ok(Self::Paired(PairedBinaryOperation::LogicalAnd( + input.parse()?, + ))) + } else if input.peek(Token![||]) { + Ok(Self::Paired(PairedBinaryOperation::LogicalOr( + input.parse()?, + ))) + } else if input.peek(Token![==]) { + Ok(Self::Paired(PairedBinaryOperation::Equal(input.parse()?))) + } else if input.peek(Token![!=]) { + Ok(Self::Paired(PairedBinaryOperation::NotEqual( + input.parse()?, + ))) + } else if input.peek(Token![>=]) { + Ok(Self::Paired(PairedBinaryOperation::GreaterThanOrEqual( + input.parse()?, + ))) + } else if input.peek(Token![<=]) { + Ok(Self::Paired(PairedBinaryOperation::LessThanOrEqual( + input.parse()?, + ))) + } else if input.peek(Token![<<]) { + Ok(Self::Integer(IntegerBinaryOperation::ShiftLeft( + input.parse()?, + ))) + } else if input.peek(Token![>>]) { + Ok(Self::Integer(IntegerBinaryOperation::ShiftRight( + input.parse()?, + ))) + } else if input.peek(Token![>]) { + Ok(Self::Paired(PairedBinaryOperation::GreaterThan( + input.parse()?, + ))) + } else if input.peek(Token![<]) { + Ok(Self::Paired(PairedBinaryOperation::LessThan( + input.parse()?, + ))) + } else if input.peek(Token![&]) { + Ok(Self::Paired(PairedBinaryOperation::BitAnd(input.parse()?))) + } else if input.peek(Token![|]) { + Ok(Self::Paired(PairedBinaryOperation::BitOr(input.parse()?))) + } else if input.peek(Token![^]) { + Ok(Self::Paired(PairedBinaryOperation::BitXor(input.parse()?))) + } else { + input.parse_err("Expected one of + - * / % && || ^ & | == < <= != >= > << or >>") } } +} - pub(super) fn for_binary_operator(syn_operator: syn::BinOp) -> ParseResult { - let operator = match syn_operator { - syn::BinOp::Add(_) => BinaryOperator::Paired(PairedBinaryOperator::Addition), - syn::BinOp::Sub(_) => BinaryOperator::Paired(PairedBinaryOperator::Subtraction), - syn::BinOp::Mul(_) => BinaryOperator::Paired(PairedBinaryOperator::Multiplication), - syn::BinOp::Div(_) => BinaryOperator::Paired(PairedBinaryOperator::Division), - syn::BinOp::Rem(_) => BinaryOperator::Paired(PairedBinaryOperator::Remainder), - syn::BinOp::And(_) => BinaryOperator::Paired(PairedBinaryOperator::LogicalAnd), - syn::BinOp::Or(_) => BinaryOperator::Paired(PairedBinaryOperator::LogicalOr), - syn::BinOp::BitXor(_) => BinaryOperator::Paired(PairedBinaryOperator::BitXor), - syn::BinOp::BitAnd(_) => BinaryOperator::Paired(PairedBinaryOperator::BitAnd), - syn::BinOp::BitOr(_) => BinaryOperator::Paired(PairedBinaryOperator::BitOr), - syn::BinOp::Shl(_) => BinaryOperator::Integer(IntegerBinaryOperator::ShiftLeft), - syn::BinOp::Shr(_) => BinaryOperator::Integer(IntegerBinaryOperator::ShiftRight), - syn::BinOp::Eq(_) => BinaryOperator::Paired(PairedBinaryOperator::Equal), - syn::BinOp::Lt(_) => BinaryOperator::Paired(PairedBinaryOperator::LessThan), - syn::BinOp::Le(_) => BinaryOperator::Paired(PairedBinaryOperator::LessThanOrEqual), - syn::BinOp::Ne(_) => BinaryOperator::Paired(PairedBinaryOperator::NotEqual), - syn::BinOp::Ge(_) => BinaryOperator::Paired(PairedBinaryOperator::GreaterThanOrEqual), - syn::BinOp::Gt(_) => BinaryOperator::Paired(PairedBinaryOperator::GreaterThan), - other_binary_operation => { - return other_binary_operation - .parse_err("This operation is not supported in preinterpret expressions") - } - }; - Ok(Self { - source_span: None, - operator_span: syn_operator.span_range().join_into_span_else_start(), - operator, - }) - } - +impl BinaryOperation { pub(super) fn lazy_evaluate( &self, left: &EvaluationValue, ) -> ExecutionResult> { - match self.operator { - BinaryOperator::Paired(PairedBinaryOperator::LogicalAnd) => { + match self { + BinaryOperation::Paired(PairedBinaryOperation::LogicalAnd { .. }) => { match left.clone().into_bool() { Some(bool) => { if !bool.value { @@ -227,7 +234,7 @@ impl BinaryOperation { None => self.execution_err("The left operand was not a boolean"), } } - BinaryOperator::Paired(PairedBinaryOperator::LogicalOr) => { + BinaryOperation::Paired(PairedBinaryOperation::LogicalOr { .. }) => { match left.clone().into_bool() { Some(bool) => { if bool.value { @@ -244,115 +251,141 @@ impl BinaryOperation { } pub(super) fn evaluate( - self, + &self, left: EvaluationValue, right: EvaluationValue, ) -> ExecutionResult { - match self.operator { - BinaryOperator::Paired(operator) => { - let value_pair = left.expect_value_pair(operator, right, self.operator_span)?; - value_pair.handle_paired_binary_operation(self) + match self { + BinaryOperation::Paired(operation) => { + let value_pair = left.expect_value_pair(operation, right)?; + value_pair.handle_paired_binary_operation(operation) } - BinaryOperator::Integer(_) => { - let right = right.into_integer().ok_or_else(|| { - self.operator_span - .execution_error("The shift amount must be an integer") - })?; - left.handle_integer_binary_operation(right, self) + BinaryOperation::Integer(operation) => { + let right = right + .into_integer() + .ok_or_else(|| self.execution_error("The shift amount must be an integer"))?; + left.handle_integer_binary_operation(right, operation) } } } +} - pub(super) fn paired_operator(&self) -> PairedBinaryOperator { - match self.operator { - BinaryOperator::Paired(operator) => operator, - BinaryOperator::Integer(_) => panic!("Expected a paired operator"), +impl HasSpan for BinaryOperation { + fn span(&self) -> Span { + match self { + BinaryOperation::Paired(_) => self.span(), + BinaryOperation::Integer(_) => self.span(), } } +} - pub(super) fn integer_operator(&self) -> IntegerBinaryOperator { - match self.operator { - BinaryOperator::Paired(_) => panic!("Expected an integer operator"), - BinaryOperator::Integer(operator) => operator, - } +impl Operation for BinaryOperation { + fn source_span_for_output(&self) -> Option { + None } -} -impl HasSpan for BinaryOperation { - fn span(&self) -> Span { - self.operator_span + fn symbol(&self) -> &'static str { + match self { + BinaryOperation::Paired(paired) => paired.symbol(), + BinaryOperation::Integer(integer) => integer.symbol(), + } } } #[derive(Copy, Clone)] -pub(super) enum BinaryOperator { - Paired(PairedBinaryOperator), - Integer(IntegerBinaryOperator), +pub(super) enum PairedBinaryOperation { + Addition(Token![+]), + Subtraction(Token![-]), + Multiplication(Token![*]), + Division(Token![/]), + Remainder(Token![%]), + LogicalAnd(Token![&&]), + LogicalOr(Token![||]), + BitXor(Token![^]), + BitAnd(Token![&]), + BitOr(Token![|]), + Equal(Token![==]), + LessThan(Token![<]), + LessThanOrEqual(Token![<=]), + NotEqual(Token![!=]), + GreaterThanOrEqual(Token![>=]), + GreaterThan(Token![>]), } -impl BinaryOperator { - pub(super) fn symbol(&self) -> &'static str { +impl Operation for PairedBinaryOperation { + fn source_span_for_output(&self) -> Option { + None + } + + fn symbol(&self) -> &'static str { match self { - BinaryOperator::Paired(paired) => paired.symbol(), - BinaryOperator::Integer(integer) => integer.symbol(), + PairedBinaryOperation::Addition { .. } => "+", + PairedBinaryOperation::Subtraction { .. } => "-", + PairedBinaryOperation::Multiplication { .. } => "*", + PairedBinaryOperation::Division { .. } => "/", + PairedBinaryOperation::Remainder { .. } => "%", + PairedBinaryOperation::LogicalAnd { .. } => "&&", + PairedBinaryOperation::LogicalOr { .. } => "||", + PairedBinaryOperation::BitXor { .. } => "^", + PairedBinaryOperation::BitAnd { .. } => "&", + PairedBinaryOperation::BitOr { .. } => "|", + PairedBinaryOperation::Equal { .. } => "==", + PairedBinaryOperation::LessThan { .. } => "<", + PairedBinaryOperation::LessThanOrEqual { .. } => "<=", + PairedBinaryOperation::NotEqual { .. } => "!=", + PairedBinaryOperation::GreaterThanOrEqual { .. } => ">=", + PairedBinaryOperation::GreaterThan { .. } => ">", } } } -#[derive(Copy, Clone)] -pub(super) enum PairedBinaryOperator { - Addition, - Subtraction, - Multiplication, - Division, - Remainder, - LogicalAnd, - LogicalOr, - BitXor, - BitAnd, - BitOr, - Equal, - LessThan, - LessThanOrEqual, - NotEqual, - GreaterThanOrEqual, - GreaterThan, -} - -impl PairedBinaryOperator { - pub(super) fn symbol(&self) -> &'static str { +impl HasSpanRange for PairedBinaryOperation { + fn span_range(&self) -> SpanRange { match self { - PairedBinaryOperator::Addition => "+", - PairedBinaryOperator::Subtraction => "-", - PairedBinaryOperator::Multiplication => "*", - PairedBinaryOperator::Division => "/", - PairedBinaryOperator::Remainder => "%", - PairedBinaryOperator::LogicalAnd => "&&", - PairedBinaryOperator::LogicalOr => "||", - PairedBinaryOperator::BitXor => "^", - PairedBinaryOperator::BitAnd => "&", - PairedBinaryOperator::BitOr => "|", - PairedBinaryOperator::Equal => "==", - PairedBinaryOperator::LessThan => "<", - PairedBinaryOperator::LessThanOrEqual => "<=", - PairedBinaryOperator::NotEqual => "!=", - PairedBinaryOperator::GreaterThanOrEqual => ">=", - PairedBinaryOperator::GreaterThan => ">", + PairedBinaryOperation::Addition(plus) => plus.span_range(), + PairedBinaryOperation::Subtraction(minus) => minus.span_range(), + PairedBinaryOperation::Multiplication(star) => star.span_range(), + PairedBinaryOperation::Division(slash) => slash.span_range(), + PairedBinaryOperation::Remainder(percent) => percent.span_range(), + PairedBinaryOperation::LogicalAnd(and_and) => and_and.span_range(), + PairedBinaryOperation::LogicalOr(or_or) => or_or.span_range(), + PairedBinaryOperation::BitXor(caret) => caret.span_range(), + PairedBinaryOperation::BitAnd(and) => and.span_range(), + PairedBinaryOperation::BitOr(or) => or.span_range(), + PairedBinaryOperation::Equal(eq_eq) => eq_eq.span_range(), + PairedBinaryOperation::LessThan(lt) => lt.span_range(), + PairedBinaryOperation::LessThanOrEqual(le) => le.span_range(), + PairedBinaryOperation::NotEqual(ne) => ne.span_range(), + PairedBinaryOperation::GreaterThanOrEqual(ge) => ge.span_range(), + PairedBinaryOperation::GreaterThan(gt) => gt.span_range(), } } } #[derive(Copy, Clone)] -pub(super) enum IntegerBinaryOperator { - ShiftLeft, - ShiftRight, +pub(super) enum IntegerBinaryOperation { + ShiftLeft(Token![<<]), + ShiftRight(Token![>>]), +} + +impl Operation for IntegerBinaryOperation { + fn source_span_for_output(&self) -> Option { + None + } + + fn symbol(&self) -> &'static str { + match self { + IntegerBinaryOperation::ShiftLeft { .. } => "<<", + IntegerBinaryOperation::ShiftRight { .. } => ">>", + } + } } -impl IntegerBinaryOperator { - pub(super) fn symbol(&self) -> &'static str { +impl HasSpanRange for IntegerBinaryOperation { + fn span_range(&self) -> SpanRange { match self { - IntegerBinaryOperator::ShiftLeft => "<<", - IntegerBinaryOperator::ShiftRight => ">>", + IntegerBinaryOperation::ShiftLeft(shl) => shl.span_range(), + IntegerBinaryOperation::ShiftRight(shr) => shr.span_range(), } } } @@ -361,12 +394,12 @@ pub(super) trait HandleBinaryOperation: Sized { fn handle_paired_binary_operation( self, rhs: Self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult; fn handle_integer_binary_operation( self, rhs: EvaluationInteger, - operation: &BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult; } diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 9241a60a..83fee357 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -20,18 +20,18 @@ impl EvaluationString { self, operation: UnaryOperation, ) -> ExecutionResult { - match operation.operator { - UnaryOperator::GroupedNoOp { .. } => operation.output(self.value), - UnaryOperator::Neg { .. } | UnaryOperator::Not { .. } | UnaryOperator::Cast { .. } => { - operation.unsupported_for_value_type_err("string") - } + match operation { + UnaryOperation::GroupedNoOp { .. } => operation.output(self.value), + UnaryOperation::Neg { .. } + | UnaryOperation::Not { .. } + | UnaryOperation::Cast { .. } => operation.unsupported_for_value_type_err("string"), } } pub(super) fn handle_integer_binary_operation( self, _right: EvaluationInteger, - operation: BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { operation.unsupported_for_value_type_err("string") } @@ -39,27 +39,29 @@ impl EvaluationString { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - match operation.paired_operator() { - PairedBinaryOperator::Addition - | PairedBinaryOperator::Subtraction - | PairedBinaryOperator::Multiplication - | PairedBinaryOperator::Division - | PairedBinaryOperator::LogicalAnd - | PairedBinaryOperator::LogicalOr - | PairedBinaryOperator::Remainder - | PairedBinaryOperator::BitXor - | PairedBinaryOperator::BitAnd - | PairedBinaryOperator::BitOr => operation.unsupported_for_value_type_err("string"), - PairedBinaryOperator::Equal => operation.output(lhs == rhs), - PairedBinaryOperator::LessThan => operation.output(lhs < rhs), - PairedBinaryOperator::LessThanOrEqual => operation.output(lhs <= rhs), - PairedBinaryOperator::NotEqual => operation.output(lhs != rhs), - PairedBinaryOperator::GreaterThanOrEqual => operation.output(lhs >= rhs), - PairedBinaryOperator::GreaterThan => operation.output(lhs > rhs), + match operation { + PairedBinaryOperation::Addition { .. } + | PairedBinaryOperation::Subtraction { .. } + | PairedBinaryOperation::Multiplication { .. } + | PairedBinaryOperation::Division { .. } + | PairedBinaryOperation::LogicalAnd { .. } + | PairedBinaryOperation::LogicalOr { .. } + | PairedBinaryOperation::Remainder { .. } + | PairedBinaryOperation::BitXor { .. } + | PairedBinaryOperation::BitAnd { .. } + | PairedBinaryOperation::BitOr { .. } => { + operation.unsupported_for_value_type_err("string") + } + PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), + PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), + PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), + PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), + PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), + PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 70eca4f1..adf29613 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -32,9 +32,8 @@ impl EvaluationValue { pub(super) fn expect_value_pair( self, - operator: PairedBinaryOperator, + operation: &PairedBinaryOperation, right: Self, - operator_span: Span, ) -> ExecutionResult { Ok(match (self, right) { (EvaluationValue::Integer(left), EvaluationValue::Integer(right)) => { @@ -158,7 +157,7 @@ impl EvaluationValue { EvaluationIntegerValuePair::Isize(lhs, rhs) } (left_value, right_value) => { - return operator_span.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.describe_type(), right_value.describe_type())); } }; EvaluationLiteralPair::Integer(integer_pair) @@ -197,7 +196,7 @@ impl EvaluationValue { EvaluationFloatValuePair::F64(lhs, rhs) } (left_value, right_value) => { - return operator_span.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left_value.describe_type(), right_value.describe_type())); + return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.describe_type(), right_value.describe_type())); } }; EvaluationLiteralPair::Float(float_pair) @@ -209,7 +208,7 @@ impl EvaluationValue { EvaluationLiteralPair::CharPair(left, right) } (left, right) => { - return operator_span.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operator.symbol(), left.describe_type(), right.describe_type())); + return operation.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left.describe_type(), right.describe_type())); } }) } @@ -275,7 +274,7 @@ impl EvaluationValue { pub(super) fn handle_integer_binary_operation( self, right: EvaluationInteger, - operation: BinaryOperation, + operation: &IntegerBinaryOperation, ) -> ExecutionResult { match self { EvaluationValue::Integer(value) => { @@ -314,14 +313,14 @@ pub(super) enum EvaluationLiteralPair { impl EvaluationLiteralPair { pub(super) fn handle_paired_binary_operation( self, - operation: BinaryOperation, + operation: &PairedBinaryOperation, ) -> ExecutionResult { match self { - Self::Integer(pair) => pair.handle_paired_binary_operation(&operation), - Self::Float(pair) => pair.handle_paired_binary_operation(&operation), - Self::BooleanPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), - Self::StringPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), - Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, &operation), + Self::Integer(pair) => pair.handle_paired_binary_operation(operation), + Self::Float(pair) => pair.handle_paired_binary_operation(operation), + Self::BooleanPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::StringPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } } diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index a634209c..553946b3 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -12,7 +12,7 @@ impl SynErrorExt for syn::Error { } } -pub(crate) trait SpanErrorExt: Sized { +pub(crate) trait SpanErrorExt { fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { Err(self.error(message).into()) } @@ -37,7 +37,7 @@ pub(crate) trait SpanErrorExt: Sized { } } -impl SpanErrorExt for T { +impl SpanErrorExt for T { fn error(&self, message: impl std::fmt::Display) -> syn::Error { self.span_range().create_error(message) } @@ -59,7 +59,7 @@ pub(crate) trait HasSpanRange { fn span_range(&self) -> SpanRange; } -impl HasSpanRange for T { +impl HasSpanRange for T { fn span_range(&self) -> SpanRange { SpanRange::new_single(self.span()) } @@ -175,18 +175,6 @@ impl HasSpan for Literal { } } -impl HasSpan for Token![as] { - fn span(&self) -> Span { - self.span - } -} - -impl HasSpan for Token![in] { - fn span(&self) -> Span { - self.span - } -} - pub(crate) trait SlowSpanRange { /// This name is purposefully very long to discourage use, as it can cause nasty performance issues fn span_range_from_iterating_over_all_tokens(&self) -> SpanRange; @@ -225,4 +213,43 @@ impl_auto_span_range! { syn::BinOp, syn::UnOp, syn::token::DotDot, + syn::token::Shl, + syn::token::Shr, + syn::token::AndAnd, + syn::token::OrOr, + syn::token::EqEq, + syn::token::Lt, + syn::token::Le, + syn::token::Ne, + syn::token::Ge, + syn::token::Gt, +} + +macro_rules! single_span_token { + ($(Token![$token:tt]),* $(,)?) => { + $( + impl HasSpan for Token![$token] { + fn span(&self) -> Span { + self.span + } + } + )* + }; +} + +single_span_token! { + Token![as], + Token![in], + Token![=], + Token![!], + Token![~], + Token![?], + Token![+], + Token![-], + Token![*], + Token![/], + Token![%], + Token![^], + Token![&], + Token![|], } diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index d667f543..c7a8bea1 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -287,6 +287,25 @@ impl<'a> ParseStreamStack<'a> { self.current().parse() } + pub(crate) fn parse_with( + &mut self, + parser: impl FnOnce(ParseStream) -> ParseResult, + ) -> ParseResult { + parser(self.current()) + } + + pub(crate) fn try_parse_or_revert(&mut self) -> ParseResult { + let current = self.current(); + let fork = current.fork(); + match fork.parse::() { + Ok(output) => { + current.advance_to(&fork); + Ok(output) + } + Err(err) => Err(err), + } + } + pub(crate) fn parse_and_enter_group(&mut self) -> ParseResult<(Delimiter, DelimSpan)> { let (delimiter, delim_span, inner) = self.current().parse_any_group()?; let inner = unsafe { diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 25373c1c..8e68101d 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -11,7 +11,7 @@ pub(crate) use syn::parse::{ discouraged::Speculative, Parse as SynParse, ParseBuffer as SynParseBuffer, ParseStream as SynParseStream, Parser as SynParser, }; -pub(crate) use syn::{parse_str, Lit, LitBool, LitFloat, LitInt, Token, UnOp}; +pub(crate) use syn::{parse_str, Lit, LitBool, LitFloat, LitInt, Token}; pub(crate) use syn::{Error as SynError, Result as SynResult}; pub(crate) use crate::destructuring::*; diff --git a/tests/compilation_failures/expressions/invalid_binary_operator.rs b/tests/compilation_failures/expressions/invalid_binary_operator.rs new file mode 100644 index 00000000..181a6c4f --- /dev/null +++ b/tests/compilation_failures/expressions/invalid_binary_operator.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret! { + // The error message here isn't as good as it could be, + // because we end the Expression when we detect an invalid extension. + // This is useful to allow e.g. [!if! true == false { ... }] + // But in future, perhaps we can use different parse modes for these cases. + [!evaluate! 10 _ 10] + }; +} diff --git a/tests/compilation_failures/expressions/invalid_binary_operator.stderr b/tests/compilation_failures/expressions/invalid_binary_operator.stderr new file mode 100644 index 00000000..888b4c39 --- /dev/null +++ b/tests/compilation_failures/expressions/invalid_binary_operator.stderr @@ -0,0 +1,5 @@ +error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/invalid_binary_operator.rs:9:24 + | +9 | [!evaluate! 10 _ 10] + | ^ diff --git a/tests/compilation_failures/expressions/invalid_unary_operator.rs b/tests/compilation_failures/expressions/invalid_unary_operator.rs new file mode 100644 index 00000000..7f771285 --- /dev/null +++ b/tests/compilation_failures/expressions/invalid_unary_operator.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret! { + [!evaluate! ^10] + }; +} diff --git a/tests/compilation_failures/expressions/invalid_unary_operator.stderr b/tests/compilation_failures/expressions/invalid_unary_operator.stderr new file mode 100644 index 00000000..b8679c3f --- /dev/null +++ b/tests/compilation_failures/expressions/invalid_unary_operator.stderr @@ -0,0 +1,6 @@ +error: Expected ! or - + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/invalid_unary_operator.rs:5:21 + | +5 | [!evaluate! ^10] + | ^ diff --git a/tests/expressions.rs b/tests/expressions.rs index effcd00e..861a09ef 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -80,6 +80,22 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! "Zoo" > "Aardvark"], true); } +#[test] +fn test_expression_precedence() { + // The general rules are: + // * Operators at higher precedence should group more tightly than operators at lower precedence. + // * Operators at the same precedence should left-associate. + + // 1 + -1 + ((2 + 4) * 3) - 9 => 1 + -1 + 18 - 9 => 9 + assert_preinterpret_eq!([!evaluate! 1 + -(1) + (2 + 4) * 3 - 9], 9); + // (true > true) > true => false > true => false + assert_preinterpret_eq!([!evaluate! true > true > true], false); + // (5 - 2) - 1 => 3 - 1 => 2 + assert_preinterpret_eq!([!evaluate! 5 - 2 - 1], 2); + // ((3 * 3 - 4) < (3 << 1)) && true => 5 < 6 => true + assert_preinterpret_eq!([!evaluate! 3 * 3 - 4 < 3 << 1 && true], true); +} + #[test] fn test_very_long_expression_works() { assert_preinterpret_eq!( From 4a584858cb1b48f21cb87c4ac306abd47dea7f90 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 2 Feb 2025 19:21:32 +0000 Subject: [PATCH 072/126] refactor: Separate parsing from source and interpreted --- CHANGELOG.md | 11 + src/destructuring/destructure_group.rs | 6 +- src/destructuring/destructure_item.rs | 28 +- src/destructuring/destructure_raw.rs | 6 +- src/destructuring/destructure_segment.rs | 16 +- src/destructuring/destructure_traits.rs | 4 +- src/destructuring/destructure_variable.rs | 36 +-- src/destructuring/destructurer.rs | 20 +- src/destructuring/destructurers.rs | 20 +- src/destructuring/fields.rs | 28 +- src/expressions/evaluation.rs | 36 +-- src/expressions/expression.rs | 261 ++++++++++++++++-- src/expressions/expression_parsing.rs | 135 +++------ src/expressions/operations.rs | 85 ++++-- src/extensions/parsing.rs | 120 ++++---- src/interpretation/command.rs | 4 +- src/interpretation/command_arguments.rs | 10 +- src/interpretation/command_code_input.rs | 6 +- src/interpretation/command_field_inputs.rs | 4 +- src/interpretation/command_stream_input.rs | 26 +- src/interpretation/command_value_input.rs | 67 +++-- .../commands/control_flow_commands.rs | 6 +- src/interpretation/commands/core_commands.rs | 4 +- .../commands/destructuring_commands.rs | 2 +- .../commands/expression_commands.rs | 18 +- src/interpretation/commands/token_commands.rs | 2 +- src/interpretation/interpretation_item.rs | 124 ++------- src/interpretation/interpretation_stream.rs | 14 +- src/interpretation/interpreted_stream.rs | 15 +- src/interpretation/variable.rs | 8 +- src/lib.rs | 2 +- src/misc/errors.rs | 9 + src/misc/parse_traits.rs | 217 +++++++++++++-- 33 files changed, 858 insertions(+), 492 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14e2c4d6..e97ad2c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,15 @@ Destructuring performs parsing of a token stream. It supports: ### To come +* Consider renames: + * KindedParseStream => ParseStream + * KindedParseBuffer => ParseBuffer + * Source => Grammar + * ParseFromSource => Parse + * InterpretationX => GrammarX + * ParseFromInterpreted => Parse + * Interpreted => Output + * InterpretedX => OutputX * Complete expression rework: * Disallow expressions/commands in re-evaluated `{ .. }` blocks and command outputs * Create `ParseInterpreted` and `ParseSource` via `ParseStream`, `SourceParseStream` and `InterpretedParseStream`, and add grammar peaking to `SourceParseStream` only. Add `[!reinterpret! ...]` command for an `eval` style command. @@ -87,6 +96,8 @@ Destructuring performs parsing of a token stream. It supports: * `[!is_set! #x]` * `[!str_split! { input: Value, separator: Value, }]` * Change `(!content! ...)` to unwrap none groups, to be more permissive +* Support `'a'..'z'` in `[!range! 'a'..'z']` +* Support `#x[0]` and `#x[0..3]` and `#..x[0..3]` and other things like `[ ..=3]` * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Destructurers * `(!fields! ...)` and `(!subfields! ...)` diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs index 2ad59121..842e834c 100644 --- a/src/destructuring/destructure_group.rs +++ b/src/destructuring/destructure_group.rs @@ -6,8 +6,8 @@ pub(crate) struct DestructureGroup { inner: DestructureRemaining, } -impl Parse for DestructureGroup { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for DestructureGroup { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let (delimiter, _, content) = input.parse_any_group()?; Ok(Self { delimiter, @@ -19,7 +19,7 @@ impl Parse for DestructureGroup { impl HandleDestructure for DestructureGroup { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { let (_, inner) = input.parse_specific_group(self.delimiter)?; diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs index 1cc52cd8..541285ad 100644 --- a/src/destructuring/destructure_item.rs +++ b/src/destructuring/destructure_item.rs @@ -16,27 +16,27 @@ impl DestructureItem { /// notably the flattened command. This allows [!let! #..x = Hello => World] to parse as setting /// `x` to `Hello => World` rather than having `#..x` peeking to see it is "up to =" and then only /// parsing `Hello` into `x`. - pub(crate) fn parse_until(input: ParseStream) -> ParseResult { - Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command(Some(CommandOutputKind::None)) => { + pub(crate) fn parse_until(input: SourceParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Command(Some(CommandOutputKind::None)) => { Self::NoneOutputCommand(input.parse()?) } - PeekMatch::Command(_) => { + GrammarPeekMatch::Command(_) => { return input.parse_err( "Commands which return something are not supported in destructuring positions", ) } - PeekMatch::GroupedVariable - | PeekMatch::FlattenedVariable - | PeekMatch::AppendVariableDestructuring => { + GrammarPeekMatch::GroupedVariable + | GrammarPeekMatch::FlattenedVariable + | GrammarPeekMatch::AppendVariableDestructuring => { Self::Variable(DestructureVariable::parse_until::(input)?) } - PeekMatch::Group(_) => Self::ExactGroup(input.parse()?), - PeekMatch::Destructurer(_) => Self::Destructurer(input.parse()?), - PeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), - PeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), - PeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), - PeekMatch::End => return input.parse_err("Unexpected end"), + GrammarPeekMatch::Group(_) => Self::ExactGroup(input.parse()?), + GrammarPeekMatch::Destructurer(_) => Self::Destructurer(input.parse()?), + GrammarPeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), + GrammarPeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), + GrammarPeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), + GrammarPeekMatch::End => return input.parse_err("Unexpected end"), }) } } @@ -44,7 +44,7 @@ impl DestructureItem { impl HandleDestructure for DestructureItem { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { match self { diff --git a/src/destructuring/destructure_raw.rs b/src/destructuring/destructure_raw.rs index c9cf0199..1f4e456c 100644 --- a/src/destructuring/destructure_raw.rs +++ b/src/destructuring/destructure_raw.rs @@ -10,7 +10,7 @@ pub(crate) enum RawDestructureItem { } impl RawDestructureItem { - pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { + pub(crate) fn handle_destructure(&self, input: InterpretedParseStream) -> ExecutionResult<()> { match self { RawDestructureItem::Punct(punct) => { input.parse_punct_matching(punct.as_char())?; @@ -65,7 +65,7 @@ impl RawDestructureStream { self.inner.push(item); } - pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { + pub(crate) fn handle_destructure(&self, input: InterpretedParseStream) -> ExecutionResult<()> { for item in self.inner.iter() { item.handle_destructure(input)?; } @@ -91,7 +91,7 @@ impl RawDestructureGroup { } } - pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { + pub(crate) fn handle_destructure(&self, input: InterpretedParseStream) -> ExecutionResult<()> { let (_, inner) = input.parse_specific_group(self.delimiter)?; self.inner.handle_destructure(&inner) } diff --git a/src/destructuring/destructure_segment.rs b/src/destructuring/destructure_segment.rs index ed395aac..2c9fb714 100644 --- a/src/destructuring/destructure_segment.rs +++ b/src/destructuring/destructure_segment.rs @@ -1,19 +1,19 @@ use crate::internal_prelude::*; pub(crate) trait StopCondition: Clone { - fn should_stop(input: ParseStream) -> bool; + fn should_stop(input: SourceParseStream) -> bool; } #[derive(Clone)] pub(crate) struct UntilEnd; impl StopCondition for UntilEnd { - fn should_stop(input: ParseStream) -> bool { + fn should_stop(input: SourceParseStream) -> bool { input.is_empty() } } pub(crate) trait PeekableToken: Clone { - fn peek(input: ParseStream) -> bool; + fn peek(input: SourceParseStream) -> bool; } // This is going through such pain to ensure we stay in the public API of syn @@ -23,7 +23,7 @@ macro_rules! impl_peekable_token { ($(Token![$token:tt]),* $(,)?) => { $( impl PeekableToken for Token![$token] { - fn peek(input: ParseStream) -> bool { + fn peek(input: SourceParseStream) -> bool { input.peek(Token![$token]) } } @@ -41,7 +41,7 @@ pub(crate) struct UntilToken { token: PhantomData, } impl StopCondition for UntilToken { - fn should_stop(input: ParseStream) -> bool { + fn should_stop(input: SourceParseStream) -> bool { input.is_empty() || T::peek(input) } } @@ -55,8 +55,8 @@ pub(crate) struct DestructureSegment { inner: Vec, } -impl Parse for DestructureSegment { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for DestructureSegment { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let mut inner = vec![]; while !C::should_stop(input) { inner.push(DestructureItem::parse_until::(input)?); @@ -71,7 +71,7 @@ impl Parse for DestructureSegment { impl HandleDestructure for DestructureSegment { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { for item in self.inner.iter() { diff --git a/src/destructuring/destructure_traits.rs b/src/destructuring/destructure_traits.rs index 1349ff3d..45a141e8 100644 --- a/src/destructuring/destructure_traits.rs +++ b/src/destructuring/destructure_traits.rs @@ -10,13 +10,13 @@ pub(crate) trait HandleDestructure { // RUST-ANALYZER-SAFETY: ...this isn't generally safe... // We should only do this when we know that either the input or parser doesn't require // analysis of nested None-delimited groups. - input.syn_parse(|input| self.handle_destructure(input, interpreter)) + input.parse_with(|input| self.handle_destructure(input, interpreter)) } } fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()>; } diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index eb552556..3d2791e0 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -54,7 +54,7 @@ pub(crate) enum DestructureVariable { } impl DestructureVariable { - pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> ParseResult { + pub(crate) fn parse_only_unflattened_input(input: SourceParseStream) -> ParseResult { let variable: DestructureVariable = Self::parse_until::(input)?; if variable.is_flattened_input() { return variable @@ -64,7 +64,7 @@ impl DestructureVariable { Ok(variable) } - pub(crate) fn parse_until(input: ParseStream) -> ParseResult { + pub(crate) fn parse_until(input: SourceParseStream) -> ParseResult { let marker = input.parse()?; if input.peek(Token![..]) { let flatten = input.parse()?; @@ -180,7 +180,7 @@ impl DestructureVariable { impl HandleDestructure for DestructureVariable { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { match self { @@ -260,8 +260,8 @@ impl ParsedTokenTree { } } -impl Parse for ParsedTokenTree { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromInterpreted for ParsedTokenTree { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { Ok(match input.parse::()? { TokenTree::Group(group) if group.delimiter() == Delimiter::None => { ParsedTokenTree::NoneGroup(group) @@ -290,31 +290,31 @@ pub(crate) enum ParseUntil { impl ParseUntil { /// Peeks the next token, to discover what we should parse next - fn peek_flatten_limit(input: ParseStream) -> ParseResult { + fn peek_flatten_limit(input: SourceParseStream) -> ParseResult { if C::should_stop(input) { return Ok(ParseUntil::End); } - Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command(_) - | PeekMatch::GroupedVariable - | PeekMatch::FlattenedVariable - | PeekMatch::Destructurer(_) - | PeekMatch::AppendVariableDestructuring => { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Command(_) + | GrammarPeekMatch::GroupedVariable + | GrammarPeekMatch::FlattenedVariable + | GrammarPeekMatch::Destructurer(_) + | GrammarPeekMatch::AppendVariableDestructuring => { return input .span() .parse_err("This cannot follow a flattened destructure match"); } - PeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), - PeekMatch::Ident(ident) => ParseUntil::Ident(ident), - PeekMatch::Literal(literal) => ParseUntil::Literal(literal), - PeekMatch::Punct(punct) => ParseUntil::Punct(punct), - PeekMatch::End => ParseUntil::End, + GrammarPeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), + GrammarPeekMatch::Ident(ident) => ParseUntil::Ident(ident), + GrammarPeekMatch::Literal(literal) => ParseUntil::Literal(literal), + GrammarPeekMatch::Punct(punct) => ParseUntil::Punct(punct), + GrammarPeekMatch::End => ParseUntil::End, }) } fn handle_parse_into( &self, - input: ParseStream, + input: InterpretedParseStream, output: &mut InterpretedStream, ) -> ExecutionResult<()> { match self { diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index d077bf7d..5fd9be34 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -5,14 +5,14 @@ pub(crate) trait DestructurerDefinition: Clone { fn parse(arguments: DestructurerArguments) -> ParseResult; fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()>; } #[derive(Clone)] pub(crate) struct DestructurerArguments<'a> { - parse_stream: ParseStream<'a>, + parse_stream: SourceParseStream<'a>, destructurer_name: Ident, full_span: Span, } @@ -20,7 +20,7 @@ pub(crate) struct DestructurerArguments<'a> { #[allow(unused)] impl<'a> DestructurerArguments<'a> { pub(crate) fn new( - parse_stream: ParseStream<'a>, + parse_stream: SourceParseStream<'a>, destructurer_name: Ident, full_span: Span, ) -> Self { @@ -44,17 +44,17 @@ impl<'a> DestructurerArguments<'a> { } } - pub(crate) fn fully_parse_no_error_override(&self) -> ParseResult { + pub(crate) fn fully_parse_no_error_override(&self) -> ParseResult { self.parse_stream.parse() } pub(crate) fn fully_parse_as(&self) -> ParseResult { - self.fully_parse_or_error(T::parse, T::error_message()) + self.fully_parse_or_error(T::parse_from_source, T::error_message()) } pub(crate) fn fully_parse_or_error( &self, - parse_function: impl FnOnce(ParseStream) -> ParseResult, + parse_function: impl FnOnce(SourceParseStream) -> ParseResult, error_message: impl std::fmt::Display, ) -> ParseResult { // In future, when the diagnostic API is stable, @@ -85,8 +85,8 @@ pub(crate) struct Destructurer { source_group_span: DelimSpan, } -impl Parse for Destructurer { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for Destructurer { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Parenthesis)?; content.parse::()?; let destructurer_name = content.parse_any_ident()?; @@ -116,7 +116,7 @@ impl Parse for Destructurer { impl HandleDestructure for Destructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { self.instance.handle_destructure(input, interpreter) @@ -174,7 +174,7 @@ macro_rules! define_destructurers { } impl NamedDestructurer { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn handle_destructure(&self, input: InterpretedParseStream, interpreter: &mut Interpreter) -> ExecutionResult<()> { match self { $( Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index 8c58b717..b66a0f78 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -16,7 +16,7 @@ impl DestructurerDefinition for StreamDestructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { self.inner.handle_destructure(input, interpreter) @@ -48,7 +48,7 @@ impl DestructurerDefinition for IdentDestructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { if input.cursor().ident().is_some() { @@ -90,7 +90,7 @@ impl DestructurerDefinition for LiteralDestructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { if input.cursor().literal().is_some() { @@ -132,7 +132,7 @@ impl DestructurerDefinition for PunctDestructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { if input.cursor().any_punct().is_some() { @@ -165,7 +165,7 @@ impl DestructurerDefinition for GroupDestructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { let (_, inner) = input.parse_specific_group(Delimiter::None)?; @@ -188,7 +188,11 @@ impl DestructurerDefinition for RawDestructurer { }) } - fn handle_destructure(&self, input: ParseStream, _: &mut Interpreter) -> ExecutionResult<()> { + fn handle_destructure( + &self, + input: InterpretedParseStream, + _: &mut Interpreter, + ) -> ExecutionResult<()> { self.stream.handle_destructure(input) } } @@ -205,7 +209,7 @@ impl DestructurerDefinition for ContentDestructurer { arguments.fully_parse_or_error( |input| { Ok(Self { - stream: InterpretationStream::parse_with_context(input, arguments.full_span())?, + stream: InterpretationStream::parse_from_source(input, arguments.full_span())?, }) }, "Expected (!content! ... interpretable input ...)", @@ -214,7 +218,7 @@ impl DestructurerDefinition for ContentDestructurer { fn handle_destructure( &self, - input: ParseStream, + input: InterpretedParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { self.stream diff --git a/src/destructuring/fields.rs b/src/destructuring/fields.rs index d4d7ce06..cad0f1d4 100644 --- a/src/destructuring/fields.rs +++ b/src/destructuring/fields.rs @@ -16,24 +16,38 @@ impl FieldsParseDefinition { } } - pub(crate) fn add_required_field( + pub(crate) fn add_required_field( self, field_name: &str, example: &str, explanation: Option<&str>, set: impl Fn(&mut T, F) + 'static, ) -> Self { - self.add_field(field_name, example, explanation, true, F::parse, set) + self.add_field( + field_name, + example, + explanation, + true, + F::parse_from_source, + set, + ) } - pub(crate) fn add_optional_field( + pub(crate) fn add_optional_field( self, field_name: &str, example: &str, explanation: Option<&str>, set: impl Fn(&mut T, F) + 'static, ) -> Self { - self.add_field(field_name, example, explanation, false, F::parse, set) + self.add_field( + field_name, + example, + explanation, + false, + F::parse_from_source, + set, + ) } pub(crate) fn add_field( @@ -42,7 +56,7 @@ impl FieldsParseDefinition { example: &str, explanation: Option<&str>, is_required: bool, - parse: impl Fn(ParseStream) -> ParseResult + 'static, + parse: impl Fn(SourceParseStream) -> ParseResult + 'static, set: impl Fn(&mut T, F) + 'static, ) -> Self { if self @@ -73,7 +87,7 @@ impl FieldsParseDefinition { error_span_range: SpanRange, ) -> impl FnOnce(SynParseStream) -> ParseResult { fn inner( - input: ParseStream, + input: SourceParseStream, new_builder: T, field_definitions: &FieldDefinitions, error_span_range: SpanRange, @@ -172,5 +186,5 @@ struct FieldParseDefinition { example: String, explanation: Option, #[allow(clippy::type_complexity)] - parse_and_set: Box ParseResult<()>>, + parse_and_set: Box ParseResult<()>>, } diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 6dc8fd59..6498eed1 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -1,12 +1,12 @@ use super::*; -pub(super) struct ExpressionEvaluator<'a> { - nodes: &'a [ExpressionNode], +pub(super) struct ExpressionEvaluator<'a, K: Expressionable> { + nodes: &'a [ExpressionNode], operation_stack: Vec, } -impl<'a> ExpressionEvaluator<'a> { - pub(super) fn new(nodes: &'a [ExpressionNode]) -> Self { +impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { + pub(super) fn new(nodes: &'a [ExpressionNode]) -> Self { Self { nodes, operation_stack: Vec::new(), @@ -16,9 +16,9 @@ impl<'a> ExpressionEvaluator<'a> { pub(super) fn evaluate( mut self, root: ExpressionNodeId, - interpreter: &mut Interpreter, + evaluation_context: &mut K::EvaluationContext, ) -> ExecutionResult { - let mut next = self.begin_node_evaluation(root, interpreter)?; + let mut next = self.begin_node_evaluation(root, evaluation_context)?; loop { match next { @@ -30,7 +30,7 @@ impl<'a> ExpressionEvaluator<'a> { next = self.continue_node_evaluation(top_of_stack, evaluation_value)?; } NextAction::EnterNode(next_node) => { - next = self.begin_node_evaluation(next_node, interpreter)?; + next = self.begin_node_evaluation(next_node, evaluation_context)?; } } } @@ -39,29 +39,11 @@ impl<'a> ExpressionEvaluator<'a> { fn begin_node_evaluation( &mut self, node_id: ExpressionNodeId, - interpreter: &mut Interpreter, + evaluation_context: &mut K::EvaluationContext, ) -> ExecutionResult { Ok(match &self.nodes[node_id.0] { ExpressionNode::Leaf(leaf) => { - let interpreted = match leaf { - ExpressionLeaf::Command(command) => { - command.clone().interpret_to_new_stream(interpreter)? - } - ExpressionLeaf::GroupedVariable(grouped_variable) => { - grouped_variable.interpret_to_new_stream(interpreter)? - } - ExpressionLeaf::CodeBlock(code_block) => { - code_block.clone().interpret_to_new_stream(interpreter)? - } - ExpressionLeaf::Value(value) => { - return Ok(NextAction::HandleValue(value.clone())) - } - }; - let parsed_expression = unsafe { - // RUST-ANALYZER SAFETY: This isn't very safe, as it could have a none-delimited group in it - interpreted.syn_parse(Expression::parse)? - }; - NextAction::HandleValue(parsed_expression.evaluate_to_value(interpreter)?) + NextAction::HandleValue(K::evaluate_leaf(leaf, evaluation_context)?) } ExpressionNode::UnaryOperation { operation, input } => { self.operation_stack diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index c6a4a43c..53e61cd3 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -1,32 +1,35 @@ use super::*; +// Interpretation = Source +// ======================= + #[derive(Clone)] -pub(crate) struct Expression { - pub(super) root: ExpressionNodeId, - pub(super) span_range: SpanRange, - pub(super) nodes: std::rc::Rc<[ExpressionNode]>, +pub(crate) struct InterpretationExpression { + inner: Expression, } -impl HasSpanRange for Expression { +impl HasSpanRange for InterpretationExpression { fn span_range(&self) -> SpanRange { - self.span_range + self.inner.span_range } } -impl Parse for Expression { - fn parse(input: ParseStream) -> ParseResult { - ExpressionParser::new(input).parse() +impl ParseFromSource for InterpretationExpression { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + Ok(Self { + inner: ExpressionParser::parse(input)?, + }) } } -impl Expression { +impl InterpretationExpression { pub(crate) fn evaluate( &self, interpreter: &mut Interpreter, ) -> ExecutionResult { Ok(EvaluationOutput { value: self.evaluate_to_value(interpreter)?, - fallback_output_span: self.span_range.join_into_span_else_start(), + fallback_output_span: self.inner.span_range.join_into_span_else_start(), }) } @@ -45,15 +48,220 @@ impl Expression { &self, interpreter: &mut Interpreter, ) -> ExecutionResult { - ExpressionEvaluator::new(&self.nodes).evaluate(self.root, interpreter) + Source::evaluate_to_value(&self.inner, interpreter) + } +} + +pub(super) enum InterpretationExpressionLeaf { + Command(Command), + GroupedVariable(GroupedVariable), + CodeBlock(CommandCodeInput), + Value(EvaluationValue), +} + +impl Expressionable for Source { + type Leaf = InterpretationExpressionLeaf; + type EvaluationContext = Interpreter; + + fn leaf_end_span(leaf: &Self::Leaf) -> Option { + match leaf { + InterpretationExpressionLeaf::Command(command) => Some(command.span()), + InterpretationExpressionLeaf::GroupedVariable(variable) => { + Some(variable.span_range().end()) + } + InterpretationExpressionLeaf::CodeBlock(code_block) => Some(code_block.span()), + InterpretationExpressionLeaf::Value(value) => value.source_span(), + } + } + + fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Command(Some(output_kind)) => { + match output_kind.expression_support() { + Ok(()) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), + Err(error_message) => return input.parse_err(error_message), + } + } + GrammarPeekMatch::Command(None) => return input.parse_err("Invalid command"), + GrammarPeekMatch::GroupedVariable => UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)), + GrammarPeekMatch::FlattenedVariable => return input.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), + GrammarPeekMatch::AppendVariableDestructuring => return input.parse_err("Append variable operations are not supported in an expression"), + GrammarPeekMatch::Destructurer(_) => return input.parse_err("Destructurings are not supported in an expression"), + GrammarPeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { + let (_, delim_span) = input.parse_and_enter_group()?; + UnaryAtom::Group(delim_span) + }, + GrammarPeekMatch::Group(Delimiter::Brace) => { + let leaf = InterpretationExpressionLeaf::CodeBlock(input.parse()?); + UnaryAtom::Leaf(leaf) + } + GrammarPeekMatch::Group(Delimiter::Bracket) => return input.parse_err("Square brackets [ .. ] are not supported in an expression"), + GrammarPeekMatch::Punct(_) => { + UnaryAtom::PrefixUnaryOperation(input.parse()?) + }, + GrammarPeekMatch::Ident(_) => { + let value = EvaluationValue::Boolean(EvaluationBoolean::for_litbool(input.parse()?)); + UnaryAtom::Leaf(Self::Leaf::Value(value)) + }, + GrammarPeekMatch::Literal(_) => { + let value = EvaluationValue::for_literal(input.parse()?)?; + UnaryAtom::Leaf(Self::Leaf::Value(value)) + }, + GrammarPeekMatch::End => return input.parse_err("The expression ended in an incomplete state"), + }) + } + + fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Punct(_) => match input.try_parse_or_revert::() { + Ok(operation) => NodeExtension::BinaryOperation(operation), + Err(_) => NodeExtension::NoneMatched, + }, + GrammarPeekMatch::Ident(ident) if ident == "as" => { + let cast_operation = + UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; + NodeExtension::PostfixOperation(cast_operation) + } + _ => NodeExtension::NoneMatched, + }) + } + + fn evaluate_leaf( + leaf: &Self::Leaf, + interpreter: &mut Self::EvaluationContext, + ) -> ExecutionResult { + let interpreted = match leaf { + InterpretationExpressionLeaf::Command(command) => { + command.clone().interpret_to_new_stream(interpreter)? + } + InterpretationExpressionLeaf::GroupedVariable(grouped_variable) => { + grouped_variable.interpret_to_new_stream(interpreter)? + } + InterpretationExpressionLeaf::CodeBlock(code_block) => { + code_block.clone().interpret_to_new_stream(interpreter)? + } + InterpretationExpressionLeaf::Value(value) => return Ok(value.clone()), + }; + let parsed_expression = unsafe { + // RUST-ANALYZER SAFETY: This isn't very safe, as it could have a none-delimited group in it + interpreted.parse_as::()? + }; + parsed_expression.evaluate_to_value() + } +} + +// Interpreted +// =========== + +#[derive(Clone)] +pub(crate) struct InterpretedExpression { + inner: Expression, +} + +impl ParseFromInterpreted for InterpretedExpression { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { + Ok(Self { + inner: ExpressionParser::parse(input)?, + }) + } +} + +impl InterpretedExpression { + pub(crate) fn evaluate(&self) -> ExecutionResult { + Ok(EvaluationOutput { + value: self.evaluate_to_value()?, + fallback_output_span: self.inner.span_range.join_into_span_else_start(), + }) + } + + pub(crate) fn evaluate_to_value(&self) -> ExecutionResult { + Interpreted::evaluate_to_value(&self.inner, &mut ()) + } +} + +impl Expressionable for Interpreted { + type Leaf = EvaluationValue; + type EvaluationContext = (); + + fn leaf_end_span(leaf: &Self::Leaf) -> Option { + leaf.source_span() + } + + fn evaluate_leaf( + leaf: &Self::Leaf, + _: &mut Self::EvaluationContext, + ) -> ExecutionResult { + Ok(leaf.clone()) + } + + fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { + Ok(match input.peek_token() { + InterpretedPeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { + let (_, delim_span) = input.parse_and_enter_group()?; + UnaryAtom::Group(delim_span) + } + InterpretedPeekMatch::Group(Delimiter::Brace) => { + return input + .parse_err("Curly braces are not supported in a re-interpreted expression") + } + InterpretedPeekMatch::Group(Delimiter::Bracket) => { + return input.parse_err("Square brackets [ .. ] are not supported in an expression") + } + InterpretedPeekMatch::Ident(_) => UnaryAtom::Leaf(EvaluationValue::Boolean( + EvaluationBoolean::for_litbool(input.parse()?), + )), + InterpretedPeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), + InterpretedPeekMatch::Literal(_) => { + UnaryAtom::Leaf(EvaluationValue::for_literal(input.parse()?)?) + } + InterpretedPeekMatch::End => { + return input.parse_err("The expression ended in an incomplete state") + } + }) + } + + fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { + Ok(match input.peek_token() { + InterpretedPeekMatch::Punct(_) => { + match input.try_parse_or_revert::() { + Ok(operation) => NodeExtension::BinaryOperation(operation), + Err(_) => NodeExtension::NoneMatched, + } + } + InterpretedPeekMatch::Ident(ident) if ident == "as" => { + let cast_operation = + UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; + NodeExtension::PostfixOperation(cast_operation) + } + _ => NodeExtension::NoneMatched, + }) + } +} + +// Generic +// ======= + +pub(super) struct Expression { + pub(super) root: ExpressionNodeId, + pub(super) span_range: SpanRange, + pub(super) nodes: std::rc::Rc<[ExpressionNode]>, +} + +impl Clone for Expression { + fn clone(&self) -> Self { + Self { + root: self.root, + span_range: self.span_range, + nodes: self.nodes.clone(), + } } } #[derive(Clone, Copy)] pub(super) struct ExpressionNodeId(pub(super) usize); -pub(super) enum ExpressionNode { - Leaf(ExpressionLeaf), +pub(super) enum ExpressionNode { + Leaf(K::Leaf), UnaryOperation { operation: UnaryOperation, input: ExpressionNodeId, @@ -65,9 +273,24 @@ pub(super) enum ExpressionNode { }, } -pub(super) enum ExpressionLeaf { - Command(Command), - GroupedVariable(GroupedVariable), - CodeBlock(CommandCodeInput), - Value(EvaluationValue), +pub(super) trait Expressionable: Sized { + type Leaf; + type EvaluationContext; + + fn leaf_end_span(leaf: &Self::Leaf) -> Option; + + fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult>; + fn parse_extension(input: &mut ParseStreamStack) -> ParseResult; + + fn evaluate_leaf( + leaf: &Self::Leaf, + context: &mut Self::EvaluationContext, + ) -> ExecutionResult; + + fn evaluate_to_value( + expression: &Expression, + context: &mut Self::EvaluationContext, + ) -> ExecutionResult { + ExpressionEvaluator::new(&expression.nodes).evaluate(expression.root, context) + } } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index fc5bb605..36427dd0 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -1,32 +1,44 @@ use super::*; -pub(super) struct ExpressionParser<'a> { - streams: ParseStreamStack<'a>, - nodes: ExpressionNodes, +pub(super) struct ExpressionParser<'a, K: Expressionable> { + streams: ParseStreamStack<'a, K>, + nodes: ExpressionNodes, expression_stack: Vec, span_range: SpanRange, + kind: PhantomData, } -impl<'a> ExpressionParser<'a> { - pub(super) fn new(input: ParseStream<'a>) -> Self { +impl<'a, K: Expressionable> ExpressionParser<'a, K> { + pub(super) fn parse(input: KindedParseStream<'a, K>) -> ParseResult> { Self { streams: ParseStreamStack::new(input), nodes: ExpressionNodes::new(), expression_stack: Vec::with_capacity(10), span_range: SpanRange::new_single(input.span()), + kind: PhantomData, } + .run() } - pub(super) fn parse(mut self) -> ParseResult { + fn run(mut self) -> ParseResult> { let mut work_item = self.push_stack_frame(ExpressionStackFrame::Root); loop { work_item = match work_item { WorkItem::RequireUnaryAtom => { - let unary_atom = self.parse_unary_atom()?; + let unary_atom = K::parse_unary_atom(&mut self.streams)?; self.extend_with_unary_atom(unary_atom)? } WorkItem::TryParseAndApplyExtension { node } => { - let extension = self.parse_extension()?; + let extension = K::parse_extension(&mut self.streams)?; + match &extension { + NodeExtension::PostfixOperation(op) => { + self.span_range.set_end(op.end_span()); + } + NodeExtension::BinaryOperation(op) => { + self.span_range.set_end(op.span_range().end()); + } + NodeExtension::NoneMatched => {} + }; self.attempt_extension(node, extension)? } WorkItem::TryApplyAlreadyParsedExtension { node, extension } => { @@ -39,31 +51,19 @@ impl<'a> ExpressionParser<'a> { } } - fn extend_with_unary_atom(&mut self, unary_atom: UnaryAtom) -> ParseResult { + fn extend_with_unary_atom(&mut self, unary_atom: UnaryAtom) -> ParseResult { Ok(match unary_atom { - UnaryAtom::Command(command) => { - self.span_range.set_end(command.span()); - self.add_leaf(ExpressionLeaf::Command(command)) - } - UnaryAtom::GroupedVariable(variable) => { - self.span_range.set_end(variable.span_range().end()); - self.add_leaf(ExpressionLeaf::GroupedVariable(variable)) - } - UnaryAtom::CodeBlock(code_block) => { - self.span_range.set_end(code_block.span()); - self.add_leaf(ExpressionLeaf::CodeBlock(code_block)) - } - UnaryAtom::Value(value) => { - if let Some(span) = value.source_span() { + UnaryAtom::Leaf(leaf) => { + if let Some(span) = K::leaf_end_span(&leaf) { self.span_range.set_end(span); } - self.add_leaf(ExpressionLeaf::Value(value)) + self.add_leaf(leaf) } UnaryAtom::Group(delim_span) => { self.span_range.set_end(delim_span.close()); self.push_stack_frame(ExpressionStackFrame::Group { delim_span }) } - UnaryAtom::UnaryOperation(operation) => { + UnaryAtom::PrefixUnaryOperation(operation) => { self.span_range.set_end(operation.span()); self.push_stack_frame(ExpressionStackFrame::IncompletePrefixOperation { operation }) } @@ -121,7 +121,7 @@ impl<'a> ExpressionParser<'a> { ExpressionStackFrame::IncompletePrefixOperation { operation } => { WorkItem::TryApplyAlreadyParsedExtension { node: self.nodes.add_node(ExpressionNode::UnaryOperation { - operation, + operation: operation.into(), input: node, }), extension, @@ -141,57 +141,7 @@ impl<'a> ExpressionParser<'a> { } } - fn parse_extension(&mut self) -> ParseResult { - Ok(match self.streams.peek_grammar() { - PeekMatch::Punct(_) => match self.streams.try_parse_or_revert::() { - Ok(operation) => NodeExtension::BinaryOperation(operation), - Err(_) => NodeExtension::NoneMatched, - }, - PeekMatch::Ident(ident) if ident == "as" => { - let cast_operation = UnaryOperation::for_cast_operation(self.streams.parse()?, { - let target_type = self.streams.parse::()?; - self.span_range.set_end(target_type.span()); - target_type - })?; - NodeExtension::PostfixOperation(cast_operation) - } - _ => NodeExtension::NoneMatched, - }) - } - - fn parse_unary_atom(&mut self) -> ParseResult { - Ok(match self.streams.peek_grammar() { - PeekMatch::Command(Some(output_kind)) => { - match output_kind.expression_support() { - Ok(()) => UnaryAtom::Command(self.streams.parse()?), - Err(error_message) => return self.streams.parse_err(error_message), - } - } - PeekMatch::Command(None) => return self.streams.parse_err("Invalid command"), - PeekMatch::GroupedVariable => UnaryAtom::GroupedVariable(self.streams.parse()?), - PeekMatch::FlattenedVariable => return self.streams.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), - PeekMatch::AppendVariableDestructuring => return self.streams.parse_err("Append variable operations are not supported in an expression"), - PeekMatch::Destructurer(_) => return self.streams.parse_err("Destructurings are not supported in an expression"), - PeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { - let (_, delim_span) = self.streams.parse_and_enter_group()?; - UnaryAtom::Group(delim_span) - }, - PeekMatch::Group(Delimiter::Brace) => UnaryAtom::CodeBlock(self.streams.parse()?), - PeekMatch::Group(Delimiter::Bracket) => return self.streams.parse_err("Square brackets [ .. ] are not supported in an expression"), - PeekMatch::Punct(_) => { - UnaryAtom::UnaryOperation(self.streams.parse_with(UnaryOperation::parse_from_prefix_punct)?) - }, - PeekMatch::Ident(_) => { - UnaryAtom::Value(EvaluationValue::Boolean(EvaluationBoolean::for_litbool(self.streams.parse()?))) - }, - PeekMatch::Literal(_) => { - UnaryAtom::Value(EvaluationValue::for_literal(self.streams.parse()?)?) - }, - PeekMatch::End => return self.streams.parse_err("The expression ended in an incomplete state"), - }) - } - - fn add_leaf(&mut self, leaf: ExpressionLeaf) -> WorkItem { + fn add_leaf(&mut self, leaf: K::Leaf) -> WorkItem { let node = self.nodes.add_node(ExpressionNode::Leaf(leaf)); WorkItem::TryParseAndApplyExtension { node } } @@ -217,22 +167,22 @@ impl<'a> ExpressionParser<'a> { } } -pub(super) struct ExpressionNodes { - nodes: Vec, +pub(super) struct ExpressionNodes { + nodes: Vec>, } -impl ExpressionNodes { +impl ExpressionNodes { pub(super) fn new() -> Self { Self { nodes: Vec::new() } } - pub(super) fn add_node(&mut self, node: ExpressionNode) -> ExpressionNodeId { + pub(super) fn add_node(&mut self, node: ExpressionNode) -> ExpressionNodeId { let node_id = ExpressionNodeId(self.nodes.len()); self.nodes.push(node); node_id } - pub(super) fn complete(self, root: ExpressionNodeId, span_range: SpanRange) -> Expression { + pub(super) fn complete(self, root: ExpressionNodeId, span_range: SpanRange) -> Expression { Expression { root, span_range, @@ -295,6 +245,12 @@ enum OperatorPrecendence { impl OperatorPrecendence { const MIN: Self = OperatorPrecendence::Jump; + fn of_prefix_unary_operation(op: &PrefixUnaryOperation) -> Self { + match op { + PrefixUnaryOperation::Neg { .. } | PrefixUnaryOperation::Not { .. } => Self::Prefix, + } + } + fn of_unary_operation(op: &UnaryOperation) -> Self { match op { UnaryOperation::GroupedNoOp { .. } => Self::Unambiguous, @@ -438,7 +394,7 @@ enum ExpressionStackFrame { Group { delim_span: DelimSpan }, /// An incomplete unary prefix operation /// NB: unary postfix operations such as `as` casting go straight to ExtendableNode - IncompletePrefixOperation { operation: UnaryOperation }, + IncompletePrefixOperation { operation: PrefixUnaryOperation }, /// An incomplete binary operation IncompleteBinaryOperation { lhs: ExpressionNodeId, @@ -452,7 +408,7 @@ impl ExpressionStackFrame { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompletePrefixOperation { operation, .. } => { - OperatorPrecendence::of_unary_operation(operation) + OperatorPrecendence::of_prefix_unary_operation(operation) } ExpressionStackFrame::IncompleteBinaryOperation { operation, .. } => { OperatorPrecendence::of_binary_operation(operation) @@ -480,16 +436,13 @@ enum WorkItem { }, } -enum UnaryAtom { - Command(Command), - GroupedVariable(GroupedVariable), - CodeBlock(CommandCodeInput), - Value(EvaluationValue), +pub(super) enum UnaryAtom { + Leaf(K::Leaf), Group(DelimSpan), - UnaryOperation(UnaryOperation), + PrefixUnaryOperation(PrefixUnaryOperation), } -enum NodeExtension { +pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), NoneMatched, diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 1bbe2517..40a77abc 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -31,6 +31,41 @@ pub(super) trait Operation: HasSpanRange { fn symbol(&self) -> &'static str; } +pub(super) enum PrefixUnaryOperation { + Neg(Token![-]), + Not(Token![!]), +} + +impl SynParse for PrefixUnaryOperation { + fn parse(input: SynParseStream) -> SynResult { + if input.peek(Token![-]) { + Ok(Self::Neg(input.parse()?)) + } else if input.peek(Token![!]) { + Ok(Self::Not(input.parse()?)) + } else { + Err(input.error("Expected ! or -")) + } + } +} + +impl HasSpan for PrefixUnaryOperation { + fn span(&self) -> Span { + match self { + PrefixUnaryOperation::Neg(token) => token.span, + PrefixUnaryOperation::Not(token) => token.span, + } + } +} + +impl From for UnaryOperation { + fn from(operation: PrefixUnaryOperation) -> Self { + match operation { + PrefixUnaryOperation::Neg(token) => Self::Neg { token }, + PrefixUnaryOperation::Not(token) => Self::Not { token }, + } + } +} + #[derive(Clone)] pub(super) enum UnaryOperation { Neg { @@ -44,30 +79,17 @@ pub(super) enum UnaryOperation { }, Cast { as_token: Token![as], + target_ident: Ident, target: CastTarget, }, } impl UnaryOperation { - pub(super) fn parse_from_prefix_punct(input: ParseStream) -> ParseResult { - if input.peek(Token![-]) { - Ok(Self::Neg { - token: input.parse()?, - }) - } else if input.peek(Token![!]) { - Ok(Self::Not { - token: input.parse()?, - }) - } else { - input.parse_err("Expected ! or -") - } - } - pub(super) fn for_cast_operation( as_token: Token![as], - target_type: Ident, + target_ident: Ident, ) -> ParseResult { - let target = match target_type.to_string().as_str() { + let target = match target_ident.to_string().as_str() { "int" | "integer" => CastTarget::Integer(IntegerKind::Untyped), "u8" => CastTarget::Integer(IntegerKind::U8), "u16" => CastTarget::Integer(IntegerKind::U16), @@ -87,16 +109,29 @@ impl UnaryOperation { "bool" => CastTarget::Boolean, "char" => CastTarget::Char, _ => { - return target_type + return target_ident .parse_err("This type is not supported in preinterpret cast expressions") } }; - Ok(Self::Cast { as_token, target }) + Ok(Self::Cast { + as_token, + target, + target_ident, + }) } pub(super) fn evaluate(self, input: EvaluationValue) -> ExecutionResult { input.handle_unary_operation(self) } + + pub(super) fn end_span(&self) -> Span { + match self { + UnaryOperation::Neg { token } => token.span, + UnaryOperation::Not { token } => token.span, + UnaryOperation::GroupedNoOp { span } => *span, + UnaryOperation::Cast { target_ident, .. } => target_ident.span(), + } + } } impl Operation for UnaryOperation { @@ -141,8 +176,8 @@ pub(super) enum BinaryOperation { Integer(IntegerBinaryOperation), } -impl Parse for BinaryOperation { - fn parse(input: ParseStream) -> ParseResult { +impl SynParse for BinaryOperation { + fn parse(input: SynParseStream) -> SynResult { // In line with Syn's BinOp, we use peek instead of lookahead // ...I assume for slightly increased performance // ...Or becuase 30 alternative options in the error message is too many @@ -211,7 +246,7 @@ impl Parse for BinaryOperation { } else if input.peek(Token![^]) { Ok(Self::Paired(PairedBinaryOperation::BitXor(input.parse()?))) } else { - input.parse_err("Expected one of + - * / % && || ^ & | == < <= != >= > << or >>") + Err(input.error("Expected one of + - * / % && || ^ & | == < <= != >= > << or >>")) } } } @@ -270,11 +305,11 @@ impl BinaryOperation { } } -impl HasSpan for BinaryOperation { - fn span(&self) -> Span { +impl HasSpanRange for BinaryOperation { + fn span_range(&self) -> SpanRange { match self { - BinaryOperation::Paired(_) => self.span(), - BinaryOperation::Integer(_) => self.span(), + BinaryOperation::Paired(op) => op.span_range(), + BinaryOperation::Integer(op) => op.span_range(), } } } diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index c7a8bea1..c3912c26 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -1,16 +1,21 @@ use crate::internal_prelude::*; pub(crate) trait TokenStreamParseExt: Sized { - fn parse_with>( + fn parse_as, K>(self) -> ParseResult; + fn parse_with>( self, - parser: impl FnOnce(ParseStream) -> Result, + parser: impl FnOnce(KindedParseStream) -> Result, ) -> Result; } impl TokenStreamParseExt for TokenStream { - fn parse_with>( + fn parse_as, K>(self) -> ParseResult { + self.parse_with(T::parse) + } + + fn parse_with>( self, - parser: impl FnOnce(ParseStream) -> Result, + parser: impl FnOnce(KindedParseStream) -> Result, ) -> Result { let mut result = None; let parse_result = (|input: SynParseStream| -> SynResult<()> { @@ -91,43 +96,35 @@ impl CursorExt for Cursor<'_> { } } -pub(crate) trait ParserBufferExt { - fn parse_with(&self, context: T::Context) -> ParseResult; - fn parse_all_for_interpretation(&self, span: Span) -> ParseResult; +pub(crate) trait ParserBufferExt { fn try_parse_or_message ParseResult, M: std::fmt::Display>( &self, func: F, message: M, ) -> ParseResult; fn parse_any_ident(&self) -> ParseResult; - fn parse_any_punct(&self) -> ParseResult; fn peek_ident_matching(&self, content: &str) -> bool; fn parse_ident_matching(&self, content: &str) -> ParseResult; fn peek_punct_matching(&self, punct: char) -> bool; fn parse_punct_matching(&self, content: char) -> ParseResult; fn peek_literal_matching(&self, content: &str) -> bool; fn parse_literal_matching(&self, content: &str) -> ParseResult; - fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)>; + fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, KindedParseBuffer)>; fn peek_specific_group(&self, delimiter: Delimiter) -> bool; fn parse_group_matching( &self, matching: impl FnOnce(Delimiter) -> bool, expected_message: impl FnOnce() -> String, - ) -> ParseResult<(DelimSpan, ParseBuffer)>; - fn parse_specific_group(&self, delimiter: Delimiter) -> ParseResult<(DelimSpan, ParseBuffer)>; + ) -> ParseResult<(DelimSpan, KindedParseBuffer)>; + fn parse_specific_group( + &self, + delimiter: Delimiter, + ) -> ParseResult<(DelimSpan, KindedParseBuffer)>; fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult; fn parse_error(&self, message: impl std::fmt::Display) -> ParseError; } -impl ParserBufferExt for ParseBuffer<'_> { - fn parse_with(&self, context: T::Context) -> ParseResult { - T::parse_with_context(self, context) - } - - fn parse_all_for_interpretation(&self, span: Span) -> ParseResult { - self.parse_with(span) - } - +impl ParserBufferExt for KindedParseBuffer<'_, K> { fn try_parse_or_message ParseResult, M: std::fmt::Display>( &self, parse: F, @@ -141,14 +138,6 @@ impl ParserBufferExt for ParseBuffer<'_> { Ok(self.call(Ident::parse_any)?) } - fn parse_any_punct(&self) -> ParseResult { - // Annoyingly, ' behaves weirdly in syn, so we need to handle it - match self.parse::()? { - TokenTree::Punct(punct) => Ok(punct), - _ => self.span().parse_err("expected punctuation"), - } - } - fn peek_ident_matching(&self, content: &str) -> bool { self.cursor().ident_matching(content).is_some() } @@ -185,7 +174,7 @@ impl ParserBufferExt for ParseBuffer<'_> { })?) } - fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)> { + fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, KindedParseBuffer)> { use syn::parse::discouraged::AnyDelimiter; let (delimiter, delim_span, parse_buffer) = self.parse_any_delimiter()?; Ok((delimiter, delim_span, parse_buffer.into())) @@ -199,11 +188,10 @@ impl ParserBufferExt for ParseBuffer<'_> { &self, matching: impl FnOnce(Delimiter) -> bool, expected_message: impl FnOnce() -> String, - ) -> ParseResult<(DelimSpan, ParseBuffer)> { - use syn::parse::discouraged::AnyDelimiter; - let error_span = match self.parse_any_delimiter() { + ) -> ParseResult<(DelimSpan, KindedParseBuffer)> { + let error_span = match self.parse_any_group() { Ok((delimiter, delim_span, inner)) if matching(delimiter) => { - return Ok((delim_span, inner.into())); + return Ok((delim_span, inner)); } Ok((_, delim_span, _)) => delim_span.open(), Err(error) => error.span(), @@ -214,7 +202,7 @@ impl ParserBufferExt for ParseBuffer<'_> { fn parse_specific_group( &self, expected_delimiter: Delimiter, - ) -> ParseResult<(DelimSpan, ParseBuffer)> { + ) -> ParseResult<(DelimSpan, KindedParseBuffer)> { self.parse_group_matching( |delimiter| delimiter == expected_delimiter, || format!("Expected {}", expected_delimiter.description_of_open()), @@ -230,6 +218,27 @@ impl ParserBufferExt for ParseBuffer<'_> { } } +pub(crate) trait SourceParserBufferExt { + fn parse_with_context( + &self, + context: T::Context, + ) -> ParseResult; + fn parse_all_for_interpretation(&self, span: Span) -> ParseResult; +} + +impl SourceParserBufferExt for SourceParseBuffer<'_> { + fn parse_with_context( + &self, + context: T::Context, + ) -> ParseResult { + T::parse_from_source(self, context) + } + + fn parse_all_for_interpretation(&self, span: Span) -> ParseResult { + self.parse_with_context(span) + } +} + pub(crate) trait DelimiterExt { fn description_of_open(&self) -> &'static str; #[allow(unused)] @@ -258,20 +267,20 @@ impl DelimiterExt for Delimiter { /// Allows storing a stack of parse buffers for certain parse strategies which require /// handling multiple groups in parallel. -pub(crate) struct ParseStreamStack<'a> { - base: ParseStream<'a>, - group_stack: Vec>, +pub(crate) struct ParseStreamStack<'a, K> { + base: KindedParseStream<'a, K>, + group_stack: Vec>, } -impl<'a> ParseStreamStack<'a> { - pub(crate) fn new(base: ParseStream<'a>) -> Self { +impl<'a, K> ParseStreamStack<'a, K> { + pub(crate) fn new(base: KindedParseStream<'a, K>) -> Self { Self { base, group_stack: Vec::new(), } } - fn current(&self) -> ParseStream<'_> { + fn current(&self) -> KindedParseStream<'_, K> { self.group_stack.last().unwrap_or(self.base) } @@ -279,22 +288,15 @@ impl<'a> ParseStreamStack<'a> { self.current().parse_err(message) } - pub(crate) fn peek_grammar(&mut self) -> PeekMatch { - detect_preinterpret_grammar(self.current().cursor()) - } - - pub(crate) fn parse(&mut self) -> ParseResult { + pub(crate) fn parse>(&mut self) -> ParseResult { self.current().parse() } - pub(crate) fn parse_with( - &mut self, - parser: impl FnOnce(ParseStream) -> ParseResult, - ) -> ParseResult { - parser(self.current()) + pub(crate) fn parse_any_ident(&mut self) -> ParseResult { + self.current().parse_any_ident() } - pub(crate) fn try_parse_or_revert(&mut self) -> ParseResult { + pub(crate) fn try_parse_or_revert>(&mut self) -> ParseResult { let current = self.current(); let fork = current.fork(); match fork.parse::() { @@ -320,7 +322,7 @@ impl<'a> ParseStreamStack<'a> { // ==> exit_group() ensures the parse buffers are dropped in the correct order // ==> If a user forgets to do it (or e.g. an error path or panic causes exit_group not to be called) // Then the drop glue ensures the groups are dropped in the correct order. - std::mem::transmute::, ParseBuffer<'a>>(inner) + std::mem::transmute::, KindedParseBuffer<'a, K>>(inner) }; self.group_stack.push(inner); Ok((delimiter, delim_span)) @@ -340,7 +342,19 @@ impl<'a> ParseStreamStack<'a> { } } -impl Drop for ParseStreamStack<'_> { +impl ParseStreamStack<'_, Source> { + pub(crate) fn peek_grammar(&mut self) -> GrammarPeekMatch { + self.current().peek_grammar() + } +} + +impl ParseStreamStack<'_, Interpreted> { + pub(crate) fn peek_token(&mut self) -> InterpretedPeekMatch { + self.current().peek_token() + } +} + +impl Drop for ParseStreamStack<'_, K> { fn drop(&mut self) { while !self.group_stack.is_empty() { self.exit_group(); diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 128317dc..0f8c9352 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -408,8 +408,8 @@ pub(crate) struct Command { source_group_span: DelimSpan, } -impl Parse for Command { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for Command { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; content.parse::()?; let flattening = if content.peek(Token![.]) { diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index 2d14b9e6..44095e55 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct CommandArguments<'a> { - parse_stream: ParseStream<'a>, + parse_stream: SourceParseStream<'a>, command_name: Ident, /// The span of the [ ... ] which contained the command command_span: Span, @@ -10,7 +10,7 @@ pub(crate) struct CommandArguments<'a> { impl<'a> CommandArguments<'a> { pub(crate) fn new( - parse_stream: ParseStream<'a>, + parse_stream: SourceParseStream<'a>, command_name: Ident, command_span: Span, ) -> Self { @@ -36,12 +36,12 @@ impl<'a> CommandArguments<'a> { } pub(crate) fn fully_parse_as(&self) -> ParseResult { - self.fully_parse_or_error(T::parse, T::error_message()) + self.fully_parse_or_error(T::parse_from_source, T::error_message()) } pub(crate) fn fully_parse_or_error( &self, - parse_function: impl FnOnce(ParseStream) -> ParseResult, + parse_function: impl FnOnce(SourceParseStream) -> ParseResult, error_message: impl std::fmt::Display, ) -> ParseResult { // In future, when the diagnostic API is stable, @@ -74,6 +74,6 @@ impl<'a> CommandArguments<'a> { } } -pub(crate) trait ArgumentsContent: Parse { +pub(crate) trait ArgumentsContent: ParseFromSource { fn error_message() -> String; } diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index 18da676e..85e4ed23 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -7,10 +7,10 @@ pub(crate) struct CommandCodeInput { inner: InterpretationStream, } -impl Parse for CommandCodeInput { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for CommandCodeInput { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; - let inner = content.parse_with(delim_span.join())?; + let inner = content.parse_with_context(delim_span.join())?; Ok(Self { delim_span, inner }) } } diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index 86f576a2..56b995bd 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -24,8 +24,8 @@ macro_rules! define_field_inputs { )* } - impl Parse for $inputs_type { - fn parse(input: ParseStream) -> ParseResult { + impl ParseFromSource for $inputs_type { + fn parse_from_source(input: SourceParseStream) -> ParseResult { $( let mut $required_field: Option<$required_type> = None; )* diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 50a9048f..12a01037 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -21,14 +21,14 @@ pub(crate) enum CommandStreamInput { ExplicitStream(InterpretationGroup), } -impl Parse for CommandStreamInput { - fn parse(input: ParseStream) -> ParseResult { - Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command(_) => Self::Command(input.parse()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), - PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), +impl ParseFromSource for CommandStreamInput { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Command(_) => Self::Command(input.parse()?), + GrammarPeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + GrammarPeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + GrammarPeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), + GrammarPeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), _ => input.span() .parse_err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) @@ -115,6 +115,9 @@ impl Interpret for CommandStreamInput { } } +/// From a code neatness perspective, this would be better as `InterpretedCommandStream`... +/// However this approach avoids a round-trip to TokenStream, which causes bugs in RustAnalyzer. +/// This can be removed when we fork syn and can parse the InterpretedStream directly. fn parse_as_stream_input( input: impl Interpret + HasSpanRange, interpreter: &mut Interpreter, @@ -149,10 +152,9 @@ pub(crate) struct InterpretedCommandStream { pub(crate) stream: InterpretedStream, } -impl Parse for InterpretedCommandStream { - fn parse(input: ParseStream) -> ParseResult { - // We assume we're parsing an already interpreted raw stream here, so we replicate - // parse_as_stream_input +impl ParseFromInterpreted for InterpretedCommandStream { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { + // We replicate parse_as_stream_input let (_, inner) = input.parse_group_matching( |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None), || "Expected [...] or a transparent group from a #variable or stream-output command such as [!group! ...]".to_string(), diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index b9b87f4a..3375dc05 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -13,27 +13,33 @@ pub(crate) enum CommandValueInput { Value(T), } -impl Parse for CommandValueInput { - fn parse(input: ParseStream) -> ParseResult { - Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command(_) => Self::Command(input.parse()?), - PeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - PeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), - PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { +impl ParseFromSource for CommandValueInput { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Command(_) => Self::Command(input.parse()?), + GrammarPeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + GrammarPeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + GrammarPeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + GrammarPeekMatch::AppendVariableDestructuring | GrammarPeekMatch::Destructurer(_) => { return input .span() .parse_err("Destructurings are not supported here") } - PeekMatch::Group(_) - | PeekMatch::Punct(_) - | PeekMatch::Literal(_) - | PeekMatch::Ident(_) - | PeekMatch::End => Self::Value(input.parse()?), + GrammarPeekMatch::Group(_) + | GrammarPeekMatch::Punct(_) + | GrammarPeekMatch::Literal(_) + | GrammarPeekMatch::Ident(_) + | GrammarPeekMatch::End => Self::Value(input.parse()?), }) } } +impl ParseFromInterpreted for CommandValueInput { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { + Ok(Self::Value(input.parse()?)) + } +} + impl HasSpanRange for CommandValueInput { fn span_range(&self) -> SpanRange { match self { @@ -46,7 +52,9 @@ impl HasSpanRange for CommandValueInput { } } -impl, I: Parse> InterpretValue for CommandValueInput { +impl, I: ParseFromInterpreted> InterpretValue + for CommandValueInput +{ type InterpretedValue = I; fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { @@ -72,7 +80,7 @@ impl, I: Parse> InterpretValue for Comma // RUST-ANALYZER SAFETY: We only use I with simple parse functions so far which don't care about // none-delimited groups interpreted_stream - .syn_parse(I::parse) + .parse_with(I::parse_from_interpreted) .add_context_if_error_and_no_context(|| { format!( "Occurred whilst parsing the {} to a {}.", @@ -92,8 +100,19 @@ pub(crate) struct Grouped { pub(crate) inner: T, } -impl Parse for Grouped { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for Grouped { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + let (delimiter, delim_span, inner) = input.parse_any_group()?; + Ok(Self { + delimiter, + delim_span, + inner: inner.parse()?, + }) + } +} + +impl ParseFromInterpreted for Grouped { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { let (delimiter, delim_span, inner) = input.parse_any_group()?; Ok(Self { delimiter, @@ -126,8 +145,18 @@ pub(crate) struct Repeated { pub(crate) inner: Vec, } -impl Parse for Repeated { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for Repeated { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + let mut inner = vec![]; + while !input.is_empty() { + inner.push(input.parse::()?); + } + Ok(Self { inner }) + } +} + +impl ParseFromInterpreted for Repeated { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { let mut inner = vec![]; while !input.is_empty() { inner.push(input.parse::()?); diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index b07e0077..3af3cbc7 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -2,9 +2,9 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IfCommand { - condition: Expression, + condition: InterpretationExpression, true_code: CommandCodeInput, - else_ifs: Vec<(Expression, CommandCodeInput)>, + else_ifs: Vec<(InterpretationExpression, CommandCodeInput)>, else_code: Option, } @@ -80,7 +80,7 @@ impl ControlFlowCommandDefinition for IfCommand { #[derive(Clone)] pub(crate) struct WhileCommand { - condition: Expression, + condition: InterpretationExpression, loop_code: CommandCodeInput, } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 731f06d4..e34041d8 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -21,7 +21,7 @@ impl NoOutputCommandDefinition for SetCommand { Ok(Self { variable: input.parse()?, equals: input.parse()?, - arguments: input.parse_with(arguments.command_span())?, + arguments: input.parse_with_context(arguments.command_span())?, }) }, "Expected [!set! #variable = ..]", @@ -221,7 +221,7 @@ impl NoOutputCommandDefinition for ErrorCommand { } else { Ok(Self { inputs: EitherErrorInput::JustMessage( - input.parse_with(arguments.command_span())?, + input.parse_with_context(arguments.command_span())?, ), }) } diff --git a/src/interpretation/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs index 0ef941f3..92b534e8 100644 --- a/src/interpretation/commands/destructuring_commands.rs +++ b/src/interpretation/commands/destructuring_commands.rs @@ -21,7 +21,7 @@ impl NoOutputCommandDefinition for LetCommand { Ok(Self { destructuring: input.parse()?, equals: input.parse()?, - arguments: input.parse_with(arguments.command_span())?, + arguments: input.parse_with_context(arguments.command_span())?, }) }, "Expected [!let! = ...]", diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 55651384..23144c48 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct EvaluateCommand { - expression: Expression, + expression: InterpretationExpression, command_span: Span, } @@ -39,7 +39,7 @@ pub(crate) struct AssignCommand { operator: Option, #[allow(unused)] equals: Token![=], - expression: Expression, + expression: InterpretationExpression, command_span: Span, } @@ -91,15 +91,15 @@ impl NoOutputCommandDefinition for AssignCommand { // RUST-ANALYZER SAFETY: Hopefully it won't contain a none-delimited group variable .interpret_to_new_stream(interpreter)? - .syn_parse(Expression::parse)? - .evaluate(interpreter)? + .parse_as::()? + .evaluate()? .to_tokens(&mut calculation); }; operator.to_tokens(&mut calculation); expression .evaluate(interpreter)? .to_tokens(&mut calculation); - calculation.parse_with(Expression::parse)? + calculation.parse_as()? } else { expression }; @@ -115,9 +115,9 @@ impl NoOutputCommandDefinition for AssignCommand { #[derive(Clone)] pub(crate) struct RangeCommand { - left: Expression, + left: InterpretationExpression, range_limits: RangeLimits, - right: Expression, + right: InterpretationExpression, } impl CommandType for RangeCommand { @@ -202,8 +202,8 @@ enum RangeLimits { Closed(Token![..=]), } -impl Parse for RangeLimits { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for RangeLimits { + fn parse_from_source(input: SourceParseStream) -> ParseResult { if input.peek(Token![..=]) { Ok(RangeLimits::Closed(input.parse()?)) } else { diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index f82e954d..f00f7cbf 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -292,7 +292,7 @@ fn handle_split( unsafe { // RUST-ANALYZER SAFETY: This is as safe as we can get. // Typically the separator won't contain none-delimited groups, so we're OK - input.syn_parse(move |input| { + input.parse_with(move |input| { let mut current_item = InterpretedStream::new(); let mut drop_empty_next = drop_empty_start; while !input.is_empty() { diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index c53e3da7..36f0bec8 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -11,117 +11,25 @@ pub(crate) enum InterpretationItem { Literal(Literal), } -impl Parse for InterpretationItem { - fn parse(input: ParseStream) -> ParseResult { - Ok(match detect_preinterpret_grammar(input.cursor()) { - PeekMatch::Command(_) => InterpretationItem::Command(input.parse()?), - PeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), - PeekMatch::GroupedVariable => InterpretationItem::GroupedVariable(input.parse()?), - PeekMatch::FlattenedVariable => InterpretationItem::FlattenedVariable(input.parse()?), - PeekMatch::AppendVariableDestructuring | PeekMatch::Destructurer(_) => { - return input.parse_err("Destructurings are not supported here") - } - PeekMatch::Punct(_) => InterpretationItem::Punct(input.parse_any_punct()?), - PeekMatch::Ident(_) => InterpretationItem::Ident(input.parse_any_ident()?), - PeekMatch::Literal(_) => InterpretationItem::Literal(input.parse()?), - PeekMatch::End => return input.parse_err("Expected some item"), - }) - } -} - -#[allow(unused)] -pub(crate) enum PeekMatch { - Command(Option), - GroupedVariable, - FlattenedVariable, - AppendVariableDestructuring, - Destructurer(Option), - Group(Delimiter), - Ident(Ident), - Punct(Punct), - Literal(Literal), - End, -} - -pub(crate) fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> PeekMatch { - // We have to check groups first, so that we handle transparent groups - // and avoid the self.ignore_none() calls inside cursor - if let Some((next, delimiter, _, _)) = cursor.any_group() { - if delimiter == Delimiter::Bracket { - if let Some((_, next)) = next.punct_matching('!') { - if let Some((ident, next)) = next.ident() { - if next.punct_matching('!').is_some() { - let output_kind = - CommandKind::for_ident(&ident).map(|kind| kind.standard_output_kind()); - return PeekMatch::Command(output_kind); - } - } - if let Some((first, next)) = next.punct_matching('.') { - if let Some((_, next)) = next.punct_matching('.') { - if let Some((ident, next)) = next.ident() { - if next.punct_matching('!').is_some() { - let output_kind = CommandKind::for_ident(&ident).and_then(|kind| { - kind.flattened_output_kind(first.span_range()).ok() - }); - return PeekMatch::Command(output_kind); - } - } - } - } - } - } - if delimiter == Delimiter::Parenthesis { - if let Some((_, next)) = next.punct_matching('!') { - if let Some((ident, next)) = next.ident() { - if next.punct_matching('!').is_some() { - return PeekMatch::Destructurer(DestructurerKind::for_ident(&ident)); - } - } +impl ParseFromSource for InterpretationItem { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + GrammarPeekMatch::Command(_) => InterpretationItem::Command(input.parse()?), + GrammarPeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), + GrammarPeekMatch::GroupedVariable => { + InterpretationItem::GroupedVariable(input.parse()?) } - } - - // Ideally we'd like to detect $($tt)* substitutions from macros and interpret them as - // a Raw (uninterpreted) group, because typically that's what a user would typically intend. - // - // You'd think mapping a Delimiter::None to a PeekMatch::RawGroup would be a good way - // of doing this, but unfortunately this behaviour is very arbitrary and not in a helpful way: - // => A $tt or $($tt)* is not grouped... - // => A $literal or $($literal)* _is_ outputted in a group... - // - // So this isn't possible. It's unlikely to matter much, and a user can always do: - // [!raw! $($tt)*] anyway. - - return PeekMatch::Group(delimiter); - } - if let Some((_, next)) = cursor.punct_matching('#') { - if next.ident().is_some() { - return PeekMatch::GroupedVariable; - } - if let Some((_, next)) = next.punct_matching('.') { - if let Some((_, next)) = next.punct_matching('.') { - if next.ident().is_some() { - return PeekMatch::FlattenedVariable; - } - if let Some((_, next)) = next.punct_matching('>') { - if next.punct_matching('>').is_some() { - return PeekMatch::AppendVariableDestructuring; - } - } + GrammarPeekMatch::FlattenedVariable => { + InterpretationItem::FlattenedVariable(input.parse()?) } - } - if let Some((_, next)) = next.punct_matching('>') { - if next.punct_matching('>').is_some() { - return PeekMatch::AppendVariableDestructuring; + GrammarPeekMatch::AppendVariableDestructuring | GrammarPeekMatch::Destructurer(_) => { + return input.parse_err("Destructurings are not supported here") } - } - } - - match cursor.token_tree() { - Some((TokenTree::Ident(ident), _)) => PeekMatch::Ident(ident), - Some((TokenTree::Punct(punct), _)) => PeekMatch::Punct(punct), - Some((TokenTree::Literal(literal), _)) => PeekMatch::Literal(literal), - Some((TokenTree::Group(_), _)) => unreachable!("Already covered above"), - None => PeekMatch::End, + GrammarPeekMatch::Punct(_) => InterpretationItem::Punct(input.parse_any_punct()?), + GrammarPeekMatch::Ident(_) => InterpretationItem::Ident(input.parse_any_ident()?), + GrammarPeekMatch::Literal(_) => InterpretationItem::Literal(input.parse()?), + GrammarPeekMatch::End => return input.parse_err("Expected some item"), + }) } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index 00864ead..d60236fd 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -7,10 +7,10 @@ pub(crate) struct InterpretationStream { span: Span, } -impl ContextualParse for InterpretationStream { +impl ContextualParseFromSource for InterpretationStream { type Context = Span; - fn parse_with_context(input: ParseStream, span: Self::Context) -> ParseResult { + fn parse_from_source(input: SourceParseStream, span: Self::Context) -> ParseResult { let mut items = Vec::new(); while !input.is_empty() { items.push(input.parse()?); @@ -52,10 +52,10 @@ impl InterpretationGroup { } } -impl Parse for InterpretationGroup { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for InterpretationGroup { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_group()?; - let content = content.parse_with(delim_span.join())?; + let content = content.parse_with_context(delim_span.join())?; Ok(Self { source_delimiter: delimiter, source_delim_span: delim_span, @@ -97,8 +97,8 @@ impl RawGroup { } } -impl Parse for RawGroup { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for RawGroup { + fn parse_from_source(input: SourceParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_group()?; let content = content.parse()?; Ok(Self { diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index a514c0ad..4293cd2f 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -126,20 +126,25 @@ impl InterpretedStream { self.token_length == 0 } - /// For a type `T` which implements `syn::Parse`, you can call this as `syn_parse(self, T::parse)`. - /// For more complicated parsers, just pass the parsing function to this function. - /// /// WARNING: With rust-analyzer, this loses transparent groups which have been inserted. /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 /// /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. - pub(crate) unsafe fn syn_parse>( + pub(crate) unsafe fn parse_with>( self, - parser: impl FnOnce(ParseStream) -> Result, + parser: impl FnOnce(InterpretedParseStream) -> Result, ) -> Result { self.into_token_stream().parse_with(parser) } + /// WARNING: With rust-analyzer, this loses transparent groups which have been inserted. + /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 + /// + /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. + pub(crate) unsafe fn parse_as(self) -> ParseResult { + self.into_token_stream().parse_with(T::parse) + } + pub(crate) fn append_into(self, output: &mut InterpretedStream) { output.segments.extend(self.segments); output.token_length += self.token_length; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 5e2efe3a..baad40ec 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -10,8 +10,8 @@ pub(crate) struct GroupedVariable { variable_name: Ident, } -impl Parse for GroupedVariable { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for GroupedVariable { + fn parse_from_source(input: SourceParseStream) -> ParseResult { input.try_parse_or_message( |input| { Ok(Self { @@ -122,8 +122,8 @@ pub(crate) struct FlattenedVariable { variable_name: Ident, } -impl Parse for FlattenedVariable { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromSource for FlattenedVariable { + fn parse_from_source(input: SourceParseStream) -> ParseResult { input.try_parse_or_message( |input| { Ok(Self { diff --git a/src/lib.rs b/src/lib.rs index 0dc4b6e1..51f0a254 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -547,7 +547,7 @@ fn preinterpret_internal(input: TokenStream) -> SynResult { let mut interpreter = Interpreter::new(); let interpretation_stream = input - .parse_with(|input| InterpretationStream::parse_with_context(input, Span::call_site())) + .parse_with(|input| InterpretationStream::parse_from_source(input, Span::call_site())) .convert_to_final_result()?; let interpreted_stream = interpretation_stream diff --git a/src/misc/errors.rs b/src/misc/errors.rs index a131f681..6572ba7e 100644 --- a/src/misc/errors.rs +++ b/src/misc/errors.rs @@ -35,6 +35,15 @@ impl From for ParseError { } } +impl HasSpan for ParseError { + fn span(&self) -> Span { + match self { + ParseError::Standard(e) => e.span(), + ParseError::Contextual(e, _) => e.span(), + } + } +} + impl ParseError { /// This is not a `From` because it wants to be explicit pub(crate) fn convert_to_final_error(self) -> syn::Error { diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index efd8a271..c5a7bc8f 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -2,65 +2,242 @@ use std::ops::Deref; use crate::internal_prelude::*; -pub(crate) trait ContextualParse: Sized { +// Parsing of source code tokens +// ============================= + +pub(crate) trait ContextualParseFromSource: Sized { type Context; - fn parse_with_context(input: ParseStream, context: Self::Context) -> ParseResult; + fn parse_from_source(input: SourceParseStream, context: Self::Context) -> ParseResult; +} + +pub(crate) trait ParseFromSource: Sized { + fn parse_from_source(input: SourceParseStream) -> ParseResult; +} + +impl ParseFromSource for T { + fn parse_from_source(input: SourceParseStream) -> ParseResult { + Ok(T::parse(&input.inner)?) + } +} + +impl Parse for T { + fn parse(input: SourceParseStream) -> ParseResult { + T::parse_from_source(input) + } +} + +pub(crate) struct Source; +pub(crate) type SourceParseStream<'a> = &'a SourceParseBuffer<'a>; +pub(crate) type SourceParseBuffer<'a> = KindedParseBuffer<'a, Source>; + +impl SourceParseBuffer<'_> { + pub(crate) fn peek_grammar(&self) -> GrammarPeekMatch { + detect_preinterpret_grammar(self.cursor()) + } +} + +#[allow(unused)] +pub(crate) enum GrammarPeekMatch { + Command(Option), + GroupedVariable, + FlattenedVariable, + AppendVariableDestructuring, + Destructurer(Option), + Group(Delimiter), + Ident(Ident), + Punct(Punct), + Literal(Literal), + End, +} + +fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> GrammarPeekMatch { + // We have to check groups first, so that we handle transparent groups + // and avoid the self.ignore_none() calls inside cursor + if let Some((next, delimiter, _, _)) = cursor.any_group() { + if delimiter == Delimiter::Bracket { + if let Some((_, next)) = next.punct_matching('!') { + if let Some((ident, next)) = next.ident() { + if next.punct_matching('!').is_some() { + let output_kind = + CommandKind::for_ident(&ident).map(|kind| kind.standard_output_kind()); + return GrammarPeekMatch::Command(output_kind); + } + } + if let Some((first, next)) = next.punct_matching('.') { + if let Some((_, next)) = next.punct_matching('.') { + if let Some((ident, next)) = next.ident() { + if next.punct_matching('!').is_some() { + let output_kind = CommandKind::for_ident(&ident).and_then(|kind| { + kind.flattened_output_kind(first.span_range()).ok() + }); + return GrammarPeekMatch::Command(output_kind); + } + } + } + } + } + } + if delimiter == Delimiter::Parenthesis { + if let Some((_, next)) = next.punct_matching('!') { + if let Some((ident, next)) = next.ident() { + if next.punct_matching('!').is_some() { + return GrammarPeekMatch::Destructurer(DestructurerKind::for_ident(&ident)); + } + } + } + } + + // Ideally we'd like to detect $($tt)* substitutions from macros and interpret them as + // a Raw (uninterpreted) group, because typically that's what a user would typically intend. + // + // You'd think mapping a Delimiter::None to a GrammarPeekMatch::RawGroup would be a good way + // of doing this, but unfortunately this behaviour is very arbitrary and not in a helpful way: + // => A $tt or $($tt)* is not grouped... + // => A $literal or $($literal)* _is_ outputted in a group... + // + // So this isn't possible. It's unlikely to matter much, and a user can always do: + // [!raw! $($tt)*] anyway. + + return GrammarPeekMatch::Group(delimiter); + } + if let Some((_, next)) = cursor.punct_matching('#') { + if next.ident().is_some() { + return GrammarPeekMatch::GroupedVariable; + } + if let Some((_, next)) = next.punct_matching('.') { + if let Some((_, next)) = next.punct_matching('.') { + if next.ident().is_some() { + return GrammarPeekMatch::FlattenedVariable; + } + if let Some((_, next)) = next.punct_matching('>') { + if next.punct_matching('>').is_some() { + return GrammarPeekMatch::AppendVariableDestructuring; + } + } + } + } + if let Some((_, next)) = next.punct_matching('>') { + if next.punct_matching('>').is_some() { + return GrammarPeekMatch::AppendVariableDestructuring; + } + } + } + + match cursor.token_tree() { + Some((TokenTree::Ident(ident), _)) => GrammarPeekMatch::Ident(ident), + Some((TokenTree::Punct(punct), _)) => GrammarPeekMatch::Punct(punct), + Some((TokenTree::Literal(literal), _)) => GrammarPeekMatch::Literal(literal), + Some((TokenTree::Group(_), _)) => unreachable!("Already covered above"), + None => GrammarPeekMatch::End, + } } -pub(crate) trait Parse: Sized { - fn parse(input: ParseStream) -> ParseResult; +// Parsing of already interpreted tokens +// (e.g. destructuring) +// ===================================== + +pub(crate) trait ParseFromInterpreted: Sized { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult; } -impl Parse for T { - fn parse(input: ParseStream) -> ParseResult { +impl ParseFromInterpreted for T { + fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { Ok(T::parse(&input.inner)?) } } -pub(crate) type ParseStream<'a> = &'a ParseBuffer<'a>; +impl Parse for T { + fn parse(input: InterpretedParseStream) -> ParseResult { + T::parse_from_interpreted(input) + } +} + +pub(crate) struct Interpreted; +pub(crate) type InterpretedParseStream<'a> = &'a InterpretedParseBuffer<'a>; +pub(crate) type InterpretedParseBuffer<'a> = KindedParseBuffer<'a, Interpreted>; + +impl InterpretedParseBuffer<'_> { + pub(crate) fn peek_token(&self) -> InterpretedPeekMatch { + match self.cursor().token_tree() { + Some((TokenTree::Ident(ident), _)) => InterpretedPeekMatch::Ident(ident), + Some((TokenTree::Punct(punct), _)) => InterpretedPeekMatch::Punct(punct), + Some((TokenTree::Literal(literal), _)) => InterpretedPeekMatch::Literal(literal), + Some((TokenTree::Group(group), _)) => InterpretedPeekMatch::Group(group.delimiter()), + None => InterpretedPeekMatch::End, + } + } +} + +#[allow(unused)] +pub(crate) enum InterpretedPeekMatch { + Group(Delimiter), + Ident(Ident), + Punct(Punct), + Literal(Literal), + End, +} + +// Generic parsing +// =============== + +pub(crate) trait Parse: Sized { + fn parse(input: KindedParseStream) -> ParseResult; +} + +pub(crate) type KindedParseStream<'a, K> = &'a KindedParseBuffer<'a, K>; // We create our own ParseBuffer mostly so we can overwrite // parse to return ParseResult instead of syn::Result #[repr(transparent)] -pub(crate) struct ParseBuffer<'a> { +pub(crate) struct KindedParseBuffer<'a, K> { inner: SynParseBuffer<'a>, + _kind: PhantomData, } -impl<'a> From> for ParseBuffer<'a> { +impl<'a, K> From> for KindedParseBuffer<'a, K> { fn from(inner: syn::parse::ParseBuffer<'a>) -> Self { - Self { inner } + Self { + inner, + _kind: PhantomData, + } } } // This is From<&'a SynParseBuffer<'a>> for &'a ParseBuffer<'a> -impl<'a> From> for ParseStream<'a> { +impl<'a, K> From> for KindedParseStream<'a, K> { fn from(syn_parse_stream: SynParseStream<'a>) -> Self { unsafe { // SAFETY: This is safe because [Syn]ParseStream<'a> = &'a [Syn]ParseBuffer<'a> // And ParseBuffer<'a> is marked as #[repr(transparent)] so has identical layout to SynParseBuffer<'a> // So this is a transmute between compound types with identical layouts which is safe. - core::mem::transmute::, ParseStream<'a>>(syn_parse_stream) + core::mem::transmute::, KindedParseStream<'a, K>>(syn_parse_stream) } } } -impl<'a> ParseBuffer<'a> { - // Methods on SynParseBuffer are available courtesy of Deref below - // But the following methods are replaced, for ease of use: +impl<'a, K> KindedParseBuffer<'a, K> { + pub(crate) fn fork(&self) -> KindedParseBuffer<'a, K> { + KindedParseBuffer { + inner: self.inner.fork(), + _kind: PhantomData, + } + } - pub(crate) fn parse(&self) -> ParseResult { + pub(crate) fn parse>(&self) -> ParseResult { T::parse(self) } - pub(crate) fn fork(&self) -> ParseBuffer<'a> { - ParseBuffer { - inner: self.inner.fork(), + pub(crate) fn parse_any_punct(&self) -> ParseResult { + // Annoyingly, ' behaves weirdly in syn, so we need to handle it + match self.inner.parse::()? { + TokenTree::Punct(punct) => Ok(punct), + _ => self.span().parse_err("expected punctuation"), } } } -impl<'a> Deref for ParseBuffer<'a> { +impl<'a, K> Deref for KindedParseBuffer<'a, K> { type Target = SynParseBuffer<'a>; fn deref(&self) -> &Self::Target { From 971d529df76969c2759a8f7ad78013ea43285d67 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 4 Feb 2025 00:51:51 +0000 Subject: [PATCH 073/126] refactor: Various renames --- CHANGELOG.md | 170 +++++++++++-- README.md | 41 +--- src/destructuring/destructure_group.rs | 6 +- src/destructuring/destructure_item.rs | 4 +- src/destructuring/destructure_raw.rs | 6 +- src/destructuring/destructure_segment.rs | 16 +- src/destructuring/destructure_traits.rs | 4 +- src/destructuring/destructure_variable.rs | 32 +-- src/destructuring/destructurer.rs | 20 +- src/destructuring/destructurers.rs | 18 +- src/destructuring/fields.rs | 28 +-- src/expressions/boolean.rs | 12 +- src/expressions/character.rs | 12 +- src/expressions/evaluation.rs | 12 +- src/expressions/expression.rs | 108 ++++----- src/expressions/expression_parsing.rs | 2 +- src/expressions/float.rs | 30 +-- src/expressions/integer.rs | 34 +-- src/expressions/operations.rs | 30 +-- src/expressions/string.rs | 18 +- src/expressions/value.rs | 48 ++-- src/extensions/parsing.rs | 228 ++++-------------- src/interpretation/command.rs | 28 +-- src/interpretation/command_arguments.rs | 15 +- src/interpretation/command_code_input.rs | 20 +- src/interpretation/command_field_inputs.rs | 4 +- src/interpretation/command_stream_input.rs | 32 +-- src/interpretation/command_value_input.rs | 50 ++-- .../commands/concat_commands.rs | 14 +- .../commands/control_flow_commands.rs | 24 +- src/interpretation/commands/core_commands.rs | 18 +- .../commands/destructuring_commands.rs | 2 +- .../commands/expression_commands.rs | 20 +- src/interpretation/commands/token_commands.rs | 39 ++- src/interpretation/interpret_traits.rs | 14 +- src/interpretation/interpretation_item.rs | 60 +++-- src/interpretation/interpretation_stream.rs | 44 ++-- src/interpretation/interpreted_stream.rs | 99 ++++---- src/interpretation/interpreter.rs | 12 +- src/interpretation/variable.rs | 24 +- src/lib.rs | 2 +- src/misc/parse_traits.rs | 197 ++++++++++----- 42 files changed, 807 insertions(+), 790 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e97ad2c0..da3c81ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,34 +80,52 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* Consider renames: - * KindedParseStream => ParseStream - * KindedParseBuffer => ParseBuffer - * Source => Grammar - * ParseFromSource => Parse - * InterpretationX => GrammarX - * ParseFromInterpreted => Parse - * Interpreted => Output - * InterpretedX => OutputX -* Complete expression rework: - * Disallow expressions/commands in re-evaluated `{ .. }` blocks and command outputs - * Create `ParseInterpreted` and `ParseSource` via `ParseStream`, `SourceParseStream` and `InterpretedParseStream`, and add grammar peaking to `SourceParseStream` only. Add `[!reinterpret! ...]` command for an `eval` style command. +* Add test to disallow source stuff in re-evaluated `{ .. }` blocks and command outputs +* Add `[!reinterpret! ...]` command for an `eval` style command. * Support `[!set! #x]` to be `[!set! #x =]`. * `[!is_set! #x]` -* `[!str_split! { input: Value, separator: Value, }]` -* Change `(!content! ...)` to unwrap none groups, to be more permissive * Support `'a'..'z'` in `[!range! 'a'..'z']` -* Support `#x[0]` and `#x[0..3]` and `#..x[0..3]` and other things like `[ ..=3]` -* Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` -* Destructurers - * `(!fields! ...)` and `(!subfields! ...)` - * Add ability to fork (copy on write?) / revert the interpreter state and can then add: - * `(!optional! ...)` - * `(!repeated! ...)` also forbid `#x` bindings inside of them unless a `[!settings! { ... }]` has been overriden - * `{ item: (!stream! ...), minimum?: syn::int, maximum?: syn::int, separator?: (!stream! ...), after_each?: { ... }, before_all?: {}, after_all?: {}, }` +* Destructurers => Transformers + * Implement pivot to transformers outputting things ... `@[#x = @IDENT]`... + * `@TOKEN_TREE` + * `@REST` + * `@[UNTIL xxxx]` + * `@[EXPECT xxxx]` expects the tokens (or source grammar inc variables), and outputs the matched tokens (instead of dropping them as is the default). Replaces `(!content!)`. We should also consider ignoring/unwrapping none-groups to be more permissive? (assuming they're also unwrapped during parsing). + * `@[FIELDS { ... }]` and `@[SUBFIELDS { ... }]` + * Add ability to add scope to interpreter state (copy on write?) (and commit/revert) and can then add: + * `@[OPTIONAL ...]` and `@(...)?` + * `@(REPEATED { ... })` (see below) * `[!match! ...]` command - * `(!any! ...)` (with `#..x` as a catch-all) like the [!match!] command but without arms... - * (MAYBE) `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` but I don't like them much + * `@[ANY { ... }]` (with `#..x` as a catch-all) like the [!match!] command but without arms... + * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` +* Consider: + * Scrap `[!let!]` in favour of `[!parse! #x as #(...)]` + * Scrap `[!void! ...]` in favour of `[!set! _ = ...]` + * Destructurer needs to have different syntax. It's too confusingly similar! + * Final decision: `@[#x = @IDENT]` because destructurers output (SEE BELOW FOR MOST OF THE WORKING) + * Some other ideas considered: + * `(>ident> #x)`? `(>fields> {})`? `(>comma_repeated> Hello)` + * We can't use `<` otherwise it tries to open brackets and could be confused for rust syntax like: `()` + * We need to test it with the auto-formatter in case it really messes it up + * `#(>ident #x)` - not bad... + * Then we can drop `(!stream!)` as it's just `#( ... )` + * We need to test it with the auto-formatter in case it really messes it up + * `[>ident (#x)]` + * `` + * `{[ident] #x}` + * `(>ident> #x)` + * `#IDENT { #x }` + * `#IDENT { capture: #x }` + * `#(IDENT #x)` + * `@[#x = @IDENT]` + * Scrap `#>>x` etc in favour of `@[#x += ...]` +* Support `[!index! ..]`: + * `[!index! #x[0]]` + * `[!index! #x[0..3]]` + * `[!..index! #x[0..3]]` and other things like `[ ..=3]` + * `[!index! [Hello World][...]]` + => NB: This isn't in the grammar because of the risk of `#x[0]` wanting to mean `my_arr[0]` rather than "index my variable". +* Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed * Work on book @@ -119,15 +137,117 @@ Destructuring performs parsing of a token stream. It supports: * Those taking a stream as-is * Those taking some { fields } * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` etc + +### Destructuring Notes WIP +```rust +// FINAL DESIGN --- +// ALL THESE CAN BE SUPPORTED: +[!for! (#x #y) in [!parse! #input as @(impl @IDENT for @IDENT),*] { + +}] +// NICE +[!parse! #input as @[REPEATED { + item: @(impl @[#trait = @IDENT] for @[#type = @TYPE]), + item_output: { + impl BLAH BLAH { + ... + } + } +}]] +// ALSO NICE - although I don't know how we'd do custom inputs +[!define_parser! @INPUT = @(impl @IDENT for @IDENT),*] +[!for! (#x #y) in [!parse! #input as @INPUT] { + +}] +// MAYBE - probably not though... +[!parse_for! #input as @(impl @[#x = @IDENT] for @[#y = @IDENT]),* { + +}] + +// What if destructuring could also output something? (nb - OPTION 3 is preferred) + +// OPTION 1 +// * #(X ...) would append its output to the output stream, i.e. #(#output += X ...) +// * #(field: X ...) would append `field: output,` to the output stream. +// * #(#x = X) +// * #(void X) outputs nothing +// * #x and #..x still work in the same way (binding to variables?) + +// OPTION 2 - everything explicit +// * #(void IDENT) outputs nothing +// * #(+IDENT) appends to #output (effectively it's an opt-in) +// * #(.field = IDENT) appends `field: ___,` to #output +// * #(#x = IDENT) sets #x +// * #(#x += IDENT) extends #x +// * #(#x.field = IDENT) appends `field: ___,` to #x + +// OPTION 3 (preferred) - output by default; use @ for destructurers +// * @( ... ) destructure stream, has an output +// * @( ... )? optional destructure stream, has an output +// * Can similarly have @(...),+ which handles a trailing , +// * @X shorthand for @[X] for destructurers which can take no input, e.g. IDENT, TOKEN_TREE, TYPE etc +// => NOTE: Each destructurer should return just its tokens by default if it has no arguments. +// => It can also have its output over-written or other things outputted using e.g. @[TYPE { is_prefixed: X, parts: #(...), output: { #output } }] +// * #x is shorthand for @[#x = @TOKEN_TREE] +// * #..x) is shorthand for @[#x = @UNTIL_END] and #..x, is shorthand for @[CAPTURE #x = @[UNTIL_TOKEN ,]] +// * @[_ = ...] +// * @[#x = @IDENT for @IDENT] +// * @[#x += @IDENT for @IDENT] +// * @[REPEATED { ... }] +// * Can embed commands to output stuff too +// * Can output a group with: @[#x = @IDENT for @IDENT] [!group! #..x] + +// In this model, REPEATED is really clean and looks like this: +@[REPEATED { + item: #(...), // Captured into #item variable + separator?: #(), // Captured into #separator variable + min?: 0, + max?: 1000000, + handle_item?: { #item }, // Default is to output the grouped item. #()+ instead uses `{ (#..item) }` + handle_separator?: { }, // Default is to not output the separator +}] + +// How does optional work? +// @[#x = @(@IDENT)?] +// Along with: +// [!fields! { #x, my_var: #y, #z }] +// And if some field #z isn't set, it's outputted as null. + +// Do we want something like !parse_for!? It needs to execute lazily - how? +// > Probably by passing some `OnOutput` hook to an output stream method +[!parse_for! #input as @(impl @[#x = @IDENT] for @[#y = @IDENT]),+ { + +}] + +// OUTSTANDING QUESTION: +// => Token streams are a good stand-in for arrays +// => Do we need a good stand-in for fields / key-value maps and other structured data? +// => e.g. #x.hello = BLAH +// => Has a stream-representation as { hello: [!group! BLAH], } but is more performant, +// and can be destructured with `[@FIELDS { #hello }]` or read with `[!read! #x.hello] +// => And can transform output as #(.hello = @X) - Mixing/matching append output and field output will error +// => Debug impl is `[!fields! hello: [!group! BLAH],]` +// => Can be embedded into an output stream as a fields group +// => If necessary, can be converted to a stream and parsed back as `{ hello: [!group! BLAH], }` +``` + * Pushed to 0.4: * Fork of syn to: * Fix issues in Rust Analyzer - * Improve performance (?) + * Add support for a more general `TokenBuffer`, and ensure that Cursor can work in a backwards-compatible way with that buffer. Support: + * Storing a length + * Embedding tokens directly without putting them into a `Group` + * Possibling embedding a reference to a slice buffer inside a group + * Ability to parse to a TokenBuffer or TokenBufferSlice + * Possibly allowing some kind of embedding of Tokens whichcan be converted into a TokenStream. + * Currently, `ParseBuffer` stores `unexpected` and has drop glue which is a hacky abstraction. We'll need to think of an alternative. Perhaps we change `ParseBuffer` to operate on top of a `TokenBuffer` ?? + * Allow variables to use CoW semantics. Variables can be Owned(ParseBuffer) or `Slice(ParseBufferSlice), where a ParseBufferSlice is some form of reference counting to a ParseBufferCore, and a FromLocation and ToLocation which are assumed to be at the same level. * Permit `[!parse_while! (!stream! ...) from #x { ... }]` * Fix `any_punct()` to ignore none groups * Groups can either be: * Raw Groups * Or created groups, where we store `DelimSpan` for re-parsing and accessing the open/close delimiters (this will let us improve `invalid_content_wrong_group`) + * In future - improve performance of some other parts of syn * Better error messages * See e.g. invalid_content_too_short where ideally the error message would be on the last token in the stream. Perhaps End gets a span from the previous error? * See e.g. invalid_content_too_long where `unexpected token` is quite vague. diff --git a/README.md b/README.md index 99cd09c0..304d087f 100644 --- a/README.md +++ b/README.md @@ -414,20 +414,12 @@ preinterpret::preinterpret! { ### Possible extension: Token stream commands -* `[!split! #stream ,]` expects a token tree, and then some punct. Returns a stream split into transparent groups by the given token. Ignores a trailing empty stream. -* `[!skip! #stream 4]` expects to receive a (possibly transparent) group, and reads and drops the first 4 token trees from the group's stream, and outputs the rest * `[!ungroup! #stream]` expects `#stream` to be a single group and unwraps it once * `[!flatten! #stream]` removes all singleton groups from `#stream`, leaving a token stream of idents, literals and punctuation -* `[!index! #stream 0]` takes the 0th token tree from the stream -* `[!zip! #a #b #c]` returns a transparent group tuple of the nth token in each of `#a`, `#b` and `#c`. Errors if they are of different lengths -We could support a piped calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream as #x |> [!skip! #x 4] |> [!ungroup! #x]]` +We could support a postfix calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream > #x [!index! #x[4]] > #x [!ungroup! #x]]` or something like a scala for comprehension. But honestly, just saving things to variables might be cleaner. -#### Possible extension: Better performance - -We could tweak some commands to execute lazily: -* Replace `output: &mut InterpretedStream` with `output: &mut impl OutputSource` which could either write to the output or be buffered/streamed into other commands. -This would allow things like `[!zip! ...]` or `[!range! ...]` to execute lazily, assuming the consuming command such as `for` read lazily. +### Possible extension: Better performance via incremental parsing Incremental parsing using a fork of syn ([see issue](https://github.com/dtolnay/syn/issues/1842)) would allow: @@ -437,9 +429,15 @@ Incremental parsing using a fork of syn ([see issue](https://github.com/dtolnay/ Forking syn may also allow some parts to be made more performant. +### Possible extension: Better performance via lazy execution + +We could tweak some commands to execute lazily. We replace `output: &mut OutputStream` with `output: &mut impl OutputSource` which could either write to the output or be buffered/streamed into other commands. + +This could allow things like `[!zip! ...]` or `[!range! ...]` to execute lazily, assuming the consuming command such as `for` read lazily. + ### Possible extension: User-defined commands -* `[!define! [!my_command! ] { }]` +* `[!define_command! [!my_command! ] { }]` * Some ability to define and re-use commands across multiple invocations without being too expensive. Still unsure how to make this work. @@ -449,27 +447,10 @@ Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example: * `[!tokens_eq! (3 4) (3 4)]` outputs `true` because the token stream ignores spacing. * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. + * This can be effectively done already with `[!evaluate! [!debug! #x] == [!debug! #y]]` +* `[!str_split! { input: Value, separator: Value, }]` * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. -### Possible extension: Goto - -_This probably isn't needed, if we have `while` and `for`_. - -* `[!label! loop_start]` - defines a label which can be returned to. Effectively, it takes a clones of the remaining token stream after the label in the interpreter. -* `[!goto! loop_start]` - jumps to the last execution of `[!label! loop_start]`. It unrolls the preinterpret stack (dropping all unwritten token streams) until it finds a stackframe in which the interpreter has the defined label, and continues the token stream from there. - -```rust,ignore -// Hypothetical future syntax - not yet implemented! -// For now you can use a `[!while! #i <= 100 { ... }]` instead -preinterpret::preinterpret!{ - [!set! #i = 0] - [!label! loop] - const [!ident! AB #i]: u8 = 0; - [!assign! #i += 1] - [!if! (#i <= 100) { [!goto! loop] }] -} -``` - ### Possible extension: Eager expansion of macros When [eager expansion of macros returning literals](https://github.com/rust-lang/rust/issues/90765) is stabilized, it would be nice to include a command to do that, which could be used to include code, for example: `[!expand_literal_macros! include!("my-poem.txt")]`. diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs index 842e834c..3af3452c 100644 --- a/src/destructuring/destructure_group.rs +++ b/src/destructuring/destructure_group.rs @@ -6,8 +6,8 @@ pub(crate) struct DestructureGroup { inner: DestructureRemaining, } -impl ParseFromSource for DestructureGroup { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for DestructureGroup { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, _, content) = input.parse_any_group()?; Ok(Self { delimiter, @@ -19,7 +19,7 @@ impl ParseFromSource for DestructureGroup { impl HandleDestructure for DestructureGroup { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { let (_, inner) = input.parse_specific_group(self.delimiter)?; diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs index 541285ad..18007a17 100644 --- a/src/destructuring/destructure_item.rs +++ b/src/destructuring/destructure_item.rs @@ -16,7 +16,7 @@ impl DestructureItem { /// notably the flattened command. This allows [!let! #..x = Hello => World] to parse as setting /// `x` to `Hello => World` rather than having `#..x` peeking to see it is "up to =" and then only /// parsing `Hello` into `x`. - pub(crate) fn parse_until(input: SourceParseStream) -> ParseResult { + pub(crate) fn parse_until(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { GrammarPeekMatch::Command(Some(CommandOutputKind::None)) => { Self::NoneOutputCommand(input.parse()?) @@ -44,7 +44,7 @@ impl DestructureItem { impl HandleDestructure for DestructureItem { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { match self { diff --git a/src/destructuring/destructure_raw.rs b/src/destructuring/destructure_raw.rs index 1f4e456c..e55fb882 100644 --- a/src/destructuring/destructure_raw.rs +++ b/src/destructuring/destructure_raw.rs @@ -10,7 +10,7 @@ pub(crate) enum RawDestructureItem { } impl RawDestructureItem { - pub(crate) fn handle_destructure(&self, input: InterpretedParseStream) -> ExecutionResult<()> { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { match self { RawDestructureItem::Punct(punct) => { input.parse_punct_matching(punct.as_char())?; @@ -65,7 +65,7 @@ impl RawDestructureStream { self.inner.push(item); } - pub(crate) fn handle_destructure(&self, input: InterpretedParseStream) -> ExecutionResult<()> { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { for item in self.inner.iter() { item.handle_destructure(input)?; } @@ -91,7 +91,7 @@ impl RawDestructureGroup { } } - pub(crate) fn handle_destructure(&self, input: InterpretedParseStream) -> ExecutionResult<()> { + pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { let (_, inner) = input.parse_specific_group(self.delimiter)?; self.inner.handle_destructure(&inner) } diff --git a/src/destructuring/destructure_segment.rs b/src/destructuring/destructure_segment.rs index 2c9fb714..a9223aca 100644 --- a/src/destructuring/destructure_segment.rs +++ b/src/destructuring/destructure_segment.rs @@ -1,19 +1,19 @@ use crate::internal_prelude::*; pub(crate) trait StopCondition: Clone { - fn should_stop(input: SourceParseStream) -> bool; + fn should_stop(input: ParseStream) -> bool; } #[derive(Clone)] pub(crate) struct UntilEnd; impl StopCondition for UntilEnd { - fn should_stop(input: SourceParseStream) -> bool { + fn should_stop(input: ParseStream) -> bool { input.is_empty() } } pub(crate) trait PeekableToken: Clone { - fn peek(input: SourceParseStream) -> bool; + fn peek(input: ParseStream) -> bool; } // This is going through such pain to ensure we stay in the public API of syn @@ -23,7 +23,7 @@ macro_rules! impl_peekable_token { ($(Token![$token:tt]),* $(,)?) => { $( impl PeekableToken for Token![$token] { - fn peek(input: SourceParseStream) -> bool { + fn peek(input: ParseStream) -> bool { input.peek(Token![$token]) } } @@ -41,7 +41,7 @@ pub(crate) struct UntilToken { token: PhantomData, } impl StopCondition for UntilToken { - fn should_stop(input: SourceParseStream) -> bool { + fn should_stop(input: ParseStream) -> bool { input.is_empty() || T::peek(input) } } @@ -55,8 +55,8 @@ pub(crate) struct DestructureSegment { inner: Vec, } -impl ParseFromSource for DestructureSegment { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for DestructureSegment { + fn parse(input: ParseStream) -> ParseResult { let mut inner = vec![]; while !C::should_stop(input) { inner.push(DestructureItem::parse_until::(input)?); @@ -71,7 +71,7 @@ impl ParseFromSource for DestructureSegment { impl HandleDestructure for DestructureSegment { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { for item in self.inner.iter() { diff --git a/src/destructuring/destructure_traits.rs b/src/destructuring/destructure_traits.rs index 45a141e8..9f412a45 100644 --- a/src/destructuring/destructure_traits.rs +++ b/src/destructuring/destructure_traits.rs @@ -3,7 +3,7 @@ use crate::internal_prelude::*; pub(crate) trait HandleDestructure { fn handle_destructure_from_stream( &self, - input: InterpretedStream, + input: OutputStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { unsafe { @@ -16,7 +16,7 @@ pub(crate) trait HandleDestructure { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()>; } diff --git a/src/destructuring/destructure_variable.rs b/src/destructuring/destructure_variable.rs index 3d2791e0..c3ad76f1 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/destructuring/destructure_variable.rs @@ -54,7 +54,7 @@ pub(crate) enum DestructureVariable { } impl DestructureVariable { - pub(crate) fn parse_only_unflattened_input(input: SourceParseStream) -> ParseResult { + pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> ParseResult { let variable: DestructureVariable = Self::parse_until::(input)?; if variable.is_flattened_input() { return variable @@ -64,7 +64,7 @@ impl DestructureVariable { Ok(variable) } - pub(crate) fn parse_until(input: SourceParseStream) -> ParseResult { + pub(crate) fn parse_until(input: ParseStream) -> ParseResult { let marker = input.parse()?; if input.peek(Token![..]) { let flatten = input.parse()?; @@ -180,7 +180,7 @@ impl DestructureVariable { impl HandleDestructure for DestructureVariable { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { match self { @@ -189,7 +189,7 @@ impl HandleDestructure for DestructureVariable { interpreter.set_variable(self, content)?; } DestructureVariable::Flattened { until, .. } => { - let mut content = InterpretedStream::new(); + let mut content = OutputStream::new(); until.handle_parse_into(input, &mut content)?; interpreter.set_variable(self, content)?; } @@ -230,16 +230,16 @@ enum ParsedTokenTree { } impl ParsedTokenTree { - fn into_interpreted(self) -> InterpretedStream { + fn into_interpreted(self) -> OutputStream { match self { - ParsedTokenTree::NoneGroup(group) => InterpretedStream::raw(group.stream()), - ParsedTokenTree::Ident(ident) => InterpretedStream::raw(ident.to_token_stream()), - ParsedTokenTree::Punct(punct) => InterpretedStream::raw(punct.to_token_stream()), - ParsedTokenTree::Literal(literal) => InterpretedStream::raw(literal.to_token_stream()), + ParsedTokenTree::NoneGroup(group) => OutputStream::raw(group.stream()), + ParsedTokenTree::Ident(ident) => OutputStream::raw(ident.to_token_stream()), + ParsedTokenTree::Punct(punct) => OutputStream::raw(punct.to_token_stream()), + ParsedTokenTree::Literal(literal) => OutputStream::raw(literal.to_token_stream()), } } - fn push_as_token_tree(self, output: &mut InterpretedStream) { + fn push_as_token_tree(self, output: &mut OutputStream) { match self { ParsedTokenTree::NoneGroup(group) => { output.push_raw_token_tree(TokenTree::Group(group)) @@ -250,7 +250,7 @@ impl ParsedTokenTree { } } - fn flatten_into(self, output: &mut InterpretedStream) { + fn flatten_into(self, output: &mut OutputStream) { match self { ParsedTokenTree::NoneGroup(group) => output.extend_raw_tokens(group.stream()), ParsedTokenTree::Ident(ident) => output.push_ident(ident), @@ -260,8 +260,8 @@ impl ParsedTokenTree { } } -impl ParseFromInterpreted for ParsedTokenTree { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { +impl Parse for ParsedTokenTree { + fn parse(input: ParseStream) -> ParseResult { Ok(match input.parse::()? { TokenTree::Group(group) if group.delimiter() == Delimiter::None => { ParsedTokenTree::NoneGroup(group) @@ -290,7 +290,7 @@ pub(crate) enum ParseUntil { impl ParseUntil { /// Peeks the next token, to discover what we should parse next - fn peek_flatten_limit(input: SourceParseStream) -> ParseResult { + fn peek_flatten_limit(input: ParseStream) -> ParseResult { if C::should_stop(input) { return Ok(ParseUntil::End); } @@ -314,8 +314,8 @@ impl ParseUntil { fn handle_parse_into( &self, - input: InterpretedParseStream, - output: &mut InterpretedStream, + input: ParseStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { match self { ParseUntil::End => output.extend_raw_tokens(input.parse::()?), diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs index 5fd9be34..2c682c42 100644 --- a/src/destructuring/destructurer.rs +++ b/src/destructuring/destructurer.rs @@ -5,14 +5,14 @@ pub(crate) trait DestructurerDefinition: Clone { fn parse(arguments: DestructurerArguments) -> ParseResult; fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()>; } #[derive(Clone)] pub(crate) struct DestructurerArguments<'a> { - parse_stream: SourceParseStream<'a>, + parse_stream: ParseStream<'a, Source>, destructurer_name: Ident, full_span: Span, } @@ -20,7 +20,7 @@ pub(crate) struct DestructurerArguments<'a> { #[allow(unused)] impl<'a> DestructurerArguments<'a> { pub(crate) fn new( - parse_stream: SourceParseStream<'a>, + parse_stream: ParseStream<'a, Source>, destructurer_name: Ident, full_span: Span, ) -> Self { @@ -44,17 +44,17 @@ impl<'a> DestructurerArguments<'a> { } } - pub(crate) fn fully_parse_no_error_override(&self) -> ParseResult { + pub(crate) fn fully_parse_no_error_override>(&self) -> ParseResult { self.parse_stream.parse() } pub(crate) fn fully_parse_as(&self) -> ParseResult { - self.fully_parse_or_error(T::parse_from_source, T::error_message()) + self.fully_parse_or_error(T::parse, T::error_message()) } pub(crate) fn fully_parse_or_error( &self, - parse_function: impl FnOnce(SourceParseStream) -> ParseResult, + parse_function: impl FnOnce(ParseStream) -> ParseResult, error_message: impl std::fmt::Display, ) -> ParseResult { // In future, when the diagnostic API is stable, @@ -85,8 +85,8 @@ pub(crate) struct Destructurer { source_group_span: DelimSpan, } -impl ParseFromSource for Destructurer { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for Destructurer { + fn parse(input: ParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Parenthesis)?; content.parse::()?; let destructurer_name = content.parse_any_ident()?; @@ -116,7 +116,7 @@ impl ParseFromSource for Destructurer { impl HandleDestructure for Destructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { self.instance.handle_destructure(input, interpreter) @@ -174,7 +174,7 @@ macro_rules! define_destructurers { } impl NamedDestructurer { - fn handle_destructure(&self, input: InterpretedParseStream, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> ExecutionResult<()> { match self { $( Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs index b66a0f78..93b7df89 100644 --- a/src/destructuring/destructurers.rs +++ b/src/destructuring/destructurers.rs @@ -16,7 +16,7 @@ impl DestructurerDefinition for StreamDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { self.inner.handle_destructure(input, interpreter) @@ -48,7 +48,7 @@ impl DestructurerDefinition for IdentDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { if input.cursor().ident().is_some() { @@ -90,7 +90,7 @@ impl DestructurerDefinition for LiteralDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { if input.cursor().literal().is_some() { @@ -132,7 +132,7 @@ impl DestructurerDefinition for PunctDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { if input.cursor().any_punct().is_some() { @@ -165,7 +165,7 @@ impl DestructurerDefinition for GroupDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { let (_, inner) = input.parse_specific_group(Delimiter::None)?; @@ -190,7 +190,7 @@ impl DestructurerDefinition for RawDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, _: &mut Interpreter, ) -> ExecutionResult<()> { self.stream.handle_destructure(input) @@ -199,7 +199,7 @@ impl DestructurerDefinition for RawDestructurer { #[derive(Clone)] pub(crate) struct ContentDestructurer { - stream: InterpretationStream, + stream: SourceStream, } impl DestructurerDefinition for ContentDestructurer { @@ -209,7 +209,7 @@ impl DestructurerDefinition for ContentDestructurer { arguments.fully_parse_or_error( |input| { Ok(Self { - stream: InterpretationStream::parse_from_source(input, arguments.full_span())?, + stream: SourceStream::parse(input, arguments.full_span())?, }) }, "Expected (!content! ... interpretable input ...)", @@ -218,7 +218,7 @@ impl DestructurerDefinition for ContentDestructurer { fn handle_destructure( &self, - input: InterpretedParseStream, + input: ParseStream, interpreter: &mut Interpreter, ) -> ExecutionResult<()> { self.stream diff --git a/src/destructuring/fields.rs b/src/destructuring/fields.rs index cad0f1d4..bc05e28f 100644 --- a/src/destructuring/fields.rs +++ b/src/destructuring/fields.rs @@ -16,38 +16,24 @@ impl FieldsParseDefinition { } } - pub(crate) fn add_required_field( + pub(crate) fn add_required_field + 'static>( self, field_name: &str, example: &str, explanation: Option<&str>, set: impl Fn(&mut T, F) + 'static, ) -> Self { - self.add_field( - field_name, - example, - explanation, - true, - F::parse_from_source, - set, - ) + self.add_field(field_name, example, explanation, true, F::parse, set) } - pub(crate) fn add_optional_field( + pub(crate) fn add_optional_field + 'static>( self, field_name: &str, example: &str, explanation: Option<&str>, set: impl Fn(&mut T, F) + 'static, ) -> Self { - self.add_field( - field_name, - example, - explanation, - false, - F::parse_from_source, - set, - ) + self.add_field(field_name, example, explanation, false, F::parse, set) } pub(crate) fn add_field( @@ -56,7 +42,7 @@ impl FieldsParseDefinition { example: &str, explanation: Option<&str>, is_required: bool, - parse: impl Fn(SourceParseStream) -> ParseResult + 'static, + parse: impl Fn(ParseStream) -> ParseResult + 'static, set: impl Fn(&mut T, F) + 'static, ) -> Self { if self @@ -87,7 +73,7 @@ impl FieldsParseDefinition { error_span_range: SpanRange, ) -> impl FnOnce(SynParseStream) -> ParseResult { fn inner( - input: SourceParseStream, + input: ParseStream, new_builder: T, field_definitions: &FieldDefinitions, error_span_range: SpanRange, @@ -186,5 +172,5 @@ struct FieldParseDefinition { example: String, explanation: Option, #[allow(clippy::type_complexity)] - parse_and_set: Box ParseResult<()>>, + parse_and_set: Box) -> ParseResult<()>>, } diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 46823d98..04306d2f 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -19,7 +19,7 @@ impl EvaluationBoolean { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let input = self.value; match operation { UnaryOperation::Neg { .. } => operation.unsupported_for_value_type_err("boolean"), @@ -53,7 +53,7 @@ impl EvaluationBoolean { self, _right: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { @@ -66,7 +66,7 @@ impl EvaluationBoolean { self, rhs: Self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation { @@ -98,9 +98,9 @@ impl EvaluationBoolean { } } -impl ToEvaluationValue for bool { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::Boolean(EvaluationBoolean { +impl ToExpressionValue for bool { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::Boolean(EvaluationBoolean { value: self, source_span, }) diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 53a7ef3b..6eca640e 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -19,7 +19,7 @@ impl EvaluationChar { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let char = self.value; match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(char), @@ -54,7 +54,7 @@ impl EvaluationChar { self, _right: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { operation.unsupported_for_value_type_err("char") } @@ -62,7 +62,7 @@ impl EvaluationChar { self, rhs: Self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation { @@ -92,9 +92,9 @@ impl EvaluationChar { } } -impl ToEvaluationValue for char { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::Char(EvaluationChar { +impl ToExpressionValue for char { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::Char(EvaluationChar { value: self, source_span, }) diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 6498eed1..79270bdf 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -17,7 +17,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { mut self, root: ExpressionNodeId, evaluation_context: &mut K::EvaluationContext, - ) -> ExecutionResult { + ) -> ExecutionResult { let mut next = self.begin_node_evaluation(root, evaluation_context)?; loop { @@ -72,7 +72,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { fn continue_node_evaluation( &mut self, top_of_stack: EvaluationStackFrame, - evaluation_value: EvaluationValue, + evaluation_value: ExpressionValue, ) -> ExecutionResult { Ok(match top_of_stack { EvaluationStackFrame::UnaryOperation { operation } => { @@ -104,7 +104,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { } enum NextAction { - HandleValue(EvaluationValue), + HandleValue(ExpressionValue), EnterNode(ExpressionNodeId), } @@ -120,17 +120,17 @@ enum EvaluationStackFrame { enum BinaryPath { OnLeftBranch { right: ExpressionNodeId }, - OnRightBranch { left: EvaluationValue }, + OnRightBranch { left: ExpressionValue }, } pub(crate) struct EvaluationOutput { - pub(super) value: EvaluationValue, + pub(super) value: ExpressionValue, pub(super) fallback_output_span: Span, } impl EvaluationOutput { #[allow(unused)] - pub(super) fn into_value(self) -> EvaluationValue { + pub(super) fn into_value(self) -> ExpressionValue { self.value } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 53e61cd3..c9878ebf 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -1,28 +1,28 @@ use super::*; -// Interpretation = Source +// Source // ======================= #[derive(Clone)] -pub(crate) struct InterpretationExpression { +pub(crate) struct SourceExpression { inner: Expression, } -impl HasSpanRange for InterpretationExpression { +impl HasSpanRange for SourceExpression { fn span_range(&self) -> SpanRange { self.inner.span_range } } -impl ParseFromSource for InterpretationExpression { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for SourceExpression { + fn parse(input: ParseStream) -> ParseResult { Ok(Self { inner: ExpressionParser::parse(input)?, }) } } -impl InterpretationExpression { +impl SourceExpression { pub(crate) fn evaluate( &self, interpreter: &mut Interpreter, @@ -47,30 +47,28 @@ impl InterpretationExpression { pub(crate) fn evaluate_to_value( &self, interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { Source::evaluate_to_value(&self.inner, interpreter) } } -pub(super) enum InterpretationExpressionLeaf { +pub(super) enum SourceExpressionLeaf { Command(Command), GroupedVariable(GroupedVariable), - CodeBlock(CommandCodeInput), - Value(EvaluationValue), + CodeBlock(SourceCodeBlock), + Value(ExpressionValue), } impl Expressionable for Source { - type Leaf = InterpretationExpressionLeaf; + type Leaf = SourceExpressionLeaf; type EvaluationContext = Interpreter; fn leaf_end_span(leaf: &Self::Leaf) -> Option { match leaf { - InterpretationExpressionLeaf::Command(command) => Some(command.span()), - InterpretationExpressionLeaf::GroupedVariable(variable) => { - Some(variable.span_range().end()) - } - InterpretationExpressionLeaf::CodeBlock(code_block) => Some(code_block.span()), - InterpretationExpressionLeaf::Value(value) => value.source_span(), + SourceExpressionLeaf::Command(command) => Some(command.span()), + SourceExpressionLeaf::GroupedVariable(variable) => Some(variable.span_range().end()), + SourceExpressionLeaf::CodeBlock(code_block) => Some(code_block.span()), + SourceExpressionLeaf::Value(value) => value.source_span(), } } @@ -92,7 +90,7 @@ impl Expressionable for Source { UnaryAtom::Group(delim_span) }, GrammarPeekMatch::Group(Delimiter::Brace) => { - let leaf = InterpretationExpressionLeaf::CodeBlock(input.parse()?); + let leaf = SourceExpressionLeaf::CodeBlock(input.parse()?); UnaryAtom::Leaf(leaf) } GrammarPeekMatch::Group(Delimiter::Bracket) => return input.parse_err("Square brackets [ .. ] are not supported in an expression"), @@ -100,11 +98,11 @@ impl Expressionable for Source { UnaryAtom::PrefixUnaryOperation(input.parse()?) }, GrammarPeekMatch::Ident(_) => { - let value = EvaluationValue::Boolean(EvaluationBoolean::for_litbool(input.parse()?)); + let value = ExpressionValue::Boolean(EvaluationBoolean::for_litbool(input.parse()?)); UnaryAtom::Leaf(Self::Leaf::Value(value)) }, GrammarPeekMatch::Literal(_) => { - let value = EvaluationValue::for_literal(input.parse()?)?; + let value = ExpressionValue::for_literal(input.parse()?)?; UnaryAtom::Leaf(Self::Leaf::Value(value)) }, GrammarPeekMatch::End => return input.parse_err("The expression ended in an incomplete state"), @@ -129,44 +127,44 @@ impl Expressionable for Source { fn evaluate_leaf( leaf: &Self::Leaf, interpreter: &mut Self::EvaluationContext, - ) -> ExecutionResult { + ) -> ExecutionResult { let interpreted = match leaf { - InterpretationExpressionLeaf::Command(command) => { + SourceExpressionLeaf::Command(command) => { command.clone().interpret_to_new_stream(interpreter)? } - InterpretationExpressionLeaf::GroupedVariable(grouped_variable) => { + SourceExpressionLeaf::GroupedVariable(grouped_variable) => { grouped_variable.interpret_to_new_stream(interpreter)? } - InterpretationExpressionLeaf::CodeBlock(code_block) => { + SourceExpressionLeaf::CodeBlock(code_block) => { code_block.clone().interpret_to_new_stream(interpreter)? } - InterpretationExpressionLeaf::Value(value) => return Ok(value.clone()), + SourceExpressionLeaf::Value(value) => return Ok(value.clone()), }; let parsed_expression = unsafe { // RUST-ANALYZER SAFETY: This isn't very safe, as it could have a none-delimited group in it - interpreted.parse_as::()? + interpreted.parse_as::()? }; parsed_expression.evaluate_to_value() } } -// Interpreted +// Output // =========== #[derive(Clone)] -pub(crate) struct InterpretedExpression { - inner: Expression, +pub(crate) struct OutputExpression { + inner: Expression, } -impl ParseFromInterpreted for InterpretedExpression { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { +impl Parse for OutputExpression { + fn parse(input: ParseStream) -> ParseResult { Ok(Self { inner: ExpressionParser::parse(input)?, }) } } -impl InterpretedExpression { +impl OutputExpression { pub(crate) fn evaluate(&self) -> ExecutionResult { Ok(EvaluationOutput { value: self.evaluate_to_value()?, @@ -174,13 +172,13 @@ impl InterpretedExpression { }) } - pub(crate) fn evaluate_to_value(&self) -> ExecutionResult { - Interpreted::evaluate_to_value(&self.inner, &mut ()) + pub(crate) fn evaluate_to_value(&self) -> ExecutionResult { + Output::evaluate_to_value(&self.inner, &mut ()) } } -impl Expressionable for Interpreted { - type Leaf = EvaluationValue; +impl Expressionable for Output { + type Leaf = ExpressionValue; type EvaluationContext = (); fn leaf_end_span(leaf: &Self::Leaf) -> Option { @@ -190,45 +188,43 @@ impl Expressionable for Interpreted { fn evaluate_leaf( leaf: &Self::Leaf, _: &mut Self::EvaluationContext, - ) -> ExecutionResult { + ) -> ExecutionResult { Ok(leaf.clone()) } fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { - Ok(match input.peek_token() { - InterpretedPeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { + Ok(match input.peek_grammar() { + OutputPeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { let (_, delim_span) = input.parse_and_enter_group()?; UnaryAtom::Group(delim_span) } - InterpretedPeekMatch::Group(Delimiter::Brace) => { + OutputPeekMatch::Group(Delimiter::Brace) => { return input .parse_err("Curly braces are not supported in a re-interpreted expression") } - InterpretedPeekMatch::Group(Delimiter::Bracket) => { + OutputPeekMatch::Group(Delimiter::Bracket) => { return input.parse_err("Square brackets [ .. ] are not supported in an expression") } - InterpretedPeekMatch::Ident(_) => UnaryAtom::Leaf(EvaluationValue::Boolean( + OutputPeekMatch::Ident(_) => UnaryAtom::Leaf(ExpressionValue::Boolean( EvaluationBoolean::for_litbool(input.parse()?), )), - InterpretedPeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), - InterpretedPeekMatch::Literal(_) => { - UnaryAtom::Leaf(EvaluationValue::for_literal(input.parse()?)?) + OutputPeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), + OutputPeekMatch::Literal(_) => { + UnaryAtom::Leaf(ExpressionValue::for_literal(input.parse()?)?) } - InterpretedPeekMatch::End => { + OutputPeekMatch::End => { return input.parse_err("The expression ended in an incomplete state") } }) } fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { - Ok(match input.peek_token() { - InterpretedPeekMatch::Punct(_) => { - match input.try_parse_or_revert::() { - Ok(operation) => NodeExtension::BinaryOperation(operation), - Err(_) => NodeExtension::NoneMatched, - } - } - InterpretedPeekMatch::Ident(ident) if ident == "as" => { + Ok(match input.peek_grammar() { + OutputPeekMatch::Punct(_) => match input.try_parse_or_revert::() { + Ok(operation) => NodeExtension::BinaryOperation(operation), + Err(_) => NodeExtension::NoneMatched, + }, + OutputPeekMatch::Ident(ident) if ident == "as" => { let cast_operation = UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; NodeExtension::PostfixOperation(cast_operation) @@ -285,12 +281,12 @@ pub(super) trait Expressionable: Sized { fn evaluate_leaf( leaf: &Self::Leaf, context: &mut Self::EvaluationContext, - ) -> ExecutionResult; + ) -> ExecutionResult; fn evaluate_to_value( expression: &Expression, context: &mut Self::EvaluationContext, - ) -> ExecutionResult { + ) -> ExecutionResult { ExpressionEvaluator::new(&expression.nodes).evaluate(expression.root, context) } } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 36427dd0..5a76045b 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -9,7 +9,7 @@ pub(super) struct ExpressionParser<'a, K: Expressionable> { } impl<'a, K: Expressionable> ExpressionParser<'a, K> { - pub(super) fn parse(input: KindedParseStream<'a, K>) -> ParseResult> { + pub(super) fn parse(input: ParseStream<'a, K>) -> ParseResult> { Self { streams: ParseStreamStack::new(input), nodes: ExpressionNodes::new(), diff --git a/src/expressions/float.rs b/src/expressions/float.rs index acc328be..d254645c 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -21,7 +21,7 @@ impl EvaluationFloat { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => input.handle_unary_operation(&operation), EvaluationFloatValue::F32(input) => input.handle_unary_operation(&operation), @@ -33,7 +33,7 @@ impl EvaluationFloat { self, right: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationFloatValue::Untyped(input) => { input.handle_integer_binary_operation(right, operation) @@ -64,7 +64,7 @@ impl EvaluationFloatValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::F32(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -137,7 +137,7 @@ impl UntypedFloat { pub(super) fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let input = self.parse_fallback()?; match operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), @@ -175,7 +175,7 @@ impl UntypedFloat { self, _rhs: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { @@ -188,7 +188,7 @@ impl UntypedFloat { self, rhs: Self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; match operation { @@ -257,9 +257,9 @@ impl UntypedFloat { } } -impl ToEvaluationValue for UntypedFloat { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::Float(EvaluationFloat { +impl ToExpressionValue for UntypedFloat { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::Float(EvaluationFloat { value: EvaluationFloatValue::Untyped(self), source_span, }) @@ -270,9 +270,9 @@ macro_rules! impl_float_operations { ( $($float_enum_variant:ident($float_type:ident)),* $(,)? ) => {$( - impl ToEvaluationValue for $float_type { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::Float(EvaluationFloat { + impl ToExpressionValue for $float_type { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::Float(EvaluationFloat { value: EvaluationFloatValue::$float_enum_variant(self), source_span }) @@ -280,7 +280,7 @@ macro_rules! impl_float_operations { } impl HandleUnaryOperation for $float_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation { UnaryOperation::Neg { .. } => operation.output(-self), UnaryOperation::Not { .. } => operation.unsupported_for_value_type_err(stringify!($float_type)), @@ -309,7 +309,7 @@ macro_rules! impl_float_operations { } impl HandleBinaryOperation for $float_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { // Unlike integer arithmetic, float arithmetic does not overflow // and instead falls back to NaN or infinity. In future we could // allow trapping on these codes, but for now this is good enough @@ -342,7 +342,7 @@ macro_rules! impl_float_operations { self, _rhs: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err(stringify!($float_type)) diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index d5c481c1..a2835647 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -38,7 +38,7 @@ impl EvaluationInteger { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), EvaluationIntegerValue::U8(input) => input.handle_unary_operation(&operation), @@ -60,7 +60,7 @@ impl EvaluationInteger { self, right: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self.value { EvaluationIntegerValue::Untyped(input) => { input.handle_integer_binary_operation(right, operation) @@ -131,7 +131,7 @@ impl EvaluationIntegerValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::U8(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -264,7 +264,7 @@ impl UntypedInteger { pub(super) fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let input = self.parse_fallback()?; match operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), @@ -304,7 +304,7 @@ impl UntypedInteger { self, rhs: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; match operation { IntegerBinaryOperation::ShiftLeft { .. } => match rhs.value { @@ -348,7 +348,7 @@ impl UntypedInteger { self, rhs: Self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; let overflow_error = || { @@ -432,9 +432,9 @@ impl UntypedInteger { } } -impl ToEvaluationValue for UntypedInteger { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::Integer(EvaluationInteger { +impl ToExpressionValue for UntypedInteger { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::Integer(EvaluationInteger { value: EvaluationIntegerValue::Untyped(self), source_span, }) @@ -446,9 +446,9 @@ macro_rules! impl_int_operations_except_unary { ( $($integer_enum_variant:ident($integer_type:ident)),* $(,)? ) => {$( - impl ToEvaluationValue for $integer_type { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::Integer(EvaluationInteger { + impl ToExpressionValue for $integer_type { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::Integer(EvaluationInteger { value: EvaluationIntegerValue::$integer_enum_variant(self), source_span, }) @@ -456,7 +456,7 @@ macro_rules! impl_int_operations_except_unary { } impl HandleBinaryOperation for $integer_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { let lhs = self; let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.symbol(), rhs); match operation { @@ -483,7 +483,7 @@ macro_rules! impl_int_operations_except_unary { self, rhs: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self; match operation { IntegerBinaryOperation::ShiftLeft { .. } => { @@ -529,7 +529,7 @@ macro_rules! impl_int_operations_except_unary { macro_rules! impl_unsigned_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Neg { .. } @@ -564,7 +564,7 @@ macro_rules! impl_unsigned_unary_operations { macro_rules! impl_signed_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { + fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Neg { .. } => operation.output(-self), @@ -600,7 +600,7 @@ impl HandleUnaryOperation for u8 { fn handle_unary_operation( self, operation: &UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 40a77abc..9293efe0 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -1,15 +1,15 @@ use super::*; pub(super) trait Operation: HasSpanRange { - fn output(&self, output_value: impl ToEvaluationValue) -> ExecutionResult { + fn output(&self, output_value: impl ToExpressionValue) -> ExecutionResult { Ok(output_value.to_value(self.source_span_for_output())) } fn output_if_some( &self, - output_value: Option, + output_value: Option, error_message: impl FnOnce() -> String, - ) -> ExecutionResult { + ) -> ExecutionResult { match output_value { Some(output_value) => self.output(output_value), None => self.execution_err(error_message()), @@ -19,7 +19,7 @@ pub(super) trait Operation: HasSpanRange { fn unsupported_for_value_type_err( &self, value_type: &'static str, - ) -> ExecutionResult { + ) -> ExecutionResult { Err(self.execution_error(format!( "The {} operator is not supported for {} values", self.symbol(), @@ -120,7 +120,7 @@ impl UnaryOperation { }) } - pub(super) fn evaluate(self, input: EvaluationValue) -> ExecutionResult { + pub(super) fn evaluate(self, input: ExpressionValue) -> ExecutionResult { input.handle_unary_operation(self) } @@ -167,7 +167,7 @@ impl HasSpan for UnaryOperation { pub(super) trait HandleUnaryOperation: Sized { fn handle_unary_operation(self, operation: &UnaryOperation) - -> ExecutionResult; + -> ExecutionResult; } #[derive(Clone)] @@ -254,14 +254,14 @@ impl SynParse for BinaryOperation { impl BinaryOperation { pub(super) fn lazy_evaluate( &self, - left: &EvaluationValue, - ) -> ExecutionResult> { + left: &ExpressionValue, + ) -> ExecutionResult> { match self { BinaryOperation::Paired(PairedBinaryOperation::LogicalAnd { .. }) => { match left.clone().into_bool() { Some(bool) => { if !bool.value { - Ok(Some(EvaluationValue::Boolean(bool))) + Ok(Some(ExpressionValue::Boolean(bool))) } else { Ok(None) } @@ -273,7 +273,7 @@ impl BinaryOperation { match left.clone().into_bool() { Some(bool) => { if bool.value { - Ok(Some(EvaluationValue::Boolean(bool))) + Ok(Some(ExpressionValue::Boolean(bool))) } else { Ok(None) } @@ -287,9 +287,9 @@ impl BinaryOperation { pub(super) fn evaluate( &self, - left: EvaluationValue, - right: EvaluationValue, - ) -> ExecutionResult { + left: ExpressionValue, + right: ExpressionValue, + ) -> ExecutionResult { match self { BinaryOperation::Paired(operation) => { let value_pair = left.expect_value_pair(operation, right)?; @@ -430,11 +430,11 @@ pub(super) trait HandleBinaryOperation: Sized { self, rhs: Self, operation: &PairedBinaryOperation, - ) -> ExecutionResult; + ) -> ExecutionResult; fn handle_integer_binary_operation( self, rhs: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult; + ) -> ExecutionResult; } diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 83fee357..e1f72185 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -19,7 +19,7 @@ impl EvaluationString { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self.value), UnaryOperation::Neg { .. } @@ -32,7 +32,7 @@ impl EvaluationString { self, _right: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { operation.unsupported_for_value_type_err("string") } @@ -40,7 +40,7 @@ impl EvaluationString { self, rhs: Self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; match operation { @@ -70,18 +70,18 @@ impl EvaluationString { } } -impl ToEvaluationValue for String { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::String(EvaluationString { +impl ToExpressionValue for String { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::String(EvaluationString { value: self, source_span, }) } } -impl ToEvaluationValue for &str { - fn to_value(self, source_span: Option) -> EvaluationValue { - EvaluationValue::String(EvaluationString { +impl ToExpressionValue for &str { + fn to_value(self, source_span: Option) -> ExpressionValue { + ExpressionValue::String(EvaluationString { value: self.to_string(), source_span, }) diff --git a/src/expressions/value.rs b/src/expressions/value.rs index adf29613..0ab98a06 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -1,7 +1,7 @@ use super::*; #[derive(Clone)] -pub(crate) enum EvaluationValue { +pub(crate) enum ExpressionValue { Integer(EvaluationInteger), Float(EvaluationFloat), Boolean(EvaluationBoolean), @@ -9,11 +9,11 @@ pub(crate) enum EvaluationValue { Char(EvaluationChar), } -pub(super) trait ToEvaluationValue: Sized { - fn to_value(self, source_span: Option) -> EvaluationValue; +pub(super) trait ToExpressionValue: Sized { + fn to_value(self, source_span: Option) -> ExpressionValue; } -impl EvaluationValue { +impl ExpressionValue { pub(super) fn for_literal(lit: syn::Lit) -> ParseResult { // https://docs.rs/syn/latest/syn/enum.Lit.html Ok(match lit { @@ -36,7 +36,7 @@ impl EvaluationValue { right: Self, ) -> ExecutionResult { Ok(match (self, right) { - (EvaluationValue::Integer(left), EvaluationValue::Integer(right)) => { + (ExpressionValue::Integer(left), ExpressionValue::Integer(right)) => { let integer_pair = match (left.value, right.value) { (EvaluationIntegerValue::Untyped(untyped_lhs), rhs) => match rhs { EvaluationIntegerValue::Untyped(untyped_rhs) => { @@ -162,10 +162,10 @@ impl EvaluationValue { }; EvaluationLiteralPair::Integer(integer_pair) } - (EvaluationValue::Boolean(left), EvaluationValue::Boolean(right)) => { + (ExpressionValue::Boolean(left), ExpressionValue::Boolean(right)) => { EvaluationLiteralPair::BooleanPair(left, right) } - (EvaluationValue::Float(left), EvaluationValue::Float(right)) => { + (ExpressionValue::Float(left), ExpressionValue::Float(right)) => { let float_pair = match (left.value, right.value) { (EvaluationFloatValue::Untyped(untyped_lhs), rhs) => match rhs { EvaluationFloatValue::Untyped(untyped_rhs) => { @@ -201,10 +201,10 @@ impl EvaluationValue { }; EvaluationLiteralPair::Float(float_pair) } - (EvaluationValue::String(left), EvaluationValue::String(right)) => { + (ExpressionValue::String(left), ExpressionValue::String(right)) => { EvaluationLiteralPair::StringPair(left, right) } - (EvaluationValue::Char(left), EvaluationValue::Char(right)) => { + (ExpressionValue::Char(left), ExpressionValue::Char(right)) => { EvaluationLiteralPair::CharPair(left, right) } (left, right) => { @@ -215,14 +215,14 @@ impl EvaluationValue { pub(crate) fn into_integer(self) -> Option { match self { - EvaluationValue::Integer(value) => Some(value), + ExpressionValue::Integer(value) => Some(value), _ => None, } } pub(crate) fn into_bool(self) -> Option { match self { - EvaluationValue::Boolean(value) => Some(value), + ExpressionValue::Boolean(value) => Some(value), _ => None, } } @@ -261,13 +261,13 @@ impl EvaluationValue { pub(super) fn handle_unary_operation( self, operation: UnaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { - EvaluationValue::Integer(value) => value.handle_unary_operation(operation), - EvaluationValue::Float(value) => value.handle_unary_operation(operation), - EvaluationValue::Boolean(value) => value.handle_unary_operation(operation), - EvaluationValue::String(value) => value.handle_unary_operation(operation), - EvaluationValue::Char(value) => value.handle_unary_operation(operation), + ExpressionValue::Integer(value) => value.handle_unary_operation(operation), + ExpressionValue::Float(value) => value.handle_unary_operation(operation), + ExpressionValue::Boolean(value) => value.handle_unary_operation(operation), + ExpressionValue::String(value) => value.handle_unary_operation(operation), + ExpressionValue::Char(value) => value.handle_unary_operation(operation), } } @@ -275,21 +275,21 @@ impl EvaluationValue { self, right: EvaluationInteger, operation: &IntegerBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { - EvaluationValue::Integer(value) => { + ExpressionValue::Integer(value) => { value.handle_integer_binary_operation(right, operation) } - EvaluationValue::Float(value) => { + ExpressionValue::Float(value) => { value.handle_integer_binary_operation(right, operation) } - EvaluationValue::Boolean(value) => { + ExpressionValue::Boolean(value) => { value.handle_integer_binary_operation(right, operation) } - EvaluationValue::String(value) => { + ExpressionValue::String(value) => { value.handle_integer_binary_operation(right, operation) } - EvaluationValue::Char(value) => value.handle_integer_binary_operation(right, operation), + ExpressionValue::Char(value) => value.handle_integer_binary_operation(right, operation), } } } @@ -314,7 +314,7 @@ impl EvaluationLiteralPair { pub(super) fn handle_paired_binary_operation( self, operation: &PairedBinaryOperation, - ) -> ExecutionResult { + ) -> ExecutionResult { match self { Self::Integer(pair) => pair.handle_paired_binary_operation(operation), Self::Float(pair) => pair.handle_paired_binary_operation(operation), diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index c3912c26..75e38789 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -1,42 +1,61 @@ use crate::internal_prelude::*; pub(crate) trait TokenStreamParseExt: Sized { - fn parse_as, K>(self) -> ParseResult; - fn parse_with>( + fn source_parse_as>(self) -> ParseResult; + fn source_parse_with>( self, - parser: impl FnOnce(KindedParseStream) -> Result, + parser: impl FnOnce(ParseStream) -> Result, + ) -> Result; + + fn interpreted_parse_with>( + self, + parser: impl FnOnce(ParseStream) -> Result, ) -> Result; } impl TokenStreamParseExt for TokenStream { - fn parse_as, K>(self) -> ParseResult { - self.parse_with(T::parse) + fn source_parse_as>(self) -> ParseResult { + self.source_parse_with(T::parse) } - fn parse_with>( + fn source_parse_with>( self, - parser: impl FnOnce(KindedParseStream) -> Result, + parser: impl FnOnce(ParseStream) -> Result, ) -> Result { - let mut result = None; - let parse_result = (|input: SynParseStream| -> SynResult<()> { - result = Some(parser(input.into())); - match &result { - // Some fallback error to ensure that we don't go down the unexpected branch inside parse2 - Some(Err(_)) => Err(SynError::new(Span::call_site(), "")), - _ => Ok(()), - } - }) - .parse2(self); + parse_with(self, parser) + } - match (result, parse_result) { - (Some(Ok(value)), Ok(())) => Ok(value), - (Some(Err(error)), _) => Err(error), - // If the inner result was Ok, but the parse result was an error, this indicates that the parse2 - // hit the "unexpected" path, indicating that some parse buffer (i.e. group) wasn't fully consumed. - // So we propagate this error. - (Some(Ok(_)), Err(error)) => Err(error.into()), - (None, _) => unreachable!(), + fn interpreted_parse_with>( + self, + parser: impl FnOnce(ParseStream) -> Result, + ) -> Result { + parse_with(self, parser) + } +} + +fn parse_with>( + stream: TokenStream, + parser: impl FnOnce(ParseStream) -> Result, +) -> Result { + let mut result = None; + let parse_result = (|input: SynParseStream| -> SynResult<()> { + result = Some(parser(input.into())); + match &result { + // Some fallback error to ensure that we don't go down the unexpected branch inside parse2 + Some(Err(_)) => Err(SynError::new(Span::call_site(), "")), + _ => Ok(()), } + }) + .parse2(stream); + + match (result, parse_result) { + (Some(Ok(value)), Ok(())) => Ok(value), + (Some(Err(error)), _) => Err(error), + // If the inner result was Ok, but the parse result was an error, this indicates that the parse2 + // hit the "unexpected" path, indicating that some parse buffer (i.e. group) wasn't fully consumed. + // So we propagate this error. + (Some(Ok(_)), Err(error)) => Err(error.into()), + (None, _) => unreachable!(), } } @@ -96,149 +115,6 @@ impl CursorExt for Cursor<'_> { } } -pub(crate) trait ParserBufferExt { - fn try_parse_or_message ParseResult, M: std::fmt::Display>( - &self, - func: F, - message: M, - ) -> ParseResult; - fn parse_any_ident(&self) -> ParseResult; - fn peek_ident_matching(&self, content: &str) -> bool; - fn parse_ident_matching(&self, content: &str) -> ParseResult; - fn peek_punct_matching(&self, punct: char) -> bool; - fn parse_punct_matching(&self, content: char) -> ParseResult; - fn peek_literal_matching(&self, content: &str) -> bool; - fn parse_literal_matching(&self, content: &str) -> ParseResult; - fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, KindedParseBuffer)>; - fn peek_specific_group(&self, delimiter: Delimiter) -> bool; - fn parse_group_matching( - &self, - matching: impl FnOnce(Delimiter) -> bool, - expected_message: impl FnOnce() -> String, - ) -> ParseResult<(DelimSpan, KindedParseBuffer)>; - fn parse_specific_group( - &self, - delimiter: Delimiter, - ) -> ParseResult<(DelimSpan, KindedParseBuffer)>; - fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult; - fn parse_error(&self, message: impl std::fmt::Display) -> ParseError; -} - -impl ParserBufferExt for KindedParseBuffer<'_, K> { - fn try_parse_or_message ParseResult, M: std::fmt::Display>( - &self, - parse: F, - message: M, - ) -> ParseResult { - let error_span = self.span(); - parse(self).map_err(|_| error_span.error(message).into()) - } - - fn parse_any_ident(&self) -> ParseResult { - Ok(self.call(Ident::parse_any)?) - } - - fn peek_ident_matching(&self, content: &str) -> bool { - self.cursor().ident_matching(content).is_some() - } - - fn parse_ident_matching(&self, content: &str) -> ParseResult { - Ok(self.step(|cursor| { - cursor - .ident_matching(content) - .ok_or_else(|| cursor.span().error(format!("expected {}", content))) - })?) - } - - fn peek_punct_matching(&self, punct: char) -> bool { - self.cursor().punct_matching(punct).is_some() - } - - fn parse_punct_matching(&self, punct: char) -> ParseResult { - Ok(self.step(|cursor| { - cursor - .punct_matching(punct) - .ok_or_else(|| cursor.span().error(format!("expected {}", punct))) - })?) - } - - fn peek_literal_matching(&self, content: &str) -> bool { - self.cursor().literal_matching(content).is_some() - } - - fn parse_literal_matching(&self, content: &str) -> ParseResult { - Ok(self.step(|cursor| { - cursor - .literal_matching(content) - .ok_or_else(|| cursor.span().error(format!("expected {}", content))) - })?) - } - - fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, KindedParseBuffer)> { - use syn::parse::discouraged::AnyDelimiter; - let (delimiter, delim_span, parse_buffer) = self.parse_any_delimiter()?; - Ok((delimiter, delim_span, parse_buffer.into())) - } - - fn peek_specific_group(&self, delimiter: Delimiter) -> bool { - self.cursor().group_matching(delimiter).is_some() - } - - fn parse_group_matching( - &self, - matching: impl FnOnce(Delimiter) -> bool, - expected_message: impl FnOnce() -> String, - ) -> ParseResult<(DelimSpan, KindedParseBuffer)> { - let error_span = match self.parse_any_group() { - Ok((delimiter, delim_span, inner)) if matching(delimiter) => { - return Ok((delim_span, inner)); - } - Ok((_, delim_span, _)) => delim_span.open(), - Err(error) => error.span(), - }; - error_span.parse_err(expected_message()) - } - - fn parse_specific_group( - &self, - expected_delimiter: Delimiter, - ) -> ParseResult<(DelimSpan, KindedParseBuffer)> { - self.parse_group_matching( - |delimiter| delimiter == expected_delimiter, - || format!("Expected {}", expected_delimiter.description_of_open()), - ) - } - - fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { - Err(self.parse_error(message)) - } - - fn parse_error(&self, message: impl std::fmt::Display) -> ParseError { - self.span().parse_error(message) - } -} - -pub(crate) trait SourceParserBufferExt { - fn parse_with_context( - &self, - context: T::Context, - ) -> ParseResult; - fn parse_all_for_interpretation(&self, span: Span) -> ParseResult; -} - -impl SourceParserBufferExt for SourceParseBuffer<'_> { - fn parse_with_context( - &self, - context: T::Context, - ) -> ParseResult { - T::parse_from_source(self, context) - } - - fn parse_all_for_interpretation(&self, span: Span) -> ParseResult { - self.parse_with_context(span) - } -} - pub(crate) trait DelimiterExt { fn description_of_open(&self) -> &'static str; #[allow(unused)] @@ -268,19 +144,19 @@ impl DelimiterExt for Delimiter { /// Allows storing a stack of parse buffers for certain parse strategies which require /// handling multiple groups in parallel. pub(crate) struct ParseStreamStack<'a, K> { - base: KindedParseStream<'a, K>, - group_stack: Vec>, + base: ParseStream<'a, K>, + group_stack: Vec>, } impl<'a, K> ParseStreamStack<'a, K> { - pub(crate) fn new(base: KindedParseStream<'a, K>) -> Self { + pub(crate) fn new(base: ParseStream<'a, K>) -> Self { Self { base, group_stack: Vec::new(), } } - fn current(&self) -> KindedParseStream<'_, K> { + fn current(&self) -> ParseStream<'_, K> { self.group_stack.last().unwrap_or(self.base) } @@ -322,7 +198,7 @@ impl<'a, K> ParseStreamStack<'a, K> { // ==> exit_group() ensures the parse buffers are dropped in the correct order // ==> If a user forgets to do it (or e.g. an error path or panic causes exit_group not to be called) // Then the drop glue ensures the groups are dropped in the correct order. - std::mem::transmute::, KindedParseBuffer<'a, K>>(inner) + std::mem::transmute::, ParseBuffer<'a, K>>(inner) }; self.group_stack.push(inner); Ok((delimiter, delim_span)) @@ -348,9 +224,9 @@ impl ParseStreamStack<'_, Source> { } } -impl ParseStreamStack<'_, Interpreted> { - pub(crate) fn peek_token(&mut self) -> InterpretedPeekMatch { - self.current().peek_token() +impl ParseStreamStack<'_, Output> { + pub(crate) fn peek_grammar(&mut self) -> OutputPeekMatch { + self.current().peek_grammar() } } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 0f8c9352..247261aa 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -45,7 +45,7 @@ trait CommandInvocation { fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()>; } @@ -71,7 +71,7 @@ trait CommandInvocationAs { fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()>; } @@ -79,7 +79,7 @@ impl> CommandInvocation for fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { >::execute_into(self, context, output) } @@ -114,7 +114,7 @@ impl CommandInvocationAs for C { fn execute_into( self: Box, context: ExecutionContext, - _: &mut InterpretedStream, + _: &mut OutputStream, ) -> ExecutionResult<()> { self.execute(context.interpreter)?; Ok(()) @@ -151,7 +151,7 @@ impl CommandInvocationAs for C { fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { output.push_raw_token_tree(self.execute(context.interpreter)?); Ok(()) @@ -188,7 +188,7 @@ impl CommandInvocationAs for C { fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { output.push_ident(self.execute(context.interpreter)?); Ok(()) @@ -201,7 +201,7 @@ impl CommandInvocationAs for C { pub(crate) struct OutputKindStream; impl OutputKind for OutputKindStream { - type Output = InterpretedStream; + type Output = OutputStream; fn resolve_standard() -> CommandOutputKind { CommandOutputKind::GroupedStream @@ -220,7 +220,7 @@ pub(crate) trait StreamCommandDefinition: fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()>; } @@ -228,7 +228,7 @@ impl CommandInvocationAs for C { fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { match context.output_kind { CommandOutputKind::FlattenedStream => self.execute(context.interpreter, output), @@ -267,7 +267,7 @@ pub(crate) trait ControlFlowCommandDefinition: fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()>; } @@ -275,7 +275,7 @@ impl CommandInvocationAs fn execute_into( self: Box, context: ExecutionContext, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { self.execute(context.interpreter, output) } @@ -408,8 +408,8 @@ pub(crate) struct Command { source_group_span: DelimSpan, } -impl ParseFromSource for Command { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for Command { + fn parse(input: ParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; content.parse::()?; let flattening = if content.peek(Token![.]) { @@ -469,7 +469,7 @@ impl Interpret for Command { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let context = ExecutionContext { interpreter, diff --git a/src/interpretation/command_arguments.rs b/src/interpretation/command_arguments.rs index 44095e55..90b31e03 100644 --- a/src/interpretation/command_arguments.rs +++ b/src/interpretation/command_arguments.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct CommandArguments<'a> { - parse_stream: SourceParseStream<'a>, + parse_stream: ParseStream<'a, Source>, command_name: Ident, /// The span of the [ ... ] which contained the command command_span: Span, @@ -10,7 +10,7 @@ pub(crate) struct CommandArguments<'a> { impl<'a> CommandArguments<'a> { pub(crate) fn new( - parse_stream: SourceParseStream<'a>, + parse_stream: ParseStream<'a, Source>, command_name: Ident, command_span: Span, ) -> Self { @@ -36,12 +36,12 @@ impl<'a> CommandArguments<'a> { } pub(crate) fn fully_parse_as(&self) -> ParseResult { - self.fully_parse_or_error(T::parse_from_source, T::error_message()) + self.fully_parse_or_error(T::parse, T::error_message()) } pub(crate) fn fully_parse_or_error( &self, - parse_function: impl FnOnce(SourceParseStream) -> ParseResult, + parse_function: impl FnOnce(ParseStream) -> ParseResult, error_message: impl std::fmt::Display, ) -> ParseResult { // In future, when the diagnostic API is stable, @@ -64,9 +64,8 @@ impl<'a> CommandArguments<'a> { Ok(parsed) } - pub(crate) fn parse_all_for_interpretation(&self) -> ParseResult { - self.parse_stream - .parse_all_for_interpretation(self.command_span) + pub(crate) fn parse_all_as_source(&self) -> ParseResult { + self.parse_stream.parse_with_context(self.command_span) } pub(crate) fn read_all_as_raw_token_stream(&self) -> TokenStream { @@ -74,6 +73,6 @@ impl<'a> CommandArguments<'a> { } } -pub(crate) trait ArgumentsContent: ParseFromSource { +pub(crate) trait ArgumentsContent: Parse { fn error_message() -> String; } diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/command_code_input.rs index 85e4ed23..b7399a14 100644 --- a/src/interpretation/command_code_input.rs +++ b/src/interpretation/command_code_input.rs @@ -1,31 +1,31 @@ use crate::internal_prelude::*; -/// Parses a group { .. } for interpretation +/// A group { .. } representing code which can be interpreted #[derive(Clone)] -pub(crate) struct CommandCodeInput { +pub(crate) struct SourceCodeBlock { delim_span: DelimSpan, - inner: InterpretationStream, + inner: SourceStream, } -impl ParseFromSource for CommandCodeInput { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for SourceCodeBlock { + fn parse(input: ParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; let inner = content.parse_with_context(delim_span.join())?; Ok(Self { delim_span, inner }) } } -impl HasSpan for CommandCodeInput { +impl HasSpan for SourceCodeBlock { fn span(&self) -> Span { self.delim_span.join() } } -impl CommandCodeInput { +impl SourceCodeBlock { pub(crate) fn interpret_loop_content_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult> { match self.inner.interpret_into(interpreter, output) { Ok(()) => Ok(None), @@ -37,11 +37,11 @@ impl CommandCodeInput { } } -impl Interpret for CommandCodeInput { +impl Interpret for SourceCodeBlock { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { self.inner.interpret_into(interpreter, output) } diff --git a/src/interpretation/command_field_inputs.rs b/src/interpretation/command_field_inputs.rs index 56b995bd..02d36ff9 100644 --- a/src/interpretation/command_field_inputs.rs +++ b/src/interpretation/command_field_inputs.rs @@ -24,8 +24,8 @@ macro_rules! define_field_inputs { )* } - impl ParseFromSource for $inputs_type { - fn parse_from_source(input: SourceParseStream) -> ParseResult { + impl Parse for $inputs_type { + fn parse(input: ParseStream) -> ParseResult { $( let mut $required_field: Option<$required_type> = None; )* diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/command_stream_input.rs index 12a01037..b5620048 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/command_stream_input.rs @@ -17,12 +17,12 @@ pub(crate) enum CommandStreamInput { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), - Code(CommandCodeInput), - ExplicitStream(InterpretationGroup), + Code(SourceCodeBlock), + ExplicitStream(SourceGroup), } -impl ParseFromSource for CommandStreamInput { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for CommandStreamInput { + fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { GrammarPeekMatch::Command(_) => Self::Command(input.parse()?), GrammarPeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), @@ -51,7 +51,7 @@ impl Interpret for CommandStreamInput { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { match self { CommandStreamInput::Command(mut command) => { @@ -115,14 +115,14 @@ impl Interpret for CommandStreamInput { } } -/// From a code neatness perspective, this would be better as `InterpretedCommandStream`... +/// From a code neatness perspective, this would be better as `OutputCommandStream`... /// However this approach avoids a round-trip to TokenStream, which causes bugs in RustAnalyzer. -/// This can be removed when we fork syn and can parse the InterpretedStream directly. +/// This can be removed when we fork syn and can parse the OutputStream directly. fn parse_as_stream_input( input: impl Interpret + HasSpanRange, interpreter: &mut Interpreter, error_message: impl FnOnce() -> String, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let span = input.span_range(); input @@ -136,31 +136,31 @@ fn parse_as_stream_input( } impl InterpretValue for CommandStreamInput { - type InterpretedValue = InterpretedCommandStream; + type OutputValue = OutputCommandStream; fn interpret_to_value( self, interpreter: &mut Interpreter, - ) -> ExecutionResult { - Ok(InterpretedCommandStream { + ) -> ExecutionResult { + Ok(OutputCommandStream { stream: self.interpret_to_new_stream(interpreter)?, }) } } -pub(crate) struct InterpretedCommandStream { - pub(crate) stream: InterpretedStream, +pub(crate) struct OutputCommandStream { + pub(crate) stream: OutputStream, } -impl ParseFromInterpreted for InterpretedCommandStream { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { +impl Parse for OutputCommandStream { + fn parse(input: ParseStream) -> ParseResult { // We replicate parse_as_stream_input let (_, inner) = input.parse_group_matching( |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None), || "Expected [...] or a transparent group from a #variable or stream-output command such as [!group! ...]".to_string(), )?; Ok(Self { - stream: InterpretedStream::raw(inner.parse()?), + stream: OutputStream::raw(inner.parse()?), }) } } diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/command_value_input.rs index 3375dc05..d980e763 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/command_value_input.rs @@ -9,12 +9,12 @@ pub(crate) enum CommandValueInput { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), - Code(CommandCodeInput), + Code(SourceCodeBlock), Value(T), } -impl ParseFromSource for CommandValueInput { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl> Parse for CommandValueInput { + fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { GrammarPeekMatch::Command(_) => Self::Command(input.parse()?), GrammarPeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), @@ -34,8 +34,8 @@ impl ParseFromSource for CommandValueInput { } } -impl ParseFromInterpreted for CommandValueInput { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { +impl> Parse for CommandValueInput { + fn parse(input: ParseStream) -> ParseResult { Ok(Self::Value(input.parse()?)) } } @@ -52,10 +52,8 @@ impl HasSpanRange for CommandValueInput { } } -impl, I: ParseFromInterpreted> InterpretValue - for CommandValueInput -{ - type InterpretedValue = I; +impl, I: Parse> InterpretValue for CommandValueInput { + type OutputValue = I; fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { let descriptor = match self { @@ -77,10 +75,10 @@ impl, I: ParseFromInterpreted> Interpret CommandValueInput::Value(value) => return value.interpret_to_value(interpreter), }; unsafe { - // RUST-ANALYZER SAFETY: We only use I with simple parse functions so far which don't care about - // none-delimited groups + // RUST-ANALYZER SAFETY: If I is a very simple parse function, this is safe. + // Zip uses it with a parse function which does care about none-delimited groups however. interpreted_stream - .parse_with(I::parse_from_interpreted) + .parse_with(I::parse) .add_context_if_error_and_no_context(|| { format!( "Occurred whilst parsing the {} to a {}.", @@ -100,8 +98,8 @@ pub(crate) struct Grouped { pub(crate) inner: T, } -impl ParseFromSource for Grouped { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl> Parse for Grouped { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, inner) = input.parse_any_group()?; Ok(Self { delimiter, @@ -111,8 +109,8 @@ impl ParseFromSource for Grouped { } } -impl ParseFromInterpreted for Grouped { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { +impl> Parse for Grouped { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, inner) = input.parse_any_group()?; Ok(Self { delimiter, @@ -124,14 +122,14 @@ impl ParseFromInterpreted for Grouped { impl InterpretValue for Grouped where - T: InterpretValue, + T: InterpretValue, { - type InterpretedValue = Grouped; + type OutputValue = Grouped; fn interpret_to_value( self, interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { Ok(Grouped { delimiter: self.delimiter, delim_span: self.delim_span, @@ -145,8 +143,8 @@ pub(crate) struct Repeated { pub(crate) inner: Vec, } -impl ParseFromSource for Repeated { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl> Parse for Repeated { + fn parse(input: ParseStream) -> ParseResult { let mut inner = vec![]; while !input.is_empty() { inner.push(input.parse::()?); @@ -155,8 +153,8 @@ impl ParseFromSource for Repeated { } } -impl ParseFromInterpreted for Repeated { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { +impl> Parse for Repeated { + fn parse(input: ParseStream) -> ParseResult { let mut inner = vec![]; while !input.is_empty() { inner.push(input.parse::()?); @@ -167,14 +165,14 @@ impl ParseFromInterpreted for Repeated { impl InterpretValue for Repeated where - T: InterpretValue, + T: InterpretValue, { - type InterpretedValue = Repeated; + type OutputValue = Repeated; fn interpret_to_value( self, interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { let mut interpreted = Vec::with_capacity(self.inner.len()); for item in self.inner.into_iter() { interpreted.push(item.interpret_to_value(interpreter)?); diff --git a/src/interpretation/commands/concat_commands.rs b/src/interpretation/commands/concat_commands.rs index 6c1fb1f9..b4d6adf3 100644 --- a/src/interpretation/commands/concat_commands.rs +++ b/src/interpretation/commands/concat_commands.rs @@ -5,7 +5,7 @@ use crate::internal_prelude::*; //======== fn concat_into_string( - input: InterpretationStream, + input: SourceStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, ) -> ExecutionResult { @@ -18,7 +18,7 @@ fn concat_into_string( } fn concat_into_ident( - input: InterpretationStream, + input: SourceStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, ) -> ExecutionResult { @@ -34,7 +34,7 @@ fn concat_into_ident( } fn concat_into_literal( - input: InterpretationStream, + input: SourceStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, ) -> ExecutionResult { @@ -57,7 +57,7 @@ macro_rules! define_literal_concat_command { ) => { #[derive(Clone)] pub(crate) struct $command { - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for $command { @@ -69,7 +69,7 @@ macro_rules! define_literal_concat_command { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, + arguments: arguments.parse_all_as_source()?, }) } @@ -89,7 +89,7 @@ macro_rules! define_ident_concat_command { ) => { #[derive(Clone)] pub(crate) struct $command { - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for $command { @@ -101,7 +101,7 @@ macro_rules! define_ident_concat_command { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, + arguments: arguments.parse_all_as_source()?, }) } diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 3af3cbc7..e1d902d6 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -2,10 +2,10 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IfCommand { - condition: InterpretationExpression, - true_code: CommandCodeInput, - else_ifs: Vec<(InterpretationExpression, CommandCodeInput)>, - else_code: Option, + condition: SourceExpression, + true_code: SourceCodeBlock, + else_ifs: Vec<(SourceExpression, SourceCodeBlock)>, + else_code: Option, } impl CommandType for IfCommand { @@ -49,7 +49,7 @@ impl ControlFlowCommandDefinition for IfCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let evaluated_condition = self .condition @@ -80,8 +80,8 @@ impl ControlFlowCommandDefinition for IfCommand { #[derive(Clone)] pub(crate) struct WhileCommand { - condition: InterpretationExpression, - loop_code: CommandCodeInput, + condition: SourceExpression, + loop_code: SourceCodeBlock, } impl CommandType for WhileCommand { @@ -106,7 +106,7 @@ impl ControlFlowCommandDefinition for WhileCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let mut iteration_counter = interpreter.start_iteration_counter(&self.condition); loop { @@ -139,7 +139,7 @@ impl ControlFlowCommandDefinition for WhileCommand { #[derive(Clone)] pub(crate) struct LoopCommand { - loop_code: CommandCodeInput, + loop_code: SourceCodeBlock, } impl CommandType for LoopCommand { @@ -163,7 +163,7 @@ impl ControlFlowCommandDefinition for LoopCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let mut iteration_counter = interpreter.start_iteration_counter(&self.loop_code); @@ -189,7 +189,7 @@ pub(crate) struct ForCommand { #[allow(unused)] in_token: Token![in], input: CommandStreamInput, - loop_code: CommandCodeInput, + loop_code: SourceCodeBlock, } impl CommandType for ForCommand { @@ -216,7 +216,7 @@ impl ControlFlowCommandDefinition for ForCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let stream = self.input.interpret_to_new_stream(interpreter)?; diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index e34041d8..82011841 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -5,7 +5,7 @@ pub(crate) struct SetCommand { variable: GroupedVariable, #[allow(unused)] equals: Token![=], - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for SetCommand { @@ -40,7 +40,7 @@ pub(crate) struct ExtendCommand { variable: GroupedVariable, #[allow(unused)] plus_equals: Token![+=], - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for ExtendCommand { @@ -56,7 +56,7 @@ impl NoOutputCommandDefinition for ExtendCommand { Ok(Self { variable: input.parse()?, plus_equals: input.parse()?, - arguments: input.parse_all_for_interpretation(arguments.command_span())?, + arguments: input.parse_with_context(arguments.command_span())?, }) }, "Expected [!extend! #variable += ..]", @@ -94,7 +94,7 @@ impl StreamCommandDefinition for RawCommand { fn execute( self: Box, _interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { output.extend_raw_tokens(self.token_stream); Ok(()) @@ -124,7 +124,7 @@ impl NoOutputCommandDefinition for IgnoreCommand { #[derive(Clone)] pub(crate) struct VoidCommand { - inner: InterpretationStream, + inner: SourceStream, } impl CommandType for VoidCommand { @@ -136,7 +136,7 @@ impl NoOutputCommandDefinition for VoidCommand { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - inner: arguments.parse_all_for_interpretation()?, + inner: arguments.parse_all_as_source()?, }) } @@ -194,7 +194,7 @@ impl CommandType for ErrorCommand { #[derive(Clone)] enum EitherErrorInput { Fields(ErrorInputs), - JustMessage(InterpretationStream), + JustMessage(SourceStream), } define_field_inputs! { @@ -293,7 +293,7 @@ impl NoOutputCommandDefinition for ErrorCommand { #[derive(Clone)] pub(crate) struct DebugCommand { - inner: InterpretationStream, + inner: SourceStream, } impl CommandType for DebugCommand { @@ -305,7 +305,7 @@ impl ValueCommandDefinition for DebugCommand { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - inner: arguments.parse_all_for_interpretation()?, + inner: arguments.parse_all_as_source()?, }) } diff --git a/src/interpretation/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs index 92b534e8..ef6e7f76 100644 --- a/src/interpretation/commands/destructuring_commands.rs +++ b/src/interpretation/commands/destructuring_commands.rs @@ -5,7 +5,7 @@ pub(crate) struct LetCommand { destructuring: DestructureUntil, #[allow(unused)] equals: Token![=], - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for LetCommand { diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 23144c48..9d9a1080 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct EvaluateCommand { - expression: InterpretationExpression, + expression: SourceExpression, command_span: Span, } @@ -39,7 +39,7 @@ pub(crate) struct AssignCommand { operator: Option, #[allow(unused)] equals: Token![=], - expression: InterpretationExpression, + expression: SourceExpression, command_span: Span, } @@ -91,7 +91,7 @@ impl NoOutputCommandDefinition for AssignCommand { // RUST-ANALYZER SAFETY: Hopefully it won't contain a none-delimited group variable .interpret_to_new_stream(interpreter)? - .parse_as::()? + .parse_as::()? .evaluate()? .to_tokens(&mut calculation); }; @@ -99,7 +99,7 @@ impl NoOutputCommandDefinition for AssignCommand { expression .evaluate(interpreter)? .to_tokens(&mut calculation); - calculation.parse_as()? + calculation.source_parse_as()? } else { expression }; @@ -115,9 +115,9 @@ impl NoOutputCommandDefinition for AssignCommand { #[derive(Clone)] pub(crate) struct RangeCommand { - left: InterpretationExpression, + left: SourceExpression, range_limits: RangeLimits, - right: InterpretationExpression, + right: SourceExpression, } impl CommandType for RangeCommand { @@ -143,7 +143,7 @@ impl StreamCommandDefinition for RangeCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let range_span_range = self.range_limits.span_range(); let range_span = range_span_range.join_into_span_else_start(); @@ -185,7 +185,7 @@ impl StreamCommandDefinition for RangeCommand { } } -fn output_range(iter: impl Iterator, span: Span, output: &mut InterpretedStream) { +fn output_range(iter: impl Iterator, span: Span, output: &mut OutputStream) { output.extend_raw_tokens(iter.map(|value| { let literal = Literal::i128_unsuffixed(value).with_span(span); TokenTree::Literal(literal) @@ -202,8 +202,8 @@ enum RangeLimits { Closed(Token![..=]), } -impl ParseFromSource for RangeLimits { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for RangeLimits { + fn parse(input: ParseStream) -> ParseResult { if input.peek(Token![..=]) { Ok(RangeLimits::Closed(input.parse()?)) } else { diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index f00f7cbf..ac8480ca 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -2,7 +2,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct IsEmptyCommand { - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for IsEmptyCommand { @@ -14,7 +14,7 @@ impl ValueCommandDefinition for IsEmptyCommand { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, + arguments: arguments.parse_all_as_source()?, }) } @@ -27,7 +27,7 @@ impl ValueCommandDefinition for IsEmptyCommand { #[derive(Clone)] pub(crate) struct LengthCommand { - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for LengthCommand { @@ -39,7 +39,7 @@ impl ValueCommandDefinition for LengthCommand { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, + arguments: arguments.parse_all_as_source()?, }) } @@ -53,7 +53,7 @@ impl ValueCommandDefinition for LengthCommand { #[derive(Clone)] pub(crate) struct GroupCommand { - arguments: InterpretationStream, + arguments: SourceStream, } impl CommandType for GroupCommand { @@ -65,14 +65,14 @@ impl StreamCommandDefinition for GroupCommand { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - arguments: arguments.parse_all_for_interpretation()?, + arguments: arguments.parse_all_as_source()?, }) } fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { // The grouping happens automatically because a non-flattened // stream command is outputted in a group. @@ -114,7 +114,7 @@ impl StreamCommandDefinition for IntersperseCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let items = self.inputs.items.interpret_to_new_stream(interpreter)?; let add_trailing = match self.inputs.add_trailing { @@ -169,7 +169,7 @@ impl SeparatorAppender { &mut self, interpreter: &mut Interpreter, remaining: RemainingItemCount, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { match self.separator(remaining) { TrailingSeparator::Normal => self.separator.clone().interpret_into(interpreter, output), @@ -249,7 +249,7 @@ impl StreamCommandDefinition for SplitCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let output_span = self.inputs.stream.span_range().join_into_span_else_start(); let stream = self.inputs.stream.interpret_to_new_stream(interpreter)?; @@ -281,8 +281,8 @@ impl StreamCommandDefinition for SplitCommand { } fn handle_split( - input: InterpretedStream, - output: &mut InterpretedStream, + input: OutputStream, + output: &mut OutputStream, output_span: Span, separator: RawDestructureStream, drop_empty_start: bool, @@ -293,7 +293,7 @@ fn handle_split( // RUST-ANALYZER SAFETY: This is as safe as we can get. // Typically the separator won't contain none-delimited groups, so we're OK input.parse_with(move |input| { - let mut current_item = InterpretedStream::new(); + let mut current_item = OutputStream::new(); let mut drop_empty_next = drop_empty_start; while !input.is_empty() { let separator_fork = input.fork(); @@ -303,8 +303,7 @@ fn handle_split( } input.advance_to(&separator_fork); if !(current_item.is_empty() && drop_empty_next) { - let complete_item = - core::mem::replace(&mut current_item, InterpretedStream::new()); + let complete_item = core::mem::replace(&mut current_item, OutputStream::new()); output.push_new_group(complete_item, Delimiter::None, output_span); } drop_empty_next = drop_empty_middle; @@ -319,7 +318,7 @@ fn handle_split( #[derive(Clone)] pub(crate) struct CommaSplitCommand { - input: InterpretationStream, + input: SourceStream, } impl CommandType for CommaSplitCommand { @@ -331,14 +330,14 @@ impl StreamCommandDefinition for CommaSplitCommand { fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { - input: arguments.parse_all_for_interpretation()?, + input: arguments.parse_all_as_source()?, }) } fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let output_span = self.input.span_range().join_into_span_else_start(); let stream = self.input.interpret_to_new_stream(interpreter)?; @@ -408,7 +407,7 @@ impl StreamCommandDefinition for ZipCommand { fn execute( self: Box, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let (grouped_streams, error_on_length_mismatch) = match self.inputs { EitherZipInput::Fields(inputs) => (inputs.streams, inputs.error_on_length_mismatch), @@ -448,7 +447,7 @@ impl StreamCommandDefinition for ZipCommand { .map(|stream| stream.stream.into_iter()) .collect(); for _ in 0..min_stream_length { - let mut inner = InterpretedStream::new(); + let mut inner = OutputStream::new(); for iter in iters.iter_mut() { inner.push_interpreted_item(iter.next().unwrap()); } diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index b02693da..35e5457d 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -4,35 +4,35 @@ pub(crate) trait Interpret: Sized { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()>; fn interpret_to_new_stream( self, interpreter: &mut Interpreter, - ) -> ExecutionResult { - let mut output = InterpretedStream::new(); + ) -> ExecutionResult { + let mut output = OutputStream::new(); self.interpret_into(interpreter, &mut output)?; Ok(output) } } pub(crate) trait InterpretValue: Sized { - type InterpretedValue; + type OutputValue; fn interpret_to_value( self, interpreter: &mut Interpreter, - ) -> ExecutionResult; + ) -> ExecutionResult; } impl InterpretValue for T { - type InterpretedValue = Self; + type OutputValue = Self; fn interpret_to_value( self, _interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { Ok(self) } } diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs index 36f0bec8..5fffed2f 100644 --- a/src/interpretation/interpretation_item.rs +++ b/src/interpretation/interpretation_item.rs @@ -1,75 +1,71 @@ use crate::internal_prelude::*; #[derive(Clone)] -pub(crate) enum InterpretationItem { +pub(crate) enum SourceItem { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), - InterpretationGroup(InterpretationGroup), + SourceGroup(SourceGroup), Punct(Punct), Ident(Ident), Literal(Literal), } -impl ParseFromSource for InterpretationItem { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for SourceItem { + fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(_) => InterpretationItem::Command(input.parse()?), - GrammarPeekMatch::Group(_) => InterpretationItem::InterpretationGroup(input.parse()?), - GrammarPeekMatch::GroupedVariable => { - InterpretationItem::GroupedVariable(input.parse()?) - } - GrammarPeekMatch::FlattenedVariable => { - InterpretationItem::FlattenedVariable(input.parse()?) - } + GrammarPeekMatch::Command(_) => SourceItem::Command(input.parse()?), + GrammarPeekMatch::Group(_) => SourceItem::SourceGroup(input.parse()?), + GrammarPeekMatch::GroupedVariable => SourceItem::GroupedVariable(input.parse()?), + GrammarPeekMatch::FlattenedVariable => SourceItem::FlattenedVariable(input.parse()?), GrammarPeekMatch::AppendVariableDestructuring | GrammarPeekMatch::Destructurer(_) => { return input.parse_err("Destructurings are not supported here") } - GrammarPeekMatch::Punct(_) => InterpretationItem::Punct(input.parse_any_punct()?), - GrammarPeekMatch::Ident(_) => InterpretationItem::Ident(input.parse_any_ident()?), - GrammarPeekMatch::Literal(_) => InterpretationItem::Literal(input.parse()?), + GrammarPeekMatch::Punct(_) => SourceItem::Punct(input.parse_any_punct()?), + GrammarPeekMatch::Ident(_) => SourceItem::Ident(input.parse_any_ident()?), + GrammarPeekMatch::Literal(_) => SourceItem::Literal(input.parse()?), GrammarPeekMatch::End => return input.parse_err("Expected some item"), }) } } -impl Interpret for InterpretationItem { +impl Interpret for SourceItem { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { match self { - InterpretationItem::Command(command_invocation) => { + SourceItem::Command(command_invocation) => { command_invocation.interpret_into(interpreter, output)?; } - InterpretationItem::GroupedVariable(variable) => { + SourceItem::GroupedVariable(variable) => { variable.interpret_into(interpreter, output)?; } - InterpretationItem::FlattenedVariable(variable) => { + SourceItem::FlattenedVariable(variable) => { variable.interpret_into(interpreter, output)?; } - InterpretationItem::InterpretationGroup(group) => { + SourceItem::SourceGroup(group) => { group.interpret_into(interpreter, output)?; } - InterpretationItem::Punct(punct) => output.push_punct(punct), - InterpretationItem::Ident(ident) => output.push_ident(ident), - InterpretationItem::Literal(literal) => output.push_literal(literal), + SourceItem::Punct(punct) => output.push_punct(punct), + SourceItem::Ident(ident) => output.push_ident(ident), + SourceItem::Literal(literal) => output.push_literal(literal), } Ok(()) } } -impl HasSpanRange for InterpretationItem { +impl HasSpanRange for SourceItem { fn span_range(&self) -> SpanRange { match self { - InterpretationItem::Command(command_invocation) => command_invocation.span_range(), - InterpretationItem::FlattenedVariable(variable) => variable.span_range(), - InterpretationItem::GroupedVariable(variable) => variable.span_range(), - InterpretationItem::InterpretationGroup(group) => group.span_range(), - InterpretationItem::Punct(punct) => punct.span_range(), - InterpretationItem::Ident(ident) => ident.span_range(), - InterpretationItem::Literal(literal) => literal.span_range(), + SourceItem::Command(command_invocation) => command_invocation.span_range(), + SourceItem::FlattenedVariable(variable) => variable.span_range(), + SourceItem::GroupedVariable(variable) => variable.span_range(), + SourceItem::SourceGroup(group) => group.span_range(), + SourceItem::Punct(punct) => punct.span_range(), + SourceItem::Ident(ident) => ident.span_range(), + SourceItem::Literal(literal) => literal.span_range(), } } } diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs index d60236fd..688beaec 100644 --- a/src/interpretation/interpretation_stream.rs +++ b/src/interpretation/interpretation_stream.rs @@ -2,15 +2,15 @@ use crate::internal_prelude::*; /// A parsed stream ready for interpretation #[derive(Clone)] -pub(crate) struct InterpretationStream { - items: Vec, +pub(crate) struct SourceStream { + items: Vec, span: Span, } -impl ContextualParseFromSource for InterpretationStream { +impl ContextualParse for SourceStream { type Context = Span; - fn parse_from_source(input: SourceParseStream, span: Self::Context) -> ParseResult { + fn parse(input: ParseStream, span: Self::Context) -> ParseResult { let mut items = Vec::new(); while !input.is_empty() { items.push(input.parse()?); @@ -19,11 +19,11 @@ impl ContextualParseFromSource for InterpretationStream { } } -impl Interpret for InterpretationStream { +impl Interpret for SourceStream { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { for item in self.items { item.interpret_into(interpreter, output)?; @@ -32,7 +32,7 @@ impl Interpret for InterpretationStream { } } -impl HasSpan for InterpretationStream { +impl HasSpan for SourceStream { fn span(&self) -> Span { self.span } @@ -40,20 +40,20 @@ impl HasSpan for InterpretationStream { /// A parsed group ready for interpretation #[derive(Clone)] -pub(crate) struct InterpretationGroup { +pub(crate) struct SourceGroup { source_delimiter: Delimiter, source_delim_span: DelimSpan, - content: InterpretationStream, + content: SourceStream, } -impl InterpretationGroup { - pub(crate) fn into_content(self) -> InterpretationStream { +impl SourceGroup { + pub(crate) fn into_content(self) -> SourceStream { self.content } } -impl ParseFromSource for InterpretationGroup { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for SourceGroup { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_group()?; let content = content.parse_with_context(delim_span.join())?; Ok(Self { @@ -64,11 +64,11 @@ impl ParseFromSource for InterpretationGroup { } } -impl Interpret for InterpretationGroup { +impl Interpret for SourceGroup { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { let inner = self.content.interpret_to_new_stream(interpreter)?; output.push_new_group(inner, self.source_delimiter, self.source_delim_span.join()); @@ -76,7 +76,7 @@ impl Interpret for InterpretationGroup { } } -impl HasSpan for InterpretationGroup { +impl HasSpan for SourceGroup { fn span(&self) -> Span { self.source_delim_span.join() } @@ -97,8 +97,8 @@ impl RawGroup { } } -impl ParseFromSource for RawGroup { - fn parse_from_source(input: SourceParseStream) -> ParseResult { +impl Parse for RawGroup { + fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_group()?; let content = content.parse()?; Ok(Self { @@ -110,13 +110,9 @@ impl ParseFromSource for RawGroup { } impl Interpret for RawGroup { - fn interpret_into( - self, - _: &mut Interpreter, - output: &mut InterpretedStream, - ) -> ExecutionResult<()> { + fn interpret_into(self, _: &mut Interpreter, output: &mut OutputStream) -> ExecutionResult<()> { output.push_new_group( - InterpretedStream::raw(self.content), + OutputStream::raw(self.content), self.source_delimeter, self.source_delim_span.join(), ); diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 4293cd2f..5a400806 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -1,7 +1,7 @@ use crate::internal_prelude::*; #[derive(Clone)] -pub(crate) struct InterpretedStream { +pub(crate) struct OutputStream { /// Currently, even ~inside~ macro executions, round-tripping [`Delimiter::None`] groups to a TokenStream /// breaks in rust-analyzer. This causes various spurious errors in the IDE. /// See: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 @@ -10,7 +10,7 @@ pub(crate) struct InterpretedStream { /// This ensures any non-delimited groups are not round-tripped to a TokenStream. /// /// In future, we may wish to internally use some kind of `TokenBuffer` type, which I've started on below. - segments: Vec, + segments: Vec, token_length: usize, } @@ -18,25 +18,25 @@ pub(crate) struct InterpretedStream { // This was primarily implemented to avoid this issue: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 // But it doesn't actually help because the `syn::parse` mechanism only operates on a TokenStream, // so we have to convert back into a TokenStream. -enum InterpretedSegment { +enum OutputSegment { TokenVec(Vec), // Cheaper than a TokenStream (probably) - InterpretedGroup(Delimiter, Span, InterpretedStream), + OutputGroup(Delimiter, Span, OutputStream), } -pub(crate) enum InterpretedTokenTree { +pub(crate) enum OutputTokenTree { TokenTree(TokenTree), - InterpretedGroup(Delimiter, Span, InterpretedStream), + OutputGroup(Delimiter, Span, OutputStream), } -impl From for InterpretedStream { - fn from(value: InterpretedTokenTree) -> Self { +impl From for OutputStream { + fn from(value: OutputTokenTree) -> Self { let mut new = Self::new(); new.push_interpreted_item(value); new } } -impl InterpretedStream { +impl OutputStream { pub(crate) fn new() -> Self { Self { segments: vec![], @@ -76,24 +76,21 @@ impl InterpretedStream { pub(crate) fn push_new_group( &mut self, - inner_tokens: InterpretedStream, + inner_tokens: OutputStream, delimiter: Delimiter, span: Span, ) { - self.segments.push(InterpretedSegment::InterpretedGroup( - delimiter, - span, - inner_tokens, - )); + self.segments + .push(OutputSegment::OutputGroup(delimiter, span, inner_tokens)); self.token_length += 1; } - pub(crate) fn push_interpreted_item(&mut self, segment_item: InterpretedTokenTree) { + pub(crate) fn push_interpreted_item(&mut self, segment_item: OutputTokenTree) { match segment_item { - InterpretedTokenTree::TokenTree(token_tree) => { + OutputTokenTree::TokenTree(token_tree) => { self.push_raw_token_tree(token_tree); } - InterpretedTokenTree::InterpretedGroup(delimiter, span, inner_tokens) => { + OutputTokenTree::OutputGroup(delimiter, span, inner_tokens) => { self.push_new_group(inner_tokens, delimiter, span); } } @@ -104,12 +101,12 @@ impl InterpretedStream { } pub(crate) fn extend_raw_tokens(&mut self, tokens: impl IntoIterator) { - if !matches!(self.segments.last(), Some(InterpretedSegment::TokenVec(_))) { - self.segments.push(InterpretedSegment::TokenVec(vec![])); + if !matches!(self.segments.last(), Some(OutputSegment::TokenVec(_))) { + self.segments.push(OutputSegment::TokenVec(vec![])); } match self.segments.last_mut() { - Some(InterpretedSegment::TokenVec(token_vec)) => { + Some(OutputSegment::TokenVec(token_vec)) => { let before_length = token_vec.len(); token_vec.extend(tokens); self.token_length += token_vec.len() - before_length; @@ -132,25 +129,25 @@ impl InterpretedStream { /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. pub(crate) unsafe fn parse_with>( self, - parser: impl FnOnce(InterpretedParseStream) -> Result, + parser: impl FnOnce(ParseStream) -> Result, ) -> Result { - self.into_token_stream().parse_with(parser) + self.into_token_stream().interpreted_parse_with(parser) } /// WARNING: With rust-analyzer, this loses transparent groups which have been inserted. /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 /// /// Annotate usages with // RUST-ANALYZER SAFETY: ... to explain why the use of this function is OK. - pub(crate) unsafe fn parse_as(self) -> ParseResult { - self.into_token_stream().parse_with(T::parse) + pub(crate) unsafe fn parse_as>(self) -> ParseResult { + self.into_token_stream().interpreted_parse_with(T::parse) } - pub(crate) fn append_into(self, output: &mut InterpretedStream) { + pub(crate) fn append_into(self, output: &mut OutputStream) { output.segments.extend(self.segments); output.token_length += self.token_length; } - pub(crate) fn append_cloned_into(&self, output: &mut InterpretedStream) { + pub(crate) fn append_cloned_into(&self, output: &mut OutputStream) { self.clone().append_into(output); } @@ -167,10 +164,10 @@ impl InterpretedStream { unsafe fn append_to_token_stream(self, output: &mut TokenStream) { for segment in self.segments { match segment { - InterpretedSegment::TokenVec(vec) => { + OutputSegment::TokenVec(vec) => { output.extend(vec); } - InterpretedSegment::InterpretedGroup(delimiter, span, inner) => { + OutputSegment::OutputGroup(delimiter, span, inner) => { output.extend(iter::once(TokenTree::Group( Group::new(delimiter, inner.into_token_stream()).with_span(span), ))) @@ -188,7 +185,7 @@ impl InterpretedStream { fn append_to_token_stream_without_transparent_groups(self, output: &mut TokenStream) { for segment in self.segments { match segment { - InterpretedSegment::TokenVec(vec) => { + OutputSegment::TokenVec(vec) => { for token in vec { match token { TokenTree::Group(group) if group.delimiter() == Delimiter::None => { @@ -198,7 +195,7 @@ impl InterpretedStream { } } } - InterpretedSegment::InterpretedGroup(delimiter, span, interpreted_stream) => { + OutputSegment::OutputGroup(delimiter, span, interpreted_stream) => { if delimiter == Delimiter::None { interpreted_stream .append_to_token_stream_without_transparent_groups(output); @@ -219,16 +216,16 @@ impl InterpretedStream { self, check_group: impl FnOnce(Delimiter) -> bool, create_error: impl FnOnce() -> SynError, - ) -> ParseResult { + ) -> ParseResult { let mut item_vec = self.into_item_vec(); if item_vec.len() == 1 { match item_vec.pop().unwrap() { - InterpretedTokenTree::InterpretedGroup(delimiter, _, inner) => { + OutputTokenTree::OutputGroup(delimiter, _, inner) => { if check_group(delimiter) { return Ok(inner); } } - InterpretedTokenTree::TokenTree(TokenTree::Group(group)) => { + OutputTokenTree::TokenTree(TokenTree::Group(group)) => { if check_group(group.delimiter()) { return Ok(Self::raw(group.stream())); } @@ -239,15 +236,15 @@ impl InterpretedStream { Err(create_error().into()) } - pub(crate) fn into_item_vec(self) -> Vec { + pub(crate) fn into_item_vec(self) -> Vec { let mut output = Vec::with_capacity(self.token_length); for segment in self.segments { match segment { - InterpretedSegment::TokenVec(vec) => { - output.extend(vec.into_iter().map(InterpretedTokenTree::TokenTree)); + OutputSegment::TokenVec(vec) => { + output.extend(vec.into_iter().map(OutputTokenTree::TokenTree)); } - InterpretedSegment::InterpretedGroup(delimiter, span, interpreted_stream) => { - output.push(InterpretedTokenTree::InterpretedGroup( + OutputSegment::OutputGroup(delimiter, span, interpreted_stream) => { + output.push(OutputTokenTree::OutputGroup( delimiter, span, interpreted_stream, @@ -262,10 +259,10 @@ impl InterpretedStream { let mut output = RawDestructureStream::empty(); for segment in self.segments { match segment { - InterpretedSegment::TokenVec(vec) => { + OutputSegment::TokenVec(vec) => { output.append_from_token_stream(vec); } - InterpretedSegment::InterpretedGroup(delimiter, _, inner) => { + OutputSegment::OutputGroup(delimiter, _, inner) => { output.push_item(RawDestructureItem::Group(RawDestructureGroup::new( delimiter, inner.into_raw_destructure_stream(), @@ -281,15 +278,15 @@ impl InterpretedStream { behaviour: &ConcatBehaviour, output: &mut String, prefix_spacing: Spacing, - stream: InterpretedStream, + stream: OutputStream, ) { let mut spacing = prefix_spacing; for segment in stream.segments { spacing = match segment { - InterpretedSegment::TokenVec(vec) => { + OutputSegment::TokenVec(vec) => { concat_recursive_token_stream(behaviour, output, spacing, vec) } - InterpretedSegment::InterpretedGroup(delimiter, _, interpreted_stream) => { + OutputSegment::OutputGroup(delimiter, _, interpreted_stream) => { behaviour.before_token_tree(output, spacing); behaviour.wrap_delimiters( output, @@ -360,9 +357,9 @@ impl InterpretedStream { } } -impl IntoIterator for InterpretedStream { - type IntoIter = std::vec::IntoIter; - type Item = InterpretedTokenTree; +impl IntoIterator for OutputStream { + type IntoIter = std::vec::IntoIter; + type Item = OutputTokenTree; fn into_iter(self) -> Self::IntoIter { self.into_item_vec().into_iter() @@ -453,9 +450,9 @@ impl ConcatBehaviour { } } -impl From for InterpretedStream { +impl From for OutputStream { fn from(value: TokenTree) -> Self { - InterpretedStream::raw(value.into()) + OutputStream::raw(value.into()) } } @@ -470,7 +467,7 @@ impl From for InterpretedStream { // There are a few places where we support (or might wish to support) parsing // as part of interpretation: // * e.g. of a token stream in `CommandValueInput` -// * e.g. as part of a PARSER, from an InterpretedStream +// * e.g. as part of a PARSER, from an OutputStream // * e.g. of a variable, as part of incremental parsing (while_parse style loops) // // I spent quite a while considering whether this could be wrapping a @@ -500,7 +497,7 @@ impl From for InterpretedStream { // converting a Cursor into an indexed based cursor, which can safely be stored separately // from the TokenBuffer. // -// We could use this abstraction for InterpretedStream; and our variables could store a +// We could use this abstraction for OutputStream; and our variables could store a // tuple of (IndexCursor, PreinterpretTokenBuffer) /// Inspired/ forked from [`syn::buffer::TokenBuffer`], in order to support appending tokens, diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 309a9317..700657f8 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -12,11 +12,11 @@ pub(crate) struct Interpreter { #[derive(Clone)] pub(crate) struct VariableData { - value: Rc>, + value: Rc>, } impl VariableData { - fn new(tokens: InterpretedStream) -> Self { + fn new(tokens: OutputStream) -> Self { Self { value: Rc::new(RefCell::new(tokens)), } @@ -25,7 +25,7 @@ impl VariableData { pub(crate) fn get<'d>( &'d self, variable: &impl IsVariable, - ) -> ExecutionResult> { + ) -> ExecutionResult> { self.value.try_borrow().map_err(|_| { variable .error("The variable cannot be read if it is currently being modified") @@ -36,7 +36,7 @@ impl VariableData { pub(crate) fn get_mut<'d>( &'d self, variable: &impl IsVariable, - ) -> ExecutionResult> { + ) -> ExecutionResult> { self.value.try_borrow_mut().map_err(|_| { variable.execution_error( "The variable cannot be modified if it is already currently being modified", @@ -47,7 +47,7 @@ impl VariableData { pub(crate) fn set( &self, variable: &impl IsVariable, - content: InterpretedStream, + content: OutputStream, ) -> ExecutionResult<()> { *self.get_mut(variable)? = content; Ok(()) @@ -71,7 +71,7 @@ impl Interpreter { pub(crate) fn set_variable( &mut self, variable: &impl IsVariable, - tokens: InterpretedStream, + tokens: OutputStream, ) -> ExecutionResult<()> { match self.variable_data.entry(variable.get_name()) { Entry::Occupied(mut entry) => { diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index baad40ec..d8199a69 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -10,9 +10,9 @@ pub(crate) struct GroupedVariable { variable_name: Ident, } -impl ParseFromSource for GroupedVariable { - fn parse_from_source(input: SourceParseStream) -> ParseResult { - input.try_parse_or_message( +impl Parse for GroupedVariable { + fn parse(input: ParseStream) -> ParseResult { + input.try_parse_or_error( |input| { Ok(Self { marker: input.parse()?, @@ -28,7 +28,7 @@ impl GroupedVariable { pub(crate) fn set( &self, interpreter: &mut Interpreter, - value: InterpretedStream, + value: OutputStream, ) -> ExecutionResult<()> { interpreter.set_variable(self, value) } @@ -47,7 +47,7 @@ impl GroupedVariable { pub(crate) fn substitute_ungrouped_contents_into( &self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? @@ -58,7 +58,7 @@ impl GroupedVariable { pub(crate) fn substitute_grouped_into( &self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { output.push_new_group( self.read_existing(interpreter)?.get(self)?.clone(), @@ -90,7 +90,7 @@ impl Interpret for &GroupedVariable { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { self.substitute_grouped_into(interpreter, output) } @@ -122,9 +122,9 @@ pub(crate) struct FlattenedVariable { variable_name: Ident, } -impl ParseFromSource for FlattenedVariable { - fn parse_from_source(input: SourceParseStream) -> ParseResult { - input.try_parse_or_message( +impl Parse for FlattenedVariable { + fn parse(input: ParseStream) -> ParseResult { + input.try_parse_or_error( |input| { Ok(Self { marker: input.parse()?, @@ -141,7 +141,7 @@ impl FlattenedVariable { pub(crate) fn substitute_into( &self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? @@ -175,7 +175,7 @@ impl Interpret for &FlattenedVariable { fn interpret_into( self, interpreter: &mut Interpreter, - output: &mut InterpretedStream, + output: &mut OutputStream, ) -> ExecutionResult<()> { self.substitute_into(interpreter, output) } diff --git a/src/lib.rs b/src/lib.rs index 51f0a254..a1db817c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -547,7 +547,7 @@ fn preinterpret_internal(input: TokenStream) -> SynResult { let mut interpreter = Interpreter::new(); let interpretation_stream = input - .parse_with(|input| InterpretationStream::parse_from_source(input, Span::call_site())) + .source_parse_with(|input| SourceStream::parse(input, Span::call_site())) .convert_to_final_result()?; let interpreted_stream = interpretation_stream diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index c5a7bc8f..3cd2ca51 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -5,33 +5,9 @@ use crate::internal_prelude::*; // Parsing of source code tokens // ============================= -pub(crate) trait ContextualParseFromSource: Sized { - type Context; - - fn parse_from_source(input: SourceParseStream, context: Self::Context) -> ParseResult; -} - -pub(crate) trait ParseFromSource: Sized { - fn parse_from_source(input: SourceParseStream) -> ParseResult; -} - -impl ParseFromSource for T { - fn parse_from_source(input: SourceParseStream) -> ParseResult { - Ok(T::parse(&input.inner)?) - } -} - -impl Parse for T { - fn parse(input: SourceParseStream) -> ParseResult { - T::parse_from_source(input) - } -} - pub(crate) struct Source; -pub(crate) type SourceParseStream<'a> = &'a SourceParseBuffer<'a>; -pub(crate) type SourceParseBuffer<'a> = KindedParseBuffer<'a, Source>; -impl SourceParseBuffer<'_> { +impl ParseBuffer<'_, Source> { pub(crate) fn peek_grammar(&self) -> GrammarPeekMatch { detect_preinterpret_grammar(self.cursor()) } @@ -137,40 +113,22 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> GrammarPeekMatch // (e.g. destructuring) // ===================================== -pub(crate) trait ParseFromInterpreted: Sized { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult; -} +pub(crate) struct Output; -impl ParseFromInterpreted for T { - fn parse_from_interpreted(input: InterpretedParseStream) -> ParseResult { - Ok(T::parse(&input.inner)?) - } -} - -impl Parse for T { - fn parse(input: InterpretedParseStream) -> ParseResult { - T::parse_from_interpreted(input) - } -} - -pub(crate) struct Interpreted; -pub(crate) type InterpretedParseStream<'a> = &'a InterpretedParseBuffer<'a>; -pub(crate) type InterpretedParseBuffer<'a> = KindedParseBuffer<'a, Interpreted>; - -impl InterpretedParseBuffer<'_> { - pub(crate) fn peek_token(&self) -> InterpretedPeekMatch { +impl ParseBuffer<'_, Output> { + pub(crate) fn peek_grammar(&self) -> OutputPeekMatch { match self.cursor().token_tree() { - Some((TokenTree::Ident(ident), _)) => InterpretedPeekMatch::Ident(ident), - Some((TokenTree::Punct(punct), _)) => InterpretedPeekMatch::Punct(punct), - Some((TokenTree::Literal(literal), _)) => InterpretedPeekMatch::Literal(literal), - Some((TokenTree::Group(group), _)) => InterpretedPeekMatch::Group(group.delimiter()), - None => InterpretedPeekMatch::End, + Some((TokenTree::Ident(ident), _)) => OutputPeekMatch::Ident(ident), + Some((TokenTree::Punct(punct), _)) => OutputPeekMatch::Punct(punct), + Some((TokenTree::Literal(literal), _)) => OutputPeekMatch::Literal(literal), + Some((TokenTree::Group(group), _)) => OutputPeekMatch::Group(group.delimiter()), + None => OutputPeekMatch::End, } } } #[allow(unused)] -pub(crate) enum InterpretedPeekMatch { +pub(crate) enum OutputPeekMatch { Group(Delimiter), Ident(Ident), Punct(Punct), @@ -182,20 +140,32 @@ pub(crate) enum InterpretedPeekMatch { // =============== pub(crate) trait Parse: Sized { - fn parse(input: KindedParseStream) -> ParseResult; + fn parse(input: ParseStream) -> ParseResult; } -pub(crate) type KindedParseStream<'a, K> = &'a KindedParseBuffer<'a, K>; +pub(crate) trait ContextualParse: Sized { + type Context; + + fn parse(input: ParseStream, context: Self::Context) -> ParseResult; +} + +impl Parse for T { + fn parse(input: ParseStream) -> ParseResult { + Ok(T::parse(&input.inner)?) + } +} + +pub(crate) type ParseStream<'a, K> = &'a ParseBuffer<'a, K>; // We create our own ParseBuffer mostly so we can overwrite // parse to return ParseResult instead of syn::Result #[repr(transparent)] -pub(crate) struct KindedParseBuffer<'a, K> { +pub(crate) struct ParseBuffer<'a, K> { inner: SynParseBuffer<'a>, _kind: PhantomData, } -impl<'a, K> From> for KindedParseBuffer<'a, K> { +impl<'a, K> From> for ParseBuffer<'a, K> { fn from(inner: syn::parse::ParseBuffer<'a>) -> Self { Self { inner, @@ -205,20 +175,20 @@ impl<'a, K> From> for KindedParseBuffer<'a, K> { } // This is From<&'a SynParseBuffer<'a>> for &'a ParseBuffer<'a> -impl<'a, K> From> for KindedParseStream<'a, K> { +impl<'a, K> From> for ParseStream<'a, K> { fn from(syn_parse_stream: SynParseStream<'a>) -> Self { unsafe { // SAFETY: This is safe because [Syn]ParseStream<'a> = &'a [Syn]ParseBuffer<'a> // And ParseBuffer<'a> is marked as #[repr(transparent)] so has identical layout to SynParseBuffer<'a> // So this is a transmute between compound types with identical layouts which is safe. - core::mem::transmute::, KindedParseStream<'a, K>>(syn_parse_stream) + core::mem::transmute::, ParseStream<'a, K>>(syn_parse_stream) } } } -impl<'a, K> KindedParseBuffer<'a, K> { - pub(crate) fn fork(&self) -> KindedParseBuffer<'a, K> { - KindedParseBuffer { +impl<'a, K> ParseBuffer<'a, K> { + pub(crate) fn fork(&self) -> ParseBuffer<'a, K> { + ParseBuffer { inner: self.inner.fork(), _kind: PhantomData, } @@ -228,6 +198,26 @@ impl<'a, K> KindedParseBuffer<'a, K> { T::parse(self) } + pub(crate) fn parse_with_context>( + &self, + context: T::Context, + ) -> ParseResult { + T::parse(self, context) + } + + pub(crate) fn try_parse_or_error< + T, + F: FnOnce(&Self) -> ParseResult, + M: std::fmt::Display, + >( + &self, + parse: F, + message: M, + ) -> ParseResult { + let error_span = self.span(); + parse(self).map_err(|_| error_span.error(message).into()) + } + pub(crate) fn parse_any_punct(&self) -> ParseResult { // Annoyingly, ' behaves weirdly in syn, so we need to handle it match self.inner.parse::()? { @@ -235,9 +225,92 @@ impl<'a, K> KindedParseBuffer<'a, K> { _ => self.span().parse_err("expected punctuation"), } } + + pub(crate) fn parse_any_ident(&self) -> ParseResult { + Ok(self.call(Ident::parse_any)?) + } + + pub(crate) fn peek_ident_matching(&self, content: &str) -> bool { + self.cursor().ident_matching(content).is_some() + } + + pub(crate) fn parse_ident_matching(&self, content: &str) -> ParseResult { + Ok(self.step(|cursor| { + cursor + .ident_matching(content) + .ok_or_else(|| cursor.span().error(format!("expected {}", content))) + })?) + } + + pub(crate) fn peek_punct_matching(&self, punct: char) -> bool { + self.cursor().punct_matching(punct).is_some() + } + + pub(crate) fn parse_punct_matching(&self, punct: char) -> ParseResult { + Ok(self.step(|cursor| { + cursor + .punct_matching(punct) + .ok_or_else(|| cursor.span().error(format!("expected {}", punct))) + })?) + } + + pub(crate) fn peek_literal_matching(&self, content: &str) -> bool { + self.cursor().literal_matching(content).is_some() + } + + pub(crate) fn parse_literal_matching(&self, content: &str) -> ParseResult { + Ok(self.step(|cursor| { + cursor + .literal_matching(content) + .ok_or_else(|| cursor.span().error(format!("expected {}", content))) + })?) + } + + pub(crate) fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)> { + use syn::parse::discouraged::AnyDelimiter; + let (delimiter, delim_span, parse_buffer) = self.parse_any_delimiter()?; + Ok((delimiter, delim_span, parse_buffer.into())) + } + + pub(crate) fn peek_specific_group(&self, delimiter: Delimiter) -> bool { + self.cursor().group_matching(delimiter).is_some() + } + + pub(crate) fn parse_group_matching( + &self, + matching: impl FnOnce(Delimiter) -> bool, + expected_message: impl FnOnce() -> String, + ) -> ParseResult<(DelimSpan, ParseBuffer)> { + let error_span = match self.parse_any_group() { + Ok((delimiter, delim_span, inner)) if matching(delimiter) => { + return Ok((delim_span, inner)); + } + Ok((_, delim_span, _)) => delim_span.open(), + Err(error) => error.span(), + }; + error_span.parse_err(expected_message()) + } + + pub(crate) fn parse_specific_group( + &self, + expected_delimiter: Delimiter, + ) -> ParseResult<(DelimSpan, ParseBuffer)> { + self.parse_group_matching( + |delimiter| delimiter == expected_delimiter, + || format!("Expected {}", expected_delimiter.description_of_open()), + ) + } + + pub(crate) fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { + Err(self.parse_error(message)) + } + + pub(crate) fn parse_error(&self, message: impl std::fmt::Display) -> ParseError { + self.span().parse_error(message) + } } -impl<'a, K> Deref for KindedParseBuffer<'a, K> { +impl<'a, K> Deref for ParseBuffer<'a, K> { type Target = SynParseBuffer<'a>; fn deref(&self) -> &Self::Target { From 5432a9b6708a2d92b304975460125807af98b950 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 4 Feb 2025 17:12:06 +0000 Subject: [PATCH 074/126] test: Add some more code block tests --- CHANGELOG.md | 26 +++++++------------ .../code_blocks_are_not_reevaluated.rs | 9 +++++++ .../code_blocks_are_not_reevaluated.stderr | 5 ++++ tests/tokens.rs | 13 ++++++++++ 4 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs create mode 100644 tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index da3c81ea..c508aef2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,13 +80,12 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* Add test to disallow source stuff in re-evaluated `{ .. }` blocks and command outputs -* Add `[!reinterpret! ...]` command for an `eval` style command. -* Support `[!set! #x]` to be `[!set! #x =]`. +* Support `[!set! #x]` to be `[!set! #x =]`, and allow `[!set! _ = ]` to replace `[!void! ]`. * `[!is_set! #x]` * Support `'a'..'z'` in `[!range! 'a'..'z']` * Destructurers => Transformers * Implement pivot to transformers outputting things ... `@[#x = @IDENT]`... + * Scrap `[!let!]` in favour of `[!parse! #x as #(...)]` * `@TOKEN_TREE` * `@REST` * `@[UNTIL xxxx]` @@ -99,8 +98,6 @@ Destructuring performs parsing of a token stream. It supports: * `@[ANY { ... }]` (with `#..x` as a catch-all) like the [!match!] command but without arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` * Consider: - * Scrap `[!let!]` in favour of `[!parse! #x as #(...)]` - * Scrap `[!void! ...]` in favour of `[!set! _ = ...]` * Destructurer needs to have different syntax. It's too confusingly similar! * Final decision: `@[#x = @IDENT]` because destructurers output (SEE BELOW FOR MOST OF THE WORKING) * Some other ideas considered: @@ -125,6 +122,7 @@ Destructuring performs parsing of a token stream. It supports: * `[!..index! #x[0..3]]` and other things like `[ ..=3]` * `[!index! [Hello World][...]]` => NB: This isn't in the grammar because of the risk of `#x[0]` wanting to mean `my_arr[0]` rather than "index my variable". +* Add `[!reinterpret! ...]` command for an `eval` style command. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed @@ -218,20 +216,16 @@ Destructuring performs parsing of a token stream. It supports: [!parse_for! #input as @(impl @[#x = @IDENT] for @[#y = @IDENT]),+ { }] - -// OUTSTANDING QUESTION: -// => Token streams are a good stand-in for arrays -// => Do we need a good stand-in for fields / key-value maps and other structured data? -// => e.g. #x.hello = BLAH -// => Has a stream-representation as { hello: [!group! BLAH], } but is more performant, -// and can be destructured with `[@FIELDS { #hello }]` or read with `[!read! #x.hello] -// => And can transform output as #(.hello = @X) - Mixing/matching append output and field output will error -// => Debug impl is `[!fields! hello: [!group! BLAH],]` -// => Can be embedded into an output stream as a fields group -// => If necessary, can be converted to a stream and parsed back as `{ hello: [!group! BLAH], }` ``` * Pushed to 0.4: + * Map/Field style variable type, like a JS object: + * e.g. #x.hello = BLAH + * Has a stream-representation as `{ hello: [!group! BLAH], }` but is more performant, and can be destructured with `[@FIELDS { #hello }]` or read with `[!read! #x.hello]` or `[!read! #x["hello"]]` + * And can transform output as `#(.hello = @X)` (mixing/matching append output and field output will error) + * Debug impl is `[!fields! hello: [!group! BLAH],]` + * Can be embedded into an output stream as a fields group + * If necessary, can be converted to a stream and parsed back as `{ hello: [!group! BLAH], }` * Fork of syn to: * Fix issues in Rust Analyzer * Add support for a more general `TokenBuffer`, and ensure that Cursor can work in a backwards-compatible way with that buffer. Support: diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs new file mode 100644 index 00000000..c5c1bc94 --- /dev/null +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs @@ -0,0 +1,9 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + [!set! #value = 1] + [!set! #indirect = [!raw! #value]] + [!evaluate! { #indirect }] + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr new file mode 100644 index 00000000..19b4b2c0 --- /dev/null +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr @@ -0,0 +1,5 @@ +error: Expected ! or - + --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:6:35 + | +6 | [!set! #indirect = [!raw! #value]] + | ^ diff --git a/tests/tokens.rs b/tests/tokens.rs index 297d93b6..efb21089 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -381,6 +381,19 @@ fn test_split() { }, "[!group!] [!group! A] [!group! B] [!group! E] [!group!]" ); + // Code blocks are only evaluated once + // (i.e. no "unknown variable B is output in the below") + assert_preinterpret_eq!( + { + [!debug! [!..split! { + stream: { + [A [!raw! #] B [!raw! #] C] + }, + separator: [#], + }]] + }, + "[!group! A] [!group! B] [!group! C]" + ); } #[test] From 117812e789140f4554f9212d1c8d86138e2fa24f Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 4 Feb 2025 17:38:15 +0000 Subject: [PATCH 075/126] tweak: Merge !void! and !extend! into !set! --- CHANGELOG.md | 5 +- src/interpretation/command.rs | 2 - src/interpretation/commands/core_commands.rs | 158 ++++++++++-------- .../core/extend_flattened_variable.rs | 2 +- .../core/extend_flattened_variable.stderr | 8 +- .../core/extend_non_existing_variable.rs | 2 +- .../core/extend_non_existing_variable.stderr | 6 +- ...xtend_variable_and_then_extend_it_again.rs | 2 +- ...d_variable_and_then_extend_it_again.stderr | 6 +- .../core/extend_variable_and_then_read_it.rs | 2 +- .../extend_variable_and_then_read_it.stderr | 6 +- .../core/extend_variable_and_then_set_it.rs | 2 +- .../extend_variable_and_then_set_it.stderr | 6 +- .../core/set_flattened_variable.stderr | 2 +- tests/core.rs | 32 +++- tests/expressions.rs | 4 +- 16 files changed, 138 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c508aef2..6f8abae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ * Core commands: * `[!error! ...]` to output a compile error - * `[!extend! #x += ...]` to performantly add extra characters to the stream + * `[!set! #x += ...]` to performantly add extra characters to the stream * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. * `[!void! ...]` interprets its arguments but then ignores any outputs. It can be used inside destructurings. * Expression commands: @@ -68,7 +68,7 @@ Destructuring performs parsing of a token stream. It supports: * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) * `#..>>x` - Reads a stream, appends a group (can be read back with `!for! #y in #x { ... }`) * `#..>>..x` - Reads a stream, appends a stream -* Commands which don't output a value, like `[!set! ...]` or `[!extend! ...]` +* Commands which don't output a value, like `[!set! ...]` * Named destructurings: * `(!stream! ...)` (TODO - decide if this is a good name) * `(!ident! ...)` @@ -80,7 +80,6 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* Support `[!set! #x]` to be `[!set! #x =]`, and allow `[!set! _ = ]` to replace `[!void! ]`. * `[!is_set! #x]` * Support `'a'..'z'` in `[!range! 'a'..'z']` * Destructurers => Transformers diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 247261aa..334f020a 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -347,10 +347,8 @@ define_command_kind! { // Core Commands SetCommand, LetCommand, - ExtendCommand, RawCommand, IgnoreCommand, - VoidCommand, SettingsCommand, ErrorCommand, DebugCommand, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 82011841..3ead998d 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -2,10 +2,30 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct SetCommand { - variable: GroupedVariable, - #[allow(unused)] - equals: Token![=], - arguments: SourceStream, + arguments: SetArguments, +} + +#[allow(unused)] +#[derive(Clone)] +enum SetArguments { + SetVariable { + variable: GroupedVariable, + equals: Token![=], + content: SourceStream, + }, + ExtendVariable { + variable: GroupedVariable, + plus_equals: Token![+=], + content: SourceStream, + }, + SetVariablesEmpty { + variables: Vec, + }, + Discard { + discard: Token![_], + equals: Token![=], + content: SourceStream, + }, } impl CommandType for SetCommand { @@ -18,57 +38,73 @@ impl NoOutputCommandDefinition for SetCommand { fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { - Ok(Self { - variable: input.parse()?, - equals: input.parse()?, - arguments: input.parse_with_context(arguments.command_span())?, - }) + if input.peek(Token![_]) { + return Ok(SetCommand { + arguments: SetArguments::Discard { + discard: input.parse()?, + equals: input.parse()?, + content: input.parse_with_context(arguments.command_span())?, + }, + }); + } + let variable = input.parse()?; + if input.peek(Token![+=]) { + return Ok(SetCommand { + arguments: SetArguments::ExtendVariable { + variable, + plus_equals: input.parse()?, + content: input.parse_with_context(arguments.command_span())?, + }, + }) + } + if input.peek(Token![=]) { + return Ok(SetCommand { + arguments: SetArguments::SetVariable { + variable, + equals: input.parse()?, + content: input.parse_with_context(arguments.command_span())?, + }, + }) + } + let mut variables = vec![variable]; + loop { + if !input.is_empty() { + input.parse::()?; + } + if input.is_empty() { + return Ok(SetCommand { + arguments: SetArguments::SetVariablesEmpty { variables }, + }); + } + variables.push(input.parse()?); + } }, - "Expected [!set! #variable = ..]", + "Expected [!set! #var1 = ...] or [!set! #var1 += ...] or [!set! _ = ...] or [!set! #var1, #var2]", ) } fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; - self.variable.set(interpreter, result_tokens)?; - Ok(()) - } -} - -#[derive(Clone)] -pub(crate) struct ExtendCommand { - variable: GroupedVariable, - #[allow(unused)] - plus_equals: Token![+=], - arguments: SourceStream, -} - -impl CommandType for ExtendCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for ExtendCommand { - const COMMAND_NAME: &'static str = "extend"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - variable: input.parse()?, - plus_equals: input.parse()?, - arguments: input.parse_with_context(arguments.command_span())?, - }) + match self.arguments { + SetArguments::SetVariable { variable, content, .. } => { + let content = content.interpret_to_new_stream(interpreter)?; + variable.set(interpreter, content)?; }, - "Expected [!extend! #variable += ..]", - ) - } - - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let variable_data = self.variable.get_existing_for_mutation(interpreter)?; - self.arguments.interpret_into( - interpreter, - variable_data.get_mut(&self.variable)?.deref_mut(), - )?; + SetArguments::ExtendVariable { variable, content, .. } => { + let variable_data = variable.get_existing_for_mutation(interpreter)?; + content.interpret_into( + interpreter, + variable_data.get_mut(&variable)?.deref_mut(), + )?; + }, + SetArguments::SetVariablesEmpty { variables } => { + for variable in variables { + variable.set(interpreter, OutputStream::new())?; + } + }, + SetArguments::Discard { content, .. } => { + let _ = content.interpret_to_new_stream(interpreter)?; + }, + } Ok(()) } } @@ -122,30 +158,6 @@ impl NoOutputCommandDefinition for IgnoreCommand { } } -#[derive(Clone)] -pub(crate) struct VoidCommand { - inner: SourceStream, -} - -impl CommandType for VoidCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for VoidCommand { - const COMMAND_NAME: &'static str = "void"; - - fn parse(arguments: CommandArguments) -> ParseResult { - Ok(Self { - inner: arguments.parse_all_as_source()?, - }) - } - - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let _ = self.inner.interpret_to_new_stream(interpreter)?; - Ok(()) - } -} - #[derive(Clone)] pub(crate) struct SettingsCommand { inputs: SettingsInputs, diff --git a/tests/compilation_failures/core/extend_flattened_variable.rs b/tests/compilation_failures/core/extend_flattened_variable.rs index fff05516..b8004fe0 100644 --- a/tests/compilation_failures/core/extend_flattened_variable.rs +++ b/tests/compilation_failures/core/extend_flattened_variable.rs @@ -3,6 +3,6 @@ use preinterpret::*; fn main() { preinterpret! { [!set! #variable = 1] - [!extend! #..variable += 2] + [!set! #..variable += 2] } } \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_flattened_variable.stderr b/tests/compilation_failures/core/extend_flattened_variable.stderr index f3d07f43..28fcacf2 100644 --- a/tests/compilation_failures/core/extend_flattened_variable.stderr +++ b/tests/compilation_failures/core/extend_flattened_variable.stderr @@ -1,6 +1,6 @@ error: Expected #variable - Occurred whilst parsing [!extend! ...] - Expected [!extend! #variable += ..] - --> tests/compilation_failures/core/extend_flattened_variable.rs:6:19 + Occurred whilst parsing [!set! ...] - Expected [!set! #var1 = ...] or [!set! #var1 += ...] or [!set! _ = ...] or [!set! #var1, #var2] + --> tests/compilation_failures/core/extend_flattened_variable.rs:6:16 | -6 | [!extend! #..variable += 2] - | ^ +6 | [!set! #..variable += 2] + | ^ diff --git a/tests/compilation_failures/core/extend_non_existing_variable.rs b/tests/compilation_failures/core/extend_non_existing_variable.rs index 2f53b841..eafb285c 100644 --- a/tests/compilation_failures/core/extend_non_existing_variable.rs +++ b/tests/compilation_failures/core/extend_non_existing_variable.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { preinterpret! { - [!extend! #variable += 2] + [!set! #variable += 2] } } \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_non_existing_variable.stderr b/tests/compilation_failures/core/extend_non_existing_variable.stderr index 17c75804..a03f9546 100644 --- a/tests/compilation_failures/core/extend_non_existing_variable.stderr +++ b/tests/compilation_failures/core/extend_non_existing_variable.stderr @@ -1,5 +1,5 @@ error: The variable #variable wasn't already set - --> tests/compilation_failures/core/extend_non_existing_variable.rs:5:19 + --> tests/compilation_failures/core/extend_non_existing_variable.rs:5:16 | -5 | [!extend! #variable += 2] - | ^^^^^^^^^ +5 | [!set! #variable += 2] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs index 5ada7c6b..2019d567 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs +++ b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs @@ -3,6 +3,6 @@ use preinterpret::*; fn main() { preinterpret! { [!set! #variable = Hello] - [!extend! #variable += World [!extend! #variable += !]] + [!set! #variable += World [!set! #variable += !]] } } \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr index 0d5d52f0..05749f29 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr @@ -1,5 +1,5 @@ error: The variable cannot be modified if it is already currently being modified - --> tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs:6:48 + --> tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs:6:42 | -6 | [!extend! #variable += World [!extend! #variable += !]] - | ^^^^^^^^^ +6 | [!set! #variable += World [!set! #variable += !]] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/extend_variable_and_then_read_it.rs b/tests/compilation_failures/core/extend_variable_and_then_read_it.rs index 72881695..b016d73a 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_read_it.rs +++ b/tests/compilation_failures/core/extend_variable_and_then_read_it.rs @@ -3,6 +3,6 @@ use preinterpret::*; fn main() { preinterpret! { [!set! #variable = Hello] - [!extend! #variable += World #variable] + [!set! #variable += World #variable] } } \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr index 5217d807..8dd508e7 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr @@ -1,5 +1,5 @@ error: The variable cannot be read if it is currently being modified - --> tests/compilation_failures/core/extend_variable_and_then_read_it.rs:6:38 + --> tests/compilation_failures/core/extend_variable_and_then_read_it.rs:6:35 | -6 | [!extend! #variable += World #variable] - | ^^^^^^^^^ +6 | [!set! #variable += World #variable] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.rs b/tests/compilation_failures/core/extend_variable_and_then_set_it.rs index c9dc4472..8ca1f9b2 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_set_it.rs +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.rs @@ -3,6 +3,6 @@ use preinterpret::*; fn main() { preinterpret! { [!set! #variable = Hello] - [!extend! #variable += World [!set! #variable = Hello2]] + [!set! #variable += World [!set! #variable = Hello2]] } } \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr index f5f5475a..eba38ba1 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr @@ -1,5 +1,5 @@ error: The variable cannot be modified if it is already currently being modified - --> tests/compilation_failures/core/extend_variable_and_then_set_it.rs:6:45 + --> tests/compilation_failures/core/extend_variable_and_then_set_it.rs:6:42 | -6 | [!extend! #variable += World [!set! #variable = Hello2]] - | ^^^^^^^^^ +6 | [!set! #variable += World [!set! #variable = Hello2]] + | ^^^^^^^^^ diff --git a/tests/compilation_failures/core/set_flattened_variable.stderr b/tests/compilation_failures/core/set_flattened_variable.stderr index 07e7cb05..c26ff84e 100644 --- a/tests/compilation_failures/core/set_flattened_variable.stderr +++ b/tests/compilation_failures/core/set_flattened_variable.stderr @@ -1,5 +1,5 @@ error: Expected #variable - Occurred whilst parsing [!set! ...] - Expected [!set! #variable = ..] + Occurred whilst parsing [!set! ...] - Expected [!set! #var1 = ...] or [!set! #var1 += ...] or [!set! _ = ...] or [!set! #var1, #var2] --> tests/compilation_failures/core/set_flattened_variable.rs:5:16 | 5 | [!set! #..variable = 1] diff --git a/tests/core.rs b/tests/core.rs index fdc1c6a9..274127f6 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -41,7 +41,7 @@ fn test_extend() { assert_preinterpret_eq!( { [!set! #variable = "Hello"] - [!extend! #variable += " World!"] + [!set! #variable += " World!"] [!string! #variable] }, "Hello World!" @@ -51,9 +51,9 @@ fn test_extend() { [!set! #i = 1] [!set! #output = [!..group!]] [!while! (#i <= 4) { - [!extend! #output += #i] + [!set! #output += #i] [!if! (#i <= 3) { - [!extend! #output += ", "] + [!set! #output += ", "] }] [!assign! #i += 1] }] @@ -72,11 +72,33 @@ fn test_ignore() { }, false); } + +#[test] +fn test_empty_set() { + assert_preinterpret_eq!({ + [!set! #x] + [!set! #x += "hello"] + #x + }, "hello"); + assert_preinterpret_eq!({ + [!set! #x, #y] + [!set! #x += "hello"] + [!set! #y += "world"] + [!string! #x " " #y] + }, "hello world"); + assert_preinterpret_eq!({ + [!set! #x, #y, #z,] + [!set! #x += "hello"] + [!set! #y += "world"] + [!string! #x " " #y #z] + }, "hello world"); +} + #[test] -fn test_void() { +fn test_discard_set() { assert_preinterpret_eq!({ [!set! #x = false] - [!void! [!set! #x = true] things _are_ interpreted, but the result is ignored...] + [!set! _ = [!set! #x = true] things _are_ interpreted, but the result is ignored...] #x }, true); } diff --git a/tests/expressions.rs b/tests/expressions.rs index 861a09ef..2a69506b 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -116,7 +116,7 @@ fn boolean_operators_short_circuit() { assert_preinterpret_eq!( { [!set! #is_lazy = true] - [!void! [!evaluate! false && { [!set! #is_lazy = false] true }]] + [!set! _ = [!evaluate! false && { [!set! #is_lazy = false] true }]] #is_lazy }, true @@ -125,7 +125,7 @@ fn boolean_operators_short_circuit() { assert_preinterpret_eq!( { [!set! #is_lazy = true] - [!void! [!evaluate! true || { [!set! #is_lazy = false] true }]] + [!set! _ = [!evaluate! true || { [!set! #is_lazy = false] true }]] #is_lazy }, true From 56334d4a7999bed7bab97af40c5d25d42d07a0bc Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 4 Feb 2025 20:33:28 +0000 Subject: [PATCH 076/126] feature: !range! now supports characters --- CHANGELOG.md | 4 +- src/expressions/boolean.rs | 26 +- src/expressions/character.rs | 37 +- src/expressions/evaluation.rs | 24 +- src/expressions/expression.rs | 16 +- src/expressions/float.rs | 88 ++-- src/expressions/integer.rs | 424 ++++++++++-------- src/expressions/mod.rs | 1 + src/expressions/operations.rs | 48 +- src/expressions/string.rs | 24 +- src/expressions/value.rs | 255 ++++++----- src/interpretation/commands/core_commands.rs | 22 +- .../commands/expression_commands.rs | 111 ++--- tests/core.rs | 1 - tests/expressions.rs | 7 +- 15 files changed, 566 insertions(+), 522 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f8abae4..44360bfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,8 +80,6 @@ Destructuring performs parsing of a token stream. It supports: ### To come -* `[!is_set! #x]` -* Support `'a'..'z'` in `[!range! 'a'..'z']` * Destructurers => Transformers * Implement pivot to transformers outputting things ... `@[#x = @IDENT]`... * Scrap `[!let!]` in favour of `[!parse! #x as #(...)]` @@ -115,12 +113,14 @@ Destructuring performs parsing of a token stream. It supports: * `#(IDENT #x)` * `@[#x = @IDENT]` * Scrap `#>>x` etc in favour of `@[#x += ...]` +* `[!is_set! #x]` * Support `[!index! ..]`: * `[!index! #x[0]]` * `[!index! #x[0..3]]` * `[!..index! #x[0..3]]` and other things like `[ ..=3]` * `[!index! [Hello World][...]]` => NB: This isn't in the grammar because of the risk of `#x[0]` wanting to mean `my_arr[0]` rather than "index my variable". +* Have UntypedInteger have an inner representation of either i128 or literal (and same with float) * Add `[!reinterpret! ...]` command for an `eval` style command. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * TODO check diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 04306d2f..9c276448 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -1,14 +1,14 @@ use super::*; #[derive(Clone)] -pub(crate) struct EvaluationBoolean { +pub(crate) struct ExpressionBoolean { pub(super) value: bool, /// The span of the source code that generated this boolean value. /// It may not have a value if generated from a complex expression. pub(super) source_span: Option, } -impl EvaluationBoolean { +impl ExpressionBoolean { pub(super) fn for_litbool(lit: syn::LitBool) -> Self { Self { source_span: Some(lit.span()), @@ -21,8 +21,10 @@ impl EvaluationBoolean { operation: UnaryOperation, ) -> ExecutionResult { let input = self.value; - match operation { - UnaryOperation::Neg { .. } => operation.unsupported_for_value_type_err("boolean"), + Ok(match operation { + UnaryOperation::Neg { .. } => { + return operation.unsupported_for_value_type_err("boolean") + } UnaryOperation::Not { .. } => operation.output(!input), UnaryOperation::GroupedNoOp { .. } => operation.output(input), UnaryOperation::Cast { target, .. } => match target { @@ -42,16 +44,16 @@ impl EvaluationBoolean { CastTarget::Integer(IntegerKind::U128) => operation.output(input as u128), CastTarget::Integer(IntegerKind::Usize) => operation.output(input as usize), CastTarget::Float(_) | CastTarget::Char => { - operation.execution_err("This cast is not supported") + return operation.execution_err("This cast is not supported") } CastTarget::Boolean => operation.output(self.value), }, - } + }) } pub(super) fn handle_integer_binary_operation( self, - _right: EvaluationInteger, + _right: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { match operation { @@ -69,17 +71,17 @@ impl EvaluationBoolean { ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - match operation { + Ok(match operation { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } | PairedBinaryOperation::Division { .. } => { - operation.unsupported_for_value_type_err("boolean") + return operation.unsupported_for_value_type_err("boolean") } PairedBinaryOperation::LogicalAnd { .. } => operation.output(lhs && rhs), PairedBinaryOperation::LogicalOr { .. } => operation.output(lhs || rhs), PairedBinaryOperation::Remainder { .. } => { - operation.unsupported_for_value_type_err("boolean") + return operation.unsupported_for_value_type_err("boolean") } PairedBinaryOperation::BitXor { .. } => operation.output(lhs ^ rhs), PairedBinaryOperation::BitAnd { .. } => operation.output(lhs & rhs), @@ -90,7 +92,7 @@ impl EvaluationBoolean { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs & !rhs), - } + }) } pub(super) fn to_ident(&self, fallback_span: Span) -> Ident { @@ -100,7 +102,7 @@ impl EvaluationBoolean { impl ToExpressionValue for bool { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::Boolean(EvaluationBoolean { + ExpressionValue::Boolean(ExpressionBoolean { value: self, source_span, }) diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 6eca640e..3d5d38f1 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -1,14 +1,14 @@ use super::*; #[derive(Clone)] -pub(crate) struct EvaluationChar { +pub(crate) struct ExpressionChar { pub(super) value: char, /// The span of the source code that generated this boolean value. /// It may not have a value if generated from a complex expression. pub(super) source_span: Option, } -impl EvaluationChar { +impl ExpressionChar { pub(super) fn for_litchar(lit: syn::LitChar) -> Self { Self { value: lit.value(), @@ -21,10 +21,10 @@ impl EvaluationChar { operation: UnaryOperation, ) -> ExecutionResult { let char = self.value; - match operation { + Ok(match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(char), UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - operation.unsupported_for_value_type_err("char") + return operation.unsupported_for_value_type_err("char") } UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { @@ -44,15 +44,32 @@ impl EvaluationChar { CastTarget::Integer(IntegerKind::Usize) => operation.output(char as usize), CastTarget::Char => operation.output(char), CastTarget::Boolean | CastTarget::Float(_) => { - operation.unsupported_for_value_type_err("char") + return operation.unsupported_for_value_type_err("char") } }, + }) + } + + pub(super) fn create_range( + self, + right: Self, + range_limits: &syn::RangeLimits, + ) -> Box + '_> { + let left = self.value; + let right = right.value; + match range_limits { + syn::RangeLimits::HalfOpen { .. } => { + Box::new((left..right).map(|x| range_limits.output(x))) + } + syn::RangeLimits::Closed { .. } => { + Box::new((left..=right).map(|x| range_limits.output(x))) + } } } pub(super) fn handle_integer_binary_operation( self, - _right: EvaluationInteger, + _right: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { operation.unsupported_for_value_type_err("char") @@ -65,7 +82,7 @@ impl EvaluationChar { ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - match operation { + Ok(match operation { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } @@ -76,7 +93,7 @@ impl EvaluationChar { | PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } | PairedBinaryOperation::BitOr { .. } => { - operation.unsupported_for_value_type_err("char") + return operation.unsupported_for_value_type_err("char") } PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), @@ -84,7 +101,7 @@ impl EvaluationChar { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), - } + }) } pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { @@ -94,7 +111,7 @@ impl EvaluationChar { impl ToExpressionValue for char { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::Char(EvaluationChar { + ExpressionValue::Char(ExpressionChar { value: self, source_span, }) diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 79270bdf..ccfd052c 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -123,19 +123,19 @@ enum BinaryPath { OnRightBranch { left: ExpressionValue }, } -pub(crate) struct EvaluationOutput { +pub(crate) struct ExpressionOutput { pub(super) value: ExpressionValue, pub(super) fallback_output_span: Span, } -impl EvaluationOutput { +impl ExpressionOutput { #[allow(unused)] - pub(super) fn into_value(self) -> ExpressionValue { + pub(crate) fn into_value(self) -> ExpressionValue { self.value } #[allow(unused)] - pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { + pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { let error_span = self.span(); match self.value.into_integer() { Some(integer) => Ok(integer), @@ -143,18 +143,6 @@ impl EvaluationOutput { } } - pub(crate) fn try_into_i128(self, error_message: &str) -> ExecutionResult { - let error_span = self.span(); - match self - .value - .into_integer() - .and_then(|integer| integer.try_into_i128()) - { - Some(integer) => Ok(integer), - None => error_span.execution_err(error_message), - } - } - pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { let error_span = self.span(); match self.value.into_bool() { @@ -168,7 +156,7 @@ impl EvaluationOutput { } } -impl HasSpan for EvaluationOutput { +impl HasSpan for ExpressionOutput { fn span(&self) -> Span { self.value .source_span() @@ -176,7 +164,7 @@ impl HasSpan for EvaluationOutput { } } -impl quote::ToTokens for EvaluationOutput { +impl quote::ToTokens for ExpressionOutput { fn to_tokens(&self, tokens: &mut TokenStream) { self.to_token_tree().to_tokens(tokens); } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index c9878ebf..6fef7937 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -26,8 +26,8 @@ impl SourceExpression { pub(crate) fn evaluate( &self, interpreter: &mut Interpreter, - ) -> ExecutionResult { - Ok(EvaluationOutput { + ) -> ExecutionResult { + Ok(ExpressionOutput { value: self.evaluate_to_value(interpreter)?, fallback_output_span: self.inner.span_range.join_into_span_else_start(), }) @@ -37,8 +37,8 @@ impl SourceExpression { &self, interpreter: &mut Interpreter, fallback_output_span: Span, - ) -> ExecutionResult { - Ok(EvaluationOutput { + ) -> ExecutionResult { + Ok(ExpressionOutput { value: self.evaluate_to_value(interpreter)?, fallback_output_span, }) @@ -98,7 +98,7 @@ impl Expressionable for Source { UnaryAtom::PrefixUnaryOperation(input.parse()?) }, GrammarPeekMatch::Ident(_) => { - let value = ExpressionValue::Boolean(EvaluationBoolean::for_litbool(input.parse()?)); + let value = ExpressionValue::Boolean(ExpressionBoolean::for_litbool(input.parse()?)); UnaryAtom::Leaf(Self::Leaf::Value(value)) }, GrammarPeekMatch::Literal(_) => { @@ -165,8 +165,8 @@ impl Parse for OutputExpression { } impl OutputExpression { - pub(crate) fn evaluate(&self) -> ExecutionResult { - Ok(EvaluationOutput { + pub(crate) fn evaluate(&self) -> ExecutionResult { + Ok(ExpressionOutput { value: self.evaluate_to_value()?, fallback_output_span: self.inner.span_range.join_into_span_else_start(), }) @@ -206,7 +206,7 @@ impl Expressionable for Output { return input.parse_err("Square brackets [ .. ] are not supported in an expression") } OutputPeekMatch::Ident(_) => UnaryAtom::Leaf(ExpressionValue::Boolean( - EvaluationBoolean::for_litbool(input.parse()?), + ExpressionBoolean::for_litbool(input.parse()?), )), OutputPeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), OutputPeekMatch::Literal(_) => { diff --git a/src/expressions/float.rs b/src/expressions/float.rs index d254645c..db3eb2c9 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -2,18 +2,18 @@ use super::*; use crate::internal_prelude::*; #[derive(Clone)] -pub(crate) struct EvaluationFloat { - pub(super) value: EvaluationFloatValue, +pub(crate) struct ExpressionFloat { + pub(super) value: ExpressionFloatValue, /// The span of the source code that generated this boolean value. /// It may not have a value if generated from a complex expression. pub(super) source_span: Option, } -impl EvaluationFloat { +impl ExpressionFloat { pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { let source_span = Some(lit.span()); Ok(Self { - value: EvaluationFloatValue::for_litfloat(lit)?, + value: ExpressionFloatValue::for_litfloat(lit)?, source_span, }) } @@ -23,25 +23,25 @@ impl EvaluationFloat { operation: UnaryOperation, ) -> ExecutionResult { match self.value { - EvaluationFloatValue::Untyped(input) => input.handle_unary_operation(&operation), - EvaluationFloatValue::F32(input) => input.handle_unary_operation(&operation), - EvaluationFloatValue::F64(input) => input.handle_unary_operation(&operation), + ExpressionFloatValue::Untyped(input) => input.handle_unary_operation(&operation), + ExpressionFloatValue::F32(input) => input.handle_unary_operation(&operation), + ExpressionFloatValue::F64(input) => input.handle_unary_operation(&operation), } } pub(super) fn handle_integer_binary_operation( self, - right: EvaluationInteger, + right: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { match self.value { - EvaluationFloatValue::Untyped(input) => { + ExpressionFloatValue::Untyped(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationFloatValue::F32(input) => { + ExpressionFloatValue::F32(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationFloatValue::F64(input) => { + ExpressionFloatValue::F64(input) => { input.handle_integer_binary_operation(right, operation) } } @@ -54,13 +54,13 @@ impl EvaluationFloat { } } -pub(super) enum EvaluationFloatValuePair { +pub(super) enum ExpressionFloatValuePair { Untyped(UntypedFloat, UntypedFloat), F32(f32, f32), F64(f64, f64), } -impl EvaluationFloatValuePair { +impl ExpressionFloatValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &PairedBinaryOperation, @@ -74,13 +74,13 @@ impl EvaluationFloatValuePair { } #[derive(Clone)] -pub(super) enum EvaluationFloatValue { +pub(super) enum ExpressionFloatValue { Untyped(UntypedFloat), F32(f32), F64(f64), } -impl EvaluationFloatValue { +impl ExpressionFloatValue { pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { Ok(match lit.suffix() { "" => Self::Untyped(UntypedFloat::new_from_lit_float(lit)), @@ -96,17 +96,17 @@ impl EvaluationFloatValue { pub(super) fn describe_type(&self) -> &'static str { match self { - EvaluationFloatValue::Untyped(_) => "untyped float", - EvaluationFloatValue::F32(_) => "f32", - EvaluationFloatValue::F64(_) => "f64", + ExpressionFloatValue::Untyped(_) => "untyped float", + ExpressionFloatValue::F32(_) => "f32", + ExpressionFloatValue::F64(_) => "f64", } } fn to_unspanned_literal(&self) -> Literal { match self { - EvaluationFloatValue::Untyped(float) => float.to_unspanned_literal(), - EvaluationFloatValue::F32(float) => Literal::f32_suffixed(*float), - EvaluationFloatValue::F64(float) => Literal::f64_suffixed(*float), + ExpressionFloatValue::Untyped(float) => float.to_unspanned_literal(), + ExpressionFloatValue::F32(float) => Literal::f32_suffixed(*float), + ExpressionFloatValue::F64(float) => Literal::f64_suffixed(*float), } } } @@ -139,9 +139,11 @@ impl UntypedFloat { operation: &UnaryOperation, ) -> ExecutionResult { let input = self.parse_fallback()?; - match operation { + Ok(match operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), - UnaryOperation::Not { .. } => operation.unsupported_for_value_type_err("untyped float"), + UnaryOperation::Not { .. } => { + return operation.unsupported_for_value_type_err("untyped float") + } UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { @@ -165,15 +167,15 @@ impl UntypedFloat { CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input), CastTarget::Boolean | CastTarget::Char => { - operation.execution_err("This cast is not supported") + return operation.execution_err("This cast is not supported") } }, - } + }) } pub(super) fn handle_integer_binary_operation( self, - _rhs: EvaluationInteger, + _rhs: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { match operation { @@ -191,7 +193,7 @@ impl UntypedFloat { ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; - match operation { + Ok(match operation { PairedBinaryOperation::Addition { .. } => { operation.output(Self::from_fallback(lhs + rhs)) } @@ -205,7 +207,7 @@ impl UntypedFloat { operation.output(Self::from_fallback(lhs / rhs)) } PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { - operation.unsupported_for_value_type_err(stringify!($float_type)) + return operation.unsupported_for_value_type_err(stringify!($float_type)) } PairedBinaryOperation::Remainder { .. } => { operation.output(Self::from_fallback(lhs % rhs)) @@ -213,7 +215,7 @@ impl UntypedFloat { PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } | PairedBinaryOperation::BitOr { .. } => { - operation.unsupported_for_value_type_err("untyped float") + return operation.unsupported_for_value_type_err("untyped float") } PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), @@ -221,7 +223,7 @@ impl UntypedFloat { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), - } + }) } pub(super) fn from_fallback(value: FallbackFloat) -> Self { @@ -259,8 +261,8 @@ impl UntypedFloat { impl ToExpressionValue for UntypedFloat { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::Float(EvaluationFloat { - value: EvaluationFloatValue::Untyped(self), + ExpressionValue::Float(ExpressionFloat { + value: ExpressionFloatValue::Untyped(self), source_span, }) } @@ -272,8 +274,8 @@ macro_rules! impl_float_operations { ) => {$( impl ToExpressionValue for $float_type { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::Float(EvaluationFloat { - value: EvaluationFloatValue::$float_enum_variant(self), + ExpressionValue::Float(ExpressionFloat { + value: ExpressionFloatValue::$float_enum_variant(self), source_span }) } @@ -281,9 +283,9 @@ macro_rules! impl_float_operations { impl HandleUnaryOperation for $float_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - match operation { + Ok(match operation { UnaryOperation::Neg { .. } => operation.output(-self), - UnaryOperation::Not { .. } => operation.unsupported_for_value_type_err(stringify!($float_type)), + UnaryOperation::Not { .. } => return operation.unsupported_for_value_type_err(stringify!($float_type)), UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), @@ -302,9 +304,9 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), - CastTarget::Boolean | CastTarget::Char => operation.execution_err("This cast is not supported"), + CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), } - } + }) } } @@ -314,20 +316,20 @@ macro_rules! impl_float_operations { // and instead falls back to NaN or infinity. In future we could // allow trapping on these codes, but for now this is good enough let lhs = self; - match operation { + Ok(match operation { PairedBinaryOperation::Addition { .. } => operation.output(lhs + rhs), PairedBinaryOperation::Subtraction { .. } => operation.output(lhs - rhs), PairedBinaryOperation::Multiplication { .. } => operation.output(lhs * rhs), PairedBinaryOperation::Division { .. } => operation.output(lhs / rhs), PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { - operation.unsupported_for_value_type_err(stringify!($float_type)) + return operation.unsupported_for_value_type_err(stringify!($float_type)) } PairedBinaryOperation::Remainder { .. } => operation.output(lhs % rhs), PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } | PairedBinaryOperation::BitOr { .. } => { - operation.unsupported_for_value_type_err(stringify!($float_type)) + return operation.unsupported_for_value_type_err(stringify!($float_type)) } PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), @@ -335,12 +337,12 @@ macro_rules! impl_float_operations { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), - } + }) } fn handle_integer_binary_operation( self, - _rhs: EvaluationInteger, + _rhs: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { match operation { diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index a2835647..ecf87884 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -1,104 +1,86 @@ use super::*; #[derive(Clone)] -pub(crate) struct EvaluationInteger { - pub(super) value: EvaluationIntegerValue, +pub(crate) struct ExpressionInteger { + pub(super) value: ExpressionIntegerValue, /// The span of the source code that generated this boolean value. /// It may not have a value if generated from a complex expression. pub(super) source_span: Option, } -impl EvaluationInteger { +impl ExpressionInteger { pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { let source_span = Some(lit.span()); Ok(Self { - value: EvaluationIntegerValue::for_litint(lit)?, + value: ExpressionIntegerValue::for_litint(lit)?, source_span, }) } - pub(crate) fn try_into_i128(self) -> Option { - match self.value { - EvaluationIntegerValue::Untyped(x) => x.parse_fallback().ok(), - EvaluationIntegerValue::U8(x) => Some(x.into()), - EvaluationIntegerValue::U16(x) => Some(x.into()), - EvaluationIntegerValue::U32(x) => Some(x.into()), - EvaluationIntegerValue::U64(x) => Some(x.into()), - EvaluationIntegerValue::U128(x) => x.try_into().ok(), - EvaluationIntegerValue::Usize(x) => x.try_into().ok(), - EvaluationIntegerValue::I8(x) => Some(x.into()), - EvaluationIntegerValue::I16(x) => Some(x.into()), - EvaluationIntegerValue::I32(x) => Some(x.into()), - EvaluationIntegerValue::I64(x) => Some(x.into()), - EvaluationIntegerValue::I128(x) => Some(x), - EvaluationIntegerValue::Isize(x) => x.try_into().ok(), - } - } - pub(super) fn handle_unary_operation( self, operation: UnaryOperation, ) -> ExecutionResult { match self.value { - EvaluationIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::U8(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::U16(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::U32(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::U64(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::U128(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::Usize(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::I8(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::I16(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::I32(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::I64(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::I128(input) => input.handle_unary_operation(&operation), - EvaluationIntegerValue::Isize(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::U8(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::U16(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::U32(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::U64(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::U128(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::Usize(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::I8(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::I16(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::I32(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::I64(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::I128(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::Isize(input) => input.handle_unary_operation(&operation), } } pub(super) fn handle_integer_binary_operation( self, - right: EvaluationInteger, + right: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { match self.value { - EvaluationIntegerValue::Untyped(input) => { + ExpressionIntegerValue::Untyped(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::U8(input) => { + ExpressionIntegerValue::U8(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::U16(input) => { + ExpressionIntegerValue::U16(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::U32(input) => { + ExpressionIntegerValue::U32(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::U64(input) => { + ExpressionIntegerValue::U64(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::U128(input) => { + ExpressionIntegerValue::U128(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::Usize(input) => { + ExpressionIntegerValue::Usize(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::I8(input) => { + ExpressionIntegerValue::I8(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::I16(input) => { + ExpressionIntegerValue::I16(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::I32(input) => { + ExpressionIntegerValue::I32(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::I64(input) => { + ExpressionIntegerValue::I64(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::I128(input) => { + ExpressionIntegerValue::I128(input) => { input.handle_integer_binary_operation(right, operation) } - EvaluationIntegerValue::Isize(input) => { + ExpressionIntegerValue::Isize(input) => { input.handle_integer_binary_operation(right, operation) } } @@ -111,7 +93,7 @@ impl EvaluationInteger { } } -pub(super) enum EvaluationIntegerValuePair { +pub(super) enum ExpressionIntegerValuePair { Untyped(UntypedInteger, UntypedInteger), U8(u8, u8), U16(u16, u16), @@ -127,7 +109,7 @@ pub(super) enum EvaluationIntegerValuePair { Isize(isize, isize), } -impl EvaluationIntegerValuePair { +impl ExpressionIntegerValuePair { pub(super) fn handle_paired_binary_operation( self, operation: &PairedBinaryOperation, @@ -148,6 +130,27 @@ impl EvaluationIntegerValuePair { Self::Isize(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } + + pub(crate) fn create_range( + self, + range_limits: &syn::RangeLimits, + ) -> ExecutionResult + '_>> { + Ok(match self { + Self::Untyped(lhs, rhs) => return lhs.create_range(rhs, range_limits), + Self::U8(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::U16(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::U32(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::U64(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::U128(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::Usize(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::I8(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::I16(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::I32(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::I64(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::I128(lhs, rhs) => lhs.create_range(rhs, range_limits), + Self::Isize(lhs, rhs) => lhs.create_range(rhs, range_limits), + }) + } } #[derive(Copy, Clone)] @@ -168,7 +171,7 @@ pub(super) enum IntegerKind { } #[derive(Clone)] -pub(super) enum EvaluationIntegerValue { +pub(super) enum ExpressionIntegerValue { Untyped(UntypedInteger), U8(u8), U16(u16), @@ -184,7 +187,7 @@ pub(super) enum EvaluationIntegerValue { Isize(isize), } -impl EvaluationIntegerValue { +impl ExpressionIntegerValue { pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { Ok(match lit.suffix() { "" => Self::Untyped(UntypedInteger::new_from_lit_int(lit)), @@ -210,37 +213,37 @@ impl EvaluationIntegerValue { pub(super) fn describe_type(&self) -> &'static str { match self { - EvaluationIntegerValue::Untyped(_) => "untyped integer", - EvaluationIntegerValue::U8(_) => "u8", - EvaluationIntegerValue::U16(_) => "u16", - EvaluationIntegerValue::U32(_) => "u32", - EvaluationIntegerValue::U64(_) => "u64", - EvaluationIntegerValue::U128(_) => "u128", - EvaluationIntegerValue::Usize(_) => "usize", - EvaluationIntegerValue::I8(_) => "i8", - EvaluationIntegerValue::I16(_) => "i16", - EvaluationIntegerValue::I32(_) => "i32", - EvaluationIntegerValue::I64(_) => "i64", - EvaluationIntegerValue::I128(_) => "i128", - EvaluationIntegerValue::Isize(_) => "isize", + ExpressionIntegerValue::Untyped(_) => "untyped integer", + ExpressionIntegerValue::U8(_) => "u8", + ExpressionIntegerValue::U16(_) => "u16", + ExpressionIntegerValue::U32(_) => "u32", + ExpressionIntegerValue::U64(_) => "u64", + ExpressionIntegerValue::U128(_) => "u128", + ExpressionIntegerValue::Usize(_) => "usize", + ExpressionIntegerValue::I8(_) => "i8", + ExpressionIntegerValue::I16(_) => "i16", + ExpressionIntegerValue::I32(_) => "i32", + ExpressionIntegerValue::I64(_) => "i64", + ExpressionIntegerValue::I128(_) => "i128", + ExpressionIntegerValue::Isize(_) => "isize", } } fn to_unspanned_literal(&self) -> Literal { match self { - EvaluationIntegerValue::Untyped(int) => int.to_unspanned_literal(), - EvaluationIntegerValue::U8(int) => Literal::u8_suffixed(*int), - EvaluationIntegerValue::U16(int) => Literal::u16_suffixed(*int), - EvaluationIntegerValue::U32(int) => Literal::u32_suffixed(*int), - EvaluationIntegerValue::U64(int) => Literal::u64_suffixed(*int), - EvaluationIntegerValue::U128(int) => Literal::u128_suffixed(*int), - EvaluationIntegerValue::Usize(int) => Literal::usize_suffixed(*int), - EvaluationIntegerValue::I8(int) => Literal::i8_suffixed(*int), - EvaluationIntegerValue::I16(int) => Literal::i16_suffixed(*int), - EvaluationIntegerValue::I32(int) => Literal::i32_suffixed(*int), - EvaluationIntegerValue::I64(int) => Literal::i64_suffixed(*int), - EvaluationIntegerValue::I128(int) => Literal::i128_suffixed(*int), - EvaluationIntegerValue::Isize(int) => Literal::isize_suffixed(*int), + ExpressionIntegerValue::Untyped(int) => int.to_unspanned_literal(), + ExpressionIntegerValue::U8(int) => Literal::u8_suffixed(*int), + ExpressionIntegerValue::U16(int) => Literal::u16_suffixed(*int), + ExpressionIntegerValue::U32(int) => Literal::u32_suffixed(*int), + ExpressionIntegerValue::U64(int) => Literal::u64_suffixed(*int), + ExpressionIntegerValue::U128(int) => Literal::u128_suffixed(*int), + ExpressionIntegerValue::Usize(int) => Literal::usize_suffixed(*int), + ExpressionIntegerValue::I8(int) => Literal::i8_suffixed(*int), + ExpressionIntegerValue::I16(int) => Literal::i16_suffixed(*int), + ExpressionIntegerValue::I32(int) => Literal::i32_suffixed(*int), + ExpressionIntegerValue::I64(int) => Literal::i64_suffixed(*int), + ExpressionIntegerValue::I128(int) => Literal::i128_suffixed(*int), + ExpressionIntegerValue::Isize(int) => Literal::isize_suffixed(*int), } } } @@ -266,10 +269,10 @@ impl UntypedInteger { operation: &UnaryOperation, ) -> ExecutionResult { let input = self.parse_fallback()?; - match operation { + Ok(match operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), UnaryOperation::Not { .. } => { - operation.unsupported_for_value_type_err("untyped integer") + return operation.unsupported_for_value_type_err("untyped integer") } UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Cast { target, .. } => match target { @@ -294,54 +297,54 @@ impl UntypedInteger { CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input as f64), CastTarget::Boolean | CastTarget::Char => { - operation.execution_err("This cast is not supported") + return operation.execution_err("This cast is not supported") } }, - } + }) } pub(super) fn handle_integer_binary_operation( self, - rhs: EvaluationInteger, + rhs: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { let lhs = self.parse_fallback()?; - match operation { + Ok(match operation { IntegerBinaryOperation::ShiftLeft { .. } => match rhs.value { - EvaluationIntegerValue::Untyped(rhs) => { + ExpressionIntegerValue::Untyped(rhs) => { operation.output(lhs << rhs.parse_fallback()?) } - EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U16(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U32(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U64(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U128(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::Usize(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I8(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I16(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I32(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I64(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U8(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U16(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U32(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U64(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U128(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::Usize(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I8(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I16(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I32(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I64(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I128(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::Isize(rhs) => operation.output(lhs << rhs), }, IntegerBinaryOperation::ShiftRight { .. } => match rhs.value { - EvaluationIntegerValue::Untyped(rhs) => { + ExpressionIntegerValue::Untyped(rhs) => { operation.output(lhs >> rhs.parse_fallback()?) } - EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U16(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U32(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U64(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U128(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I8(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I16(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I32(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I64(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I128(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U8(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U16(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U32(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U64(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U128(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I8(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I16(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I32(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I64(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I128(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), }, - } + }) } pub(super) fn handle_paired_binary_operation( @@ -359,30 +362,40 @@ impl UntypedInteger { rhs ) }; - match operation { - PairedBinaryOperation::Addition { .. } => operation.output_if_some( - lhs.checked_add(rhs).map(Self::from_fallback), - overflow_error, - ), - PairedBinaryOperation::Subtraction { .. } => operation.output_if_some( - lhs.checked_sub(rhs).map(Self::from_fallback), - overflow_error, - ), - PairedBinaryOperation::Multiplication { .. } => operation.output_if_some( - lhs.checked_mul(rhs).map(Self::from_fallback), - overflow_error, - ), - PairedBinaryOperation::Division { .. } => operation.output_if_some( - lhs.checked_div(rhs).map(Self::from_fallback), - overflow_error, - ), + Ok(match operation { + PairedBinaryOperation::Addition { .. } => { + return operation.output_if_some( + lhs.checked_add(rhs).map(Self::from_fallback), + overflow_error, + ) + } + PairedBinaryOperation::Subtraction { .. } => { + return operation.output_if_some( + lhs.checked_sub(rhs).map(Self::from_fallback), + overflow_error, + ) + } + PairedBinaryOperation::Multiplication { .. } => { + return operation.output_if_some( + lhs.checked_mul(rhs).map(Self::from_fallback), + overflow_error, + ) + } + PairedBinaryOperation::Division { .. } => { + return operation.output_if_some( + lhs.checked_div(rhs).map(Self::from_fallback), + overflow_error, + ) + } PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { - operation.unsupported_for_value_type_err("untyped integer") + return operation.unsupported_for_value_type_err("untyped integer"); + } + PairedBinaryOperation::Remainder { .. } => { + return operation.output_if_some( + lhs.checked_rem(rhs).map(Self::from_fallback), + overflow_error, + ) } - PairedBinaryOperation::Remainder { .. } => operation.output_if_some( - lhs.checked_rem(rhs).map(Self::from_fallback), - overflow_error, - ), PairedBinaryOperation::BitXor { .. } => { operation.output(Self::from_fallback(lhs ^ rhs)) } @@ -396,7 +409,24 @@ impl UntypedInteger { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), - } + }) + } + + pub(super) fn create_range( + self, + right: Self, + range_limits: &syn::RangeLimits, + ) -> ExecutionResult + '_>> { + let left = self.parse_fallback()?; + let right = right.parse_fallback()?; + Ok(match range_limits { + syn::RangeLimits::HalfOpen { .. } => { + Box::new((left..right).map(|x| range_limits.output(Self::from_fallback(x)))) + } + syn::RangeLimits::Closed { .. } => { + Box::new((left..=right).map(|x| range_limits.output(Self::from_fallback(x)))) + } + }) } pub(super) fn from_fallback(value: FallbackInteger) -> Self { @@ -434,8 +464,8 @@ impl UntypedInteger { impl ToExpressionValue for UntypedInteger { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::Integer(EvaluationInteger { - value: EvaluationIntegerValue::Untyped(self), + ExpressionValue::Integer(ExpressionInteger { + value: ExpressionIntegerValue::Untyped(self), source_span, }) } @@ -448,8 +478,8 @@ macro_rules! impl_int_operations_except_unary { ) => {$( impl ToExpressionValue for $integer_type { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::Integer(EvaluationInteger { - value: EvaluationIntegerValue::$integer_enum_variant(self), + ExpressionValue::Integer(ExpressionInteger { + value: ExpressionIntegerValue::$integer_enum_variant(self), source_span, }) } @@ -459,14 +489,14 @@ macro_rules! impl_int_operations_except_unary { fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { let lhs = self; let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.symbol(), rhs); - match operation { - PairedBinaryOperation::Addition { .. } => operation.output_if_some(lhs.checked_add(rhs), overflow_error), - PairedBinaryOperation::Subtraction { .. } => operation.output_if_some(lhs.checked_sub(rhs), overflow_error), - PairedBinaryOperation::Multiplication { .. } => operation.output_if_some(lhs.checked_mul(rhs), overflow_error), - PairedBinaryOperation::Division { .. } => operation.output_if_some(lhs.checked_div(rhs), overflow_error), + Ok(match operation { + PairedBinaryOperation::Addition { .. } => return operation.output_if_some(lhs.checked_add(rhs), overflow_error), + PairedBinaryOperation::Subtraction { .. } => return operation.output_if_some(lhs.checked_sub(rhs), overflow_error), + PairedBinaryOperation::Multiplication { .. } => return operation.output_if_some(lhs.checked_mul(rhs), overflow_error), + PairedBinaryOperation::Division { .. } => return operation.output_if_some(lhs.checked_div(rhs), overflow_error), PairedBinaryOperation::LogicalAnd { .. } - | PairedBinaryOperation::LogicalOr { .. } => operation.unsupported_for_value_type_err(stringify!($integer_type)), - PairedBinaryOperation::Remainder { .. } => operation.output_if_some(lhs.checked_rem(rhs), overflow_error), + | PairedBinaryOperation::LogicalOr { .. } => return operation.unsupported_for_value_type_err(stringify!($integer_type)), + PairedBinaryOperation::Remainder { .. } => return operation.output_if_some(lhs.checked_rem(rhs), overflow_error), PairedBinaryOperation::BitXor { .. } => operation.output(lhs ^ rhs), PairedBinaryOperation::BitAnd { .. } => operation.output(lhs & rhs), PairedBinaryOperation::BitOr { .. } => operation.output(lhs | rhs), @@ -476,50 +506,68 @@ macro_rules! impl_int_operations_except_unary { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), - } + }) } fn handle_integer_binary_operation( self, - rhs: EvaluationInteger, + rhs: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { let lhs = self; - match operation { + Ok(match operation { IntegerBinaryOperation::ShiftLeft { .. } => { match rhs.value { - EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), - EvaluationIntegerValue::U8(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U16(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U32(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U64(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::U128(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::Usize(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I8(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I16(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I32(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I64(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::I128(rhs) => operation.output(lhs << rhs), - EvaluationIntegerValue::Isize(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), + ExpressionIntegerValue::U8(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U16(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U32(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U64(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::U128(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::Usize(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I8(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I16(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I32(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I64(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::I128(rhs) => operation.output(lhs << rhs), + ExpressionIntegerValue::Isize(rhs) => operation.output(lhs << rhs), } }, IntegerBinaryOperation::ShiftRight { .. } => { match rhs.value { - EvaluationIntegerValue::Untyped(rhs) => operation.output(lhs >> rhs.parse_fallback()?), - EvaluationIntegerValue::U8(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U16(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U32(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U64(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::U128(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I8(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I16(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I32(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I64(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::I128(rhs) => operation.output(lhs >> rhs), - EvaluationIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::Untyped(rhs) => operation.output(lhs >> rhs.parse_fallback()?), + ExpressionIntegerValue::U8(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U16(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U32(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U64(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::U128(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::Usize(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I8(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I16(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I32(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I64(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::I128(rhs) => operation.output(lhs >> rhs), + ExpressionIntegerValue::Isize(rhs) => operation.output(lhs >> rhs), } }, + }) + } + } + + impl HandleCreateRange for $integer_type { + fn create_range( + self, + right: Self, + range_limits: &syn::RangeLimits, + ) -> Box + '_> { + let left = self; + match range_limits { + syn::RangeLimits::HalfOpen { .. } => { + Box::new((left..right).map(|x| range_limits.output(x))) + }, + syn::RangeLimits::Closed { .. } => { + Box::new((left..=right).map(|x| range_limits.output(x))) + } } } } @@ -530,11 +578,11 @@ macro_rules! impl_unsigned_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - match operation { + Ok(match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - operation.unsupported_for_value_type_err(stringify!($integer_type)) + return operation.unsupported_for_value_type_err(stringify!($integer_type)) }, UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), @@ -553,9 +601,9 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), - CastTarget::Boolean | CastTarget::Char => operation.execution_err("This cast is not supported"), + CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), } - } + }) } } )*}; @@ -565,11 +613,11 @@ macro_rules! impl_signed_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - match operation { + Ok(match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Neg { .. } => operation.output(-self), UnaryOperation::Not { .. } => { - operation.unsupported_for_value_type_err(stringify!($integer_type)) + return operation.unsupported_for_value_type_err(stringify!($integer_type)) }, UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), @@ -588,9 +636,9 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), - CastTarget::Boolean | CastTarget::Char => operation.execution_err("This cast is not supported"), + CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), } - } + }) } } )*}; @@ -601,10 +649,10 @@ impl HandleUnaryOperation for u8 { self, operation: &UnaryOperation, ) -> ExecutionResult { - match operation { + Ok(match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - operation.unsupported_for_value_type_err("u8") + return operation.unsupported_for_value_type_err("u8") } UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { @@ -628,9 +676,11 @@ impl HandleUnaryOperation for u8 { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Char => operation.output(self as char), - CastTarget::Boolean => operation.execution_err("This cast is not supported"), + CastTarget::Boolean => { + return operation.execution_err("This cast is not supported") + } }, - } + }) } } diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index f35e8f14..f9751222 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -22,3 +22,4 @@ use string::*; use value::*; pub(crate) use expression::*; +pub(crate) use value::*; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 9293efe0..e894c9fc 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -1,8 +1,8 @@ use super::*; pub(super) trait Operation: HasSpanRange { - fn output(&self, output_value: impl ToExpressionValue) -> ExecutionResult { - Ok(output_value.to_value(self.source_span_for_output())) + fn output(&self, output_value: impl ToExpressionValue) -> ExpressionValue { + output_value.to_value(self.source_span_for_output()) } fn output_if_some( @@ -11,7 +11,7 @@ pub(super) trait Operation: HasSpanRange { error_message: impl FnOnce() -> String, ) -> ExecutionResult { match output_value { - Some(output_value) => self.output(output_value), + Some(output_value) => Ok(self.output(output_value)), None => self.execution_err(error_message()), } } @@ -27,7 +27,10 @@ pub(super) trait Operation: HasSpanRange { ))) } - fn source_span_for_output(&self) -> Option; + fn source_span_for_output(&self) -> Option { + None + } + fn symbol(&self) -> &'static str; } @@ -315,10 +318,6 @@ impl HasSpanRange for BinaryOperation { } impl Operation for BinaryOperation { - fn source_span_for_output(&self) -> Option { - None - } - fn symbol(&self) -> &'static str { match self { BinaryOperation::Paired(paired) => paired.symbol(), @@ -348,10 +347,6 @@ pub(super) enum PairedBinaryOperation { } impl Operation for PairedBinaryOperation { - fn source_span_for_output(&self) -> Option { - None - } - fn symbol(&self) -> &'static str { match self { PairedBinaryOperation::Addition { .. } => "+", @@ -404,10 +399,6 @@ pub(super) enum IntegerBinaryOperation { } impl Operation for IntegerBinaryOperation { - fn source_span_for_output(&self) -> Option { - None - } - fn symbol(&self) -> &'static str { match self { IntegerBinaryOperation::ShiftLeft { .. } => "<<", @@ -434,7 +425,30 @@ pub(super) trait HandleBinaryOperation: Sized { fn handle_integer_binary_operation( self, - rhs: EvaluationInteger, + rhs: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult; } + +pub(super) trait HandleCreateRange: Sized { + fn create_range( + self, + right: Self, + range_limits: &syn::RangeLimits, + ) -> Box + '_>; +} + +impl Operation for syn::RangeLimits { + fn symbol(&self) -> &'static str { + match self { + syn::RangeLimits::HalfOpen(_) => "..", + syn::RangeLimits::Closed(_) => "..=", + } + } +} + +impl HasSpanRange for syn::RangeLimits { + fn span_range(&self) -> SpanRange { + self.span_range_from_iterating_over_all_tokens() + } +} diff --git a/src/expressions/string.rs b/src/expressions/string.rs index e1f72185..b09eb399 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -1,14 +1,14 @@ use super::*; #[derive(Clone)] -pub(crate) struct EvaluationString { +pub(crate) struct ExpressionString { pub(super) value: String, /// The span of the source code that generated this boolean value. /// It may not have a value if generated from a complex expression. pub(super) source_span: Option, } -impl EvaluationString { +impl ExpressionString { pub(super) fn for_litstr(lit: syn::LitStr) -> Self { Self { value: lit.value(), @@ -20,17 +20,19 @@ impl EvaluationString { self, operation: UnaryOperation, ) -> ExecutionResult { - match operation { + Ok(match operation { UnaryOperation::GroupedNoOp { .. } => operation.output(self.value), UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } - | UnaryOperation::Cast { .. } => operation.unsupported_for_value_type_err("string"), - } + | UnaryOperation::Cast { .. } => { + return operation.unsupported_for_value_type_err("string") + } + }) } pub(super) fn handle_integer_binary_operation( self, - _right: EvaluationInteger, + _right: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { operation.unsupported_for_value_type_err("string") @@ -43,7 +45,7 @@ impl EvaluationString { ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - match operation { + Ok(match operation { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } @@ -54,7 +56,7 @@ impl EvaluationString { | PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } | PairedBinaryOperation::BitOr { .. } => { - operation.unsupported_for_value_type_err("string") + return operation.unsupported_for_value_type_err("string") } PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), @@ -62,7 +64,7 @@ impl EvaluationString { PairedBinaryOperation::NotEqual { .. } => operation.output(lhs != rhs), PairedBinaryOperation::GreaterThanOrEqual { .. } => operation.output(lhs >= rhs), PairedBinaryOperation::GreaterThan { .. } => operation.output(lhs > rhs), - } + }) } pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { @@ -72,7 +74,7 @@ impl EvaluationString { impl ToExpressionValue for String { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::String(EvaluationString { + ExpressionValue::String(ExpressionString { value: self, source_span, }) @@ -81,7 +83,7 @@ impl ToExpressionValue for String { impl ToExpressionValue for &str { fn to_value(self, source_span: Option) -> ExpressionValue { - ExpressionValue::String(EvaluationString { + ExpressionValue::String(ExpressionString { value: self.to_string(), source_span, }) diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 0ab98a06..a1be10a5 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -2,11 +2,11 @@ use super::*; #[derive(Clone)] pub(crate) enum ExpressionValue { - Integer(EvaluationInteger), - Float(EvaluationFloat), - Boolean(EvaluationBoolean), - String(EvaluationString), - Char(EvaluationChar), + Integer(ExpressionInteger), + Float(ExpressionFloat), + Boolean(ExpressionBoolean), + String(ExpressionString), + Char(ExpressionChar), } pub(super) trait ToExpressionValue: Sized { @@ -17,11 +17,11 @@ impl ExpressionValue { pub(super) fn for_literal(lit: syn::Lit) -> ParseResult { // https://docs.rs/syn/latest/syn/enum.Lit.html Ok(match lit { - Lit::Int(lit) => Self::Integer(EvaluationInteger::for_litint(lit)?), - Lit::Float(lit) => Self::Float(EvaluationFloat::for_litfloat(lit)?), - Lit::Bool(lit) => Self::Boolean(EvaluationBoolean::for_litbool(lit)), - Lit::Str(lit) => Self::String(EvaluationString::for_litstr(lit)), - Lit::Char(lit) => Self::Char(EvaluationChar::for_litchar(lit)), + Lit::Int(lit) => Self::Integer(ExpressionInteger::for_litint(lit)?), + Lit::Float(lit) => Self::Float(ExpressionFloat::for_litfloat(lit)?), + Lit::Bool(lit) => Self::Boolean(ExpressionBoolean::for_litbool(lit)), + Lit::Str(lit) => Self::String(ExpressionString::for_litstr(lit)), + Lit::Char(lit) => Self::Char(ExpressionChar::for_litchar(lit)), other_literal => { return other_literal .span() @@ -32,129 +32,129 @@ impl ExpressionValue { pub(super) fn expect_value_pair( self, - operation: &PairedBinaryOperation, + operation: &impl Operation, right: Self, ) -> ExecutionResult { Ok(match (self, right) { (ExpressionValue::Integer(left), ExpressionValue::Integer(right)) => { let integer_pair = match (left.value, right.value) { - (EvaluationIntegerValue::Untyped(untyped_lhs), rhs) => match rhs { - EvaluationIntegerValue::Untyped(untyped_rhs) => { - EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) + (ExpressionIntegerValue::Untyped(untyped_lhs), rhs) => match rhs { + ExpressionIntegerValue::Untyped(untyped_rhs) => { + ExpressionIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) } - EvaluationIntegerValue::U8(rhs) => { - EvaluationIntegerValuePair::U8(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::U8(rhs) => { + ExpressionIntegerValuePair::U8(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::U16(rhs) => { - EvaluationIntegerValuePair::U16(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::U16(rhs) => { + ExpressionIntegerValuePair::U16(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::U32(rhs) => { - EvaluationIntegerValuePair::U32(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::U32(rhs) => { + ExpressionIntegerValuePair::U32(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::U64(rhs) => { - EvaluationIntegerValuePair::U64(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::U64(rhs) => { + ExpressionIntegerValuePair::U64(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::U128(rhs) => { - EvaluationIntegerValuePair::U128(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::U128(rhs) => { + ExpressionIntegerValuePair::U128(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::Usize(rhs) => { - EvaluationIntegerValuePair::Usize(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::Usize(rhs) => { + ExpressionIntegerValuePair::Usize(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::I8(rhs) => { - EvaluationIntegerValuePair::I8(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::I8(rhs) => { + ExpressionIntegerValuePair::I8(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::I16(rhs) => { - EvaluationIntegerValuePair::I16(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::I16(rhs) => { + ExpressionIntegerValuePair::I16(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::I32(rhs) => { - EvaluationIntegerValuePair::I32(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::I32(rhs) => { + ExpressionIntegerValuePair::I32(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::I64(rhs) => { - EvaluationIntegerValuePair::I64(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::I64(rhs) => { + ExpressionIntegerValuePair::I64(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::I128(rhs) => { - EvaluationIntegerValuePair::I128(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::I128(rhs) => { + ExpressionIntegerValuePair::I128(untyped_lhs.parse_as()?, rhs) } - EvaluationIntegerValue::Isize(rhs) => { - EvaluationIntegerValuePair::Isize(untyped_lhs.parse_as()?, rhs) + ExpressionIntegerValue::Isize(rhs) => { + ExpressionIntegerValuePair::Isize(untyped_lhs.parse_as()?, rhs) } }, - (lhs, EvaluationIntegerValue::Untyped(untyped_rhs)) => match lhs { - EvaluationIntegerValue::Untyped(untyped_lhs) => { - EvaluationIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) + (lhs, ExpressionIntegerValue::Untyped(untyped_rhs)) => match lhs { + ExpressionIntegerValue::Untyped(untyped_lhs) => { + ExpressionIntegerValuePair::Untyped(untyped_lhs, untyped_rhs) } - EvaluationIntegerValue::U8(lhs) => { - EvaluationIntegerValuePair::U8(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::U8(lhs) => { + ExpressionIntegerValuePair::U8(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::U16(lhs) => { - EvaluationIntegerValuePair::U16(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::U16(lhs) => { + ExpressionIntegerValuePair::U16(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::U32(lhs) => { - EvaluationIntegerValuePair::U32(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::U32(lhs) => { + ExpressionIntegerValuePair::U32(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::U64(lhs) => { - EvaluationIntegerValuePair::U64(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::U64(lhs) => { + ExpressionIntegerValuePair::U64(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::U128(lhs) => { - EvaluationIntegerValuePair::U128(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::U128(lhs) => { + ExpressionIntegerValuePair::U128(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::Usize(lhs) => { - EvaluationIntegerValuePair::Usize(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::Usize(lhs) => { + ExpressionIntegerValuePair::Usize(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::I8(lhs) => { - EvaluationIntegerValuePair::I8(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::I8(lhs) => { + ExpressionIntegerValuePair::I8(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::I16(lhs) => { - EvaluationIntegerValuePair::I16(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::I16(lhs) => { + ExpressionIntegerValuePair::I16(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::I32(lhs) => { - EvaluationIntegerValuePair::I32(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::I32(lhs) => { + ExpressionIntegerValuePair::I32(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::I64(lhs) => { - EvaluationIntegerValuePair::I64(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::I64(lhs) => { + ExpressionIntegerValuePair::I64(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::I128(lhs) => { - EvaluationIntegerValuePair::I128(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::I128(lhs) => { + ExpressionIntegerValuePair::I128(lhs, untyped_rhs.parse_as()?) } - EvaluationIntegerValue::Isize(lhs) => { - EvaluationIntegerValuePair::Isize(lhs, untyped_rhs.parse_as()?) + ExpressionIntegerValue::Isize(lhs) => { + ExpressionIntegerValuePair::Isize(lhs, untyped_rhs.parse_as()?) } }, - (EvaluationIntegerValue::U8(lhs), EvaluationIntegerValue::U8(rhs)) => { - EvaluationIntegerValuePair::U8(lhs, rhs) + (ExpressionIntegerValue::U8(lhs), ExpressionIntegerValue::U8(rhs)) => { + ExpressionIntegerValuePair::U8(lhs, rhs) } - (EvaluationIntegerValue::U16(lhs), EvaluationIntegerValue::U16(rhs)) => { - EvaluationIntegerValuePair::U16(lhs, rhs) + (ExpressionIntegerValue::U16(lhs), ExpressionIntegerValue::U16(rhs)) => { + ExpressionIntegerValuePair::U16(lhs, rhs) } - (EvaluationIntegerValue::U32(lhs), EvaluationIntegerValue::U32(rhs)) => { - EvaluationIntegerValuePair::U32(lhs, rhs) + (ExpressionIntegerValue::U32(lhs), ExpressionIntegerValue::U32(rhs)) => { + ExpressionIntegerValuePair::U32(lhs, rhs) } - (EvaluationIntegerValue::U64(lhs), EvaluationIntegerValue::U64(rhs)) => { - EvaluationIntegerValuePair::U64(lhs, rhs) + (ExpressionIntegerValue::U64(lhs), ExpressionIntegerValue::U64(rhs)) => { + ExpressionIntegerValuePair::U64(lhs, rhs) } - (EvaluationIntegerValue::U128(lhs), EvaluationIntegerValue::U128(rhs)) => { - EvaluationIntegerValuePair::U128(lhs, rhs) + (ExpressionIntegerValue::U128(lhs), ExpressionIntegerValue::U128(rhs)) => { + ExpressionIntegerValuePair::U128(lhs, rhs) } - (EvaluationIntegerValue::Usize(lhs), EvaluationIntegerValue::Usize(rhs)) => { - EvaluationIntegerValuePair::Usize(lhs, rhs) + (ExpressionIntegerValue::Usize(lhs), ExpressionIntegerValue::Usize(rhs)) => { + ExpressionIntegerValuePair::Usize(lhs, rhs) } - (EvaluationIntegerValue::I8(lhs), EvaluationIntegerValue::I8(rhs)) => { - EvaluationIntegerValuePair::I8(lhs, rhs) + (ExpressionIntegerValue::I8(lhs), ExpressionIntegerValue::I8(rhs)) => { + ExpressionIntegerValuePair::I8(lhs, rhs) } - (EvaluationIntegerValue::I16(lhs), EvaluationIntegerValue::I16(rhs)) => { - EvaluationIntegerValuePair::I16(lhs, rhs) + (ExpressionIntegerValue::I16(lhs), ExpressionIntegerValue::I16(rhs)) => { + ExpressionIntegerValuePair::I16(lhs, rhs) } - (EvaluationIntegerValue::I32(lhs), EvaluationIntegerValue::I32(rhs)) => { - EvaluationIntegerValuePair::I32(lhs, rhs) + (ExpressionIntegerValue::I32(lhs), ExpressionIntegerValue::I32(rhs)) => { + ExpressionIntegerValuePair::I32(lhs, rhs) } - (EvaluationIntegerValue::I64(lhs), EvaluationIntegerValue::I64(rhs)) => { - EvaluationIntegerValuePair::I64(lhs, rhs) + (ExpressionIntegerValue::I64(lhs), ExpressionIntegerValue::I64(rhs)) => { + ExpressionIntegerValuePair::I64(lhs, rhs) } - (EvaluationIntegerValue::I128(lhs), EvaluationIntegerValue::I128(rhs)) => { - EvaluationIntegerValuePair::I128(lhs, rhs) + (ExpressionIntegerValue::I128(lhs), ExpressionIntegerValue::I128(rhs)) => { + ExpressionIntegerValuePair::I128(lhs, rhs) } - (EvaluationIntegerValue::Isize(lhs), EvaluationIntegerValue::Isize(rhs)) => { - EvaluationIntegerValuePair::Isize(lhs, rhs) + (ExpressionIntegerValue::Isize(lhs), ExpressionIntegerValue::Isize(rhs)) => { + ExpressionIntegerValuePair::Isize(lhs, rhs) } (left_value, right_value) => { return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.describe_type(), right_value.describe_type())); @@ -167,33 +167,33 @@ impl ExpressionValue { } (ExpressionValue::Float(left), ExpressionValue::Float(right)) => { let float_pair = match (left.value, right.value) { - (EvaluationFloatValue::Untyped(untyped_lhs), rhs) => match rhs { - EvaluationFloatValue::Untyped(untyped_rhs) => { - EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) + (ExpressionFloatValue::Untyped(untyped_lhs), rhs) => match rhs { + ExpressionFloatValue::Untyped(untyped_rhs) => { + ExpressionFloatValuePair::Untyped(untyped_lhs, untyped_rhs) } - EvaluationFloatValue::F32(rhs) => { - EvaluationFloatValuePair::F32(untyped_lhs.parse_as()?, rhs) + ExpressionFloatValue::F32(rhs) => { + ExpressionFloatValuePair::F32(untyped_lhs.parse_as()?, rhs) } - EvaluationFloatValue::F64(rhs) => { - EvaluationFloatValuePair::F64(untyped_lhs.parse_as()?, rhs) + ExpressionFloatValue::F64(rhs) => { + ExpressionFloatValuePair::F64(untyped_lhs.parse_as()?, rhs) } }, - (lhs, EvaluationFloatValue::Untyped(untyped_rhs)) => match lhs { - EvaluationFloatValue::Untyped(untyped_lhs) => { - EvaluationFloatValuePair::Untyped(untyped_lhs, untyped_rhs) + (lhs, ExpressionFloatValue::Untyped(untyped_rhs)) => match lhs { + ExpressionFloatValue::Untyped(untyped_lhs) => { + ExpressionFloatValuePair::Untyped(untyped_lhs, untyped_rhs) } - EvaluationFloatValue::F32(lhs) => { - EvaluationFloatValuePair::F32(lhs, untyped_rhs.parse_as()?) + ExpressionFloatValue::F32(lhs) => { + ExpressionFloatValuePair::F32(lhs, untyped_rhs.parse_as()?) } - EvaluationFloatValue::F64(lhs) => { - EvaluationFloatValuePair::F64(lhs, untyped_rhs.parse_as()?) + ExpressionFloatValue::F64(lhs) => { + ExpressionFloatValuePair::F64(lhs, untyped_rhs.parse_as()?) } }, - (EvaluationFloatValue::F32(lhs), EvaluationFloatValue::F32(rhs)) => { - EvaluationFloatValuePair::F32(lhs, rhs) + (ExpressionFloatValue::F32(lhs), ExpressionFloatValue::F32(rhs)) => { + ExpressionFloatValuePair::F32(lhs, rhs) } - (EvaluationFloatValue::F64(lhs), EvaluationFloatValue::F64(rhs)) => { - EvaluationFloatValuePair::F64(lhs, rhs) + (ExpressionFloatValue::F64(lhs), ExpressionFloatValue::F64(rhs)) => { + ExpressionFloatValuePair::F64(lhs, rhs) } (left_value, right_value) => { return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.describe_type(), right_value.describe_type())); @@ -213,14 +213,14 @@ impl ExpressionValue { }) } - pub(crate) fn into_integer(self) -> Option { + pub(crate) fn into_integer(self) -> Option { match self { ExpressionValue::Integer(value) => Some(value), _ => None, } } - pub(crate) fn into_bool(self) -> Option { + pub(crate) fn into_bool(self) -> Option { match self { ExpressionValue::Boolean(value) => Some(value), _ => None, @@ -228,7 +228,7 @@ impl ExpressionValue { } /// The span is used if there isn't already a span available - pub(super) fn to_token_tree(&self, fallback_output_span: Span) -> TokenTree { + pub(crate) fn to_token_tree(&self, fallback_output_span: Span) -> TokenTree { match self { Self::Integer(int) => int.to_literal(fallback_output_span).into(), Self::Float(float) => float.to_literal(fallback_output_span).into(), @@ -273,7 +273,7 @@ impl ExpressionValue { pub(super) fn handle_integer_binary_operation( self, - right: EvaluationInteger, + right: ExpressionInteger, operation: &IntegerBinaryOperation, ) -> ExecutionResult { match self { @@ -292,6 +292,15 @@ impl ExpressionValue { ExpressionValue::Char(value) => value.handle_integer_binary_operation(right, operation), } } + + pub(crate) fn create_range( + self, + other: Self, + range_limits: &syn::RangeLimits, + ) -> ExecutionResult + '_>> { + self.expect_value_pair(range_limits, other)? + .create_range(range_limits) + } } #[derive(Copy, Clone)] @@ -303,11 +312,11 @@ pub(super) enum CastTarget { } pub(super) enum EvaluationLiteralPair { - Integer(EvaluationIntegerValuePair), - Float(EvaluationFloatValuePair), - BooleanPair(EvaluationBoolean, EvaluationBoolean), - StringPair(EvaluationString, EvaluationString), - CharPair(EvaluationChar, EvaluationChar), + Integer(ExpressionIntegerValuePair), + Float(ExpressionFloatValuePair), + BooleanPair(ExpressionBoolean, ExpressionBoolean), + StringPair(ExpressionString, ExpressionString), + CharPair(ExpressionChar, ExpressionChar), } impl EvaluationLiteralPair { @@ -323,4 +332,18 @@ impl EvaluationLiteralPair { Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } + + pub(super) fn create_range( + self, + range_limits: &syn::RangeLimits, + ) -> ExecutionResult + '_>> { + Ok(match self { + EvaluationLiteralPair::Integer(pair) => return pair.create_range(range_limits), + EvaluationLiteralPair::CharPair(left, right) => left.create_range(right, range_limits), + _ => { + return range_limits + .execution_err("The range must be between two integers or two characters") + } + }) + } } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 3ead998d..83008a63 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -85,25 +85,27 @@ impl NoOutputCommandDefinition for SetCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { match self.arguments { - SetArguments::SetVariable { variable, content, .. } => { + SetArguments::SetVariable { + variable, content, .. + } => { let content = content.interpret_to_new_stream(interpreter)?; variable.set(interpreter, content)?; - }, - SetArguments::ExtendVariable { variable, content, .. } => { + } + SetArguments::ExtendVariable { + variable, content, .. + } => { let variable_data = variable.get_existing_for_mutation(interpreter)?; - content.interpret_into( - interpreter, - variable_data.get_mut(&variable)?.deref_mut(), - )?; - }, + content + .interpret_into(interpreter, variable_data.get_mut(&variable)?.deref_mut())?; + } SetArguments::SetVariablesEmpty { variables } => { for variable in variables { variable.set(interpreter, OutputStream::new())?; } - }, + } SetArguments::Discard { content, .. } => { let _ = content.interpret_to_new_stream(interpreter)?; - }, + } } Ok(()) } diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 9d9a1080..31f93cd7 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -116,7 +116,7 @@ impl NoOutputCommandDefinition for AssignCommand { #[derive(Clone)] pub(crate) struct RangeCommand { left: SourceExpression, - range_limits: RangeLimits, + range_limits: syn::RangeLimits, right: SourceExpression, } @@ -145,95 +145,34 @@ impl StreamCommandDefinition for RangeCommand { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - let range_span_range = self.range_limits.span_range(); - let range_span = range_span_range.join_into_span_else_start(); - - let left = self - .left - .evaluate(interpreter)? - .try_into_i128("The left side of the range must be an i128-compatible integer")?; - let right = self - .right - .evaluate(interpreter)? - .try_into_i128("The right side of the range must be an i128-compatible integer")?; - - if left > right { - return Ok(()); - } - - let length = self - .range_limits - .length_of_range(left, right) - .ok_or_else(|| { - range_span_range.error("The range is too large to be represented as a usize") - })?; - - interpreter - .start_iteration_counter(&range_span_range) - .add_and_check(length)?; - - match self.range_limits { - RangeLimits::HalfOpen(_) => { - output_range(left..right, range_span, output); + let range_limits = self.range_limits; + let left = self.left.evaluate_to_value(interpreter)?; + let right = self.right.evaluate_to_value(interpreter)?; + + let range_iterator = left.create_range(right, &range_limits)?; + + let (_, length) = range_iterator.size_hint(); + match length { + Some(length) => { + interpreter + .start_iteration_counter(&range_limits) + .add_and_check(length)?; } - RangeLimits::Closed(_) => { - output_range(left..=right, range_span, output); + None => { + return range_limits + .execution_err("The range must be between two integers or two characters"); } - }; - - Ok(()) - } -} - -fn output_range(iter: impl Iterator, span: Span, output: &mut OutputStream) { - output.extend_raw_tokens(iter.map(|value| { - let literal = Literal::i128_unsuffixed(value).with_span(span); - TokenTree::Literal(literal) - // We wrap it in a singleton group to ensure that negative - // numbers are treated as single items in other stream commands - .into_singleton_group(Delimiter::None) - })) -} - -// A copy of syn::RangeLimits to avoid needing a `full` dependency on syn -#[derive(Clone)] -enum RangeLimits { - HalfOpen(Token![..]), - Closed(Token![..=]), -} - -impl Parse for RangeLimits { - fn parse(input: ParseStream) -> ParseResult { - if input.peek(Token![..=]) { - Ok(RangeLimits::Closed(input.parse()?)) - } else { - Ok(RangeLimits::HalfOpen(input.parse()?)) } - } -} -impl ToTokens for RangeLimits { - fn to_tokens(&self, tokens: &mut TokenStream) { - match self { - RangeLimits::HalfOpen(token) => token.to_tokens(tokens), - RangeLimits::Closed(token) => token.to_tokens(tokens), - } - } -} + let output_span = range_limits.span_range().start(); + output.extend_raw_tokens(range_iterator.map(|value| { + value + .to_token_tree(output_span) + // We wrap it in a singleton group to ensure that negative + // numbers are treated as single items in other stream commands + .into_singleton_group(Delimiter::None) + })); -impl HasSpanRange for RangeLimits { - fn span_range(&self) -> SpanRange { - self.span_range_from_iterating_over_all_tokens() - } -} - -impl RangeLimits { - fn length_of_range(&self, left: i128, right: i128) -> Option { - match self { - RangeLimits::HalfOpen(_) => usize::try_from(right.checked_sub(left)?).ok(), - RangeLimits::Closed(_) => { - usize::try_from(right.checked_sub(left)?.checked_add(1)?).ok() - } - } + Ok(()) } } diff --git a/tests/core.rs b/tests/core.rs index 274127f6..c62ca6e1 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -72,7 +72,6 @@ fn test_ignore() { }, false); } - #[test] fn test_empty_set() { assert_preinterpret_eq!({ diff --git a/tests/expressions.rs b/tests/expressions.rs index 2a69506b..74dbb014 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -163,7 +163,7 @@ fn assign_works() { } #[test] -fn range_works() { +fn test_range() { assert_preinterpret_eq!( [!string![!intersperse! { items: [!range! -2..5], @@ -197,4 +197,9 @@ fn range_works() { }, "" ); + assert_preinterpret_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); + assert_preinterpret_eq!( + { [!debug! [!..range! -1i8..3i8]] }, + "[!group! - 1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]" + ); } From 9da3e57cff2c44901bf592967097c38da751f680 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 6 Feb 2025 01:01:33 +0000 Subject: [PATCH 077/126] feature: Generalized destructurers to transformers --- CHANGELOG.md | 98 +++---- src/destructuring/destructure_group.rs | 28 -- src/destructuring/destructure_item.rs | 75 ----- src/destructuring/destructure_raw.rs | 98 ------- src/destructuring/destructure_segment.rs | 82 ------ src/destructuring/destructurer.rs | 196 ------------- src/destructuring/destructurers.rs | 230 ---------------- src/destructuring/mod.rs | 20 -- src/expressions/expression.rs | 30 +- src/extensions/parsing.rs | 2 +- src/internal_prelude.rs | 2 +- src/interpretation/command.rs | 35 ++- .../commands/control_flow_commands.rs | 28 +- src/interpretation/commands/core_commands.rs | 37 ++- .../commands/destructuring_commands.rs | 36 --- .../commands/expression_commands.rs | 4 +- src/interpretation/commands/mod.rs | 4 +- src/interpretation/commands/token_commands.rs | 88 +++--- .../commands/transforming_commands.rs | 79 ++++++ src/interpretation/interpretation_item.rs | 71 ----- src/interpretation/interpretation_stream.rs | 127 --------- src/interpretation/interpreted_stream.rs | 27 +- src/interpretation/mod.rs | 20 +- ...and_code_input.rs => source_code_block.rs} | 0 src/interpretation/source_stream.rs | 156 +++++++++++ ...stream_input.rs => source_stream_input.rs} | 42 +-- ...command_value_input.rs => source_value.rs} | 64 ++--- src/lib.rs | 2 +- src/misc/errors.rs | 7 + .../field_inputs.rs} | 0 src/misc/mod.rs | 2 + src/misc/parse_traits.rs | 65 +++-- src/transformation/exact_stream.rs | 190 +++++++++++++ .../fields.rs | 0 src/transformation/mod.rs | 18 ++ src/transformation/parse_utilities.rs | 134 +++++++++ src/transformation/transform_stream.rs | 259 ++++++++++++++++++ .../transformation_traits.rs} | 10 +- src/transformation/transformer.rs | 231 ++++++++++++++++ src/transformation/transformers.rs | 130 +++++++++ .../variable_binding.rs} | 144 +++------- .../destructure_with_flattened_command.stderr | 6 - ...cture_with_ident_flattened_variable.stderr | 6 - ...ure_with_literal_flattened_variable.stderr | 6 - ...destructure_with_outputting_command.stderr | 6 - ...cture_with_punct_flattened_variable.stderr | 6 - .../invalid_content_wrong_group_2.stderr | 5 - .../append_before_set.rs | 0 .../append_before_set.stderr | 2 +- .../destructure_with_flattened_command.rs | 0 .../destructure_with_flattened_command.stderr | 5 + ...structure_with_ident_flattened_variable.rs | 0 ...cture_with_ident_flattened_variable.stderr | 5 + ...ructure_with_literal_flattened_variable.rs | 0 ...ure_with_literal_flattened_variable.stderr | 5 + .../destructure_with_outputting_command.rs | 0 ...destructure_with_outputting_command.stderr | 5 + ...structure_with_punct_flattened_variable.rs | 0 ...cture_with_punct_flattened_variable.stderr | 5 + .../double_flattened_variable.rs | 0 .../double_flattened_variable.stderr | 4 +- .../invalid_content_too_long.rs | 0 .../invalid_content_too_long.stderr | 2 +- .../invalid_content_too_short.rs | 0 .../invalid_content_too_short.stderr | 2 +- .../invalid_content_wrong_group.rs | 0 .../invalid_content_wrong_group.stderr | 2 +- .../invalid_content_wrong_group_2.rs | 0 .../invalid_content_wrong_group_2.stderr | 5 + .../invalid_content_wrong_ident.rs | 0 .../invalid_content_wrong_ident.stderr | 2 +- .../invalid_content_wrong_punct.rs | 0 .../invalid_content_wrong_punct.stderr | 2 +- .../invalid_group_content_too_long.rs | 0 .../invalid_group_content_too_long.stderr | 2 +- .../invalid_group_content_too_short.rs | 0 .../invalid_group_content_too_short.stderr | 2 +- .../transforming/mistaken_at_symbol.rs | 11 + .../transforming/mistaken_at_symbol.stderr | 5 + tests/core.rs | 4 +- tests/tokens.rs | 4 +- tests/{destructuring.rs => transforming.rs} | 85 ++++-- 82 files changed, 1661 insertions(+), 1404 deletions(-) delete mode 100644 src/destructuring/destructure_group.rs delete mode 100644 src/destructuring/destructure_item.rs delete mode 100644 src/destructuring/destructure_raw.rs delete mode 100644 src/destructuring/destructure_segment.rs delete mode 100644 src/destructuring/destructurer.rs delete mode 100644 src/destructuring/destructurers.rs delete mode 100644 src/destructuring/mod.rs delete mode 100644 src/interpretation/commands/destructuring_commands.rs create mode 100644 src/interpretation/commands/transforming_commands.rs delete mode 100644 src/interpretation/interpretation_item.rs delete mode 100644 src/interpretation/interpretation_stream.rs rename src/interpretation/{command_code_input.rs => source_code_block.rs} (100%) create mode 100644 src/interpretation/source_stream.rs rename src/interpretation/{command_stream_input.rs => source_stream_input.rs} (80%) rename src/interpretation/{command_value_input.rs => source_value.rs} (66%) rename src/{interpretation/command_field_inputs.rs => misc/field_inputs.rs} (100%) create mode 100644 src/transformation/exact_stream.rs rename src/{destructuring => transformation}/fields.rs (100%) create mode 100644 src/transformation/mod.rs create mode 100644 src/transformation/parse_utilities.rs create mode 100644 src/transformation/transform_stream.rs rename src/{destructuring/destructure_traits.rs => transformation/transformation_traits.rs} (67%) create mode 100644 src/transformation/transformer.rs create mode 100644 src/transformation/transformers.rs rename src/{destructuring/destructure_variable.rs => transformation/variable_binding.rs} (64%) delete mode 100644 tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr delete mode 100644 tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr delete mode 100644 tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr delete mode 100644 tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr delete mode 100644 tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr delete mode 100644 tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr rename tests/compilation_failures/{destructuring => transforming}/append_before_set.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/append_before_set.stderr (63%) rename tests/compilation_failures/{destructuring => transforming}/destructure_with_flattened_command.rs (100%) create mode 100644 tests/compilation_failures/transforming/destructure_with_flattened_command.stderr rename tests/compilation_failures/{destructuring => transforming}/destructure_with_ident_flattened_variable.rs (100%) create mode 100644 tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr rename tests/compilation_failures/{destructuring => transforming}/destructure_with_literal_flattened_variable.rs (100%) create mode 100644 tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr rename tests/compilation_failures/{destructuring => transforming}/destructure_with_outputting_command.rs (100%) create mode 100644 tests/compilation_failures/transforming/destructure_with_outputting_command.stderr rename tests/compilation_failures/{destructuring => transforming}/destructure_with_punct_flattened_variable.rs (100%) create mode 100644 tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr rename tests/compilation_failures/{destructuring => transforming}/double_flattened_variable.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/double_flattened_variable.stderr (56%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_too_long.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_too_long.stderr (64%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_too_short.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_too_short.stderr (76%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_group.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_group.stderr (61%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_group_2.rs (100%) create mode 100644 tests/compilation_failures/transforming/invalid_content_wrong_group_2.stderr rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_ident.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_ident.stderr (62%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_punct.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_content_wrong_punct.stderr (62%) rename tests/compilation_failures/{destructuring => transforming}/invalid_group_content_too_long.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_group_content_too_long.stderr (69%) rename tests/compilation_failures/{destructuring => transforming}/invalid_group_content_too_short.rs (100%) rename tests/compilation_failures/{destructuring => transforming}/invalid_group_content_too_short.stderr (67%) create mode 100644 tests/compilation_failures/transforming/mistaken_at_symbol.rs create mode 100644 tests/compilation_failures/transforming/mistaken_at_symbol.stderr rename tests/{destructuring.rs => transforming.rs} (57%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44360bfc..d5023fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,12 @@ ### New Commands * Core commands: - * `[!error! ...]` to output a compile error - * `[!set! #x += ...]` to performantly add extra characters to the stream + * `[!error! ...]` to output a compile error. + * `[!set! #x += ...]` to performantly add extra characters to the stream. + * `[!set! _ = ...]` interprets its arguments but then ignores any outputs. * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. - * `[!void! ...]` interprets its arguments but then ignores any outputs. It can be used inside destructurings. + * `[!output! ...]` can be used to just output its interpreted contents. Normally it's a no-op, but it can be useful inside a transformer. + * `[!settings! { ... }]` can be used to adjust the iteration limit. * Expression commands: * `[!evaluate! ]` * `[!assign! #x += ]` for `+` and other supported operators @@ -29,12 +31,10 @@ * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. - * `[!..group! ...]` which just outputs its contents as-is, useful where the grammar - only takes a single item, but we want to output multiple tokens * `[!intersperse! { ... }]` which inserts separator tokens between each token tree in a stream. - * `[!split! ...]` - * `[!comma_split! ...]` - * `[!zip! (#countries #flags #capitals)]` which can be used to combine multiple streams together + * `[!split! ...]` which can be used to split a stream with a given separating stream. + * `[!comma_split! ...]` which can be used to split a stream on `,` tokens. + * `[!zip! (#countries #flags #capitals)]` which can be used to combine multiple streams together. * Destructuring commands: * `[!let! = ...]` does destructuring/parsing (see next section). Note `[!let! #..x = ...]` is equivalent to `[!set! #x = ...]` @@ -56,63 +56,54 @@ Currently supported are: Expressions behave intuitively as you'd expect from writing regular rust code, except they happen at compile time. -### Destructuring +### Transforming -Destructuring performs parsing of a token stream. It supports: +Transforming performs parsing of a token stream, whilst also outputting a stream. The input stream must be parsed in its entirety. -* Explicit punctuation, idents, literals and groups +Transform streams (or substreams) can be redirected to set variables or append to variables. Commands can also be injected to add to the output. + +Inside a transform stream, the following grammar is supported: + +* `@(...)`, `@(#x = ...)`, `@(#x += ...)` and `@(_ = ...)` - Explicit transform (sub)streams which either output, set, append or discard its output. +* Explicit punctuation, idents, literals and groups. These aren't output by default, except directly inside a `@[EXACT ...]` transformer. * Variable bindings: - * `#x` - Reads a token tree, writes a stream (opposite of `#x`) - * `#..x` - Reads a stream, writes a stream (opposite of `#..x`) + * `#x` - Reads a token tree, writes its content (opposite of `#x`). Equivalent to `@(#x = @TOKEN_OR_GROUP_CONTENT)` + * `#..x` - Reads a stream, writes a stream (opposite of `#..x`). + * If it's at the end of the transformer stream, it's equivalent to `@(#x = @REST)`. + * If it's followed by a token `T` in the transformer stream, it's equivalent to `@(#x = @[UNTIL T])` * `#>>x` - Reads a token tree, appends a token tree (can be read back with `!for! #y in #x { ... }`) * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) * `#..>>x` - Reads a stream, appends a group (can be read back with `!for! #y in #x { ... }`) * `#..>>..x` - Reads a stream, appends a stream -* Commands which don't output a value, like `[!set! ...]` * Named destructurings: - * `(!stream! ...)` (TODO - decide if this is a good name) - * `(!ident! ...)` - * `(!punct! ...)` - * `(!literal! ...)` - * `(!group! ...)` - * `(!raw! ...)` - * `(!content! ...)` + * `@IDENT` - Consumes and output any ident. + * `@PUNCT` - Consumes and outputs any punctation + * `@LITERAL` - Consumes and outputs any literal + * `@[GROUP ...]` - Consumes a none-delimited group. Its arguments are used to transform the group's contents. + * `@[EXACT ...]` - Interprets its arguments (i.e. variables are substituted, not bound; and command output is gathered) into an "exact match stream". And then expects to consume exactly the same stream from the input. It outputs the parsed stream. +* Commands: Their output is appended to the transform's output. Useful patterns include: + * `@(#inner = ...) [!output! #inner]` - wraps the output in a transparent group ### To come -* Destructurers => Transformers - * Implement pivot to transformers outputting things ... `@[#x = @IDENT]`... - * Scrap `[!let!]` in favour of `[!parse! #x as #(...)]` +* Destructurers => Transformers + * Scrap `[!let!]` in favour of `[!parse! #x as @(_ = ...)]` * `@TOKEN_TREE` + * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. + * `@[ANY_GROUP ...]` * `@REST` - * `@[UNTIL xxxx]` - * `@[EXPECT xxxx]` expects the tokens (or source grammar inc variables), and outputs the matched tokens (instead of dropping them as is the default). Replaces `(!content!)`. We should also consider ignoring/unwrapping none-groups to be more permissive? (assuming they're also unwrapped during parsing). + * `@[UNTIL xxxx]` - For now - takes a raw stream which is turned into an ExactStream. * `@[FIELDS { ... }]` and `@[SUBFIELDS { ... }]` * Add ability to add scope to interpreter state (copy on write?) (and commit/revert) and can then add: * `@[OPTIONAL ...]` and `@(...)?` * `@(REPEATED { ... })` (see below) + * Potentially change `@UNTIL` to take a transform stream instead of a raw stream. * `[!match! ...]` command * `@[ANY { ... }]` (with `#..x` as a catch-all) like the [!match!] command but without arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` * Consider: - * Destructurer needs to have different syntax. It's too confusingly similar! - * Final decision: `@[#x = @IDENT]` because destructurers output (SEE BELOW FOR MOST OF THE WORKING) - * Some other ideas considered: - * `(>ident> #x)`? `(>fields> {})`? `(>comma_repeated> Hello)` - * We can't use `<` otherwise it tries to open brackets and could be confused for rust syntax like: `()` - * We need to test it with the auto-formatter in case it really messes it up - * `#(>ident #x)` - not bad... - * Then we can drop `(!stream!)` as it's just `#( ... )` - * We need to test it with the auto-formatter in case it really messes it up - * `[>ident (#x)]` - * `` - * `{[ident] #x}` - * `(>ident> #x)` - * `#IDENT { #x }` - * `#IDENT { capture: #x }` - * `#(IDENT #x)` - * `@[#x = @IDENT]` - * Scrap `#>>x` etc in favour of `@[#x += ...]` + * If the `[!split!]` command should actually be a transformer? + * Scrap `#>>x` etc in favour of `@(#x += ...)` * `[!is_set! #x]` * Support `[!index! ..]`: * `[!index! #x[0]]` @@ -123,6 +114,7 @@ Destructuring performs parsing of a token stream. It supports: * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) * Add `[!reinterpret! ...]` command for an `eval` style command. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` +* Get rid of needless cloning * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed * Work on book @@ -144,7 +136,7 @@ Destructuring performs parsing of a token stream. It supports: }] // NICE [!parse! #input as @[REPEATED { - item: @(impl @[#trait = @IDENT] for @[#type = @TYPE]), + item: @(impl @(#trait = @IDENT) for @(#type = @TYPE)), item_output: { impl BLAH BLAH { ... @@ -157,7 +149,7 @@ Destructuring performs parsing of a token stream. It supports: }] // MAYBE - probably not though... -[!parse_for! #input as @(impl @[#x = @IDENT] for @[#y = @IDENT]),* { +[!parse_for! #input as @(impl @(#x = @IDENT) for @(#y = @IDENT)),* { }] @@ -185,14 +177,14 @@ Destructuring performs parsing of a token stream. It supports: // * @X shorthand for @[X] for destructurers which can take no input, e.g. IDENT, TOKEN_TREE, TYPE etc // => NOTE: Each destructurer should return just its tokens by default if it has no arguments. // => It can also have its output over-written or other things outputted using e.g. @[TYPE { is_prefixed: X, parts: #(...), output: { #output } }] -// * #x is shorthand for @[#x = @TOKEN_TREE] +// * #x is shorthand for @[#x = @TOKEN_OR_GROUP_CONTENT] // * #..x) is shorthand for @[#x = @UNTIL_END] and #..x, is shorthand for @[CAPTURE #x = @[UNTIL_TOKEN ,]] -// * @[_ = ...] -// * @[#x = @IDENT for @IDENT] -// * @[#x += @IDENT for @IDENT] +// * @(_ = ...) +// * @(#x = impl @IDENT for @IDENT) +// * @(#x += impl @IDENT for @IDENT) // * @[REPEATED { ... }] // * Can embed commands to output stuff too -// * Can output a group with: @[#x = @IDENT for @IDENT] [!group! #..x] +// * Can output a group with: @(#x = @IDENT for @IDENT) [!output! #x] // In this model, REPEATED is really clean and looks like this: @[REPEATED { @@ -205,14 +197,14 @@ Destructuring performs parsing of a token stream. It supports: }] // How does optional work? -// @[#x = @(@IDENT)?] +// @(#x = @(@IDENT)?) // Along with: // [!fields! { #x, my_var: #y, #z }] // And if some field #z isn't set, it's outputted as null. // Do we want something like !parse_for!? It needs to execute lazily - how? // > Probably by passing some `OnOutput` hook to an output stream method -[!parse_for! #input as @(impl @[#x = @IDENT] for @[#y = @IDENT]),+ { +[!parse_for! #input as @(impl @(#x = @IDENT) for @(#y = @IDENT)),+ { }] ``` diff --git a/src/destructuring/destructure_group.rs b/src/destructuring/destructure_group.rs deleted file mode 100644 index 3af3452c..00000000 --- a/src/destructuring/destructure_group.rs +++ /dev/null @@ -1,28 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) struct DestructureGroup { - delimiter: Delimiter, - inner: DestructureRemaining, -} - -impl Parse for DestructureGroup { - fn parse(input: ParseStream) -> ParseResult { - let (delimiter, _, content) = input.parse_any_group()?; - Ok(Self { - delimiter, - inner: content.parse()?, - }) - } -} - -impl HandleDestructure for DestructureGroup { - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - let (_, inner) = input.parse_specific_group(self.delimiter)?; - self.inner.handle_destructure(&inner, interpreter) - } -} diff --git a/src/destructuring/destructure_item.rs b/src/destructuring/destructure_item.rs deleted file mode 100644 index 18007a17..00000000 --- a/src/destructuring/destructure_item.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) enum DestructureItem { - NoneOutputCommand(Command), - Variable(DestructureVariable), - Destructurer(Destructurer), - ExactPunct(Punct), - ExactIdent(Ident), - ExactLiteral(Literal), - ExactGroup(DestructureGroup), -} - -impl DestructureItem { - /// We provide a stop condition so that some of the items can know when to stop consuming greedily - - /// notably the flattened command. This allows [!let! #..x = Hello => World] to parse as setting - /// `x` to `Hello => World` rather than having `#..x` peeking to see it is "up to =" and then only - /// parsing `Hello` into `x`. - pub(crate) fn parse_until(input: ParseStream) -> ParseResult { - Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(Some(CommandOutputKind::None)) => { - Self::NoneOutputCommand(input.parse()?) - } - GrammarPeekMatch::Command(_) => { - return input.parse_err( - "Commands which return something are not supported in destructuring positions", - ) - } - GrammarPeekMatch::GroupedVariable - | GrammarPeekMatch::FlattenedVariable - | GrammarPeekMatch::AppendVariableDestructuring => { - Self::Variable(DestructureVariable::parse_until::(input)?) - } - GrammarPeekMatch::Group(_) => Self::ExactGroup(input.parse()?), - GrammarPeekMatch::Destructurer(_) => Self::Destructurer(input.parse()?), - GrammarPeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), - GrammarPeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), - GrammarPeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), - GrammarPeekMatch::End => return input.parse_err("Unexpected end"), - }) - } -} - -impl HandleDestructure for DestructureItem { - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - match self { - DestructureItem::Variable(variable) => { - variable.handle_destructure(input, interpreter)?; - } - DestructureItem::NoneOutputCommand(command) => { - let _ = command.clone().interpret_to_new_stream(interpreter)?; - } - DestructureItem::Destructurer(destructurer) => { - destructurer.handle_destructure(input, interpreter)?; - } - DestructureItem::ExactPunct(punct) => { - input.parse_punct_matching(punct.as_char())?; - } - DestructureItem::ExactIdent(ident) => { - input.parse_ident_matching(&ident.to_string())?; - } - DestructureItem::ExactLiteral(literal) => { - input.parse_literal_matching(&literal.to_string())?; - } - DestructureItem::ExactGroup(group) => { - group.handle_destructure(input, interpreter)?; - } - } - Ok(()) - } -} diff --git a/src/destructuring/destructure_raw.rs b/src/destructuring/destructure_raw.rs deleted file mode 100644 index e55fb882..00000000 --- a/src/destructuring/destructure_raw.rs +++ /dev/null @@ -1,98 +0,0 @@ -use crate::internal_prelude::*; - -/// This is an *exact* destructuring, which must match item-by-item. -#[derive(Clone)] -pub(crate) enum RawDestructureItem { - Punct(Punct), - Ident(Ident), - Literal(Literal), - Group(RawDestructureGroup), -} - -impl RawDestructureItem { - pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { - match self { - RawDestructureItem::Punct(punct) => { - input.parse_punct_matching(punct.as_char())?; - } - RawDestructureItem::Ident(ident) => { - input.parse_ident_matching(&ident.to_string())?; - } - RawDestructureItem::Literal(literal) => { - input.parse_literal_matching(&literal.to_string())?; - } - RawDestructureItem::Group(group) => { - group.handle_destructure(input)?; - } - } - Ok(()) - } - - pub(crate) fn new_from_token_tree(token_tree: TokenTree) -> Self { - match token_tree { - TokenTree::Punct(punct) => Self::Punct(punct), - TokenTree::Ident(ident) => Self::Ident(ident), - TokenTree::Literal(literal) => Self::Literal(literal), - TokenTree::Group(group) => Self::Group(RawDestructureGroup::new_from_group(&group)), - } - } -} - -#[derive(Clone)] -pub(crate) struct RawDestructureStream { - inner: Vec, -} - -impl RawDestructureStream { - pub(crate) fn empty() -> Self { - Self { inner: vec![] } - } - - pub(crate) fn new_from_token_stream(tokens: impl IntoIterator) -> Self { - let mut new = Self::empty(); - new.append_from_token_stream(tokens); - new - } - - pub(crate) fn append_from_token_stream(&mut self, tokens: impl IntoIterator) { - for token_tree in tokens { - self.inner - .push(RawDestructureItem::new_from_token_tree(token_tree)); - } - } - - pub(crate) fn push_item(&mut self, item: RawDestructureItem) { - self.inner.push(item); - } - - pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { - for item in self.inner.iter() { - item.handle_destructure(input)?; - } - Ok(()) - } -} - -#[derive(Clone)] -pub(crate) struct RawDestructureGroup { - delimiter: Delimiter, - inner: RawDestructureStream, -} - -impl RawDestructureGroup { - pub(crate) fn new(delimiter: Delimiter, inner: RawDestructureStream) -> Self { - Self { delimiter, inner } - } - - pub(crate) fn new_from_group(group: &Group) -> Self { - Self { - delimiter: group.delimiter(), - inner: RawDestructureStream::new_from_token_stream(group.stream()), - } - } - - pub(crate) fn handle_destructure(&self, input: ParseStream) -> ExecutionResult<()> { - let (_, inner) = input.parse_specific_group(self.delimiter)?; - self.inner.handle_destructure(&inner) - } -} diff --git a/src/destructuring/destructure_segment.rs b/src/destructuring/destructure_segment.rs deleted file mode 100644 index a9223aca..00000000 --- a/src/destructuring/destructure_segment.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) trait StopCondition: Clone { - fn should_stop(input: ParseStream) -> bool; -} - -#[derive(Clone)] -pub(crate) struct UntilEnd; -impl StopCondition for UntilEnd { - fn should_stop(input: ParseStream) -> bool { - input.is_empty() - } -} - -pub(crate) trait PeekableToken: Clone { - fn peek(input: ParseStream) -> bool; -} - -// This is going through such pain to ensure we stay in the public API of syn -// We'd like to be able to use `input.peek::()` for some syn::token::Token -// but that's just not an API they support for some reason -macro_rules! impl_peekable_token { - ($(Token![$token:tt]),* $(,)?) => { - $( - impl PeekableToken for Token![$token] { - fn peek(input: ParseStream) -> bool { - input.peek(Token![$token]) - } - } - )* - }; -} - -impl_peekable_token! { - Token![=], - Token![in], -} - -#[derive(Clone)] -pub(crate) struct UntilToken { - token: PhantomData, -} -impl StopCondition for UntilToken { - fn should_stop(input: ParseStream) -> bool { - input.is_empty() || T::peek(input) - } -} - -pub(crate) type DestructureRemaining = DestructureSegment; -pub(crate) type DestructureUntil = DestructureSegment>; - -#[derive(Clone)] -pub(crate) struct DestructureSegment { - stop_condition: PhantomData, - inner: Vec, -} - -impl Parse for DestructureSegment { - fn parse(input: ParseStream) -> ParseResult { - let mut inner = vec![]; - while !C::should_stop(input) { - inner.push(DestructureItem::parse_until::(input)?); - } - Ok(Self { - stop_condition: PhantomData, - inner, - }) - } -} - -impl HandleDestructure for DestructureSegment { - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - for item in self.inner.iter() { - item.handle_destructure(input, interpreter)?; - } - Ok(()) - } -} diff --git a/src/destructuring/destructurer.rs b/src/destructuring/destructurer.rs deleted file mode 100644 index 2c682c42..00000000 --- a/src/destructuring/destructurer.rs +++ /dev/null @@ -1,196 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) trait DestructurerDefinition: Clone { - const DESTRUCTURER_NAME: &'static str; - fn parse(arguments: DestructurerArguments) -> ParseResult; - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()>; -} - -#[derive(Clone)] -pub(crate) struct DestructurerArguments<'a> { - parse_stream: ParseStream<'a, Source>, - destructurer_name: Ident, - full_span: Span, -} - -#[allow(unused)] -impl<'a> DestructurerArguments<'a> { - pub(crate) fn new( - parse_stream: ParseStream<'a, Source>, - destructurer_name: Ident, - full_span: Span, - ) -> Self { - Self { - parse_stream, - destructurer_name, - full_span, - } - } - - pub(crate) fn full_span(&self) -> Span { - self.full_span - } - - /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message - pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> ParseResult<()> { - if self.parse_stream.is_empty() { - Ok(()) - } else { - self.full_span.parse_err(error_message) - } - } - - pub(crate) fn fully_parse_no_error_override>(&self) -> ParseResult { - self.parse_stream.parse() - } - - pub(crate) fn fully_parse_as(&self) -> ParseResult { - self.fully_parse_or_error(T::parse, T::error_message()) - } - - pub(crate) fn fully_parse_or_error( - &self, - parse_function: impl FnOnce(ParseStream) -> ParseResult, - error_message: impl std::fmt::Display, - ) -> ParseResult { - // In future, when the diagnostic API is stable, - // we can add this context directly onto the command ident... - // Rather than just selectively adding it to the inner-most error. - // - // For now though, we can add additional context to the error message. - // But we can avoid adding this additional context if it's already been added in an - // inner error, because that's likely the correct local context to show. - let parsed = - parse_function(self.parse_stream).add_context_if_error_and_no_context(|| { - format!( - "Occurred whilst parsing (!{}! ...) - {}", - self.destructurer_name, error_message, - ) - })?; - - self.assert_empty(error_message)?; - - Ok(parsed) - } -} - -#[derive(Clone)] -pub(crate) struct Destructurer { - instance: NamedDestructurer, - #[allow(unused)] - source_group_span: DelimSpan, -} - -impl Parse for Destructurer { - fn parse(input: ParseStream) -> ParseResult { - let (delim_span, content) = input.parse_specific_group(Delimiter::Parenthesis)?; - content.parse::()?; - let destructurer_name = content.parse_any_ident()?; - let destructurer_kind = match DestructurerKind::for_ident(&destructurer_name) { - Some(destructurer_kind) => destructurer_kind, - None => destructurer_name.span().err( - format!( - "Expected `(!! ...)`, for one of: {}.\nIf this wasn't intended to be a named destructuring, you can work around this with (!raw! (!{} ... ))", - DestructurerKind::list_all(), - destructurer_name, - ), - )?, - }; - content.parse::()?; - let instance = destructurer_kind.parse_instance(DestructurerArguments::new( - &content, - destructurer_name, - delim_span.join(), - ))?; - Ok(Self { - instance, - source_group_span: delim_span, - }) - } -} - -impl HandleDestructure for Destructurer { - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - self.instance.handle_destructure(input, interpreter) - } -} - -macro_rules! define_destructurers { - ( - $( - $destructurer:ident, - )* - ) => { - #[allow(clippy::enum_variant_names)] - #[derive(Clone, Copy)] - pub(crate) enum DestructurerKind { - $( - $destructurer, - )* - } - - impl DestructurerKind { - fn parse_instance(&self, arguments: DestructurerArguments) -> ParseResult { - Ok(match self { - $( - Self::$destructurer => NamedDestructurer::$destructurer( - $destructurer::parse(arguments)? - ), - )* - }) - } - - pub(crate) fn for_ident(ident: &Ident) -> Option { - Some(match ident.to_string().as_ref() { - $( - $destructurer::DESTRUCTURER_NAME => Self::$destructurer, - )* - _ => return None, - }) - } - - const ALL_KIND_NAMES: &'static [&'static str] = &[$($destructurer::DESTRUCTURER_NAME,)*]; - - pub(crate) fn list_all() -> String { - // TODO improve to add an "and" at the end - Self::ALL_KIND_NAMES.join(", ") - } - } - - #[derive(Clone)] - #[allow(clippy::enum_variant_names)] - pub(crate) enum NamedDestructurer { - $( - $destructurer($destructurer), - )* - } - - impl NamedDestructurer { - fn handle_destructure(&self, input: ParseStream, interpreter: &mut Interpreter) -> ExecutionResult<()> { - match self { - $( - Self::$destructurer(destructurer) => destructurer.handle_destructure(input, interpreter), - )* - } - } - } - }; -} - -define_destructurers! { - StreamDestructurer, - IdentDestructurer, - LiteralDestructurer, - PunctDestructurer, - GroupDestructurer, - RawDestructurer, - ContentDestructurer, -} diff --git a/src/destructuring/destructurers.rs b/src/destructuring/destructurers.rs deleted file mode 100644 index 93b7df89..00000000 --- a/src/destructuring/destructurers.rs +++ /dev/null @@ -1,230 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) struct StreamDestructurer { - inner: DestructureRemaining, -} - -impl DestructurerDefinition for StreamDestructurer { - const DESTRUCTURER_NAME: &'static str = "stream"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - Ok(Self { - inner: arguments.fully_parse_no_error_override()?, - }) - } - - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - self.inner.handle_destructure(input, interpreter) - } -} - -#[derive(Clone)] -pub(crate) struct IdentDestructurer { - variable: Option, -} - -impl DestructurerDefinition for IdentDestructurer { - const DESTRUCTURER_NAME: &'static str = "ident"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - if input.is_empty() { - Ok(Self { variable: None }) - } else { - Ok(Self { - variable: Some(DestructureVariable::parse_only_unflattened_input(input)?), - }) - } - }, - "Expected (!ident! #x) or (!ident #>>x) or (!ident!)", - ) - } - - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - if input.cursor().ident().is_some() { - match &self.variable { - Some(variable) => variable.handle_destructure(input, interpreter), - None => { - let _ = input.parse_any_ident()?; - Ok(()) - } - } - } else { - input.parse_err("Expected an ident")? - } - } -} - -#[derive(Clone)] -pub(crate) struct LiteralDestructurer { - variable: Option, -} - -impl DestructurerDefinition for LiteralDestructurer { - const DESTRUCTURER_NAME: &'static str = "literal"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - if input.is_empty() { - Ok(Self { variable: None }) - } else { - Ok(Self { - variable: Some(DestructureVariable::parse_only_unflattened_input(input)?), - }) - } - }, - "Expected (!literal! #x) or (!literal! #>>x) or (!literal!)", - ) - } - - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - if input.cursor().literal().is_some() { - match &self.variable { - Some(variable) => variable.handle_destructure(input, interpreter), - None => { - let _ = input.parse::()?; - Ok(()) - } - } - } else { - input.parse_err("Expected a literal")? - } - } -} - -#[derive(Clone)] -pub(crate) struct PunctDestructurer { - variable: Option, -} - -impl DestructurerDefinition for PunctDestructurer { - const DESTRUCTURER_NAME: &'static str = "punct"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - if input.is_empty() { - Ok(Self { variable: None }) - } else { - Ok(Self { - variable: Some(DestructureVariable::parse_only_unflattened_input(input)?), - }) - } - }, - "Expected (!punct! #x) or (!punct! #>>x) or (!punct!)", - ) - } - - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - if input.cursor().any_punct().is_some() { - match &self.variable { - Some(variable) => variable.handle_destructure(input, interpreter), - None => { - let _ = input.parse_any_punct()?; - Ok(()) - } - } - } else { - input.parse_err("Expected a punct")? - } - } -} - -#[derive(Clone)] -pub(crate) struct GroupDestructurer { - inner: DestructureRemaining, -} - -impl DestructurerDefinition for GroupDestructurer { - const DESTRUCTURER_NAME: &'static str = "group"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - Ok(Self { - inner: arguments.fully_parse_no_error_override()?, - }) - } - - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - let (_, inner) = input.parse_specific_group(Delimiter::None)?; - self.inner.handle_destructure(&inner, interpreter) - } -} - -#[derive(Clone)] -pub(crate) struct RawDestructurer { - stream: RawDestructureStream, -} - -impl DestructurerDefinition for RawDestructurer { - const DESTRUCTURER_NAME: &'static str = "raw"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - let token_stream: TokenStream = arguments.fully_parse_no_error_override()?; - Ok(Self { - stream: RawDestructureStream::new_from_token_stream(token_stream), - }) - } - - fn handle_destructure( - &self, - input: ParseStream, - _: &mut Interpreter, - ) -> ExecutionResult<()> { - self.stream.handle_destructure(input) - } -} - -#[derive(Clone)] -pub(crate) struct ContentDestructurer { - stream: SourceStream, -} - -impl DestructurerDefinition for ContentDestructurer { - const DESTRUCTURER_NAME: &'static str = "content"; - - fn parse(arguments: DestructurerArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - stream: SourceStream::parse(input, arguments.full_span())?, - }) - }, - "Expected (!content! ... interpretable input ...)", - ) - } - - fn handle_destructure( - &self, - input: ParseStream, - interpreter: &mut Interpreter, - ) -> ExecutionResult<()> { - self.stream - .clone() - .interpret_to_new_stream(interpreter)? - .into_raw_destructure_stream() - .handle_destructure(input) - } -} diff --git a/src/destructuring/mod.rs b/src/destructuring/mod.rs deleted file mode 100644 index 0a2d15a5..00000000 --- a/src/destructuring/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -mod destructure_group; -mod destructure_item; -mod destructure_raw; -mod destructure_segment; -mod destructure_traits; -mod destructure_variable; -mod destructurer; -mod destructurers; -mod fields; - -pub(crate) use destructure_group::*; -pub(crate) use destructure_item::*; -pub(crate) use destructure_raw::*; -pub(crate) use destructure_segment::*; -pub(crate) use destructure_traits::*; -pub(crate) use destructure_variable::*; -pub(crate) use destructurer::*; -pub(crate) use destructurers::*; -#[allow(unused)] -pub(crate) use fields::*; diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 6fef7937..ead02c5f 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -74,48 +74,48 @@ impl Expressionable for Source { fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(Some(output_kind)) => { + SourcePeekMatch::Command(Some(output_kind)) => { match output_kind.expression_support() { Ok(()) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), Err(error_message) => return input.parse_err(error_message), } } - GrammarPeekMatch::Command(None) => return input.parse_err("Invalid command"), - GrammarPeekMatch::GroupedVariable => UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)), - GrammarPeekMatch::FlattenedVariable => return input.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), - GrammarPeekMatch::AppendVariableDestructuring => return input.parse_err("Append variable operations are not supported in an expression"), - GrammarPeekMatch::Destructurer(_) => return input.parse_err("Destructurings are not supported in an expression"), - GrammarPeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { + SourcePeekMatch::Command(None) => return input.parse_err("Invalid command"), + SourcePeekMatch::GroupedVariable => UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)), + SourcePeekMatch::FlattenedVariable => return input.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), + SourcePeekMatch::AppendVariableBinding => return input.parse_err("Append variable operations are not supported in an expression"), + SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => return input.parse_err("Destructurings are not supported in an expression"), + SourcePeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { let (_, delim_span) = input.parse_and_enter_group()?; UnaryAtom::Group(delim_span) }, - GrammarPeekMatch::Group(Delimiter::Brace) => { + SourcePeekMatch::Group(Delimiter::Brace) => { let leaf = SourceExpressionLeaf::CodeBlock(input.parse()?); UnaryAtom::Leaf(leaf) } - GrammarPeekMatch::Group(Delimiter::Bracket) => return input.parse_err("Square brackets [ .. ] are not supported in an expression"), - GrammarPeekMatch::Punct(_) => { + SourcePeekMatch::Group(Delimiter::Bracket) => return input.parse_err("Square brackets [ .. ] are not supported in an expression"), + SourcePeekMatch::Punct(_) => { UnaryAtom::PrefixUnaryOperation(input.parse()?) }, - GrammarPeekMatch::Ident(_) => { + SourcePeekMatch::Ident(_) => { let value = ExpressionValue::Boolean(ExpressionBoolean::for_litbool(input.parse()?)); UnaryAtom::Leaf(Self::Leaf::Value(value)) }, - GrammarPeekMatch::Literal(_) => { + SourcePeekMatch::Literal(_) => { let value = ExpressionValue::for_literal(input.parse()?)?; UnaryAtom::Leaf(Self::Leaf::Value(value)) }, - GrammarPeekMatch::End => return input.parse_err("The expression ended in an incomplete state"), + SourcePeekMatch::End => return input.parse_err("The expression ended in an incomplete state"), }) } fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { Ok(match input.peek_grammar() { - GrammarPeekMatch::Punct(_) => match input.try_parse_or_revert::() { + SourcePeekMatch::Punct(_) => match input.try_parse_or_revert::() { Ok(operation) => NodeExtension::BinaryOperation(operation), Err(_) => NodeExtension::NoneMatched, }, - GrammarPeekMatch::Ident(ident) if ident == "as" => { + SourcePeekMatch::Ident(ident) if ident == "as" => { let cast_operation = UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; NodeExtension::PostfixOperation(cast_operation) diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index 75e38789..fe02ebd6 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -219,7 +219,7 @@ impl<'a, K> ParseStreamStack<'a, K> { } impl ParseStreamStack<'_, Source> { - pub(crate) fn peek_grammar(&mut self) -> GrammarPeekMatch { + pub(crate) fn peek_grammar(&mut self) -> SourcePeekMatch { self.current().peek_grammar() } } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 8e68101d..8b45c829 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -14,8 +14,8 @@ pub(crate) use syn::parse::{ pub(crate) use syn::{parse_str, Lit, LitBool, LitFloat, LitInt, Token}; pub(crate) use syn::{Error as SynError, Result as SynResult}; -pub(crate) use crate::destructuring::*; pub(crate) use crate::expressions::*; pub(crate) use crate::extensions::*; pub(crate) use crate::interpretation::*; pub(crate) use crate::misc::*; +pub(crate) use crate::transformation::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 334f020a..b13816a3 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -10,7 +10,7 @@ pub(crate) enum CommandOutputKind { Ident, FlattenedStream, GroupedStream, - ControlFlowCodeStream, + Stream, } impl CommandOutputKind { @@ -20,7 +20,7 @@ impl CommandOutputKind { CommandOutputKind::None => Err("A command which returns nothing cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), CommandOutputKind::Ident => Err("A command which returns idents cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), CommandOutputKind::FlattenedStream => Err("A command which returns a flattened stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), - CommandOutputKind::ControlFlowCodeStream => Err("A control flow command which returns a code stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), + CommandOutputKind::Stream => Err("A control flow command which returns a code stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), } } } @@ -199,8 +199,8 @@ impl CommandInvocationAs for C { // OutputKindStream //================= -pub(crate) struct OutputKindStream; -impl OutputKind for OutputKindStream { +pub(crate) struct OutputKindGroupedStream; +impl OutputKind for OutputKindGroupedStream { type Output = OutputStream; fn resolve_standard() -> CommandOutputKind { @@ -212,8 +212,8 @@ impl OutputKind for OutputKindStream { } } -pub(crate) trait StreamCommandDefinition: - Sized + CommandType +pub(crate) trait GroupedStreamCommandDefinition: + Sized + CommandType { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; @@ -224,7 +224,7 @@ pub(crate) trait StreamCommandDefinition: ) -> ExecutionResult<()>; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self: Box, context: ExecutionContext, @@ -246,21 +246,22 @@ impl CommandInvocationAs for C { // OutputKindControlFlow //====================== -pub(crate) struct OutputKindControlFlow; -impl OutputKind for OutputKindControlFlow { +pub(crate) struct OutputKindStreaming; +impl OutputKind for OutputKindStreaming { type Output = (); fn resolve_standard() -> CommandOutputKind { - CommandOutputKind::ControlFlowCodeStream + CommandOutputKind::Stream } fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { - error_span_range.parse_err("This command is control flow, so is always flattened and cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command") + error_span_range.parse_err("This command always outputs a flattened stream and so cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command") } } -pub(crate) trait ControlFlowCommandDefinition: - Sized + CommandType +// Control Flow or a command which is unlikely to want grouped output +pub(crate) trait StreamingCommandDefinition: + Sized + CommandType { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; @@ -271,7 +272,7 @@ pub(crate) trait ControlFlowCommandDefinition: ) -> ExecutionResult<()>; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self: Box, context: ExecutionContext, @@ -346,8 +347,8 @@ macro_rules! define_command_kind { define_command_kind! { // Core Commands SetCommand, - LetCommand, RawCommand, + OutputCommand, IgnoreCommand, SettingsCommand, ErrorCommand, @@ -397,6 +398,10 @@ define_command_kind! { SplitCommand, CommaSplitCommand, ZipCommand, + + // Destructuring Commands + ParseCommand, + LetCommand, } #[derive(Clone)] diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index e1d902d6..28c31bf0 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -9,10 +9,10 @@ pub(crate) struct IfCommand { } impl CommandType for IfCommand { - type OutputKind = OutputKindControlFlow; + type OutputKind = OutputKindStreaming; } -impl ControlFlowCommandDefinition for IfCommand { +impl StreamingCommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -85,10 +85,10 @@ pub(crate) struct WhileCommand { } impl CommandType for WhileCommand { - type OutputKind = OutputKindControlFlow; + type OutputKind = OutputKindStreaming; } -impl ControlFlowCommandDefinition for WhileCommand { +impl StreamingCommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -143,10 +143,10 @@ pub(crate) struct LoopCommand { } impl CommandType for LoopCommand { - type OutputKind = OutputKindControlFlow; + type OutputKind = OutputKindStreaming; } -impl ControlFlowCommandDefinition for LoopCommand { +impl StreamingCommandDefinition for LoopCommand { const COMMAND_NAME: &'static str = "loop"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -185,18 +185,18 @@ impl ControlFlowCommandDefinition for LoopCommand { #[derive(Clone)] pub(crate) struct ForCommand { - parse_place: DestructureUntil, + parse_place: TransformStreamUntilToken, #[allow(unused)] in_token: Token![in], - input: CommandStreamInput, + input: SourceStreamInput, loop_code: SourceCodeBlock, } impl CommandType for ForCommand { - type OutputKind = OutputKindControlFlow; + type OutputKind = OutputKindStreaming; } -impl ControlFlowCommandDefinition for ForCommand { +impl StreamingCommandDefinition for ForCommand { const COMMAND_NAME: &'static str = "for"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -224,8 +224,12 @@ impl ControlFlowCommandDefinition for ForCommand { for token in stream { iteration_counter.increment_and_check()?; - self.parse_place - .handle_destructure_from_stream(token.into(), interpreter)?; + let mut ignored_transformer_output = OutputStream::new(); + self.parse_place.handle_transform_from_stream( + token.into(), + interpreter, + &mut ignored_transformer_output, + )?; match self .loop_code .clone() diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 83008a63..a4d9adf0 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -117,10 +117,10 @@ pub(crate) struct RawCommand { } impl CommandType for RawCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindStreaming; } -impl StreamCommandDefinition for RawCommand { +impl StreamingCommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -139,6 +139,33 @@ impl StreamCommandDefinition for RawCommand { } } +#[derive(Clone)] +pub(crate) struct OutputCommand { + inner: SourceStream, +} + +impl CommandType for OutputCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for OutputCommand { + const COMMAND_NAME: &'static str = "output"; + + fn parse(arguments: CommandArguments) -> ParseResult { + Ok(Self { + inner: arguments.parse_all_as_source()?, + }) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + self.inner.interpret_into(interpreter, output) + } +} + #[derive(Clone)] pub(crate) struct IgnoreCommand; @@ -173,7 +200,7 @@ define_field_inputs! { SettingsInputs { required: {}, optional: { - iteration_limit: CommandValueInput = DEFAULT_ITERATION_LIMIT ("The new iteration limit"), + iteration_limit: SourceValue = DEFAULT_ITERATION_LIMIT ("The new iteration limit"), } } } @@ -214,10 +241,10 @@ enum EitherErrorInput { define_field_inputs! { ErrorInputs { required: { - message: CommandValueInput = r#""...""# ("The error message to display"), + message: SourceValue = r#""...""# ("The error message to display"), }, optional: { - spans: CommandStreamInput = "[$abc]" ("An optional [token stream], to determine where to show the error message"), + spans: SourceStreamInput = "[$abc]" ("An optional [token stream], to determine where to show the error message"), } } } diff --git a/src/interpretation/commands/destructuring_commands.rs b/src/interpretation/commands/destructuring_commands.rs deleted file mode 100644 index ef6e7f76..00000000 --- a/src/interpretation/commands/destructuring_commands.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) struct LetCommand { - destructuring: DestructureUntil, - #[allow(unused)] - equals: Token![=], - arguments: SourceStream, -} - -impl CommandType for LetCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for LetCommand { - const COMMAND_NAME: &'static str = "let"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - destructuring: input.parse()?, - equals: input.parse()?, - arguments: input.parse_with_context(arguments.command_span())?, - }) - }, - "Expected [!let! = ...]", - ) - } - - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; - self.destructuring - .handle_destructure_from_stream(result_tokens, interpreter) - } -} diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 31f93cd7..782f3e8f 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -121,10 +121,10 @@ pub(crate) struct RangeCommand { } impl CommandType for RangeCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindGroupedStream; } -impl StreamCommandDefinition for RangeCommand { +impl GroupedStreamCommandDefinition for RangeCommand { const COMMAND_NAME: &'static str = "range"; fn parse(arguments: CommandArguments) -> ParseResult { diff --git a/src/interpretation/commands/mod.rs b/src/interpretation/commands/mod.rs index 97d4f826..8e132cf7 100644 --- a/src/interpretation/commands/mod.rs +++ b/src/interpretation/commands/mod.rs @@ -1,13 +1,13 @@ mod concat_commands; mod control_flow_commands; mod core_commands; -mod destructuring_commands; mod expression_commands; mod token_commands; +mod transforming_commands; pub(crate) use concat_commands::*; pub(crate) use control_flow_commands::*; pub(crate) use core_commands::*; -pub(crate) use destructuring_commands::*; pub(crate) use expression_commands::*; pub(crate) use token_commands::*; +pub(crate) use transforming_commands::*; diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index ac8480ca..565db7ec 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -57,10 +57,10 @@ pub(crate) struct GroupCommand { } impl CommandType for GroupCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindGroupedStream; } -impl StreamCommandDefinition for GroupCommand { +impl GroupedStreamCommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -86,23 +86,23 @@ pub(crate) struct IntersperseCommand { } impl CommandType for IntersperseCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindGroupedStream; } define_field_inputs! { IntersperseInputs { required: { - items: CommandStreamInput = "[Hello World] or #var or [!cmd! ...]", - separator: CommandStreamInput = "[,]" ("The token/s to add between each item"), + items: SourceStreamInput = "[Hello World] or #var or [!cmd! ...]", + separator: SourceStreamInput = "[,]" ("The token/s to add between each item"), }, optional: { - add_trailing: CommandValueInput = "false" ("Whether to add the separator after the last item (default: false)"), - final_separator: CommandStreamInput = "[or]" ("Define a different final separator (default: same as normal separator)"), + add_trailing: SourceValue = "false" ("Whether to add the separator after the last item (default: false)"), + final_separator: SourceStreamInput = "[or]" ("Define a different final separator (default: same as normal separator)"), } } } -impl StreamCommandDefinition for IntersperseCommand { +impl GroupedStreamCommandDefinition for IntersperseCommand { const COMMAND_NAME: &'static str = "intersperse"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -159,8 +159,8 @@ impl StreamCommandDefinition for IntersperseCommand { } struct SeparatorAppender { - separator: CommandStreamInput, - final_separator: Option, + separator: SourceStreamInput, + final_separator: Option, add_trailing: bool, } @@ -220,24 +220,24 @@ pub(crate) struct SplitCommand { } impl CommandType for SplitCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindGroupedStream; } define_field_inputs! { SplitInputs { required: { - stream: CommandStreamInput = "[...] or #var or [!cmd! ...]", - separator: CommandStreamInput = "[::]" ("The token/s to split if they match"), + stream: SourceStreamInput = "[...] or #var or [!cmd! ...]", + separator: SourceStreamInput = "[::]" ("The token/s to split if they match"), }, optional: { - drop_empty_start: CommandValueInput = "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), - drop_empty_middle: CommandValueInput = "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), - drop_empty_end: CommandValueInput = "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), + drop_empty_start: SourceValue = "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), + drop_empty_middle: SourceValue = "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), + drop_empty_end: SourceValue = "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), } } } -impl StreamCommandDefinition for SplitCommand { +impl GroupedStreamCommandDefinition for SplitCommand { const COMMAND_NAME: &'static str = "split"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -269,10 +269,14 @@ impl StreamCommandDefinition for SplitCommand { }; handle_split( + interpreter, stream, output, output_span, - separator.into_raw_destructure_stream(), + unsafe { + // RUST-ANALYZER SAFETY: This is as safe as we can get. + separator.parse_as()? + }, drop_empty_start, drop_empty_middle, drop_empty_end, @@ -280,11 +284,13 @@ impl StreamCommandDefinition for SplitCommand { } } +#[allow(clippy::too_many_arguments)] fn handle_split( + interpreter: &mut Interpreter, input: OutputStream, output: &mut OutputStream, output_span: Span, - separator: RawDestructureStream, + separator: ExactStream, drop_empty_start: bool, drop_empty_middle: bool, drop_empty_end: bool, @@ -297,7 +303,15 @@ fn handle_split( let mut drop_empty_next = drop_empty_start; while !input.is_empty() { let separator_fork = input.fork(); - if separator.handle_destructure(&separator_fork).is_err() { + let mut ignored_transformer_output = OutputStream::new(); + if separator + .handle_transform( + &separator_fork, + interpreter, + &mut ignored_transformer_output, + ) + .is_err() + { current_item.push_raw_token_tree(input.parse()?); continue; } @@ -322,10 +336,10 @@ pub(crate) struct CommaSplitCommand { } impl CommandType for CommaSplitCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindGroupedStream; } -impl StreamCommandDefinition for CommaSplitCommand { +impl GroupedStreamCommandDefinition for CommaSplitCommand { const COMMAND_NAME: &'static str = "comma_split"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -341,15 +355,21 @@ impl StreamCommandDefinition for CommaSplitCommand { ) -> ExecutionResult<()> { let output_span = self.input.span_range().join_into_span_else_start(); let stream = self.input.interpret_to_new_stream(interpreter)?; - let separator = { - let mut stream = RawDestructureStream::empty(); - stream.push_item(RawDestructureItem::Punct( - Punct::new(',', Spacing::Alone).with_span(output_span), - )); - stream - }; + let separator = Punct::new(',', Spacing::Alone) + .with_span(output_span) + .to_token_stream() + .source_parse_as()?; - handle_split(stream, output, output_span, separator, false, false, true) + handle_split( + interpreter, + stream, + output, + output_span, + separator, + false, + false, + true, + ) } } @@ -358,7 +378,7 @@ pub(crate) struct ZipCommand { inputs: EitherZipInput, } -type Streams = CommandValueInput>>; +type Streams = SourceValue>>; #[derive(Clone)] enum EitherZipInput { @@ -372,16 +392,16 @@ define_field_inputs! { streams: Streams = r#"([Hello Goodbye] [World Friend])"# ("A group of one or more streams to zip together. The outer brackets are used for the group."), }, optional: { - error_on_length_mismatch: CommandValueInput = "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), + error_on_length_mismatch: SourceValue = "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), } } } impl CommandType for ZipCommand { - type OutputKind = OutputKindStream; + type OutputKind = OutputKindGroupedStream; } -impl StreamCommandDefinition for ZipCommand { +impl GroupedStreamCommandDefinition for ZipCommand { const COMMAND_NAME: &'static str = "zip"; fn parse(arguments: CommandArguments) -> ParseResult { diff --git a/src/interpretation/commands/transforming_commands.rs b/src/interpretation/commands/transforming_commands.rs new file mode 100644 index 00000000..6fed8470 --- /dev/null +++ b/src/interpretation/commands/transforming_commands.rs @@ -0,0 +1,79 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct ParseCommand { + input: SourceStreamInput, + #[allow(unused)] + as_token: Token![as], + transformer: ExplicitTransformStream, +} + +impl CommandType for ParseCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for ParseCommand { + const COMMAND_NAME: &'static str = "parse"; + + fn parse(arguments: CommandArguments) -> ParseResult { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + input: input.parse()?, + as_token: input.parse()?, + transformer: input.parse()?, + }) + }, + "Expected [!parse! [...] as @(...)] or [!parse! #x as @(...)] where the latter is a transform stream", + ) + } + + fn execute( + self: Box, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + let input = self.input.interpret_to_new_stream(interpreter)?; + self.transformer + .handle_transform_from_stream(input, interpreter, output) + } +} + +#[derive(Clone)] +pub(crate) struct LetCommand { + destructuring: TransformStreamUntilToken, + #[allow(unused)] + equals: Token![=], + arguments: SourceStream, +} + +impl CommandType for LetCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for LetCommand { + const COMMAND_NAME: &'static str = "let"; + + fn parse(arguments: CommandArguments) -> ParseResult { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + destructuring: input.parse()?, + equals: input.parse()?, + arguments: input.parse_with_context(arguments.command_span())?, + }) + }, + "Expected [!let! = ...]", + ) + } + + fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { + let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; + let mut ignored_transformer_output = OutputStream::new(); + self.destructuring.handle_transform_from_stream( + result_tokens, + interpreter, + &mut ignored_transformer_output, + ) + } +} diff --git a/src/interpretation/interpretation_item.rs b/src/interpretation/interpretation_item.rs deleted file mode 100644 index 5fffed2f..00000000 --- a/src/interpretation/interpretation_item.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) enum SourceItem { - Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), - SourceGroup(SourceGroup), - Punct(Punct), - Ident(Ident), - Literal(Literal), -} - -impl Parse for SourceItem { - fn parse(input: ParseStream) -> ParseResult { - Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(_) => SourceItem::Command(input.parse()?), - GrammarPeekMatch::Group(_) => SourceItem::SourceGroup(input.parse()?), - GrammarPeekMatch::GroupedVariable => SourceItem::GroupedVariable(input.parse()?), - GrammarPeekMatch::FlattenedVariable => SourceItem::FlattenedVariable(input.parse()?), - GrammarPeekMatch::AppendVariableDestructuring | GrammarPeekMatch::Destructurer(_) => { - return input.parse_err("Destructurings are not supported here") - } - GrammarPeekMatch::Punct(_) => SourceItem::Punct(input.parse_any_punct()?), - GrammarPeekMatch::Ident(_) => SourceItem::Ident(input.parse_any_ident()?), - GrammarPeekMatch::Literal(_) => SourceItem::Literal(input.parse()?), - GrammarPeekMatch::End => return input.parse_err("Expected some item"), - }) - } -} - -impl Interpret for SourceItem { - fn interpret_into( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - match self { - SourceItem::Command(command_invocation) => { - command_invocation.interpret_into(interpreter, output)?; - } - SourceItem::GroupedVariable(variable) => { - variable.interpret_into(interpreter, output)?; - } - SourceItem::FlattenedVariable(variable) => { - variable.interpret_into(interpreter, output)?; - } - SourceItem::SourceGroup(group) => { - group.interpret_into(interpreter, output)?; - } - SourceItem::Punct(punct) => output.push_punct(punct), - SourceItem::Ident(ident) => output.push_ident(ident), - SourceItem::Literal(literal) => output.push_literal(literal), - } - Ok(()) - } -} - -impl HasSpanRange for SourceItem { - fn span_range(&self) -> SpanRange { - match self { - SourceItem::Command(command_invocation) => command_invocation.span_range(), - SourceItem::FlattenedVariable(variable) => variable.span_range(), - SourceItem::GroupedVariable(variable) => variable.span_range(), - SourceItem::SourceGroup(group) => group.span_range(), - SourceItem::Punct(punct) => punct.span_range(), - SourceItem::Ident(ident) => ident.span_range(), - SourceItem::Literal(literal) => literal.span_range(), - } - } -} diff --git a/src/interpretation/interpretation_stream.rs b/src/interpretation/interpretation_stream.rs deleted file mode 100644 index 688beaec..00000000 --- a/src/interpretation/interpretation_stream.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::internal_prelude::*; - -/// A parsed stream ready for interpretation -#[derive(Clone)] -pub(crate) struct SourceStream { - items: Vec, - span: Span, -} - -impl ContextualParse for SourceStream { - type Context = Span; - - fn parse(input: ParseStream, span: Self::Context) -> ParseResult { - let mut items = Vec::new(); - while !input.is_empty() { - items.push(input.parse()?); - } - Ok(Self { items, span }) - } -} - -impl Interpret for SourceStream { - fn interpret_into( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - for item in self.items { - item.interpret_into(interpreter, output)?; - } - Ok(()) - } -} - -impl HasSpan for SourceStream { - fn span(&self) -> Span { - self.span - } -} - -/// A parsed group ready for interpretation -#[derive(Clone)] -pub(crate) struct SourceGroup { - source_delimiter: Delimiter, - source_delim_span: DelimSpan, - content: SourceStream, -} - -impl SourceGroup { - pub(crate) fn into_content(self) -> SourceStream { - self.content - } -} - -impl Parse for SourceGroup { - fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, content) = input.parse_any_group()?; - let content = content.parse_with_context(delim_span.join())?; - Ok(Self { - source_delimiter: delimiter, - source_delim_span: delim_span, - content, - }) - } -} - -impl Interpret for SourceGroup { - fn interpret_into( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - let inner = self.content.interpret_to_new_stream(interpreter)?; - output.push_new_group(inner, self.source_delimiter, self.source_delim_span.join()); - Ok(()) - } -} - -impl HasSpan for SourceGroup { - fn span(&self) -> Span { - self.source_delim_span.join() - } -} - -/// A parsed group intended to be raw tokens -#[derive(Clone)] -pub(crate) struct RawGroup { - source_delimeter: Delimiter, - source_delim_span: DelimSpan, - content: TokenStream, -} - -#[allow(unused)] -impl RawGroup { - pub(crate) fn into_content(self) -> TokenStream { - self.content - } -} - -impl Parse for RawGroup { - fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, content) = input.parse_any_group()?; - let content = content.parse()?; - Ok(Self { - source_delimeter: delimiter, - source_delim_span: delim_span, - content, - }) - } -} - -impl Interpret for RawGroup { - fn interpret_into(self, _: &mut Interpreter, output: &mut OutputStream) -> ExecutionResult<()> { - output.push_new_group( - OutputStream::raw(self.content), - self.source_delimeter, - self.source_delim_span.join(), - ); - Ok(()) - } -} - -impl HasSpan for RawGroup { - fn span(&self) -> Span { - self.source_delim_span.join() - } -} diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 5a400806..45a09fd8 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -255,24 +255,6 @@ impl OutputStream { output } - pub(crate) fn into_raw_destructure_stream(self) -> RawDestructureStream { - let mut output = RawDestructureStream::empty(); - for segment in self.segments { - match segment { - OutputSegment::TokenVec(vec) => { - output.append_from_token_stream(vec); - } - OutputSegment::OutputGroup(delimiter, _, inner) => { - output.push_item(RawDestructureItem::Group(RawDestructureGroup::new( - delimiter, - inner.into_raw_destructure_stream(), - ))); - } - } - } - output - } - pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> String { fn concat_recursive_interpreted_stream( behaviour: &ConcatBehaviour, @@ -355,6 +337,13 @@ impl OutputStream { concat_recursive_interpreted_stream(behaviour, &mut output, Spacing::Joint, self); output } + + pub(crate) fn into_exact_stream(self) -> ParseResult { + unsafe { + // RUST-ANALYZER SAFETY: Can't be any safer than this for now + self.parse_as() + } + } } impl IntoIterator for OutputStream { @@ -466,7 +455,7 @@ impl From for OutputStream { // // There are a few places where we support (or might wish to support) parsing // as part of interpretation: -// * e.g. of a token stream in `CommandValueInput` +// * e.g. of a token stream in `SourceValue` // * e.g. as part of a PARSER, from an OutputStream // * e.g. of a variable, as part of incremental parsing (while_parse style loops) // diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 4b972a92..2dccad98 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,26 +1,22 @@ mod command; mod command_arguments; -mod command_code_input; -mod command_field_inputs; -mod command_stream_input; -mod command_value_input; mod commands; mod interpret_traits; -mod interpretation_item; -mod interpretation_stream; mod interpreted_stream; mod interpreter; +mod source_code_block; +mod source_stream; +mod source_stream_input; +mod source_value; mod variable; pub(crate) use command::*; pub(crate) use command_arguments::*; -pub(crate) use command_code_input::*; -pub(crate) use command_field_inputs::*; -pub(crate) use command_stream_input::*; -pub(crate) use command_value_input::*; pub(crate) use interpret_traits::*; -pub(crate) use interpretation_item::*; -pub(crate) use interpretation_stream::*; pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; +pub(crate) use source_code_block::*; +pub(crate) use source_stream::*; +pub(crate) use source_stream_input::*; +pub(crate) use source_value::*; pub(crate) use variable::*; diff --git a/src/interpretation/command_code_input.rs b/src/interpretation/source_code_block.rs similarity index 100% rename from src/interpretation/command_code_input.rs rename to src/interpretation/source_code_block.rs diff --git a/src/interpretation/source_stream.rs b/src/interpretation/source_stream.rs new file mode 100644 index 00000000..51b20ea8 --- /dev/null +++ b/src/interpretation/source_stream.rs @@ -0,0 +1,156 @@ +use crate::internal_prelude::*; + +/// A parsed stream ready for interpretation +#[derive(Clone)] +pub(crate) struct SourceStream { + items: Vec, + span: Span, +} + +impl ContextualParse for SourceStream { + type Context = Span; + + fn parse(input: ParseStream, span: Self::Context) -> ParseResult { + let mut items = Vec::new(); + while !input.is_empty() { + items.push(input.parse()?); + } + Ok(Self { items, span }) + } +} + +impl Interpret for SourceStream { + fn interpret_into( + self, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + for item in self.items { + item.interpret_into(interpreter, output)?; + } + Ok(()) + } +} + +impl HasSpan for SourceStream { + fn span(&self) -> Span { + self.span + } +} + +#[derive(Clone)] +pub(crate) enum SourceItem { + Command(Command), + GroupedVariable(GroupedVariable), + FlattenedVariable(FlattenedVariable), + SourceGroup(SourceGroup), + Punct(Punct), + Ident(Ident), + Literal(Literal), +} + +impl Parse for SourceItem { + fn parse(input: ParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + SourcePeekMatch::Command(_) => SourceItem::Command(input.parse()?), + SourcePeekMatch::Group(_) => SourceItem::SourceGroup(input.parse()?), + SourcePeekMatch::GroupedVariable => SourceItem::GroupedVariable(input.parse()?), + SourcePeekMatch::FlattenedVariable => SourceItem::FlattenedVariable(input.parse()?), + SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => { + return input.parse_err("Destructurings are not supported here. If this wasn't intended to be a destructuring, replace @ with [!raw! @]"); + } + SourcePeekMatch::AppendVariableBinding => { + return input.parse_err("Destructurings are not supported here."); + } + SourcePeekMatch::Punct(_) => SourceItem::Punct(input.parse_any_punct()?), + SourcePeekMatch::Ident(_) => SourceItem::Ident(input.parse_any_ident()?), + SourcePeekMatch::Literal(_) => SourceItem::Literal(input.parse()?), + SourcePeekMatch::End => return input.parse_err("Expected some item."), + }) + } +} + +impl Interpret for SourceItem { + fn interpret_into( + self, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + match self { + SourceItem::Command(command_invocation) => { + command_invocation.interpret_into(interpreter, output)?; + } + SourceItem::GroupedVariable(variable) => { + variable.interpret_into(interpreter, output)?; + } + SourceItem::FlattenedVariable(variable) => { + variable.interpret_into(interpreter, output)?; + } + SourceItem::SourceGroup(group) => { + group.interpret_into(interpreter, output)?; + } + SourceItem::Punct(punct) => output.push_punct(punct), + SourceItem::Ident(ident) => output.push_ident(ident), + SourceItem::Literal(literal) => output.push_literal(literal), + } + Ok(()) + } +} + +impl HasSpanRange for SourceItem { + fn span_range(&self) -> SpanRange { + match self { + SourceItem::Command(command_invocation) => command_invocation.span_range(), + SourceItem::FlattenedVariable(variable) => variable.span_range(), + SourceItem::GroupedVariable(variable) => variable.span_range(), + SourceItem::SourceGroup(group) => group.span_range(), + SourceItem::Punct(punct) => punct.span_range(), + SourceItem::Ident(ident) => ident.span_range(), + SourceItem::Literal(literal) => literal.span_range(), + } + } +} + +/// A parsed group ready for interpretation +#[derive(Clone)] +pub(crate) struct SourceGroup { + source_delimiter: Delimiter, + source_delim_span: DelimSpan, + content: SourceStream, +} + +impl SourceGroup { + pub(crate) fn into_content(self) -> SourceStream { + self.content + } +} + +impl Parse for SourceGroup { + fn parse(input: ParseStream) -> ParseResult { + let (delimiter, delim_span, content) = input.parse_any_group()?; + let content = content.parse_with_context(delim_span.join())?; + Ok(Self { + source_delimiter: delimiter, + source_delim_span: delim_span, + content, + }) + } +} + +impl Interpret for SourceGroup { + fn interpret_into( + self, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + let inner = self.content.interpret_to_new_stream(interpreter)?; + output.push_new_group(inner, self.source_delimiter, self.source_delim_span.join()); + Ok(()) + } +} + +impl HasSpan for SourceGroup { + fn span(&self) -> Span { + self.source_delim_span.join() + } +} diff --git a/src/interpretation/command_stream_input.rs b/src/interpretation/source_stream_input.rs similarity index 80% rename from src/interpretation/command_stream_input.rs rename to src/interpretation/source_stream_input.rs index b5620048..f499369d 100644 --- a/src/interpretation/command_stream_input.rs +++ b/src/interpretation/source_stream_input.rs @@ -13,7 +13,7 @@ use crate::internal_prelude::*; /// For example, as of Jan 2025, in declarative macro substitutions a $literal gets a wrapping group, /// but a $tt or $($tt)* does not. #[derive(Clone)] -pub(crate) enum CommandStreamInput { +pub(crate) enum SourceStreamInput { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), @@ -21,40 +21,40 @@ pub(crate) enum CommandStreamInput { ExplicitStream(SourceGroup), } -impl Parse for CommandStreamInput { +impl Parse for SourceStreamInput { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(_) => Self::Command(input.parse()?), - GrammarPeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - GrammarPeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - GrammarPeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), - GrammarPeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + SourcePeekMatch::Command(_) => Self::Command(input.parse()?), + SourcePeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + SourcePeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + SourcePeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), + SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), _ => input.span() .parse_err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, }) } } -impl HasSpanRange for CommandStreamInput { +impl HasSpanRange for SourceStreamInput { fn span_range(&self) -> SpanRange { match self { - CommandStreamInput::Command(command) => command.span_range(), - CommandStreamInput::GroupedVariable(variable) => variable.span_range(), - CommandStreamInput::FlattenedVariable(variable) => variable.span_range(), - CommandStreamInput::Code(code) => code.span_range(), - CommandStreamInput::ExplicitStream(group) => group.span_range(), + SourceStreamInput::Command(command) => command.span_range(), + SourceStreamInput::GroupedVariable(variable) => variable.span_range(), + SourceStreamInput::FlattenedVariable(variable) => variable.span_range(), + SourceStreamInput::Code(code) => code.span_range(), + SourceStreamInput::ExplicitStream(group) => group.span_range(), } } } -impl Interpret for CommandStreamInput { +impl Interpret for SourceStreamInput { fn interpret_into( self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { match self { - CommandStreamInput::Command(mut command) => { + SourceStreamInput::Command(mut command) => { match command.output_kind() { CommandOutputKind::None | CommandOutputKind::Value @@ -76,7 +76,7 @@ impl Interpret for CommandStreamInput { } command.interpret_into(interpreter, output) } - CommandOutputKind::ControlFlowCodeStream => parse_as_stream_input( + CommandOutputKind::Stream => parse_as_stream_input( command, interpreter, || { @@ -86,7 +86,7 @@ impl Interpret for CommandStreamInput { ), } } - CommandStreamInput::FlattenedVariable(variable) => parse_as_stream_input( + SourceStreamInput::FlattenedVariable(variable) => parse_as_stream_input( &variable, interpreter, || { @@ -97,10 +97,10 @@ impl Interpret for CommandStreamInput { }, output, ), - CommandStreamInput::GroupedVariable(variable) => { + SourceStreamInput::GroupedVariable(variable) => { variable.substitute_ungrouped_contents_into(interpreter, output) } - CommandStreamInput::Code(code) => parse_as_stream_input( + SourceStreamInput::Code(code) => parse_as_stream_input( code, interpreter, || { @@ -108,7 +108,7 @@ impl Interpret for CommandStreamInput { }, output, ), - CommandStreamInput::ExplicitStream(group) => { + SourceStreamInput::ExplicitStream(group) => { group.into_content().interpret_into(interpreter, output) } } @@ -135,7 +135,7 @@ fn parse_as_stream_input( Ok(()) } -impl InterpretValue for CommandStreamInput { +impl InterpretValue for SourceStreamInput { type OutputValue = OutputCommandStream; fn interpret_to_value( diff --git a/src/interpretation/command_value_input.rs b/src/interpretation/source_value.rs similarity index 66% rename from src/interpretation/command_value_input.rs rename to src/interpretation/source_value.rs index d980e763..292443c0 100644 --- a/src/interpretation/command_value_input.rs +++ b/src/interpretation/source_value.rs @@ -2,10 +2,10 @@ use crate::internal_prelude::*; /// This can be use to represent a value, or a source of that value at parse time. /// -/// For example, `CommandValueInput` could be used to represent a literal, +/// For example, `SourceValue` could be used to represent a literal, /// or a variable/command which could convert to a literal after interpretation. #[derive(Clone)] -pub(crate) enum CommandValueInput { +pub(crate) enum SourceValue { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), @@ -13,66 +13,68 @@ pub(crate) enum CommandValueInput { Value(T), } -impl> Parse for CommandValueInput { +impl> Parse for SourceValue { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(_) => Self::Command(input.parse()?), - GrammarPeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - GrammarPeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), - GrammarPeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), - GrammarPeekMatch::AppendVariableDestructuring | GrammarPeekMatch::Destructurer(_) => { + SourcePeekMatch::Command(_) => Self::Command(input.parse()?), + SourcePeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), + SourcePeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), + SourcePeekMatch::ExplicitTransformStream + | SourcePeekMatch::AppendVariableBinding + | SourcePeekMatch::Transformer(_) => { return input .span() .parse_err("Destructurings are not supported here") } - GrammarPeekMatch::Group(_) - | GrammarPeekMatch::Punct(_) - | GrammarPeekMatch::Literal(_) - | GrammarPeekMatch::Ident(_) - | GrammarPeekMatch::End => Self::Value(input.parse()?), + SourcePeekMatch::Group(_) + | SourcePeekMatch::Punct(_) + | SourcePeekMatch::Literal(_) + | SourcePeekMatch::Ident(_) + | SourcePeekMatch::End => Self::Value(input.parse()?), }) } } -impl> Parse for CommandValueInput { +impl> Parse for SourceValue { fn parse(input: ParseStream) -> ParseResult { Ok(Self::Value(input.parse()?)) } } -impl HasSpanRange for CommandValueInput { +impl HasSpanRange for SourceValue { fn span_range(&self) -> SpanRange { match self { - CommandValueInput::Command(command) => command.span_range(), - CommandValueInput::GroupedVariable(variable) => variable.span_range(), - CommandValueInput::FlattenedVariable(variable) => variable.span_range(), - CommandValueInput::Code(code) => code.span_range(), - CommandValueInput::Value(value) => value.span_range(), + SourceValue::Command(command) => command.span_range(), + SourceValue::GroupedVariable(variable) => variable.span_range(), + SourceValue::FlattenedVariable(variable) => variable.span_range(), + SourceValue::Code(code) => code.span_range(), + SourceValue::Value(value) => value.span_range(), } } } -impl, I: Parse> InterpretValue for CommandValueInput { +impl, I: Parse> InterpretValue for SourceValue { type OutputValue = I; fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { let descriptor = match self { - CommandValueInput::Command(_) => "command output", - CommandValueInput::GroupedVariable(_) => "grouped variable output", - CommandValueInput::FlattenedVariable(_) => "flattened variable output", - CommandValueInput::Code(_) => "output from the { ... } block", - CommandValueInput::Value(_) => "value", + SourceValue::Command(_) => "command output", + SourceValue::GroupedVariable(_) => "grouped variable output", + SourceValue::FlattenedVariable(_) => "flattened variable output", + SourceValue::Code(_) => "output from the { ... } block", + SourceValue::Value(_) => "value", }; let interpreted_stream = match self { - CommandValueInput::Command(command) => command.interpret_to_new_stream(interpreter)?, - CommandValueInput::GroupedVariable(variable) => { + SourceValue::Command(command) => command.interpret_to_new_stream(interpreter)?, + SourceValue::GroupedVariable(variable) => { variable.interpret_to_new_stream(interpreter)? } - CommandValueInput::FlattenedVariable(variable) => { + SourceValue::FlattenedVariable(variable) => { variable.interpret_to_new_stream(interpreter)? } - CommandValueInput::Code(code) => code.interpret_to_new_stream(interpreter)?, - CommandValueInput::Value(value) => return value.interpret_to_value(interpreter), + SourceValue::Code(code) => code.interpret_to_new_stream(interpreter)?, + SourceValue::Value(value) => return value.interpret_to_value(interpreter), }; unsafe { // RUST-ANALYZER SAFETY: If I is a very simple parse function, this is safe. diff --git a/src/lib.rs b/src/lib.rs index a1db817c..29d40c23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -510,12 +510,12 @@ //! //! Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in this crate by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. //! -mod destructuring; mod expressions; mod extensions; mod internal_prelude; mod interpretation; mod misc; +mod transformation; use internal_prelude::*; diff --git a/src/misc/errors.rs b/src/misc/errors.rs index 6572ba7e..94e0143b 100644 --- a/src/misc/errors.rs +++ b/src/misc/errors.rs @@ -59,6 +59,13 @@ impl ParseError { other => other, } } + + pub(crate) fn context(&self) -> Option<&str> { + match self { + ParseError::Standard(_) => None, + ParseError::Contextual(_, context) => Some(context), + } + } } // Ideally this would be our own enum with Completed / Interrupted variants, diff --git a/src/interpretation/command_field_inputs.rs b/src/misc/field_inputs.rs similarity index 100% rename from src/interpretation/command_field_inputs.rs rename to src/misc/field_inputs.rs diff --git a/src/misc/mod.rs b/src/misc/mod.rs index 475a8180..2dbf0503 100644 --- a/src/misc/mod.rs +++ b/src/misc/mod.rs @@ -1,8 +1,10 @@ mod errors; +mod field_inputs; mod parse_traits; mod string_conversion; pub(crate) use errors::*; +pub(crate) use field_inputs::*; pub(crate) use parse_traits::*; pub(crate) use string_conversion::*; diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 3cd2ca51..7e4fb1cc 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -8,18 +8,19 @@ use crate::internal_prelude::*; pub(crate) struct Source; impl ParseBuffer<'_, Source> { - pub(crate) fn peek_grammar(&self) -> GrammarPeekMatch { + pub(crate) fn peek_grammar(&self) -> SourcePeekMatch { detect_preinterpret_grammar(self.cursor()) } } #[allow(unused)] -pub(crate) enum GrammarPeekMatch { +pub(crate) enum SourcePeekMatch { Command(Option), GroupedVariable, FlattenedVariable, - AppendVariableDestructuring, - Destructurer(Option), + AppendVariableBinding, + ExplicitTransformStream, + Transformer(Option), Group(Delimiter), Ident(Ident), Punct(Punct), @@ -27,7 +28,7 @@ pub(crate) enum GrammarPeekMatch { End, } -fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> GrammarPeekMatch { +fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> SourcePeekMatch { // We have to check groups first, so that we handle transparent groups // and avoid the self.ignore_none() calls inside cursor if let Some((next, delimiter, _, _)) = cursor.any_group() { @@ -37,7 +38,7 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> GrammarPeekMatch if next.punct_matching('!').is_some() { let output_kind = CommandKind::for_ident(&ident).map(|kind| kind.standard_output_kind()); - return GrammarPeekMatch::Command(output_kind); + return SourcePeekMatch::Command(output_kind); } } if let Some((first, next)) = next.punct_matching('.') { @@ -47,22 +48,13 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> GrammarPeekMatch let output_kind = CommandKind::for_ident(&ident).and_then(|kind| { kind.flattened_output_kind(first.span_range()).ok() }); - return GrammarPeekMatch::Command(output_kind); + return SourcePeekMatch::Command(output_kind); } } } } } } - if delimiter == Delimiter::Parenthesis { - if let Some((_, next)) = next.punct_matching('!') { - if let Some((ident, next)) = next.ident() { - if next.punct_matching('!').is_some() { - return GrammarPeekMatch::Destructurer(DestructurerKind::for_ident(&ident)); - } - } - } - } // Ideally we'd like to detect $($tt)* substitutions from macros and interpret them as // a Raw (uninterpreted) group, because typically that's what a user would typically intend. @@ -75,42 +67,63 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> GrammarPeekMatch // So this isn't possible. It's unlikely to matter much, and a user can always do: // [!raw! $($tt)*] anyway. - return GrammarPeekMatch::Group(delimiter); + return SourcePeekMatch::Group(delimiter); } if let Some((_, next)) = cursor.punct_matching('#') { if next.ident().is_some() { - return GrammarPeekMatch::GroupedVariable; + return SourcePeekMatch::GroupedVariable; } if let Some((_, next)) = next.punct_matching('.') { if let Some((_, next)) = next.punct_matching('.') { if next.ident().is_some() { - return GrammarPeekMatch::FlattenedVariable; + return SourcePeekMatch::FlattenedVariable; } if let Some((_, next)) = next.punct_matching('>') { if next.punct_matching('>').is_some() { - return GrammarPeekMatch::AppendVariableDestructuring; + return SourcePeekMatch::AppendVariableBinding; } } } } if let Some((_, next)) = next.punct_matching('>') { if next.punct_matching('>').is_some() { - return GrammarPeekMatch::AppendVariableDestructuring; + return SourcePeekMatch::AppendVariableBinding; + } + } + } + + if let Some((_, next)) = cursor.punct_matching('@') { + if let Some((_, _, _)) = next.group_matching(Delimiter::Parenthesis) { + // @(...) or @(_ = ...) or @(#x = ...) + return SourcePeekMatch::ExplicitTransformStream; + } + if let Some((ident, _)) = next.ident() { + let name = ident.to_string(); + if name.to_uppercase() == name { + return SourcePeekMatch::Transformer(TransformerKind::for_ident(&ident)); + } + } + if let Some((_, next, _)) = next.group_matching(Delimiter::Bracket) { + if let Some((ident, _)) = next.ident() { + let name = ident.to_string(); + if name.to_uppercase() == name { + return SourcePeekMatch::Transformer(TransformerKind::for_ident(&ident)); + } } } } match cursor.token_tree() { - Some((TokenTree::Ident(ident), _)) => GrammarPeekMatch::Ident(ident), - Some((TokenTree::Punct(punct), _)) => GrammarPeekMatch::Punct(punct), - Some((TokenTree::Literal(literal), _)) => GrammarPeekMatch::Literal(literal), + Some((TokenTree::Ident(ident), _)) => SourcePeekMatch::Ident(ident), + Some((TokenTree::Punct(punct), _)) => SourcePeekMatch::Punct(punct), + Some((TokenTree::Literal(literal), _)) => SourcePeekMatch::Literal(literal), Some((TokenTree::Group(_), _)) => unreachable!("Already covered above"), - None => GrammarPeekMatch::End, + None => SourcePeekMatch::End, } } // Parsing of already interpreted tokens -// (e.g. destructuring) +// (e.g. transforming / destructuring) // ===================================== pub(crate) struct Output; diff --git a/src/transformation/exact_stream.rs b/src/transformation/exact_stream.rs new file mode 100644 index 00000000..a470c3c8 --- /dev/null +++ b/src/transformation/exact_stream.rs @@ -0,0 +1,190 @@ +use crate::internal_prelude::*; + +// An ExactStream is a transformer created with @[EXACT ...]. +// It has subtly different behaviour to a transform stream, but they can be nested inside each other. +// +// It differs in two ways: +// * Commands and Outputs are interpreted, and then matched from the parse stream +// * Each consumed item is output as-is + +pub(crate) type ExactStream = ExactSegment; + +#[derive(Clone)] +pub(crate) struct ExactSegment { + stop_condition: PhantomData, + inner: Vec, +} + +impl, K> Parse for ExactSegment +where + ExactItem: Parse, +{ + fn parse(input: ParseStream) -> ParseResult { + let mut inner = vec![]; + while !C::should_stop(input) { + inner.push(ExactItem::parse(input)?); + } + Ok(Self { + stop_condition: PhantomData, + inner, + }) + } +} + +impl HandleTransformation for ExactSegment { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + for item in self.inner.iter() { + item.handle_transform(input, interpreter, output)?; + } + Ok(()) + } +} + +/// This must match item-by-item. +#[derive(Clone)] +pub(crate) enum ExactItem { + TransformStreamInput(ExplicitTransformStream), + Transformer(Transformer), + ExactCommandOutput(Command), + ExactGroupedVariableOutput(GroupedVariable), + ExactFlattenedVariableOutput(FlattenedVariable), + ExactPunct(Punct), + ExactIdent(Ident), + ExactLiteral(Literal), + ExactGroup(ExactGroup), +} + +impl Parse for ExactItem { + fn parse(input: ParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + SourcePeekMatch::Command(_) => Self::ExactCommandOutput(input.parse()?), + SourcePeekMatch::GroupedVariable => Self::ExactGroupedVariableOutput(input.parse()?), + SourcePeekMatch::FlattenedVariable => { + Self::ExactFlattenedVariableOutput(input.parse()?) + } + SourcePeekMatch::AppendVariableBinding => { + return input + .parse_err("Append variable bindings are not supported in an EXACT stream") + } + SourcePeekMatch::ExplicitTransformStream => Self::TransformStreamInput(input.parse()?), + SourcePeekMatch::Transformer(_) => Self::Transformer(input.parse()?), + SourcePeekMatch::Group(_) => Self::ExactGroup(input.parse()?), + SourcePeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), + SourcePeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), + SourcePeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), + SourcePeekMatch::End => return input.parse_err("Unexpected end"), + }) + } +} + +impl Parse for ExactItem { + fn parse(input: ParseStream) -> ParseResult { + Ok(match input.peek_grammar() { + OutputPeekMatch::Group(_) => Self::ExactGroup(input.parse()?), + OutputPeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), + OutputPeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), + OutputPeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), + OutputPeekMatch::End => return input.parse_err("Unexpected end"), + }) + } +} + +impl HandleTransformation for ExactItem { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + match self { + ExactItem::TransformStreamInput(transform_stream_input) => { + transform_stream_input.handle_transform(input, interpreter, output)?; + } + ExactItem::Transformer(transformer) => { + transformer.handle_transform(input, interpreter, output)?; + } + ExactItem::ExactCommandOutput(command) => { + command + .clone() + .interpret_to_new_stream(interpreter)? + .into_exact_stream()? + .handle_transform(input, interpreter, output)?; + } + ExactItem::ExactGroupedVariableOutput(grouped_variable) => { + grouped_variable + .interpret_to_new_stream(interpreter)? + .into_exact_stream()? + .handle_transform(input, interpreter, output)?; + } + ExactItem::ExactFlattenedVariableOutput(flattened_variable) => { + flattened_variable + .interpret_to_new_stream(interpreter)? + .into_exact_stream()? + .handle_transform(input, interpreter, output)?; + } + ExactItem::ExactPunct(punct) => { + output.push_punct(input.parse_punct_matching(punct.as_char())?); + } + ExactItem::ExactIdent(ident) => { + output.push_ident(input.parse_ident_matching(&ident.to_string())?); + } + ExactItem::ExactLiteral(literal) => { + output.push_literal(input.parse_literal_matching(&literal.to_string())?); + } + ExactItem::ExactGroup(exact_group) => { + exact_group.handle_transform(input, interpreter, output)? + } + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct ExactGroup { + delimiter: Delimiter, + inner: ExactStream, +} + +impl Parse for ExactGroup +where + ExactStream: Parse, +{ + fn parse(input: ParseStream) -> ParseResult { + let (delimiter, _, content) = input.parse_any_group()?; + Ok(Self { + delimiter, + inner: content.parse()?, + }) + } +} + +impl HandleTransformation for ExactGroup { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + // Because `None` is ignored by Syn at parsing time, we can effectively be most permissive by ignoring them. + // This removes a bit of a footgun for users. + // If they really want to check for a None group, they can embed `@[GROUP @[EXACT ...]]` transformer. + if self.delimiter == Delimiter::None { + self.inner.handle_transform(input, interpreter, output) + } else { + let (source_span, inner_source) = input.parse_specific_group(self.delimiter)?; + output.push_grouped( + |inner_output| { + self.inner + .handle_transform(&inner_source, interpreter, inner_output) + }, + self.delimiter, + source_span.join(), + ) + } + } +} diff --git a/src/destructuring/fields.rs b/src/transformation/fields.rs similarity index 100% rename from src/destructuring/fields.rs rename to src/transformation/fields.rs diff --git a/src/transformation/mod.rs b/src/transformation/mod.rs new file mode 100644 index 00000000..ad5193ae --- /dev/null +++ b/src/transformation/mod.rs @@ -0,0 +1,18 @@ +mod exact_stream; +mod fields; +mod parse_utilities; +mod transform_stream; +mod transformation_traits; +mod transformer; +mod transformers; +mod variable_binding; + +pub(crate) use exact_stream::*; +#[allow(unused)] +pub(crate) use fields::*; +pub(crate) use parse_utilities::*; +pub(crate) use transform_stream::*; +pub(crate) use transformation_traits::*; +pub(crate) use transformer::*; +pub(crate) use transformers::*; +pub(crate) use variable_binding::*; diff --git a/src/transformation/parse_utilities.rs b/src/transformation/parse_utilities.rs new file mode 100644 index 00000000..0221583b --- /dev/null +++ b/src/transformation/parse_utilities.rs @@ -0,0 +1,134 @@ +use crate::internal_prelude::*; + +/// Designed to instruct the parser to stop parsing when a certain condition is met +pub(crate) trait StopCondition: Clone { + fn should_stop(input: ParseStream) -> bool; +} + +#[derive(Clone)] +pub(crate) struct UntilEnd; +impl StopCondition for UntilEnd { + fn should_stop(input: ParseStream) -> bool { + input.is_empty() + } +} + +pub(crate) trait PeekableToken: Clone { + fn peek(input: ParseStream) -> bool; +} + +// This is going through such pain to ensure we stay in the public API of syn +// We'd like to be able to use `input.peek::()` for some syn::token::Token +// but that's just not an API they support for some reason +macro_rules! impl_peekable_token { + ($(Token![$token:tt]),* $(,)?) => { + $( + impl PeekableToken for Token![$token] { + fn peek(input: ParseStream) -> bool { + input.peek(Token![$token]) + } + } + )* + }; +} + +impl_peekable_token! { + Token![=], + Token![in], +} + +#[derive(Clone)] +pub(crate) struct UntilToken { + token: PhantomData, +} + +impl, K> StopCondition for UntilToken { + fn should_stop(input: ParseStream) -> bool { + input.is_empty() || T::peek(input) + } +} + +/// Designed to automatically discover (via peeking) the next token, to limit the extent of a +/// match such as `#..x`. +#[derive(Clone)] +pub(crate) enum ParseUntil { + End, + Group(Delimiter), + Ident(Ident), + Punct(Punct), + Literal(Literal), +} + +impl ParseUntil { + /// Peeks the next token, to discover what we should parse next + pub(crate) fn peek_flatten_limit>( + input: ParseStream, + ) -> ParseResult { + if C::should_stop(input) { + return Ok(ParseUntil::End); + } + Ok(match input.peek_grammar() { + SourcePeekMatch::Command(_) + | SourcePeekMatch::GroupedVariable + | SourcePeekMatch::FlattenedVariable + | SourcePeekMatch::Transformer(_) + | SourcePeekMatch::ExplicitTransformStream + | SourcePeekMatch::AppendVariableBinding => { + return input + .span() + .parse_err("This cannot follow a flattened variable binding"); + } + SourcePeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), + SourcePeekMatch::Ident(ident) => ParseUntil::Ident(ident), + SourcePeekMatch::Literal(literal) => ParseUntil::Literal(literal), + SourcePeekMatch::Punct(punct) => ParseUntil::Punct(punct), + SourcePeekMatch::End => ParseUntil::End, + }) + } + + pub(crate) fn handle_parse_into( + &self, + input: ParseStream, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + match self { + ParseUntil::End => output.extend_raw_tokens(input.parse::()?), + ParseUntil::Group(delimiter) => { + while !input.is_empty() { + if input.peek_specific_group(*delimiter) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + ParseUntil::Ident(ident) => { + let content = ident.to_string(); + while !input.is_empty() { + if input.peek_ident_matching(&content) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + ParseUntil::Punct(punct) => { + let punct = punct.as_char(); + while !input.is_empty() { + if input.peek_punct_matching(punct) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + ParseUntil::Literal(literal) => { + let content = literal.to_string(); + while !input.is_empty() { + if input.peek_literal_matching(&content) { + return Ok(()); + } + output.push_raw_token_tree(input.parse()?); + } + } + } + Ok(()) + } +} diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs new file mode 100644 index 00000000..76a94e24 --- /dev/null +++ b/src/transformation/transform_stream.rs @@ -0,0 +1,259 @@ +use crate::internal_prelude::*; + +pub(crate) type TransformStream = TransformSegment; +pub(crate) type TransformStreamUntilToken = TransformSegment>; + +#[derive(Clone)] +pub(crate) struct TransformSegment { + stop_condition: PhantomData, + inner: Vec, +} + +impl> Parse for TransformSegment { + fn parse(input: ParseStream) -> ParseResult { + let mut inner = vec![]; + while !C::should_stop(input) { + inner.push(TransformItem::parse_until::(input)?); + } + Ok(Self { + stop_condition: PhantomData, + inner, + }) + } +} + +impl HandleTransformation for TransformSegment { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + for item in self.inner.iter() { + item.handle_transform(input, interpreter, output)?; + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) enum TransformItem { + Command(Command), + Variable(VariableBinding), + Transformer(Transformer), + TransformStreamInput(ExplicitTransformStream), + ExactPunct(Punct), + ExactIdent(Ident), + ExactLiteral(Literal), + ExactGroup(TransformGroup), +} + +impl TransformItem { + /// We provide a stop condition so that some of the items can know when to stop consuming greedily - + /// notably the flattened command. This allows [!let! #..x = Hello => World] to parse as setting + /// `x` to `Hello => World` rather than having `#..x` peeking to see it is "up to =" and then only + /// parsing `Hello` into `x`. + pub(crate) fn parse_until>( + input: ParseStream, + ) -> ParseResult { + Ok(match input.peek_grammar() { + SourcePeekMatch::Command(_) => Self::Command(input.parse()?), + SourcePeekMatch::GroupedVariable + | SourcePeekMatch::FlattenedVariable + | SourcePeekMatch::AppendVariableBinding => { + Self::Variable(VariableBinding::parse_until::(input)?) + } + SourcePeekMatch::Group(_) => Self::ExactGroup(input.parse()?), + SourcePeekMatch::ExplicitTransformStream => Self::TransformStreamInput(input.parse()?), + SourcePeekMatch::Transformer(_) => Self::Transformer(input.parse()?), + SourcePeekMatch::Punct(_) => Self::ExactPunct(input.parse_any_punct()?), + SourcePeekMatch::Literal(_) => Self::ExactLiteral(input.parse()?), + SourcePeekMatch::Ident(_) => Self::ExactIdent(input.parse_any_ident()?), + SourcePeekMatch::End => return input.parse_err("Unexpected end"), + }) + } +} + +impl HandleTransformation for TransformItem { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + match self { + TransformItem::Variable(variable) => { + variable.handle_transform(input, interpreter, output)?; + } + TransformItem::Command(command) => { + command.clone().interpret_into(interpreter, output)?; + } + TransformItem::Transformer(transformer) => { + transformer.handle_transform(input, interpreter, output)?; + } + TransformItem::TransformStreamInput(stream) => { + stream.handle_transform(input, interpreter, output)?; + } + TransformItem::ExactPunct(punct) => { + input.parse_punct_matching(punct.as_char())?; + } + TransformItem::ExactIdent(ident) => { + input.parse_ident_matching(&ident.to_string())?; + } + TransformItem::ExactLiteral(literal) => { + input.parse_literal_matching(&literal.to_string())?; + } + TransformItem::ExactGroup(group) => { + group.handle_transform(input, interpreter, output)?; + } + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct TransformGroup { + delimiter: Delimiter, + inner: TransformStream, +} + +impl Parse for TransformGroup { + fn parse(input: ParseStream) -> ParseResult { + let (delimiter, _, content) = input.parse_any_group()?; + Ok(Self { + delimiter, + inner: content.parse()?, + }) + } +} + +impl HandleTransformation for TransformGroup { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + let (_, inner) = input.parse_specific_group(self.delimiter)?; + self.inner.handle_transform(&inner, interpreter, output) + } +} + +#[derive(Clone)] +pub(crate) struct ExplicitTransformStream { + #[allow(unused)] + transformer_token: Token![@], + #[allow(unused)] + delim_span: DelimSpan, + arguments: ExplicitTransformStreamArguments, +} + +#[derive(Clone)] +pub(crate) enum ExplicitTransformStreamArguments { + Output { + content: TransformStream, + }, + StoreToVariable { + variable: GroupedVariable, + #[allow(unused)] + equals: Token![=], + content: TransformStream, + }, + ExtendToVariable { + variable: GroupedVariable, + #[allow(unused)] + plus_equals: Token![+=], + content: TransformStream, + }, + Discard { + #[allow(unused)] + discard: Token![_], + #[allow(unused)] + equals: Token![=], + content: TransformStream, + }, +} + +impl Parse for ExplicitTransformStreamArguments { + fn parse(input: ParseStream) -> ParseResult { + if input.peek(Token![_]) { + return Ok(Self::Discard { + discard: input.parse()?, + equals: input.parse()?, + content: input.parse()?, + }); + } + if input.peek(Token![#]) { + let variable = input.parse()?; + let lookahead = input.lookahead1(); + if lookahead.peek(Token![=]) { + return Ok(Self::StoreToVariable { + variable, + equals: input.parse()?, + content: input.parse()?, + }); + } + if lookahead.peek(Token![+=]) { + return Ok(Self::ExtendToVariable { + variable, + plus_equals: input.parse()?, + content: input.parse()?, + }); + } + Err(lookahead.error())?; + } + Ok(Self::Output { + content: input.parse()?, + }) + } +} + +impl Parse for ExplicitTransformStream { + fn parse(input: ParseStream) -> ParseResult { + let transformer_token = input.parse()?; + let (delim_span, content) = input.parse_specific_group(Delimiter::Parenthesis)?; + + Ok(Self { + transformer_token, + delim_span, + arguments: content.parse()?, + }) + } +} + +impl HandleTransformation for ExplicitTransformStream { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + match &self.arguments { + ExplicitTransformStreamArguments::Output { content } => { + content.handle_transform(input, interpreter, output)?; + } + ExplicitTransformStreamArguments::StoreToVariable { + variable, content, .. + } => { + let mut new_output = OutputStream::new(); + content.handle_transform(input, interpreter, &mut new_output)?; + variable.set(interpreter, new_output)?; + } + ExplicitTransformStreamArguments::ExtendToVariable { + variable, content, .. + } => { + let variable_data = variable.get_existing_for_mutation(interpreter)?; + content.handle_transform( + input, + interpreter, + variable_data.get_mut(variable)?.deref_mut(), + )?; + } + ExplicitTransformStreamArguments::Discard { content, .. } => { + let mut discarded = OutputStream::new(); + content.handle_transform(input, interpreter, &mut discarded)?; + } + } + Ok(()) + } +} diff --git a/src/destructuring/destructure_traits.rs b/src/transformation/transformation_traits.rs similarity index 67% rename from src/destructuring/destructure_traits.rs rename to src/transformation/transformation_traits.rs index 9f412a45..6d964469 100644 --- a/src/destructuring/destructure_traits.rs +++ b/src/transformation/transformation_traits.rs @@ -1,22 +1,24 @@ use crate::internal_prelude::*; -pub(crate) trait HandleDestructure { - fn handle_destructure_from_stream( +pub(crate) trait HandleTransformation { + fn handle_transform_from_stream( &self, input: OutputStream, interpreter: &mut Interpreter, + output: &mut OutputStream, ) -> ExecutionResult<()> { unsafe { // RUST-ANALYZER-SAFETY: ...this isn't generally safe... // We should only do this when we know that either the input or parser doesn't require // analysis of nested None-delimited groups. - input.parse_with(|input| self.handle_destructure(input, interpreter)) + input.parse_with(|input| self.handle_transform(input, interpreter, output)) } } - fn handle_destructure( + fn handle_transform( &self, input: ParseStream, interpreter: &mut Interpreter, + output: &mut OutputStream, ) -> ExecutionResult<()>; } diff --git a/src/transformation/transformer.rs b/src/transformation/transformer.rs new file mode 100644 index 00000000..0b2b4f97 --- /dev/null +++ b/src/transformation/transformer.rs @@ -0,0 +1,231 @@ +use crate::internal_prelude::*; + +pub(crate) trait TransformerDefinition: Clone { + const TRANSFORMER_NAME: &'static str; + fn parse(arguments: TransformerArguments) -> ParseResult; + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()>; +} + +#[derive(Clone)] +pub(crate) struct TransformerArguments<'a> { + parse_stream: ParseStream<'a, Source>, + transformer_name: Ident, + full_span: Span, +} + +#[allow(unused)] +impl<'a> TransformerArguments<'a> { + pub(crate) fn new( + parse_stream: ParseStream<'a, Source>, + transformer_name: Ident, + full_span: Span, + ) -> Self { + Self { + parse_stream, + transformer_name, + full_span, + } + } + + pub(crate) fn full_span(&self) -> Span { + self.full_span + } + + /// We use this instead of the "unexpected / drop glue" pattern in order to give a better error message + pub(crate) fn assert_empty(&self, error_message: impl std::fmt::Display) -> ParseResult<()> { + if self.parse_stream.is_empty() { + Ok(()) + } else { + self.full_span.parse_err(error_message) + } + } + + pub(crate) fn fully_parse_no_error_override>(&self) -> ParseResult { + self.parse_stream.parse() + } + + pub(crate) fn fully_parse_as(&self) -> ParseResult { + self.fully_parse_or_error(T::parse, T::error_message()) + } + + pub(crate) fn fully_parse_or_error( + &self, + parse_function: impl FnOnce(ParseStream) -> ParseResult, + error_message: impl std::fmt::Display, + ) -> ParseResult { + // In future, when the diagnostic API is stable, + // we can add this context directly onto the transformer ident... + // Rather than just selectively adding it to the inner-most error. + // + // For now though, we can add additional context to the error message. + // But we can avoid adding this additional context if it's already been added in an + // inner error, because that's likely the correct local context to show. + let parsed = + parse_function(self.parse_stream).add_context_if_error_and_no_context(|| { + format!( + "Occurred whilst parsing @[{} ...] - {}", + self.transformer_name, error_message, + ) + })?; + + self.assert_empty(error_message)?; + + Ok(parsed) + } +} + +#[derive(Clone)] +pub(crate) struct Transformer { + #[allow(unused)] + transformer_token: Token![@], + instance: NamedTransformer, + #[allow(unused)] + source_group_span: Option, +} + +impl Parse for Transformer { + fn parse(input: ParseStream) -> ParseResult { + let transformer_token = input.parse()?; + + let (name, arguments) = if input.cursor().ident().is_some() { + let ident = input.parse_any_ident()?; + (ident, None) + } else if input.peek_specific_group(Delimiter::Bracket) { + let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; + let ident = content.parse_any_ident()?; + (ident, Some((content, delim_span))) + } else { + return input.parse_err("Expected @TRANSFORMER or @[TRANSFORMER ...arguments...]"); + }; + + let transformer_kind = match TransformerKind::for_ident(&name) { + Some(transformer_kind) => transformer_kind, + None => name.span().err( + format!( + "Expected `@NAME` or `@[NAME ...arguments...]` for NAME one of: {}.\nIf this wasn't intended to be a named transformer, you can work around this by replacing the @ with @(_ = @[EXACT [!raw! @]])", + TransformerKind::list_all(), + ), + )?, + }; + + match arguments { + Some((buffer, delim_span)) => { + let arguments = TransformerArguments::new(&buffer, name, delim_span.join()); + Ok(Self { + transformer_token, + instance: transformer_kind.parse_instance(arguments)?, + source_group_span: Some(delim_span), + }) + } + None => { + let span = name.span(); + let instance = TokenStream::new() + .source_parse_with(|parse_stream| { + let arguments = TransformerArguments::new(parse_stream, name.clone(), span); + transformer_kind.parse_instance(arguments) + }) + .map_err(|original_error| { + let replacement = span.parse_error( + format!("This transformer requires arguments, and should be invoked as @[{} ...]", name), + ); + if let Some(context) = original_error.context() { + replacement.add_context_if_none(context) + } else { + replacement + } + })?; + Ok(Self { + transformer_token, + instance, + source_group_span: None, + }) + } + } + } +} + +impl HandleTransformation for Transformer { + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + self.instance.handle_transform(input, interpreter, output) + } +} + +macro_rules! define_transformers { + ( + $( + $transformer:ident, + )* + ) => { + #[allow(clippy::enum_variant_names)] + #[derive(Clone, Copy)] + pub(crate) enum TransformerKind { + $( + $transformer, + )* + } + + impl TransformerKind { + fn parse_instance(&self, arguments: TransformerArguments) -> ParseResult { + Ok(match self { + $( + Self::$transformer => NamedTransformer::$transformer( + $transformer::parse(arguments)? + ), + )* + }) + } + + pub(crate) fn for_ident(ident: &Ident) -> Option { + Some(match ident.to_string().as_ref() { + $( + $transformer::TRANSFORMER_NAME => Self::$transformer, + )* + _ => return None, + }) + } + + const ALL_KIND_NAMES: &'static [&'static str] = &[$($transformer::TRANSFORMER_NAME,)*]; + + pub(crate) fn list_all() -> String { + // TODO improve to add an "and" at the end + Self::ALL_KIND_NAMES.join(", ") + } + } + + #[derive(Clone)] + #[allow(clippy::enum_variant_names)] + pub(crate) enum NamedTransformer { + $( + $transformer($transformer), + )* + } + + impl NamedTransformer { + fn handle_transform(&self, input: ParseStream, interpreter: &mut Interpreter, output: &mut OutputStream) -> ExecutionResult<()> { + match self { + $( + Self::$transformer(transformer) => transformer.handle_transform(input, interpreter, output), + )* + } + } + } + }; +} + +define_transformers! { + IdentTransformer, + LiteralTransformer, + PunctTransformer, + GroupTransformer, + ExactTransformer, +} diff --git a/src/transformation/transformers.rs b/src/transformation/transformers.rs new file mode 100644 index 00000000..c29c0a4b --- /dev/null +++ b/src/transformation/transformers.rs @@ -0,0 +1,130 @@ +use crate::internal_prelude::*; + +#[derive(Clone)] +pub(crate) struct IdentTransformer; + +impl TransformerDefinition for IdentTransformer { + const TRANSFORMER_NAME: &'static str = "IDENT"; + + fn parse(arguments: TransformerArguments) -> ParseResult { + arguments.fully_parse_or_error(|_| Ok(Self), "Expected @IDENT or @[IDENT]") + } + + fn handle_transform( + &self, + input: ParseStream, + _: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + if input.cursor().ident().is_some() { + output.push_ident(input.parse_any_ident()?); + Ok(()) + } else { + input.parse_err("Expected an ident")? + } + } +} + +#[derive(Clone)] +pub(crate) struct LiteralTransformer; + +impl TransformerDefinition for LiteralTransformer { + const TRANSFORMER_NAME: &'static str = "LITERAL"; + + fn parse(arguments: TransformerArguments) -> ParseResult { + arguments.fully_parse_or_error(|_| Ok(Self), "Expected @LITERAL or @[LITERAL]") + } + + fn handle_transform( + &self, + input: ParseStream, + _: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + if input.cursor().literal().is_some() { + output.push_literal(input.parse()?); + Ok(()) + } else { + input.parse_err("Expected a literal")? + } + } +} + +#[derive(Clone)] +pub(crate) struct PunctTransformer; + +impl TransformerDefinition for PunctTransformer { + const TRANSFORMER_NAME: &'static str = "PUNCT"; + + fn parse(arguments: TransformerArguments) -> ParseResult { + arguments.fully_parse_or_error(|_| Ok(Self), "Expected @PUNCT or @[PUNCT]") + } + + fn handle_transform( + &self, + input: ParseStream, + _: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + if input.cursor().any_punct().is_some() { + output.push_punct(input.parse_any_punct()?); + Ok(()) + } else { + input.parse_err("Expected a punct")? + } + } +} + +#[derive(Clone)] +pub(crate) struct GroupTransformer { + inner: TransformStream, +} + +impl TransformerDefinition for GroupTransformer { + const TRANSFORMER_NAME: &'static str = "GROUP"; + + fn parse(arguments: TransformerArguments) -> ParseResult { + Ok(Self { + inner: arguments.fully_parse_no_error_override()?, + }) + } + + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + let (_, inner) = input.parse_specific_group(Delimiter::None)?; + self.inner.handle_transform(&inner, interpreter, output) + } +} + +#[derive(Clone)] +pub(crate) struct ExactTransformer { + stream: ExactStream, +} + +impl TransformerDefinition for ExactTransformer { + const TRANSFORMER_NAME: &'static str = "EXACT"; + + fn parse(arguments: TransformerArguments) -> ParseResult { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + stream: ExactStream::parse(input)?, + }) + }, + "Expected @[EXACT ... interpretable input to be matched exactly ...]", + ) + } + + fn handle_transform( + &self, + input: ParseStream, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + self.stream.handle_transform(input, interpreter, output) + } +} diff --git a/src/destructuring/destructure_variable.rs b/src/transformation/variable_binding.rs similarity index 64% rename from src/destructuring/destructure_variable.rs rename to src/transformation/variable_binding.rs index c3ad76f1..b1dd31c6 100644 --- a/src/destructuring/destructure_variable.rs +++ b/src/transformation/variable_binding.rs @@ -11,7 +11,7 @@ use crate::internal_prelude::*; /// * `#..>>..x` - Reads a stream, appends a stream #[derive(Clone)] #[allow(unused)] -pub(crate) enum DestructureVariable { +pub(crate) enum VariableBinding { /// #x - Reads a token tree, writes a stream (opposite of #x) Grouped { marker: Token![#], name: Ident }, /// #..x - Reads a stream, writes a stream (opposite of #..x) @@ -53,9 +53,10 @@ pub(crate) enum DestructureVariable { }, } -impl DestructureVariable { +impl VariableBinding { + #[allow(unused)] pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> ParseResult { - let variable: DestructureVariable = Self::parse_until::(input)?; + let variable: VariableBinding = Self::parse_until::(input)?; if variable.is_flattened_input() { return variable .span_range() @@ -64,7 +65,9 @@ impl DestructureVariable { Ok(variable) } - pub(crate) fn parse_until(input: ParseStream) -> ParseResult { + pub(crate) fn parse_until>( + input: ParseStream, + ) -> ParseResult { let marker = input.parse()?; if input.peek(Token![..]) { let flatten = input.parse()?; @@ -126,41 +129,41 @@ impl DestructureVariable { } } -impl IsVariable for DestructureVariable { +impl IsVariable for VariableBinding { fn get_name(&self) -> String { let name_ident = match self { - DestructureVariable::Grouped { name, .. } => name, - DestructureVariable::Flattened { name, .. } => name, - DestructureVariable::GroupedAppendGrouped { name, .. } => name, - DestructureVariable::GroupedAppendFlattened { name, .. } => name, - DestructureVariable::FlattenedAppendGrouped { name, .. } => name, - DestructureVariable::FlattenedAppendFlattened { name, .. } => name, + VariableBinding::Grouped { name, .. } => name, + VariableBinding::Flattened { name, .. } => name, + VariableBinding::GroupedAppendGrouped { name, .. } => name, + VariableBinding::GroupedAppendFlattened { name, .. } => name, + VariableBinding::FlattenedAppendGrouped { name, .. } => name, + VariableBinding::FlattenedAppendFlattened { name, .. } => name, }; name_ident.to_string() } } -impl HasSpanRange for DestructureVariable { +impl HasSpanRange for VariableBinding { fn span_range(&self) -> SpanRange { let (marker, name) = match self { - DestructureVariable::Grouped { marker, name, .. } => (marker, name), - DestructureVariable::Flattened { marker, name, .. } => (marker, name), - DestructureVariable::GroupedAppendGrouped { marker, name, .. } => (marker, name), - DestructureVariable::GroupedAppendFlattened { marker, name, .. } => (marker, name), - DestructureVariable::FlattenedAppendGrouped { marker, name, .. } => (marker, name), - DestructureVariable::FlattenedAppendFlattened { marker, name, .. } => (marker, name), + VariableBinding::Grouped { marker, name, .. } => (marker, name), + VariableBinding::Flattened { marker, name, .. } => (marker, name), + VariableBinding::GroupedAppendGrouped { marker, name, .. } => (marker, name), + VariableBinding::GroupedAppendFlattened { marker, name, .. } => (marker, name), + VariableBinding::FlattenedAppendGrouped { marker, name, .. } => (marker, name), + VariableBinding::FlattenedAppendFlattened { marker, name, .. } => (marker, name), }; SpanRange::new_between(marker.span, name.span()) } } -impl DestructureVariable { +impl VariableBinding { pub(crate) fn is_flattened_input(&self) -> bool { matches!( self, - DestructureVariable::Flattened { .. } - | DestructureVariable::FlattenedAppendGrouped { .. } - | DestructureVariable::FlattenedAppendFlattened { .. } + VariableBinding::Flattened { .. } + | VariableBinding::FlattenedAppendGrouped { .. } + | VariableBinding::FlattenedAppendFlattened { .. } ) } @@ -177,35 +180,36 @@ impl DestructureVariable { } } -impl HandleDestructure for DestructureVariable { - fn handle_destructure( +impl HandleTransformation for VariableBinding { + fn handle_transform( &self, input: ParseStream, interpreter: &mut Interpreter, + _: &mut OutputStream, ) -> ExecutionResult<()> { match self { - DestructureVariable::Grouped { .. } => { + VariableBinding::Grouped { .. } => { let content = input.parse::()?.into_interpreted(); interpreter.set_variable(self, content)?; } - DestructureVariable::Flattened { until, .. } => { + VariableBinding::Flattened { until, .. } => { let mut content = OutputStream::new(); until.handle_parse_into(input, &mut content)?; interpreter.set_variable(self, content)?; } - DestructureVariable::GroupedAppendGrouped { .. } => { + VariableBinding::GroupedAppendGrouped { .. } => { let variable_data = self.get_variable_data(interpreter)?; input .parse::()? .push_as_token_tree(variable_data.get_mut(self)?.deref_mut()); } - DestructureVariable::GroupedAppendFlattened { .. } => { + VariableBinding::GroupedAppendFlattened { .. } => { let variable_data = self.get_variable_data(interpreter)?; input .parse::()? .flatten_into(variable_data.get_mut(self)?.deref_mut()); } - DestructureVariable::FlattenedAppendGrouped { marker, until, .. } => { + VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { let variable_data = self.get_variable_data(interpreter)?; variable_data.get_mut(self)?.push_grouped( |inner| until.handle_parse_into(input, inner), @@ -213,7 +217,7 @@ impl HandleDestructure for DestructureVariable { marker.span, )?; } - DestructureVariable::FlattenedAppendFlattened { until, .. } => { + VariableBinding::FlattenedAppendFlattened { until, .. } => { let variable_data = self.get_variable_data(interpreter)?; until.handle_parse_into(input, variable_data.get_mut(self)?.deref_mut())?; } @@ -278,83 +282,3 @@ impl Parse for ParsedTokenTree { }) } } - -#[derive(Clone)] -pub(crate) enum ParseUntil { - End, - Group(Delimiter), - Ident(Ident), - Punct(Punct), - Literal(Literal), -} - -impl ParseUntil { - /// Peeks the next token, to discover what we should parse next - fn peek_flatten_limit(input: ParseStream) -> ParseResult { - if C::should_stop(input) { - return Ok(ParseUntil::End); - } - Ok(match input.peek_grammar() { - GrammarPeekMatch::Command(_) - | GrammarPeekMatch::GroupedVariable - | GrammarPeekMatch::FlattenedVariable - | GrammarPeekMatch::Destructurer(_) - | GrammarPeekMatch::AppendVariableDestructuring => { - return input - .span() - .parse_err("This cannot follow a flattened destructure match"); - } - GrammarPeekMatch::Group(delimiter) => ParseUntil::Group(delimiter), - GrammarPeekMatch::Ident(ident) => ParseUntil::Ident(ident), - GrammarPeekMatch::Literal(literal) => ParseUntil::Literal(literal), - GrammarPeekMatch::Punct(punct) => ParseUntil::Punct(punct), - GrammarPeekMatch::End => ParseUntil::End, - }) - } - - fn handle_parse_into( - &self, - input: ParseStream, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - match self { - ParseUntil::End => output.extend_raw_tokens(input.parse::()?), - ParseUntil::Group(delimiter) => { - while !input.is_empty() { - if input.peek_specific_group(*delimiter) { - return Ok(()); - } - output.push_raw_token_tree(input.parse()?); - } - } - ParseUntil::Ident(ident) => { - let content = ident.to_string(); - while !input.is_empty() { - if input.peek_ident_matching(&content) { - return Ok(()); - } - output.push_raw_token_tree(input.parse()?); - } - } - ParseUntil::Punct(punct) => { - let punct = punct.as_char(); - while !input.is_empty() { - if input.peek_punct_matching(punct) { - return Ok(()); - } - output.push_raw_token_tree(input.parse()?); - } - } - ParseUntil::Literal(literal) => { - let content = literal.to_string(); - while !input.is_empty() { - if input.peek_literal_matching(&content) { - return Ok(()); - } - output.push_raw_token_tree(input.parse()?); - } - } - } - Ok(()) - } -} diff --git a/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr b/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr deleted file mode 100644 index e518d650..00000000 --- a/tests/compilation_failures/destructuring/destructure_with_flattened_command.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Commands which return something are not supported in destructuring positions - Occurred whilst parsing [!let! ...] - Expected [!let! = ...] - --> tests/compilation_failures/destructuring/destructure_with_flattened_command.rs:4:26 - | -4 | preinterpret!([!let! [!..group! Output] = [!group! Output]]); - | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr b/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr deleted file mode 100644 index a1968590..00000000 --- a/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: A flattened input variable is not supported here - Occurred whilst parsing (!ident! ...) - Expected (!ident! #x) or (!ident #>>x) or (!ident!) - --> tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs:4:35 - | -4 | preinterpret!([!let! (!ident! #..x) = Hello]); - | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr b/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr deleted file mode 100644 index eb9e2f7d..00000000 --- a/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: A flattened input variable is not supported here - Occurred whilst parsing (!literal! ...) - Expected (!literal! #x) or (!literal! #>>x) or (!literal!) - --> tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs:4:37 - | -4 | preinterpret!([!let! (!literal! #..x) = "Hello"]); - | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr b/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr deleted file mode 100644 index 935d3c07..00000000 --- a/tests/compilation_failures/destructuring/destructure_with_outputting_command.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Commands which return something are not supported in destructuring positions - Occurred whilst parsing [!let! ...] - Expected [!let! = ...] - --> tests/compilation_failures/destructuring/destructure_with_outputting_command.rs:4:26 - | -4 | preinterpret!([!let! [!group! Output] = [!group! Output]]); - | ^ diff --git a/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr b/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr deleted file mode 100644 index 9c38ca23..00000000 --- a/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: A flattened input variable is not supported here - Occurred whilst parsing (!punct! ...) - Expected (!punct! #x) or (!punct! #>>x) or (!punct!) - --> tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs:4:35 - | -4 | preinterpret!([!let! (!punct! #..x) = @]); - | ^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr b/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr deleted file mode 100644 index 04570fa9..00000000 --- a/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Expected start of transparent group, from a grouped #variable substitution or stream-based command such as [!group! ...] - --> tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs:4:43 - | -4 | preinterpret!([!let! (!group! #..x) = [Hello World]]); - | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/destructuring/append_before_set.rs b/tests/compilation_failures/transforming/append_before_set.rs similarity index 100% rename from tests/compilation_failures/destructuring/append_before_set.rs rename to tests/compilation_failures/transforming/append_before_set.rs diff --git a/tests/compilation_failures/destructuring/append_before_set.stderr b/tests/compilation_failures/transforming/append_before_set.stderr similarity index 63% rename from tests/compilation_failures/destructuring/append_before_set.stderr rename to tests/compilation_failures/transforming/append_before_set.stderr index 236367af..30719a07 100644 --- a/tests/compilation_failures/destructuring/append_before_set.stderr +++ b/tests/compilation_failures/transforming/append_before_set.stderr @@ -1,5 +1,5 @@ error: The variable #x wasn't already set - --> tests/compilation_failures/destructuring/append_before_set.rs:4:26 + --> tests/compilation_failures/transforming/append_before_set.rs:4:26 | 4 | preinterpret!([!let! #>>x = Hello]); | ^^^^ diff --git a/tests/compilation_failures/destructuring/destructure_with_flattened_command.rs b/tests/compilation_failures/transforming/destructure_with_flattened_command.rs similarity index 100% rename from tests/compilation_failures/destructuring/destructure_with_flattened_command.rs rename to tests/compilation_failures/transforming/destructure_with_flattened_command.rs diff --git a/tests/compilation_failures/transforming/destructure_with_flattened_command.stderr b/tests/compilation_failures/transforming/destructure_with_flattened_command.stderr new file mode 100644 index 00000000..f06f18e2 --- /dev/null +++ b/tests/compilation_failures/transforming/destructure_with_flattened_command.stderr @@ -0,0 +1,5 @@ +error: unexpected token + --> tests/compilation_failures/transforming/destructure_with_flattened_command.rs:4:56 + | +4 | preinterpret!([!let! [!..group! Output] = [!group! Output]]); + | ^^^^^^ diff --git a/tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs similarity index 100% rename from tests/compilation_failures/destructuring/destructure_with_ident_flattened_variable.rs rename to tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs diff --git a/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr new file mode 100644 index 00000000..bd845f2b --- /dev/null +++ b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr @@ -0,0 +1,5 @@ +error: Expected ( + --> tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs:4:43 + | +4 | preinterpret!([!let! (!ident! #..x) = Hello]); + | ^^^^^ diff --git a/tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs similarity index 100% rename from tests/compilation_failures/destructuring/destructure_with_literal_flattened_variable.rs rename to tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs diff --git a/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr new file mode 100644 index 00000000..6cc089f9 --- /dev/null +++ b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr @@ -0,0 +1,5 @@ +error: Expected ( + --> tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs:4:45 + | +4 | preinterpret!([!let! (!literal! #..x) = "Hello"]); + | ^^^^^^^ diff --git a/tests/compilation_failures/destructuring/destructure_with_outputting_command.rs b/tests/compilation_failures/transforming/destructure_with_outputting_command.rs similarity index 100% rename from tests/compilation_failures/destructuring/destructure_with_outputting_command.rs rename to tests/compilation_failures/transforming/destructure_with_outputting_command.rs diff --git a/tests/compilation_failures/transforming/destructure_with_outputting_command.stderr b/tests/compilation_failures/transforming/destructure_with_outputting_command.stderr new file mode 100644 index 00000000..6393fdbf --- /dev/null +++ b/tests/compilation_failures/transforming/destructure_with_outputting_command.stderr @@ -0,0 +1,5 @@ +error: unexpected token + --> tests/compilation_failures/transforming/destructure_with_outputting_command.rs:4:54 + | +4 | preinterpret!([!let! [!group! Output] = [!group! Output]]); + | ^^^^^^ diff --git a/tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs similarity index 100% rename from tests/compilation_failures/destructuring/destructure_with_punct_flattened_variable.rs rename to tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs diff --git a/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr new file mode 100644 index 00000000..23f2ef3e --- /dev/null +++ b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr @@ -0,0 +1,5 @@ +error: Expected ( + --> tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs:4:43 + | +4 | preinterpret!([!let! (!punct! #..x) = @]); + | ^ diff --git a/tests/compilation_failures/destructuring/double_flattened_variable.rs b/tests/compilation_failures/transforming/double_flattened_variable.rs similarity index 100% rename from tests/compilation_failures/destructuring/double_flattened_variable.rs rename to tests/compilation_failures/transforming/double_flattened_variable.rs diff --git a/tests/compilation_failures/destructuring/double_flattened_variable.stderr b/tests/compilation_failures/transforming/double_flattened_variable.stderr similarity index 56% rename from tests/compilation_failures/destructuring/double_flattened_variable.stderr rename to tests/compilation_failures/transforming/double_flattened_variable.stderr index ca3bf2e6..35c196e9 100644 --- a/tests/compilation_failures/destructuring/double_flattened_variable.stderr +++ b/tests/compilation_failures/transforming/double_flattened_variable.stderr @@ -1,6 +1,6 @@ -error: This cannot follow a flattened destructure match +error: This cannot follow a flattened variable binding Occurred whilst parsing [!let! ...] - Expected [!let! = ...] - --> tests/compilation_failures/destructuring/double_flattened_variable.rs:4:31 + --> tests/compilation_failures/transforming/double_flattened_variable.rs:4:31 | 4 | preinterpret!([!let! #..x #..y = Hello World]); | ^ diff --git a/tests/compilation_failures/destructuring/invalid_content_too_long.rs b/tests/compilation_failures/transforming/invalid_content_too_long.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_content_too_long.rs rename to tests/compilation_failures/transforming/invalid_content_too_long.rs diff --git a/tests/compilation_failures/destructuring/invalid_content_too_long.stderr b/tests/compilation_failures/transforming/invalid_content_too_long.stderr similarity index 64% rename from tests/compilation_failures/destructuring/invalid_content_too_long.stderr rename to tests/compilation_failures/transforming/invalid_content_too_long.stderr index f83502fe..81df09f7 100644 --- a/tests/compilation_failures/destructuring/invalid_content_too_long.stderr +++ b/tests/compilation_failures/transforming/invalid_content_too_long.stderr @@ -1,5 +1,5 @@ error: unexpected token - --> tests/compilation_failures/destructuring/invalid_content_too_long.rs:4:51 + --> tests/compilation_failures/transforming/invalid_content_too_long.rs:4:51 | 4 | preinterpret!([!let! Hello World = Hello World!!!]); | ^ diff --git a/tests/compilation_failures/destructuring/invalid_content_too_short.rs b/tests/compilation_failures/transforming/invalid_content_too_short.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_content_too_short.rs rename to tests/compilation_failures/transforming/invalid_content_too_short.rs diff --git a/tests/compilation_failures/destructuring/invalid_content_too_short.stderr b/tests/compilation_failures/transforming/invalid_content_too_short.stderr similarity index 76% rename from tests/compilation_failures/destructuring/invalid_content_too_short.stderr rename to tests/compilation_failures/transforming/invalid_content_too_short.stderr index f7193556..ede603a2 100644 --- a/tests/compilation_failures/destructuring/invalid_content_too_short.stderr +++ b/tests/compilation_failures/transforming/invalid_content_too_short.stderr @@ -1,5 +1,5 @@ error: expected World - --> tests/compilation_failures/destructuring/invalid_content_too_short.rs:4:5 + --> tests/compilation_failures/transforming/invalid_content_too_short.rs:4:5 | 4 | preinterpret!([!let! Hello World = Hello]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group.rs b/tests/compilation_failures/transforming/invalid_content_wrong_group.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_content_wrong_group.rs rename to tests/compilation_failures/transforming/invalid_content_wrong_group.rs diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr b/tests/compilation_failures/transforming/invalid_content_wrong_group.stderr similarity index 61% rename from tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr rename to tests/compilation_failures/transforming/invalid_content_wrong_group.stderr index 9db00a4f..ded3e105 100644 --- a/tests/compilation_failures/destructuring/invalid_content_wrong_group.stderr +++ b/tests/compilation_failures/transforming/invalid_content_wrong_group.stderr @@ -1,5 +1,5 @@ error: Expected ( - --> tests/compilation_failures/destructuring/invalid_content_wrong_group.rs:4:35 + --> tests/compilation_failures/transforming/invalid_content_wrong_group.rs:4:35 | 4 | preinterpret!([!let! (#..x) = [Hello World]]); | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs b/tests/compilation_failures/transforming/invalid_content_wrong_group_2.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_content_wrong_group_2.rs rename to tests/compilation_failures/transforming/invalid_content_wrong_group_2.rs diff --git a/tests/compilation_failures/transforming/invalid_content_wrong_group_2.stderr b/tests/compilation_failures/transforming/invalid_content_wrong_group_2.stderr new file mode 100644 index 00000000..01c68b55 --- /dev/null +++ b/tests/compilation_failures/transforming/invalid_content_wrong_group_2.stderr @@ -0,0 +1,5 @@ +error: Expected ( + --> tests/compilation_failures/transforming/invalid_content_wrong_group_2.rs:4:43 + | +4 | preinterpret!([!let! (!group! #..x) = [Hello World]]); + | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs b/tests/compilation_failures/transforming/invalid_content_wrong_ident.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs rename to tests/compilation_failures/transforming/invalid_content_wrong_ident.rs diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr b/tests/compilation_failures/transforming/invalid_content_wrong_ident.stderr similarity index 62% rename from tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr rename to tests/compilation_failures/transforming/invalid_content_wrong_ident.stderr index 0f9c6db8..117f2c59 100644 --- a/tests/compilation_failures/destructuring/invalid_content_wrong_ident.stderr +++ b/tests/compilation_failures/transforming/invalid_content_wrong_ident.stderr @@ -1,5 +1,5 @@ error: expected World - --> tests/compilation_failures/destructuring/invalid_content_wrong_ident.rs:4:46 + --> tests/compilation_failures/transforming/invalid_content_wrong_ident.rs:4:46 | 4 | preinterpret!([!let! Hello World = Hello Earth]); | ^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs b/tests/compilation_failures/transforming/invalid_content_wrong_punct.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs rename to tests/compilation_failures/transforming/invalid_content_wrong_punct.rs diff --git a/tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr b/tests/compilation_failures/transforming/invalid_content_wrong_punct.stderr similarity index 62% rename from tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr rename to tests/compilation_failures/transforming/invalid_content_wrong_punct.stderr index 7fa785ab..f8647cfa 100644 --- a/tests/compilation_failures/destructuring/invalid_content_wrong_punct.stderr +++ b/tests/compilation_failures/transforming/invalid_content_wrong_punct.stderr @@ -1,5 +1,5 @@ error: expected _ - --> tests/compilation_failures/destructuring/invalid_content_wrong_punct.rs:4:48 + --> tests/compilation_failures/transforming/invalid_content_wrong_punct.rs:4:48 | 4 | preinterpret!([!let! Hello _ World = Hello World]); | ^^^^^ diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_long.rs b/tests/compilation_failures/transforming/invalid_group_content_too_long.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_group_content_too_long.rs rename to tests/compilation_failures/transforming/invalid_group_content_too_long.rs diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr b/tests/compilation_failures/transforming/invalid_group_content_too_long.stderr similarity index 69% rename from tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr rename to tests/compilation_failures/transforming/invalid_group_content_too_long.stderr index 14ea0a33..8593afc8 100644 --- a/tests/compilation_failures/destructuring/invalid_group_content_too_long.stderr +++ b/tests/compilation_failures/transforming/invalid_group_content_too_long.stderr @@ -1,5 +1,5 @@ error: unexpected token, expected `)` - --> tests/compilation_failures/destructuring/invalid_group_content_too_long.rs:4:68 + --> tests/compilation_failures/transforming/invalid_group_content_too_long.rs:4:68 | 4 | preinterpret!([!let! Group: (Hello World) = Group: (Hello World!!!)]); | ^ diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_short.rs b/tests/compilation_failures/transforming/invalid_group_content_too_short.rs similarity index 100% rename from tests/compilation_failures/destructuring/invalid_group_content_too_short.rs rename to tests/compilation_failures/transforming/invalid_group_content_too_short.rs diff --git a/tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr b/tests/compilation_failures/transforming/invalid_group_content_too_short.stderr similarity index 67% rename from tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr rename to tests/compilation_failures/transforming/invalid_group_content_too_short.stderr index 4db82988..4a4624bb 100644 --- a/tests/compilation_failures/destructuring/invalid_group_content_too_short.stderr +++ b/tests/compilation_failures/transforming/invalid_group_content_too_short.stderr @@ -1,5 +1,5 @@ error: expected ! - --> tests/compilation_failures/destructuring/invalid_group_content_too_short.rs:4:58 + --> tests/compilation_failures/transforming/invalid_group_content_too_short.rs:4:58 | 4 | preinterpret!([!let! Group: (Hello World!!) = Group: (Hello World)]); | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/transforming/mistaken_at_symbol.rs b/tests/compilation_failures/transforming/mistaken_at_symbol.rs new file mode 100644 index 00000000..4a6e0c1e --- /dev/null +++ b/tests/compilation_failures/transforming/mistaken_at_symbol.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +struct I; + +fn main() { + preinterpret!( + match I { + x @ I => x, + } + ); +} \ No newline at end of file diff --git a/tests/compilation_failures/transforming/mistaken_at_symbol.stderr b/tests/compilation_failures/transforming/mistaken_at_symbol.stderr new file mode 100644 index 00000000..e22d76c6 --- /dev/null +++ b/tests/compilation_failures/transforming/mistaken_at_symbol.stderr @@ -0,0 +1,5 @@ +error: Destructurings are not supported here. If this wasn't intended to be a destructuring, replace @ with [!raw! @] + --> tests/compilation_failures/transforming/mistaken_at_symbol.rs:8:15 + | +8 | x @ I => x, + | ^ diff --git a/tests/core.rs b/tests/core.rs index c62ca6e1..30149cb1 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -121,8 +121,8 @@ fn test_debug() { assert_preinterpret_eq!( { [!set! #x = Hello (World)] - [!debug! #x [!..raw! #test] "and" [!raw! ##] #..x] + [!debug! #x [!raw! #test] "and" [!raw! ##] #..x] }, - r###"[!group! Hello (World)] # test "and" [!group! ##] Hello (World)"### + r###"[!group! Hello (World)] # test "and" ## Hello (World)"### ); } diff --git a/tests/tokens.rs b/tests/tokens.rs index efb21089..1445bcba 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -269,9 +269,9 @@ fn complex_cases_for_intersperse_and_input_types() { // Grouped variable containing two groups assert_preinterpret_eq!({ [!set! #items = 0 1] - [!set! #item_groups = #items #items] // [!GROUP! 0 1] [!GROUP! 0 1] + [!set! #item_groups = #items #items] // [!group! 0 1] [!group! 0 1] [!string! [!intersperse! { - items: #item_groups, // [!GROUP! [!GROUP! 0 1] [!GROUP! 0 1]] + items: #item_groups, // [!group! [!group! 0 1] [!group! 0 1]] separator: [_], }]] }, "01_01"); diff --git a/tests/destructuring.rs b/tests/transforming.rs similarity index 57% rename from tests/destructuring.rs rename to tests/transforming.rs index 71f41f8a..939c06cb 100644 --- a/tests/destructuring.rs +++ b/tests/transforming.rs @@ -8,13 +8,13 @@ macro_rules! assert_preinterpret_eq { #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] -fn test_destructuring_compilation_failures() { +fn test_transfoming_compilation_failures() { if option_env!("TEST_RUST_MODE") == Some("nightly") { // Some of the outputs are different on nightly, so don't test these return; } let t = trybuild::TestCases::new(); - t.compile_fail("tests/compilation_failures/destructuring/*.rs"); + t.compile_fail("tests/compilation_failures/transforming/*.rs"); } #[test] @@ -66,68 +66,68 @@ fn test_variable_parsing() { } #[test] -fn test_stream_destructurer() { +fn test_explicit_transform_stream() { // It's not very exciting - preinterpret!([!let! (!stream! Hello World) = Hello World]); - preinterpret!([!let! Hello (!stream! World) = Hello World]); - preinterpret!([!let! (!stream! Hello (!stream! World)) = Hello World]); + preinterpret!([!let! @(Hello World) = Hello World]); + preinterpret!([!let! Hello @(World) = Hello World]); + preinterpret!([!let! @(Hello @(World)) = Hello World]); } #[test] -fn test_ident_destructurer() { +fn test_ident_transformer() { assert_preinterpret_eq!({ - [!let! The "quick" (!ident! #x) fox "jumps" = The "quick" brown fox "jumps"] + [!let! The "quick" @(#x = @IDENT) fox "jumps" = The "quick" brown fox "jumps"] [!string! #x] }, "brown"); assert_preinterpret_eq!({ [!set! #x =] - [!let! The quick (!ident! #>>x) fox jumps (!ident! #>>x) the lazy dog = The quick brown fox jumps over the lazy dog] + [!let! The quick @(#x += @IDENT) fox jumps @(#x += @IDENT) the lazy dog = The quick brown fox jumps over the lazy dog] [!string! [!intersperse! { items: #x, separator: ["_"] }]] }, "brown_over"); } #[test] -fn test_literal_destructurer() { +fn test_literal_transformer() { assert_preinterpret_eq!({ - [!let! The "quick" (!literal! #x) fox "jumps" = The "quick" "brown" fox "jumps"] + [!let! The "quick" @(#x = @LITERAL) fox "jumps" = The "quick" "brown" fox "jumps"] #x }, "brown"); // Lots of literals assert_preinterpret_eq!({ - [!set! #x =] - [!let! (!literal!) (!literal!) (!literal!) (!literal! #>>x) (!literal!) (!literal! #>>x) (!literal! #>>x) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] + [!set! #x] + [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] [!debug! #..x] }, "'c' 0b1010 r#\"123\"#"); } #[test] -fn test_punct_destructurer() { +fn test_punct_transformer() { assert_preinterpret_eq!({ - [!let! The "quick" brown fox "jumps" (!punct! #x) = The "quick" brown fox "jumps"!] + [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] [!debug! #..x] }, "!"); // Test for ' which is treated weirdly by syn / rustc assert_preinterpret_eq!({ - [!let! The "quick" fox isn 't brown and doesn (!punct! #x) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] + [!let! The "quick" fox isn 't brown and doesn @(#x = @PUNCT) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] [!debug! #..x] }, "'"); // Lots of punctuation, most of it ignored assert_preinterpret_eq!({ [!set! #x =] - [!let! (!punct!) (!punct!) (!punct!) (!punct!) (!punct! #>>x) (!punct!) (!punct!) (!punct!) (!punct!) (!punct!) (!punct! #>>x) (!punct!) (!punct!) (!punct!) = # ! $$ % ^ & * + = | @ : ;] + [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] [!debug! #..x] }, "% |"); } #[test] -fn test_group_destructurer() { +fn test_group_transformer() { assert_preinterpret_eq!({ - [!let! The "quick" (!group! brown #x) "jumps" = The "quick" [!group! brown fox] "jumps"] + [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] [!debug! #..x] }, "fox"); assert_preinterpret_eq!({ [!set! #x = "hello" "world"] - [!let! I said (!group! #..y)! = I said #x!] + [!let! I said @[GROUP #..y]! = I said #x!] [!debug! #..y] }, "\"hello\" \"world\""); // ... which is equivalent to this: @@ -141,32 +141,61 @@ fn test_group_destructurer() { #[test] fn test_none_output_commands_mid_parse() { assert_preinterpret_eq!({ - [!let! The "quick" (!literal! #x) fox [!let! #y = #x] (!ident! #x) = The "quick" "brown" fox jumps] + [!let! The "quick" @(#x = @LITERAL) fox [!let! #y = #x] @(#x = @IDENT) = The "quick" "brown" fox jumps] [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] }, "#x = jumps; #y = \"brown\""); } #[test] -fn test_raw_destructurer() { +fn test_raw_content_in_exact_transformer() { assert_preinterpret_eq!({ [!set! #x = true] - [!let! The (!raw! #x) = The [!raw! #] x] + [!let! The @[EXACT [!raw! #x]] = The [!raw! #] x] #x }, true); } #[test] -fn test_content_destructurer() { - // Content works +fn test_exact_transformer() { + // EXACT works assert_preinterpret_eq!({ [!set! #x = true] - [!let! The (!content! #..x) = The true] + [!let! The @[EXACT #..x] = The true] #x }, true); - // Content is evaluated at destructuring time + // EXACT is evaluated at execution time assert_preinterpret_eq!({ [!set! #x =] - [!let! The #>>..x fox is #>>..x. It 's super (!content! #..x). = The brown fox is brown. It 's super brown brown.] + [!let! The #>>..x fox is #>>..x. It 's super @[EXACT #..x]. = The brown fox is brown. It 's super brown brown.] true }, true); } + +#[test] +fn test_parse_command_and_exact_transformer() { + // The output stream is additive + assert_preinterpret_eq!( + { [!debug! [!parse! [Hello World] as @(@IDENT @IDENT)]] }, + "Hello World" + ); + // Substreams redirected to a variable are not included in the output + assert_preinterpret_eq!( + { + [!debug! [!parse! [The quick brown fox] as @( + @[EXACT The] quick @IDENT @(#x = @IDENT) + )]] + }, + "The brown" + ); + // This tests that: + // * Can nest EXACT and transform streams + // * Can discard output with @(_ = ...) + // * That EXACT ignores none-delimited groups, to make it more intuitive + assert_preinterpret_eq!({ + [!set! #x = [!group! fox]] + [!debug! [!parse! [The quick brown fox is a fox - right?!] as @( + // The outputs are only from the EXACT transformer + The quick @(_ = @IDENT) @[EXACT #x @(_ = @IDENT a) #..x - right?!] + )]] + }, "fox fox - right ?!"); +} From c36ad39d24d6d1b2ea196a6966620aacc6a35707 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 9 Feb 2025 01:24:13 +0000 Subject: [PATCH 078/126] refactor: Simplify expression span management --- CHANGELOG.md | 190 +++++++++++++----- src/expressions/boolean.rs | 30 +-- src/expressions/character.rs | 36 ++-- src/expressions/evaluation.rs | 76 ++----- src/expressions/expression.rs | 65 +----- src/expressions/expression_parsing.rs | 34 +--- src/expressions/float.rs | 61 +++--- src/expressions/integer.rs | 108 +++++----- src/expressions/operations.rs | 97 +++++---- src/expressions/string.rs | 38 ++-- src/expressions/value.rs | 76 +++++-- src/extensions/errors_and_spans.rs | 4 + .../commands/control_flow_commands.rs | 2 +- .../commands/expression_commands.rs | 13 +- .../control_flow/while_infinite_loop.stderr | 4 +- 15 files changed, 434 insertions(+), 400 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5023fa7..9f21f445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,8 +86,41 @@ Inside a transform stream, the following grammar is supported: ### To come -* Destructurers => Transformers - * Scrap `[!let!]` in favour of `[!parse! #x as @(_ = ...)]` +* Variable typing (stream / value / object to start with), including an `object` type, like a JS object: + * Separate `&mut Output` and `StreamOutput`, `ValueOutput = ExpressionOutput`, `ObjectOutput` + * Objects: + * Can be destructured with `@{ #hello, world: _, ... }` or read with `#(x.hello)` or `#(x["hello"])` + * Debug impl is `[!object! { hello: [!group! BLAH], ["world"]: Hi, }]` + * Fields can be lazy (for e.g. exposing functions on syn objects). + * They have an input object + * (Until we get custom type support into a syn fork), can be embedded into an output stream as a single token - e.g. `PREINTERPRET_OBJECT_2313` + (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default + stream for the object, or error and suggest fields the user should use instead. + * Values: + * Can output to an internal stream as `[!group! ]` so it can be read by other transformers. + * When we parse a `#x` (`@(#x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. + * Output to final output as unwrapped content + * New expression & definition syntax (replaces `[!set!]`??) + * `#(#x = [... stream ...])` + * `#(#x += [... stream ...])` // += acts like "extend" with a stream LHS + * `#(#x += [!group! ...])` + * `#(#x = 1 + 2 + 3)` // (Expression) Value + * `#(#x += 1)` // += acts as plus with a value LHS + * `#(#x = {})` // Object + * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` + * `#(#x = xxx {})` // For some other custom type `xxx`. + * `#([ ... stream ... ])` + * `#({ a: x, ... })` + * `#([ ... stream ... ].length)` ?? + * `#(let #x = 123; let #y = 123; let #z = {})`... + * Support `#(x)` syntax for indexing streams: + * `#(x[0])` + * `#(x[0..3])` + * `#..(x[0..3])` and other things like `[ ..=3]` + ... basically - this becomes one big expression. + * Equivalent to a `syn::Block` which is a `Vec` + * ...where a `Stmt` is either a local `let` binding or an `expression` +* Destructurers => Transformers cont * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. * `@[ANY_GROUP ...]` @@ -99,22 +132,20 @@ Inside a transform stream, the following grammar is supported: * `@(REPEATED { ... })` (see below) * Potentially change `@UNTIL` to take a transform stream instead of a raw stream. * `[!match! ...]` command - * `@[ANY { ... }]` (with `#..x` as a catch-all) like the [!match!] command but without arms... + * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` + * Scrap `[!let!]` in favour of `[!parse! [...] as @(_ = ...)]` * Consider: - * If the `[!split!]` command should actually be a transformer? * Scrap `#>>x` etc in favour of `@(#x += ...)` + * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty + * If the `[!split!]` command should actually be a transformer? + * Adding `!define_macro!` + * Adding `!define_command!` + * Adding `!define_transformer!` * `[!is_set! #x]` -* Support `[!index! ..]`: - * `[!index! #x[0]]` - * `[!index! #x[0..3]]` - * `[!..index! #x[0..3]]` and other things like `[ ..=3]` - * `[!index! [Hello World][...]]` - => NB: This isn't in the grammar because of the risk of `#x[0]` wanting to mean `my_arr[0]` rather than "index my variable". * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) -* Add `[!reinterpret! ...]` command for an `eval` style command. +* Add `[!reinterpret! ...]` command for an `eval` style command, and maybe a `@[REINTERPRET ..]` transformer. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` -* Get rid of needless cloning * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed * Work on book @@ -127,7 +158,91 @@ Inside a transform stream, the following grammar is supported: * Those taking some { fields } * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` etc -### Destructuring Notes WIP +### Transformers Revisited +```rust +// * The current situation feels unclear/arbitrary/inflexible. +// * Better would be slightly more explicit. +// +// PROPOSAL (subject to the object proposal above): +// * Four modes: +// * Output stream mode +// * Outputs stream, does not parse +// * Can embed [!commands! ...] and #() +// * #([...]) gets appended to the stream +// * #( ... ) = Expression mode. +// * One or more statements, terminated by ; (except maybe the last) +// * Idents mean variables. +// * `let #x; let _ = ; ; if {} else {}` +// * Error not to discard a non-None value with `let _ = ` +// * Not allowed to parse. +// * Only last statement can (optionally) output, with a value model of: +// * Leaf(Bool | Int | Float | String) +// * Object +// * Stream +// * None +// * The actual type is only known at evaluation time. +// * #var is equivalent to #(var) +// * @[XXX] or @[hello.world = XXX] +// * Can parse; has no output. +// * XXX is: +// * A named destructurer (possibly taking further input) +// * An { .. } object destructurer +// * A @(...) transformer stream (output = input) +// * @() = Transformer stream +// * Can parse +// * Its only output is an input stream reference +// * ...and only if it's redirected inside a @[x = $(...)] +// * [!command! ...] +// * Can output but does not parse. +// * Every transformer or transport stream has a typed output, which is either: +// * EITHER just its token tree / token stream (for simple matchers) +// * OR an OBJECT with at least two properties: +// * input => all matched characters (a slice reference which can be dropped...) +// (it might only be possible to performantly capture this after the syn fork) +// * stream => A lazy function, used to handle the output when #x is in the final output... +// likely `input` or an error depending on the case. +// * ... other properties, depending on the TRANSFORMER: +// * ITEM might have quite a few +// * STREAM has an `output` property (discussed below) +// * Drop @XXX syntax. Require: @[ ... ] instead, one of: +// * @[XXX] or equivalently @[let _ = XXX ...] +// * @[let x = XXX] or @[let x = XXX { ... }] +// * @[let x.field = XXX ...] +// * @[x = XXX] +// * @[x.field += IDENT] +// * Explicit stream: @(...) +/// * Has an explicit output property via command output, e.g. [!output! ...] which appends both: +// * Command output +// * Output of inner transformer streams. +// * It can be treated as a transformer and its output can be redirected with @[#x = @(...)] syntax. +// But, on trying a few examples, we don't want to require such redirection. +// * QUESTION: Do we actually use square brackets, i.e. @[#x = ...] (i.e. it's just without the IDENT) +// * OR maybe as an explicit [ ... ] stream: @[#x = [...]] +// * For inline destructurings (e.g. in for loops), we allow an implicit inner ... stream +// * EXACT then is just used for some sections where we're reading from variables. +// +// EXAMPLE +#(parsed = []) +[!parse! [...] as + // OPTION 1 + @( + #(let item = {}) + impl @[item.trait = IDENT] for @[item.type = IDENT] + #(parsed += item) + ),* + // OPTION 2 + @[COMMA_REPEATED { + before: #(let item = {}), + item: @(impl @[#(item.trait) = IDENT] for @[#(item.type) = IDENT]), + after: #(parsed += item), + }] +] +[!for! @[{ #trait, #type }] in #(parsed.output) { + impl #trait for #type {} +}] + +``` +### Transformer Notes WIP ```rust // FINAL DESIGN --- // ALL THESE CAN BE SUPPORTED: @@ -143,8 +258,8 @@ Inside a transform stream, the following grammar is supported: } } }]] -// ALSO NICE - although I don't know how we'd do custom inputs -[!define_parser! @INPUT = @(impl @IDENT for @IDENT),*] +// ALSO NICE +[!define_transformer! @INPUT = @(impl @IDENT for @IDENT),*] [!for! (#x #y) in [!parse! #input as @INPUT] { }] @@ -152,33 +267,21 @@ Inside a transform stream, the following grammar is supported: [!parse_for! #input as @(impl @(#x = @IDENT) for @(#y = @IDENT)),* { }] +// WHAT WIZARDRY IS THIS +[!define_transformer! @[REPEAT_EXACT @[FIELDS { + matcher: @(#matcher = $(@REST)), + repetitions: @(#repetitions = @LITERAL), +}]] = @[REINTERPRET [!for! #_ in [!range! 0..#repetitions] { $(#..matcher) }]]] -// What if destructuring could also output something? (nb - OPTION 3 is preferred) - -// OPTION 1 -// * #(X ...) would append its output to the output stream, i.e. #(#output += X ...) -// * #(field: X ...) would append `field: output,` to the output stream. -// * #(#x = X) -// * #(void X) outputs nothing -// * #x and #..x still work in the same way (binding to variables?) - -// OPTION 2 - everything explicit -// * #(void IDENT) outputs nothing -// * #(+IDENT) appends to #output (effectively it's an opt-in) -// * #(.field = IDENT) appends `field: ___,` to #output -// * #(#x = IDENT) sets #x -// * #(#x += IDENT) extends #x -// * #(#x.field = IDENT) appends `field: ___,` to #x - -// OPTION 3 (preferred) - output by default; use @ for destructurers +// SYNTAX PREFERENCE - output by default; use @ for destructurers // * @( ... ) destructure stream, has an output // * @( ... )? optional destructure stream, has an output // * Can similarly have @(...),+ which handles a trailing , // * @X shorthand for @[X] for destructurers which can take no input, e.g. IDENT, TOKEN_TREE, TYPE etc // => NOTE: Each destructurer should return just its tokens by default if it has no arguments. // => It can also have its output over-written or other things outputted using e.g. @[TYPE { is_prefixed: X, parts: #(...), output: { #output } }] -// * #x is shorthand for @[#x = @TOKEN_OR_GROUP_CONTENT] -// * #..x) is shorthand for @[#x = @UNTIL_END] and #..x, is shorthand for @[CAPTURE #x = @[UNTIL_TOKEN ,]] +// * #x is shorthand for @(#x = @TOKEN_OR_GROUP_CONTENT) +// * #..x) is shorthand for @(#x = @REST) and #..x, is shorthand for @(#x = @[UNTIL_TOKEN ,]) // * @(_ = ...) // * @(#x = impl @IDENT for @IDENT) // * @(#x += impl @IDENT for @IDENT) @@ -188,8 +291,8 @@ Inside a transform stream, the following grammar is supported: // In this model, REPEATED is really clean and looks like this: @[REPEATED { - item: #(...), // Captured into #item variable - separator?: #(), // Captured into #separator variable + item: @(...), // Captured into #item variable + separator?: @(), // Captured into #separator variable min?: 0, max?: 1000000, handle_item?: { #item }, // Default is to output the grouped item. #()+ instead uses `{ (#..item) }` @@ -199,8 +302,9 @@ Inside a transform stream, the following grammar is supported: // How does optional work? // @(#x = @(@IDENT)?) // Along with: -// [!fields! { #x, my_var: #y, #z }] +// [!object! { #x, my_var: #y, #z }] // And if some field #z isn't set, it's outputted as null. +// Field access can be with #(variable.field) // Do we want something like !parse_for!? It needs to execute lazily - how? // > Probably by passing some `OnOutput` hook to an output stream method @@ -210,13 +314,9 @@ Inside a transform stream, the following grammar is supported: ``` * Pushed to 0.4: - * Map/Field style variable type, like a JS object: - * e.g. #x.hello = BLAH - * Has a stream-representation as `{ hello: [!group! BLAH], }` but is more performant, and can be destructured with `[@FIELDS { #hello }]` or read with `[!read! #x.hello]` or `[!read! #x["hello"]]` - * And can transform output as `#(.hello = @X)` (mixing/matching append output and field output will error) - * Debug impl is `[!fields! hello: [!group! BLAH],]` - * Can be embedded into an output stream as a fields group - * If necessary, can be converted to a stream and parsed back as `{ hello: [!group! BLAH], }` + * Get rid of needless cloning of commands/variables etc + * Iterator variable type + * Trial making `for` lazily read its input somehow? Some callback on `OutputStream` I guess... * Fork of syn to: * Fix issues in Rust Analyzer * Add support for a more general `TokenBuffer`, and ensure that Cursor can work in a backwards-compatible way with that buffer. Support: diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 9c276448..63776982 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -3,30 +3,30 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionBoolean { pub(super) value: bool, - /// The span of the source code that generated this boolean value. - /// It may not have a value if generated from a complex expression. - pub(super) source_span: Option, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(super) span_range: SpanRange, } impl ExpressionBoolean { pub(super) fn for_litbool(lit: syn::LitBool) -> Self { Self { - source_span: Some(lit.span()), + span_range: lit.span().span_range(), value: lit.value, } } pub(super) fn handle_unary_operation( self, - operation: UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let input = self.value; - Ok(match operation { + Ok(match operation.operation { UnaryOperation::Neg { .. } => { return operation.unsupported_for_value_type_err("boolean") } UnaryOperation::Not { .. } => operation.output(!input), - UnaryOperation::GroupedNoOp { .. } => operation.output(input), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) @@ -54,9 +54,9 @@ impl ExpressionBoolean { pub(super) fn handle_integer_binary_operation( self, _right: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { - match operation { + match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err("boolean") @@ -67,11 +67,11 @@ impl ExpressionBoolean { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } @@ -95,16 +95,16 @@ impl ExpressionBoolean { }) } - pub(super) fn to_ident(&self, fallback_span: Span) -> Ident { - Ident::new_bool(self.value, self.source_span.unwrap_or(fallback_span)) + pub(super) fn to_ident(&self) -> Ident { + Ident::new_bool(self.value, self.span_range.join_into_span_else_start()) } } impl ToExpressionValue for bool { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Boolean(ExpressionBoolean { value: self, - source_span, + span_range, }) } } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 3d5d38f1..8bb3c293 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -3,26 +3,26 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionChar { pub(super) value: char, - /// The span of the source code that generated this boolean value. - /// It may not have a value if generated from a complex expression. - pub(super) source_span: Option, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(super) span_range: SpanRange, } impl ExpressionChar { pub(super) fn for_litchar(lit: syn::LitChar) -> Self { Self { value: lit.value(), - source_span: Some(lit.span()), + span_range: lit.span().span_range(), } } pub(super) fn handle_unary_operation( self, - operation: UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let char = self.value; - Ok(match operation { - UnaryOperation::GroupedNoOp { .. } => operation.output(char), + Ok(match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { return operation.unsupported_for_value_type_err("char") } @@ -53,16 +53,16 @@ impl ExpressionChar { pub(super) fn create_range( self, right: Self, - range_limits: &syn::RangeLimits, + range_limits: OutputSpanned, ) -> Box + '_> { let left = self.value; let right = right.value; - match range_limits { + match range_limits.operation { syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(|x| range_limits.output(x))) + Box::new((left..right).map(move |x| range_limits.output(x))) } syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(|x| range_limits.output(x))) + Box::new((left..=right).map(move |x| range_limits.output(x))) } } } @@ -70,7 +70,7 @@ impl ExpressionChar { pub(super) fn handle_integer_binary_operation( self, _right: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { operation.unsupported_for_value_type_err("char") } @@ -78,11 +78,11 @@ impl ExpressionChar { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } @@ -104,16 +104,16 @@ impl ExpressionChar { }) } - pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { - Literal::character(self.value).with_span(self.source_span.unwrap_or(fallback_span)) + pub(super) fn to_literal(&self) -> Literal { + Literal::character(self.value).with_span(self.span_range.join_into_span_else_start()) } } impl ToExpressionValue for char { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Char(ExpressionChar { value: self, - source_span, + span_range, }) } } diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index ccfd052c..1dd51b09 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -22,12 +22,12 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { loop { match next { - NextAction::HandleValue(evaluation_value) => { + NextAction::HandleValue(value) => { let top_of_stack = match self.operation_stack.pop() { Some(top) => top, - None => return Ok(evaluation_value), + None => return Ok(value), }; - next = self.continue_node_evaluation(top_of_stack, evaluation_value)?; + next = self.continue_node_evaluation(top_of_stack, value)?; } NextAction::EnterNode(next_node) => { next = self.begin_node_evaluation(next_node, evaluation_context)?; @@ -45,6 +45,12 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { ExpressionNode::Leaf(leaf) => { NextAction::HandleValue(K::evaluate_leaf(leaf, evaluation_context)?) } + ExpressionNode::Grouped { delim_span, inner } => { + self.operation_stack.push(EvaluationStackFrame::Group { + span: delim_span.join(), + }); + NextAction::EnterNode(*inner) + } ExpressionNode::UnaryOperation { operation, input } => { self.operation_stack .push(EvaluationStackFrame::UnaryOperation { @@ -72,30 +78,28 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { fn continue_node_evaluation( &mut self, top_of_stack: EvaluationStackFrame, - evaluation_value: ExpressionValue, + value: ExpressionValue, ) -> ExecutionResult { Ok(match top_of_stack { + EvaluationStackFrame::Group { span } => NextAction::HandleValue(value.with_span(span)), EvaluationStackFrame::UnaryOperation { operation } => { - let result = operation.evaluate(evaluation_value)?; - NextAction::HandleValue(result) + NextAction::HandleValue(operation.evaluate(value)?) } EvaluationStackFrame::BinaryOperation { operation, state } => match state { BinaryPath::OnLeftBranch { right } => { - if let Some(result) = operation.lazy_evaluate(&evaluation_value)? { + if let Some(result) = operation.lazy_evaluate(&value)? { NextAction::HandleValue(result) } else { self.operation_stack .push(EvaluationStackFrame::BinaryOperation { operation, - state: BinaryPath::OnRightBranch { - left: evaluation_value, - }, + state: BinaryPath::OnRightBranch { left: value }, }); NextAction::EnterNode(right) } } BinaryPath::OnRightBranch { left } => { - let result = operation.evaluate(left, evaluation_value)?; + let result = operation.evaluate(left, value)?; NextAction::HandleValue(result) } }, @@ -109,6 +113,9 @@ enum NextAction { } enum EvaluationStackFrame { + Group { + span: Span, + }, UnaryOperation { operation: UnaryOperation, }, @@ -122,50 +129,3 @@ enum BinaryPath { OnLeftBranch { right: ExpressionNodeId }, OnRightBranch { left: ExpressionValue }, } - -pub(crate) struct ExpressionOutput { - pub(super) value: ExpressionValue, - pub(super) fallback_output_span: Span, -} - -impl ExpressionOutput { - #[allow(unused)] - pub(crate) fn into_value(self) -> ExpressionValue { - self.value - } - - #[allow(unused)] - pub(crate) fn expect_integer(self, error_message: &str) -> ExecutionResult { - let error_span = self.span(); - match self.value.into_integer() { - Some(integer) => Ok(integer), - None => error_span.execution_err(error_message), - } - } - - pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { - let error_span = self.span(); - match self.value.into_bool() { - Some(boolean) => Ok(boolean.value), - None => error_span.execution_err(error_message), - } - } - - pub(crate) fn to_token_tree(&self) -> TokenTree { - self.value.to_token_tree(self.fallback_output_span) - } -} - -impl HasSpan for ExpressionOutput { - fn span(&self) -> Span { - self.value - .source_span() - .unwrap_or(self.fallback_output_span) - } -} - -impl quote::ToTokens for ExpressionOutput { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_token_tree().to_tokens(tokens); - } -} diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index ead02c5f..091b0c01 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -8,12 +8,6 @@ pub(crate) struct SourceExpression { inner: Expression, } -impl HasSpanRange for SourceExpression { - fn span_range(&self) -> SpanRange { - self.inner.span_range - } -} - impl Parse for SourceExpression { fn parse(input: ParseStream) -> ParseResult { Ok(Self { @@ -26,29 +20,8 @@ impl SourceExpression { pub(crate) fn evaluate( &self, interpreter: &mut Interpreter, - ) -> ExecutionResult { - Ok(ExpressionOutput { - value: self.evaluate_to_value(interpreter)?, - fallback_output_span: self.inner.span_range.join_into_span_else_start(), - }) - } - - pub(crate) fn evaluate_with_span( - &self, - interpreter: &mut Interpreter, - fallback_output_span: Span, - ) -> ExecutionResult { - Ok(ExpressionOutput { - value: self.evaluate_to_value(interpreter)?, - fallback_output_span, - }) - } - - pub(crate) fn evaluate_to_value( - &self, - interpreter: &mut Interpreter, ) -> ExecutionResult { - Source::evaluate_to_value(&self.inner, interpreter) + Source::evaluate(&self.inner, interpreter) } } @@ -63,15 +36,6 @@ impl Expressionable for Source { type Leaf = SourceExpressionLeaf; type EvaluationContext = Interpreter; - fn leaf_end_span(leaf: &Self::Leaf) -> Option { - match leaf { - SourceExpressionLeaf::Command(command) => Some(command.span()), - SourceExpressionLeaf::GroupedVariable(variable) => Some(variable.span_range().end()), - SourceExpressionLeaf::CodeBlock(code_block) => Some(code_block.span()), - SourceExpressionLeaf::Value(value) => value.source_span(), - } - } - fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { Ok(match input.peek_grammar() { SourcePeekMatch::Command(Some(output_kind)) => { @@ -144,7 +108,7 @@ impl Expressionable for Source { // RUST-ANALYZER SAFETY: This isn't very safe, as it could have a none-delimited group in it interpreted.parse_as::()? }; - parsed_expression.evaluate_to_value() + parsed_expression.evaluate() } } @@ -165,15 +129,8 @@ impl Parse for OutputExpression { } impl OutputExpression { - pub(crate) fn evaluate(&self) -> ExecutionResult { - Ok(ExpressionOutput { - value: self.evaluate_to_value()?, - fallback_output_span: self.inner.span_range.join_into_span_else_start(), - }) - } - - pub(crate) fn evaluate_to_value(&self) -> ExecutionResult { - Output::evaluate_to_value(&self.inner, &mut ()) + pub(crate) fn evaluate(&self) -> ExecutionResult { + Output::evaluate(&self.inner, &mut ()) } } @@ -181,10 +138,6 @@ impl Expressionable for Output { type Leaf = ExpressionValue; type EvaluationContext = (); - fn leaf_end_span(leaf: &Self::Leaf) -> Option { - leaf.source_span() - } - fn evaluate_leaf( leaf: &Self::Leaf, _: &mut Self::EvaluationContext, @@ -239,7 +192,6 @@ impl Expressionable for Output { pub(super) struct Expression { pub(super) root: ExpressionNodeId, - pub(super) span_range: SpanRange, pub(super) nodes: std::rc::Rc<[ExpressionNode]>, } @@ -247,7 +199,6 @@ impl Clone for Expression { fn clone(&self) -> Self { Self { root: self.root, - span_range: self.span_range, nodes: self.nodes.clone(), } } @@ -258,6 +209,10 @@ pub(super) struct ExpressionNodeId(pub(super) usize); pub(super) enum ExpressionNode { Leaf(K::Leaf), + Grouped { + delim_span: DelimSpan, + inner: ExpressionNodeId, + }, UnaryOperation { operation: UnaryOperation, input: ExpressionNodeId, @@ -273,8 +228,6 @@ pub(super) trait Expressionable: Sized { type Leaf; type EvaluationContext; - fn leaf_end_span(leaf: &Self::Leaf) -> Option; - fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult>; fn parse_extension(input: &mut ParseStreamStack) -> ParseResult; @@ -283,7 +236,7 @@ pub(super) trait Expressionable: Sized { context: &mut Self::EvaluationContext, ) -> ExecutionResult; - fn evaluate_to_value( + fn evaluate( expression: &Expression, context: &mut Self::EvaluationContext, ) -> ExecutionResult { diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 5a76045b..04e5f077 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -4,7 +4,6 @@ pub(super) struct ExpressionParser<'a, K: Expressionable> { streams: ParseStreamStack<'a, K>, nodes: ExpressionNodes, expression_stack: Vec, - span_range: SpanRange, kind: PhantomData, } @@ -14,7 +13,6 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { streams: ParseStreamStack::new(input), nodes: ExpressionNodes::new(), expression_stack: Vec::with_capacity(10), - span_range: SpanRange::new_single(input.span()), kind: PhantomData, } .run() @@ -30,22 +28,13 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } WorkItem::TryParseAndApplyExtension { node } => { let extension = K::parse_extension(&mut self.streams)?; - match &extension { - NodeExtension::PostfixOperation(op) => { - self.span_range.set_end(op.end_span()); - } - NodeExtension::BinaryOperation(op) => { - self.span_range.set_end(op.span_range().end()); - } - NodeExtension::NoneMatched => {} - }; self.attempt_extension(node, extension)? } WorkItem::TryApplyAlreadyParsedExtension { node, extension } => { self.attempt_extension(node, extension)? } WorkItem::Finished { root } => { - return Ok(self.nodes.complete(root, self.span_range)); + return Ok(self.nodes.complete(root)); } } } @@ -53,18 +42,11 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { fn extend_with_unary_atom(&mut self, unary_atom: UnaryAtom) -> ParseResult { Ok(match unary_atom { - UnaryAtom::Leaf(leaf) => { - if let Some(span) = K::leaf_end_span(&leaf) { - self.span_range.set_end(span); - } - self.add_leaf(leaf) - } + UnaryAtom::Leaf(leaf) => self.add_leaf(leaf), UnaryAtom::Group(delim_span) => { - self.span_range.set_end(delim_span.close()); self.push_stack_frame(ExpressionStackFrame::Group { delim_span }) } UnaryAtom::PrefixUnaryOperation(operation) => { - self.span_range.set_end(operation.span()); self.push_stack_frame(ExpressionStackFrame::IncompletePrefixOperation { operation }) } }) @@ -110,11 +92,9 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { assert!(matches!(extension, NodeExtension::NoneMatched)); self.streams.exit_group(); WorkItem::TryParseAndApplyExtension { - node: self.nodes.add_node(ExpressionNode::UnaryOperation { - operation: UnaryOperation::GroupedNoOp { - span: delim_span.join(), - }, - input: node, + node: self.nodes.add_node(ExpressionNode::Grouped { + delim_span, + inner: node, }), } } @@ -182,10 +162,9 @@ impl ExpressionNodes { node_id } - pub(super) fn complete(self, root: ExpressionNodeId, span_range: SpanRange) -> Expression { + pub(super) fn complete(self, root: ExpressionNodeId) -> Expression { Expression { root, - span_range, nodes: self.nodes.into(), } } @@ -253,7 +232,6 @@ impl OperatorPrecendence { fn of_unary_operation(op: &UnaryOperation) -> Self { match op { - UnaryOperation::GroupedNoOp { .. } => Self::Unambiguous, UnaryOperation::Cast { .. } => Self::Cast, UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => Self::Prefix, } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index db3eb2c9..872e280b 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -4,35 +4,36 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct ExpressionFloat { pub(super) value: ExpressionFloatValue, - /// The span of the source code that generated this boolean value. - /// It may not have a value if generated from a complex expression. - pub(super) source_span: Option, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(super) span_range: SpanRange, } impl ExpressionFloat { pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { - let source_span = Some(lit.span()); + let span_range = lit.span().span_range(); Ok(Self { value: ExpressionFloatValue::for_litfloat(lit)?, - source_span, + span_range, }) } pub(super) fn handle_unary_operation( self, - operation: UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self.value { - ExpressionFloatValue::Untyped(input) => input.handle_unary_operation(&operation), - ExpressionFloatValue::F32(input) => input.handle_unary_operation(&operation), - ExpressionFloatValue::F64(input) => input.handle_unary_operation(&operation), + ExpressionFloatValue::Untyped(input) => input.handle_unary_operation(operation), + ExpressionFloatValue::F32(input) => input.handle_unary_operation(operation), + ExpressionFloatValue::F64(input) => input.handle_unary_operation(operation), } } pub(super) fn handle_integer_binary_operation( self, right: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self.value { ExpressionFloatValue::Untyped(input) => { @@ -47,10 +48,10 @@ impl ExpressionFloat { } } - pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { + pub(super) fn to_literal(&self) -> Literal { self.value .to_unspanned_literal() - .with_span(self.source_span.unwrap_or(fallback_span)) + .with_span(self.span_range.join_into_span_else_start()) } } @@ -63,7 +64,7 @@ pub(super) enum ExpressionFloatValuePair { impl ExpressionFloatValuePair { pub(super) fn handle_paired_binary_operation( self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -136,15 +137,14 @@ impl UntypedFloat { pub(super) fn handle_unary_operation( self, - operation: &UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let input = self.parse_fallback()?; - Ok(match operation { + Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), UnaryOperation::Not { .. } => { return operation.unsupported_for_value_type_err("untyped float") } - UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) @@ -176,9 +176,9 @@ impl UntypedFloat { pub(super) fn handle_integer_binary_operation( self, _rhs: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { - match operation { + match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err("untyped float") @@ -189,11 +189,11 @@ impl UntypedFloat { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } => { operation.output(Self::from_fallback(lhs + rhs)) } @@ -260,10 +260,10 @@ impl UntypedFloat { } impl ToExpressionValue for UntypedFloat { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Float(ExpressionFloat { value: ExpressionFloatValue::Untyped(self), - source_span, + span_range, }) } } @@ -273,20 +273,19 @@ macro_rules! impl_float_operations { $($float_enum_variant:ident($float_type:ident)),* $(,)? ) => {$( impl ToExpressionValue for $float_type { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Float(ExpressionFloat { value: ExpressionFloatValue::$float_enum_variant(self), - source_span + span_range, }) } } impl HandleUnaryOperation for $float_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - Ok(match operation { + fn handle_unary_operation(self, operation: OutputSpanned) -> ExecutionResult { + Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(-self), UnaryOperation::Not { .. } => return operation.unsupported_for_value_type_err(stringify!($float_type)), - UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), @@ -311,12 +310,12 @@ macro_rules! impl_float_operations { } impl HandleBinaryOperation for $float_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: OutputSpanned) -> ExecutionResult { // Unlike integer arithmetic, float arithmetic does not overflow // and instead falls back to NaN or infinity. In future we could // allow trapping on these codes, but for now this is good enough let lhs = self; - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } => operation.output(lhs + rhs), PairedBinaryOperation::Subtraction { .. } => operation.output(lhs - rhs), PairedBinaryOperation::Multiplication { .. } => operation.output(lhs * rhs), @@ -343,9 +342,9 @@ macro_rules! impl_float_operations { fn handle_integer_binary_operation( self, _rhs: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { - match operation { + match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { operation.unsupported_for_value_type_err(stringify!($float_type)) }, diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index ecf87884..61e5b4db 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -3,45 +3,45 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionInteger { pub(super) value: ExpressionIntegerValue, - /// The span of the source code that generated this boolean value. - /// It may not have a value if generated from a complex expression. - pub(super) source_span: Option, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(super) span_range: SpanRange, } impl ExpressionInteger { pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { - let source_span = Some(lit.span()); Ok(Self { + span_range: lit.span().span_range(), value: ExpressionIntegerValue::for_litint(lit)?, - source_span, }) } pub(super) fn handle_unary_operation( self, - operation: UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self.value { - ExpressionIntegerValue::Untyped(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::U8(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::U16(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::U32(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::U64(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::U128(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::Usize(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::I8(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::I16(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::I32(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::I64(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::I128(input) => input.handle_unary_operation(&operation), - ExpressionIntegerValue::Isize(input) => input.handle_unary_operation(&operation), + ExpressionIntegerValue::Untyped(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::U8(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::U16(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::U32(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::U64(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::U128(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::Usize(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::I8(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::I16(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::I32(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::I64(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::I128(input) => input.handle_unary_operation(operation), + ExpressionIntegerValue::Isize(input) => input.handle_unary_operation(operation), } } pub(super) fn handle_integer_binary_operation( self, right: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self.value { ExpressionIntegerValue::Untyped(input) => { @@ -86,10 +86,10 @@ impl ExpressionInteger { } } - pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { + pub(super) fn to_literal(&self) -> Literal { self.value .to_unspanned_literal() - .with_span(self.source_span.unwrap_or(fallback_span)) + .with_span(self.span_range.join_into_span_else_start()) } } @@ -112,7 +112,7 @@ pub(super) enum ExpressionIntegerValuePair { impl ExpressionIntegerValuePair { pub(super) fn handle_paired_binary_operation( self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self { Self::Untyped(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), @@ -133,7 +133,7 @@ impl ExpressionIntegerValuePair { pub(crate) fn create_range( self, - range_limits: &syn::RangeLimits, + range_limits: OutputSpanned, ) -> ExecutionResult + '_>> { Ok(match self { Self::Untyped(lhs, rhs) => return lhs.create_range(rhs, range_limits), @@ -266,15 +266,14 @@ impl UntypedInteger { pub(super) fn handle_unary_operation( self, - operation: &UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let input = self.parse_fallback()?; - Ok(match operation { + Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), UnaryOperation::Not { .. } => { return operation.unsupported_for_value_type_err("untyped integer") } - UnaryOperation::GroupedNoOp { .. } => operation.output(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) @@ -306,10 +305,10 @@ impl UntypedInteger { pub(super) fn handle_integer_binary_operation( self, rhs: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self.parse_fallback()?; - Ok(match operation { + Ok(match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } => match rhs.value { ExpressionIntegerValue::Untyped(rhs) => { operation.output(lhs << rhs.parse_fallback()?) @@ -350,7 +349,7 @@ impl UntypedInteger { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self.parse_fallback()?; let rhs = rhs.parse_fallback()?; @@ -362,7 +361,7 @@ impl UntypedInteger { rhs ) }; - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } => { return operation.output_if_some( lhs.checked_add(rhs).map(Self::from_fallback), @@ -415,16 +414,16 @@ impl UntypedInteger { pub(super) fn create_range( self, right: Self, - range_limits: &syn::RangeLimits, + range_limits: OutputSpanned, ) -> ExecutionResult + '_>> { let left = self.parse_fallback()?; let right = right.parse_fallback()?; - Ok(match range_limits { + Ok(match range_limits.operation { syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(|x| range_limits.output(Self::from_fallback(x)))) + Box::new((left..right).map(move |x| range_limits.output(Self::from_fallback(x)))) } syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(|x| range_limits.output(Self::from_fallback(x)))) + Box::new((left..=right).map(move |x| range_limits.output(Self::from_fallback(x)))) } }) } @@ -463,10 +462,10 @@ impl UntypedInteger { } impl ToExpressionValue for UntypedInteger { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::Untyped(self), - source_span, + span_range, }) } } @@ -477,19 +476,19 @@ macro_rules! impl_int_operations_except_unary { $($integer_enum_variant:ident($integer_type:ident)),* $(,)? ) => {$( impl ToExpressionValue for $integer_type { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::$integer_enum_variant(self), - source_span, + span_range, }) } } impl HandleBinaryOperation for $integer_type { - fn handle_paired_binary_operation(self, rhs: Self, operation: &PairedBinaryOperation) -> ExecutionResult { + fn handle_paired_binary_operation(self, rhs: Self, operation: OutputSpanned) -> ExecutionResult { let lhs = self; let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.symbol(), rhs); - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } => return operation.output_if_some(lhs.checked_add(rhs), overflow_error), PairedBinaryOperation::Subtraction { .. } => return operation.output_if_some(lhs.checked_sub(rhs), overflow_error), PairedBinaryOperation::Multiplication { .. } => return operation.output_if_some(lhs.checked_mul(rhs), overflow_error), @@ -512,10 +511,10 @@ macro_rules! impl_int_operations_except_unary { fn handle_integer_binary_operation( self, rhs: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self; - Ok(match operation { + Ok(match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } => { match rhs.value { ExpressionIntegerValue::Untyped(rhs) => operation.output(lhs << rhs.parse_fallback()?), @@ -558,15 +557,15 @@ macro_rules! impl_int_operations_except_unary { fn create_range( self, right: Self, - range_limits: &syn::RangeLimits, + range_limits: OutputSpanned, ) -> Box + '_> { let left = self; - match range_limits { + match range_limits.operation { syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(|x| range_limits.output(x))) + Box::new((left..right).map(move |x| range_limits.output(x))) }, syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(|x| range_limits.output(x))) + Box::new((left..=right).map(move |x| range_limits.output(x))) } } } @@ -577,9 +576,8 @@ macro_rules! impl_int_operations_except_unary { macro_rules! impl_unsigned_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - Ok(match operation { - UnaryOperation::GroupedNoOp { .. } => operation.output(self), + fn handle_unary_operation(self, operation: OutputSpanned) -> ExecutionResult { + Ok(match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { return operation.unsupported_for_value_type_err(stringify!($integer_type)) @@ -612,9 +610,8 @@ macro_rules! impl_unsigned_unary_operations { macro_rules! impl_signed_unary_operations { ($($integer_type:ident),* $(,)?) => {$( impl HandleUnaryOperation for $integer_type { - fn handle_unary_operation(self, operation: &UnaryOperation) -> ExecutionResult { - Ok(match operation { - UnaryOperation::GroupedNoOp { .. } => operation.output(self), + fn handle_unary_operation(self, operation: OutputSpanned) -> ExecutionResult { + Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(-self), UnaryOperation::Not { .. } => { return operation.unsupported_for_value_type_err(stringify!($integer_type)) @@ -647,10 +644,9 @@ macro_rules! impl_signed_unary_operations { impl HandleUnaryOperation for u8 { fn handle_unary_operation( self, - operation: &UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { - Ok(match operation { - UnaryOperation::GroupedNoOp { .. } => operation.output(self), + Ok(match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { return operation.unsupported_for_value_type_err("u8") } diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index e894c9fc..28603716 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -1,37 +1,59 @@ use super::*; pub(super) trait Operation: HasSpanRange { - fn output(&self, output_value: impl ToExpressionValue) -> ExpressionValue { - output_value.to_value(self.source_span_for_output()) + fn with_output_span_range(&self, output_span_range: SpanRange) -> OutputSpanned<'_, Self> { + OutputSpanned { + output_span_range, + operation: self, + } + } + + fn symbol(&self) -> &'static str; +} + +pub(super) struct OutputSpanned<'a, T: Operation + ?Sized> { + pub(super) output_span_range: SpanRange, + pub(super) operation: &'a T, +} + +impl OutputSpanned<'_, T> { + pub(super) fn symbol(&self) -> &'static str { + self.operation.symbol() + } + + pub(super) fn output(&self, output_value: impl ToExpressionValue) -> ExpressionValue { + output_value.to_value(self.output_span_range) } - fn output_if_some( + pub(super) fn output_if_some( &self, output_value: Option, error_message: impl FnOnce() -> String, ) -> ExecutionResult { match output_value { Some(output_value) => Ok(self.output(output_value)), - None => self.execution_err(error_message()), + None => self.operation.execution_err(error_message()), } } - fn unsupported_for_value_type_err( + pub(super) fn unsupported_for_value_type_err( &self, value_type: &'static str, ) -> ExecutionResult { - Err(self.execution_error(format!( + Err(self.operation.execution_error(format!( "The {} operator is not supported for {} values", - self.symbol(), + self.operation.symbol(), value_type, ))) } +} - fn source_span_for_output(&self) -> Option { - None +impl HasSpanRange for OutputSpanned<'_, T> { + fn span_range(&self) -> SpanRange { + // This is used for errors of the operation, so should be targetted + // to the span range of the _operator_, not the output. + self.operation.span_range() } - - fn symbol(&self) -> &'static str; } pub(super) enum PrefixUnaryOperation { @@ -77,9 +99,6 @@ pub(super) enum UnaryOperation { Not { token: Token![!], }, - GroupedNoOp { - span: Span, - }, Cast { as_token: Token![as], target_ident: Ident, @@ -124,34 +143,21 @@ impl UnaryOperation { } pub(super) fn evaluate(self, input: ExpressionValue) -> ExecutionResult { - input.handle_unary_operation(self) - } - - pub(super) fn end_span(&self) -> Span { - match self { - UnaryOperation::Neg { token } => token.span, - UnaryOperation::Not { token } => token.span, - UnaryOperation::GroupedNoOp { span } => *span, - UnaryOperation::Cast { target_ident, .. } => target_ident.span(), - } + let mut span_range = input.span_range(); + match &self { + UnaryOperation::Neg { token } => span_range.set_start(token.span), + UnaryOperation::Not { token } => span_range.set_start(token.span), + UnaryOperation::Cast { target_ident, .. } => span_range.set_end(target_ident.span()), + }; + input.handle_unary_operation(self.with_output_span_range(span_range)) } } impl Operation for UnaryOperation { - fn source_span_for_output(&self) -> Option { - match self { - UnaryOperation::Neg { .. } => None, - UnaryOperation::Not { .. } => None, - UnaryOperation::GroupedNoOp { span } => Some(*span), - UnaryOperation::Cast { .. } => None, - } - } - fn symbol(&self) -> &'static str { match self { UnaryOperation::Neg { .. } => "-", UnaryOperation::Not { .. } => "!", - UnaryOperation::GroupedNoOp { .. } => "", UnaryOperation::Cast { .. } => "as", } } @@ -162,15 +168,16 @@ impl HasSpan for UnaryOperation { match self { UnaryOperation::Neg { token } => token.span, UnaryOperation::Not { token } => token.span, - UnaryOperation::GroupedNoOp { span } => *span, UnaryOperation::Cast { as_token, .. } => as_token.span, } } } pub(super) trait HandleUnaryOperation: Sized { - fn handle_unary_operation(self, operation: &UnaryOperation) - -> ExecutionResult; + fn handle_unary_operation( + self, + operation: OutputSpanned, + ) -> ExecutionResult; } #[derive(Clone)] @@ -293,16 +300,22 @@ impl BinaryOperation { left: ExpressionValue, right: ExpressionValue, ) -> ExecutionResult { + let span_range = + SpanRange::new_between(left.span_range().start(), right.span_range().end()); match self { BinaryOperation::Paired(operation) => { let value_pair = left.expect_value_pair(operation, right)?; - value_pair.handle_paired_binary_operation(operation) + value_pair + .handle_paired_binary_operation(operation.with_output_span_range(span_range)) } BinaryOperation::Integer(operation) => { let right = right .into_integer() .ok_or_else(|| self.execution_error("The shift amount must be an integer"))?; - left.handle_integer_binary_operation(right, operation) + left.handle_integer_binary_operation( + right, + operation.with_output_span_range(span_range), + ) } } } @@ -420,13 +433,13 @@ pub(super) trait HandleBinaryOperation: Sized { fn handle_paired_binary_operation( self, rhs: Self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult; fn handle_integer_binary_operation( self, rhs: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult; } @@ -434,7 +447,7 @@ pub(super) trait HandleCreateRange: Sized { fn create_range( self, right: Self, - range_limits: &syn::RangeLimits, + range_limits: OutputSpanned, ) -> Box + '_>; } diff --git a/src/expressions/string.rs b/src/expressions/string.rs index b09eb399..c78aaf5c 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -3,37 +3,35 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionString { pub(super) value: String, - /// The span of the source code that generated this boolean value. - /// It may not have a value if generated from a complex expression. - pub(super) source_span: Option, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(super) span_range: SpanRange, } impl ExpressionString { pub(super) fn for_litstr(lit: syn::LitStr) -> Self { Self { value: lit.value(), - source_span: Some(lit.span()), + span_range: lit.span().span_range(), } } pub(super) fn handle_unary_operation( self, - operation: UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { - Ok(match operation { - UnaryOperation::GroupedNoOp { .. } => operation.output(self.value), + match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } - | UnaryOperation::Cast { .. } => { - return operation.unsupported_for_value_type_err("string") - } - }) + | UnaryOperation::Cast { .. } => operation.unsupported_for_value_type_err("string"), + } } pub(super) fn handle_integer_binary_operation( self, _right: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { operation.unsupported_for_value_type_err("string") } @@ -41,11 +39,11 @@ impl ExpressionString { pub(super) fn handle_paired_binary_operation( self, rhs: Self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { let lhs = self.value; let rhs = rhs.value; - Ok(match operation { + Ok(match operation.operation { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } @@ -67,25 +65,25 @@ impl ExpressionString { }) } - pub(super) fn to_literal(&self, fallback_span: Span) -> Literal { - Literal::string(&self.value).with_span(self.source_span.unwrap_or(fallback_span)) + pub(super) fn to_literal(&self) -> Literal { + Literal::string(&self.value).with_span(self.span_range.join_into_span_else_start()) } } impl ToExpressionValue for String { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::String(ExpressionString { value: self, - source_span, + span_range, }) } } impl ToExpressionValue for &str { - fn to_value(self, source_span: Option) -> ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::String(ExpressionString { value: self.to_string(), - source_span, + span_range, }) } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index a1be10a5..7e98e3c5 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -10,7 +10,7 @@ pub(crate) enum ExpressionValue { } pub(super) trait ToExpressionValue: Sized { - fn to_value(self, source_span: Option) -> ExpressionValue; + fn to_value(self, span_range: SpanRange) -> ExpressionValue; } impl ExpressionValue { @@ -227,14 +227,22 @@ impl ExpressionValue { } } + pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { + let error_span = self.span_range(); + match self.into_bool() { + Some(boolean) => Ok(boolean.value), + None => error_span.execution_err(error_message), + } + } + /// The span is used if there isn't already a span available - pub(crate) fn to_token_tree(&self, fallback_output_span: Span) -> TokenTree { + pub(crate) fn to_token_tree(&self) -> TokenTree { match self { - Self::Integer(int) => int.to_literal(fallback_output_span).into(), - Self::Float(float) => float.to_literal(fallback_output_span).into(), - Self::Boolean(bool) => bool.to_ident(fallback_output_span).into(), - Self::String(string) => string.to_literal(fallback_output_span).into(), - Self::Char(char) => char.to_literal(fallback_output_span).into(), + Self::Integer(int) => int.to_literal().into(), + Self::Float(float) => float.to_literal().into(), + Self::Boolean(bool) => bool.to_ident().into(), + Self::String(string) => string.to_literal().into(), + Self::Char(char) => char.to_literal().into(), } } @@ -248,19 +256,9 @@ impl ExpressionValue { } } - pub(super) fn source_span(&self) -> Option { - match self { - Self::Integer(int) => int.source_span, - Self::Float(float) => float.source_span, - Self::Boolean(bool) => bool.source_span, - Self::String(str) => str.source_span, - Self::Char(char) => char.source_span, - } - } - pub(super) fn handle_unary_operation( self, - operation: UnaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self { ExpressionValue::Integer(value) => value.handle_unary_operation(operation), @@ -274,7 +272,7 @@ impl ExpressionValue { pub(super) fn handle_integer_binary_operation( self, right: ExpressionInteger, - operation: &IntegerBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self { ExpressionValue::Integer(value) => { @@ -298,8 +296,42 @@ impl ExpressionValue { other: Self, range_limits: &syn::RangeLimits, ) -> ExecutionResult + '_>> { + let span_range = SpanRange::new_between(self.span_range().start(), self.span_range().end()); self.expect_value_pair(range_limits, other)? - .create_range(range_limits) + .create_range(range_limits.with_output_span_range(span_range)) + } + + fn span_range_mut(&mut self) -> &mut SpanRange { + match self { + Self::Integer(value) => &mut value.span_range, + Self::Float(value) => &mut value.span_range, + Self::Boolean(value) => &mut value.span_range, + Self::String(value) => &mut value.span_range, + Self::Char(value) => &mut value.span_range, + } + } + + pub(crate) fn with_span(mut self, source_span: Span) -> ExpressionValue { + *self.span_range_mut() = source_span.span_range(); + self + } +} + +impl HasSpanRange for ExpressionValue { + fn span_range(&self) -> SpanRange { + match self { + Self::Integer(int) => int.span_range, + Self::Float(float) => float.span_range, + Self::Boolean(bool) => bool.span_range, + Self::String(str) => str.span_range, + Self::Char(char) => char.span_range, + } + } +} + +impl ToTokens for ExpressionValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.to_token_tree().to_tokens(tokens); } } @@ -322,7 +354,7 @@ pub(super) enum EvaluationLiteralPair { impl EvaluationLiteralPair { pub(super) fn handle_paired_binary_operation( self, - operation: &PairedBinaryOperation, + operation: OutputSpanned, ) -> ExecutionResult { match self { Self::Integer(pair) => pair.handle_paired_binary_operation(operation), @@ -335,7 +367,7 @@ impl EvaluationLiteralPair { pub(super) fn create_range( self, - range_limits: &syn::RangeLimits, + range_limits: OutputSpanned, ) -> ExecutionResult + '_>> { Ok(match self { EvaluationLiteralPair::Integer(pair) => return pair.create_range(range_limits), diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index 553946b3..e76ea984 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -102,6 +102,10 @@ impl SpanRange { ::span(self) } + pub(crate) fn set_start(&mut self, start: Span) { + self.start = start; + } + pub(crate) fn set_end(&mut self, end: Span) { self.end = end; } diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 28c31bf0..16cb4e80 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -108,7 +108,7 @@ impl StreamingCommandDefinition for WhileCommand { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - let mut iteration_counter = interpreter.start_iteration_counter(&self.condition); + let mut iteration_counter = interpreter.start_iteration_counter(&self.loop_code); loop { iteration_counter.increment_and_check()?; diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 782f3e8f..5e25d6c0 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -28,7 +28,8 @@ impl ValueCommandDefinition for EvaluateCommand { fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { Ok(self .expression - .evaluate_with_span(interpreter, self.command_span)? + .evaluate(interpreter)? + .with_span(self.command_span) .to_token_tree()) } } @@ -105,7 +106,8 @@ impl NoOutputCommandDefinition for AssignCommand { }; let output = expression - .evaluate_with_span(interpreter, command_span)? + .evaluate(interpreter)? + .with_span(command_span) .to_token_tree(); variable.set(interpreter, output.into())?; @@ -146,8 +148,8 @@ impl GroupedStreamCommandDefinition for RangeCommand { output: &mut OutputStream, ) -> ExecutionResult<()> { let range_limits = self.range_limits; - let left = self.left.evaluate_to_value(interpreter)?; - let right = self.right.evaluate_to_value(interpreter)?; + let left = self.left.evaluate(interpreter)?; + let right = self.right.evaluate(interpreter)?; let range_iterator = left.create_range(right, &range_limits)?; @@ -164,10 +166,9 @@ impl GroupedStreamCommandDefinition for RangeCommand { } } - let output_span = range_limits.span_range().start(); output.extend_raw_tokens(range_iterator.map(|value| { value - .to_token_tree(output_span) + .to_token_tree() // We wrap it in a singleton group to ensure that negative // numbers are treated as single items in other stream commands .into_singleton_group(Delimiter::None) diff --git a/tests/compilation_failures/control_flow/while_infinite_loop.stderr b/tests/compilation_failures/control_flow/while_infinite_loop.stderr index 50698898..a9138322 100644 --- a/tests/compilation_failures/control_flow/while_infinite_loop.stderr +++ b/tests/compilation_failures/control_flow/while_infinite_loop.stderr @@ -1,6 +1,6 @@ error: Iteration limit of 1000 exceeded. If needed, the limit can be reconfigured with [!settings! { iteration_limit: X }] - --> tests/compilation_failures/control_flow/while_infinite_loop.rs:4:28 + --> tests/compilation_failures/control_flow/while_infinite_loop.rs:4:33 | 4 | preinterpret!([!while! true {}]); - | ^^^^ + | ^^ From 0efe855eac646f2f392f2a4866d42a1a63032ea6 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 9 Feb 2025 10:04:44 +0000 Subject: [PATCH 079/126] refactor: Command is nested in enum rather than Box --- src/interpretation/command.rs | 69 ++++++++++++------- .../commands/concat_commands.rs | 7 +- .../commands/control_flow_commands.rs | 12 ++-- src/interpretation/commands/core_commands.rs | 14 ++-- .../commands/expression_commands.rs | 8 +-- src/interpretation/commands/token_commands.rs | 14 ++-- .../commands/transforming_commands.rs | 4 +- 7 files changed, 71 insertions(+), 57 deletions(-) diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index b13816a3..a84a27e1 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -5,7 +5,6 @@ use crate::internal_prelude::*; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum CommandOutputKind { None, - /// LiteralOrBool Value, Ident, FlattenedStream, @@ -43,7 +42,7 @@ struct ExecutionContext<'a> { trait CommandInvocation { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()>; @@ -69,7 +68,7 @@ impl Clone for Box { // implementations, conditioned on an associated type trait CommandInvocationAs { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()>; @@ -77,7 +76,7 @@ trait CommandInvocationAs { impl> CommandInvocation for C { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -107,15 +106,11 @@ pub(crate) trait NoOutputCommandDefinition: { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()>; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()>; } impl CommandInvocationAs for C { - fn execute_into( - self: Box, - context: ExecutionContext, - _: &mut OutputStream, - ) -> ExecutionResult<()> { + fn execute_into(self, context: ExecutionContext, _: &mut OutputStream) -> ExecutionResult<()> { self.execute(context.interpreter)?; Ok(()) } @@ -144,12 +139,12 @@ pub(crate) trait ValueCommandDefinition: { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult; } impl CommandInvocationAs for C { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -181,12 +176,12 @@ pub(crate) trait IdentCommandDefinition: { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult; } impl CommandInvocationAs for C { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -218,7 +213,7 @@ pub(crate) trait GroupedStreamCommandDefinition: const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()>; @@ -226,7 +221,7 @@ pub(crate) trait GroupedStreamCommandDefinition: impl CommandInvocationAs for C { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -266,7 +261,7 @@ pub(crate) trait StreamingCommandDefinition: const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()>; @@ -274,7 +269,7 @@ pub(crate) trait StreamingCommandDefinition: impl CommandInvocationAs for C { fn execute_into( - self: Box, + self, context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -284,7 +279,7 @@ impl CommandInvocationAs for //========================= -macro_rules! define_command_kind { +macro_rules! define_command_enums { ( $( $command:ident, @@ -299,10 +294,10 @@ macro_rules! define_command_kind { } impl CommandKind { - fn parse_invocation(&self, arguments: CommandArguments) -> ParseResult> { + fn parse_command(&self, arguments: CommandArguments) -> ParseResult { Ok(match self { $( - Self::$command => Box::new( + Self::$command => TypedCommand::$command( $command::parse(arguments)? ), )* @@ -341,10 +336,32 @@ macro_rules! define_command_kind { Self::ALL_KIND_NAMES.join(", ") } } + + #[allow(clippy::enum_variant_names)] + #[derive(Clone)] + enum TypedCommand { + $( + $command($command), + )* + } + + impl TypedCommand { + fn execute_into( + self, + context: ExecutionContext, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + match self { + $( + Self::$command(command) => <$command as CommandInvocation>::execute_into(command, context, output), + )* + } + } + } }; } -define_command_kind! { +define_command_enums! { // Core Commands SetCommand, RawCommand, @@ -406,7 +423,7 @@ define_command_kind! { #[derive(Clone)] pub(crate) struct Command { - invocation: Box, + typed: Box, output_kind: CommandOutputKind, source_group_span: DelimSpan, } @@ -438,13 +455,13 @@ impl Parse for Command { )?, }; content.parse::()?; - let invocation = command_kind.parse_invocation(CommandArguments::new( + let typed = command_kind.parse_command(CommandArguments::new( &content, command_name, delim_span.join(), ))?; Ok(Self { - invocation, + typed: Box::new(typed), output_kind, source_group_span: delim_span, }) @@ -479,6 +496,6 @@ impl Interpret for Command { output_kind: self.output_kind, delim_span: self.source_group_span, }; - self.invocation.execute_into(context, output) + self.typed.execute_into(context, output) } } diff --git a/src/interpretation/commands/concat_commands.rs b/src/interpretation/commands/concat_commands.rs index b4d6adf3..fc7e2021 100644 --- a/src/interpretation/commands/concat_commands.rs +++ b/src/interpretation/commands/concat_commands.rs @@ -73,10 +73,7 @@ macro_rules! define_literal_concat_command { }) } - fn execute( - self: Box, - interpreter: &mut Interpreter, - ) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { Ok($output_fn(self.arguments, interpreter, $conversion_fn)?.into()) } } @@ -105,7 +102,7 @@ macro_rules! define_ident_concat_command { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { $output_fn(self.arguments, interpreter, $conversion_fn).into() } } diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 16cb4e80..567950b6 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -47,7 +47,7 @@ impl StreamingCommandDefinition for IfCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -104,7 +104,7 @@ impl StreamingCommandDefinition for WhileCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -161,7 +161,7 @@ impl StreamingCommandDefinition for LoopCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -214,7 +214,7 @@ impl StreamingCommandDefinition for ForCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -264,7 +264,7 @@ impl NoOutputCommandDefinition for ContinueCommand { }) } - fn execute(self: Box, _: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, _: &mut Interpreter) -> ExecutionResult<()> { ExecutionResult::Err(ExecutionInterrupt::ControlFlow( ControlFlowInterrupt::Continue, self.span, @@ -291,7 +291,7 @@ impl NoOutputCommandDefinition for BreakCommand { }) } - fn execute(self: Box, _: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, _: &mut Interpreter) -> ExecutionResult<()> { ExecutionResult::Err(ExecutionInterrupt::ControlFlow( ControlFlowInterrupt::Break, self.span, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index a4d9adf0..ecc5d1fc 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -83,7 +83,7 @@ impl NoOutputCommandDefinition for SetCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { match self.arguments { SetArguments::SetVariable { variable, content, .. @@ -130,7 +130,7 @@ impl StreamingCommandDefinition for RawCommand { } fn execute( - self: Box, + self, _interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -158,7 +158,7 @@ impl StreamingCommandDefinition for OutputCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -182,7 +182,7 @@ impl NoOutputCommandDefinition for IgnoreCommand { Ok(Self) } - fn execute(self: Box, _interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, _interpreter: &mut Interpreter) -> ExecutionResult<()> { Ok(()) } } @@ -214,7 +214,7 @@ impl NoOutputCommandDefinition for SettingsCommand { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { if let Some(limit) = self.inputs.iteration_limit { let limit: usize = limit.interpret_to_value(interpreter)?.base10_parse()?; interpreter.set_iteration_limit(Some(limit)); @@ -274,7 +274,7 @@ impl NoOutputCommandDefinition for ErrorCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { let fields = match self.inputs { EitherErrorInput::Fields(error_inputs) => error_inputs, EitherErrorInput::JustMessage(stream) => { @@ -350,7 +350,7 @@ impl ValueCommandDefinition for DebugCommand { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { let span = self.inner.span(); let debug_string = self .inner diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 5e25d6c0..ba2c9140 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -25,7 +25,7 @@ impl ValueCommandDefinition for EvaluateCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { Ok(self .expression .evaluate(interpreter)? @@ -77,14 +77,14 @@ impl NoOutputCommandDefinition for AssignCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { let Self { variable, operator, equals: _, expression, command_span, - } = *self; + } = self; let expression = if let Some(operator) = operator { let mut calculation = TokenStream::new(); @@ -143,7 +143,7 @@ impl GroupedStreamCommandDefinition for RangeCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 565db7ec..3e8c8a65 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -18,7 +18,7 @@ impl ValueCommandDefinition for IsEmptyCommand { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { let output_span = self.arguments.span_range().join_into_span_else_start(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; Ok(Ident::new_bool(interpreted.is_empty(), output_span).into()) @@ -43,7 +43,7 @@ impl ValueCommandDefinition for LengthCommand { }) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { let output_span = self.arguments.span_range().join_into_span_else_start(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; let length_literal = Literal::usize_unsuffixed(interpreted.len()).with_span(output_span); @@ -70,7 +70,7 @@ impl GroupedStreamCommandDefinition for GroupCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -112,7 +112,7 @@ impl GroupedStreamCommandDefinition for IntersperseCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -247,7 +247,7 @@ impl GroupedStreamCommandDefinition for SplitCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -349,7 +349,7 @@ impl GroupedStreamCommandDefinition for CommaSplitCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -425,7 +425,7 @@ impl GroupedStreamCommandDefinition for ZipCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { diff --git a/src/interpretation/commands/transforming_commands.rs b/src/interpretation/commands/transforming_commands.rs index 6fed8470..eeaf9339 100644 --- a/src/interpretation/commands/transforming_commands.rs +++ b/src/interpretation/commands/transforming_commands.rs @@ -29,7 +29,7 @@ impl StreamingCommandDefinition for ParseCommand { } fn execute( - self: Box, + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { @@ -67,7 +67,7 @@ impl NoOutputCommandDefinition for LetCommand { ) } - fn execute(self: Box, interpreter: &mut Interpreter) -> ExecutionResult<()> { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { let result_tokens = self.arguments.interpret_to_new_stream(interpreter)?; let mut ignored_transformer_output = OutputStream::new(); self.destructuring.handle_transform_from_stream( From 882d797d55f4085a6850801a20a21b459cf0012b Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 9 Feb 2025 11:41:13 +0000 Subject: [PATCH 080/126] refactor: Minor tweak to unsupported error messages --- src/expressions/boolean.rs | 22 +++++------ src/expressions/character.rs | 18 +++++---- src/expressions/float.rs | 58 ++++++++++++++++++----------- src/expressions/integer.rs | 70 ++++++++++++++++++++++------------- src/expressions/mod.rs | 2 + src/expressions/operations.rs | 7 +--- src/expressions/string.rs | 20 +++++++--- src/expressions/value.rs | 18 +++++---- 8 files changed, 130 insertions(+), 85 deletions(-) diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 63776982..acbf7328 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -23,9 +23,7 @@ impl ExpressionBoolean { ) -> ExecutionResult { let input = self.value; Ok(match operation.operation { - UnaryOperation::Neg { .. } => { - return operation.unsupported_for_value_type_err("boolean") - } + UnaryOperation::Neg { .. } => return operation.unsupported(self), UnaryOperation::Not { .. } => operation.output(!input), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { @@ -58,9 +56,7 @@ impl ExpressionBoolean { ) -> ExecutionResult { match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } - | IntegerBinaryOperation::ShiftRight { .. } => { - operation.unsupported_for_value_type_err("boolean") - } + | IntegerBinaryOperation::ShiftRight { .. } => operation.unsupported(self), } } @@ -75,14 +71,10 @@ impl ExpressionBoolean { PairedBinaryOperation::Addition { .. } | PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } - | PairedBinaryOperation::Division { .. } => { - return operation.unsupported_for_value_type_err("boolean") - } + | PairedBinaryOperation::Division { .. } => return operation.unsupported(self), PairedBinaryOperation::LogicalAnd { .. } => operation.output(lhs && rhs), PairedBinaryOperation::LogicalOr { .. } => operation.output(lhs || rhs), - PairedBinaryOperation::Remainder { .. } => { - return operation.unsupported_for_value_type_err("boolean") - } + PairedBinaryOperation::Remainder { .. } => return operation.unsupported(self), PairedBinaryOperation::BitXor { .. } => operation.output(lhs ^ rhs), PairedBinaryOperation::BitAnd { .. } => operation.output(lhs & rhs), PairedBinaryOperation::BitOr { .. } => operation.output(lhs | rhs), @@ -100,6 +92,12 @@ impl ExpressionBoolean { } } +impl HasValueType for ExpressionBoolean { + fn value_type(&self) -> &'static str { + "bool" + } +} + impl ToExpressionValue for bool { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Boolean(ExpressionBoolean { diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 8bb3c293..51e8d42f 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -24,7 +24,7 @@ impl ExpressionChar { let char = self.value; Ok(match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - return operation.unsupported_for_value_type_err("char") + return operation.unsupported(self) } UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { @@ -43,9 +43,7 @@ impl ExpressionChar { CastTarget::Integer(IntegerKind::U128) => operation.output(char as u128), CastTarget::Integer(IntegerKind::Usize) => operation.output(char as usize), CastTarget::Char => operation.output(char), - CastTarget::Boolean | CastTarget::Float(_) => { - return operation.unsupported_for_value_type_err("char") - } + CastTarget::Boolean | CastTarget::Float(_) => return operation.unsupported(self), }, }) } @@ -72,7 +70,7 @@ impl ExpressionChar { _right: ExpressionInteger, operation: OutputSpanned, ) -> ExecutionResult { - operation.unsupported_for_value_type_err("char") + operation.unsupported(self) } pub(super) fn handle_paired_binary_operation( @@ -92,9 +90,7 @@ impl ExpressionChar { | PairedBinaryOperation::Remainder { .. } | PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } - | PairedBinaryOperation::BitOr { .. } => { - return operation.unsupported_for_value_type_err("char") - } + | PairedBinaryOperation::BitOr { .. } => return operation.unsupported(self), PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), @@ -109,6 +105,12 @@ impl ExpressionChar { } } +impl HasValueType for ExpressionChar { + fn value_type(&self) -> &'static str { + "char" + } +} + impl ToExpressionValue for char { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Char(ExpressionChar { diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 872e280b..72825735 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -55,6 +55,12 @@ impl ExpressionFloat { } } +impl HasValueType for ExpressionFloat { + fn value_type(&self) -> &'static str { + self.value.value_type() + } +} + pub(super) enum ExpressionFloatValuePair { Untyped(UntypedFloat, UntypedFloat), F32(f32, f32), @@ -95,14 +101,6 @@ impl ExpressionFloatValue { }) } - pub(super) fn describe_type(&self) -> &'static str { - match self { - ExpressionFloatValue::Untyped(_) => "untyped float", - ExpressionFloatValue::F32(_) => "f32", - ExpressionFloatValue::F64(_) => "f64", - } - } - fn to_unspanned_literal(&self) -> Literal { match self { ExpressionFloatValue::Untyped(float) => float.to_unspanned_literal(), @@ -112,6 +110,16 @@ impl ExpressionFloatValue { } } +impl HasValueType for ExpressionFloatValue { + fn value_type(&self) -> &'static str { + match self { + ExpressionFloatValue::Untyped(_) => "untyped float", + ExpressionFloatValue::F32(_) => "f32", + ExpressionFloatValue::F64(_) => "f64", + } + } +} + #[derive(Copy, Clone)] pub(super) enum FloatKind { Untyped, @@ -142,9 +150,7 @@ impl UntypedFloat { let input = self.parse_fallback()?; Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), - UnaryOperation::Not { .. } => { - return operation.unsupported_for_value_type_err("untyped float") - } + UnaryOperation::Not { .. } => return operation.unsupported(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) @@ -180,9 +186,7 @@ impl UntypedFloat { ) -> ExecutionResult { match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } - | IntegerBinaryOperation::ShiftRight { .. } => { - operation.unsupported_for_value_type_err("untyped float") - } + | IntegerBinaryOperation::ShiftRight { .. } => operation.unsupported(self), } } @@ -207,16 +211,14 @@ impl UntypedFloat { operation.output(Self::from_fallback(lhs / rhs)) } PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { - return operation.unsupported_for_value_type_err(stringify!($float_type)) + return operation.unsupported(self) } PairedBinaryOperation::Remainder { .. } => { operation.output(Self::from_fallback(lhs % rhs)) } PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } - | PairedBinaryOperation::BitOr { .. } => { - return operation.unsupported_for_value_type_err("untyped float") - } + | PairedBinaryOperation::BitOr { .. } => return operation.unsupported(self), PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), @@ -259,6 +261,12 @@ impl UntypedFloat { } } +impl HasValueType for UntypedFloat { + fn value_type(&self) -> &'static str { + "untyped float" + } +} + impl ToExpressionValue for UntypedFloat { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Float(ExpressionFloat { @@ -272,6 +280,12 @@ macro_rules! impl_float_operations { ( $($float_enum_variant:ident($float_type:ident)),* $(,)? ) => {$( + impl HasValueType for $float_type { + fn value_type(&self) -> &'static str { + stringify!($float_type) + } + } + impl ToExpressionValue for $float_type { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Float(ExpressionFloat { @@ -285,7 +299,7 @@ macro_rules! impl_float_operations { fn handle_unary_operation(self, operation: OutputSpanned) -> ExecutionResult { Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(-self), - UnaryOperation::Not { .. } => return operation.unsupported_for_value_type_err(stringify!($float_type)), + UnaryOperation::Not { .. } => return operation.unsupported(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), CastTarget::Integer(IntegerKind::I8) => operation.output(self as i8), @@ -322,13 +336,13 @@ macro_rules! impl_float_operations { PairedBinaryOperation::Division { .. } => operation.output(lhs / rhs), PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { - return operation.unsupported_for_value_type_err(stringify!($float_type)) + return operation.unsupported(self) } PairedBinaryOperation::Remainder { .. } => operation.output(lhs % rhs), PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } | PairedBinaryOperation::BitOr { .. } => { - return operation.unsupported_for_value_type_err(stringify!($float_type)) + return operation.unsupported(self) } PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), @@ -346,7 +360,7 @@ macro_rules! impl_float_operations { ) -> ExecutionResult { match operation.operation { IntegerBinaryOperation::ShiftLeft { .. } | IntegerBinaryOperation::ShiftRight { .. } => { - operation.unsupported_for_value_type_err(stringify!($float_type)) + operation.unsupported(self) }, } } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 61e5b4db..5c53e5a6 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -93,6 +93,12 @@ impl ExpressionInteger { } } +impl HasValueType for ExpressionInteger { + fn value_type(&self) -> &'static str { + self.value.value_type() + } +} + pub(super) enum ExpressionIntegerValuePair { Untyped(UntypedInteger, UntypedInteger), U8(u8, u8), @@ -211,24 +217,6 @@ impl ExpressionIntegerValue { }) } - pub(super) fn describe_type(&self) -> &'static str { - match self { - ExpressionIntegerValue::Untyped(_) => "untyped integer", - ExpressionIntegerValue::U8(_) => "u8", - ExpressionIntegerValue::U16(_) => "u16", - ExpressionIntegerValue::U32(_) => "u32", - ExpressionIntegerValue::U64(_) => "u64", - ExpressionIntegerValue::U128(_) => "u128", - ExpressionIntegerValue::Usize(_) => "usize", - ExpressionIntegerValue::I8(_) => "i8", - ExpressionIntegerValue::I16(_) => "i16", - ExpressionIntegerValue::I32(_) => "i32", - ExpressionIntegerValue::I64(_) => "i64", - ExpressionIntegerValue::I128(_) => "i128", - ExpressionIntegerValue::Isize(_) => "isize", - } - } - fn to_unspanned_literal(&self) -> Literal { match self { ExpressionIntegerValue::Untyped(int) => int.to_unspanned_literal(), @@ -248,6 +236,32 @@ impl ExpressionIntegerValue { } } +impl HasValueType for ExpressionIntegerValue { + fn value_type(&self) -> &'static str { + match self { + ExpressionIntegerValue::Untyped(value) => value.value_type(), + ExpressionIntegerValue::U8(value) => value.value_type(), + ExpressionIntegerValue::U16(value) => value.value_type(), + ExpressionIntegerValue::U32(value) => value.value_type(), + ExpressionIntegerValue::U64(value) => value.value_type(), + ExpressionIntegerValue::U128(value) => value.value_type(), + ExpressionIntegerValue::Usize(value) => value.value_type(), + ExpressionIntegerValue::I8(value) => value.value_type(), + ExpressionIntegerValue::I16(value) => value.value_type(), + ExpressionIntegerValue::I32(value) => value.value_type(), + ExpressionIntegerValue::I64(value) => value.value_type(), + ExpressionIntegerValue::I128(value) => value.value_type(), + ExpressionIntegerValue::Isize(value) => value.value_type(), + } + } +} + +impl HasValueType for UntypedInteger { + fn value_type(&self) -> &'static str { + "untyped integer" + } +} + #[derive(Clone)] pub(super) struct UntypedInteger( /// The span of the literal is ignored, and will be set when converted to an output. @@ -271,9 +285,7 @@ impl UntypedInteger { let input = self.parse_fallback()?; Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(Self::from_fallback(-input)), - UnaryOperation::Not { .. } => { - return operation.unsupported_for_value_type_err("untyped integer") - } + UnaryOperation::Not { .. } => return operation.unsupported(self), UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { operation.output(UntypedInteger::from_fallback(input as FallbackInteger)) @@ -387,7 +399,7 @@ impl UntypedInteger { ) } PairedBinaryOperation::LogicalAnd { .. } | PairedBinaryOperation::LogicalOr { .. } => { - return operation.unsupported_for_value_type_err("untyped integer"); + return operation.unsupported(self); } PairedBinaryOperation::Remainder { .. } => { return operation.output_if_some( @@ -475,6 +487,12 @@ macro_rules! impl_int_operations_except_unary { ( $($integer_enum_variant:ident($integer_type:ident)),* $(,)? ) => {$( + impl HasValueType for $integer_type { + fn value_type(&self) -> &'static str { + stringify!($integer_type) + } + } + impl ToExpressionValue for $integer_type { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Integer(ExpressionInteger { @@ -494,7 +512,7 @@ macro_rules! impl_int_operations_except_unary { PairedBinaryOperation::Multiplication { .. } => return operation.output_if_some(lhs.checked_mul(rhs), overflow_error), PairedBinaryOperation::Division { .. } => return operation.output_if_some(lhs.checked_div(rhs), overflow_error), PairedBinaryOperation::LogicalAnd { .. } - | PairedBinaryOperation::LogicalOr { .. } => return operation.unsupported_for_value_type_err(stringify!($integer_type)), + | PairedBinaryOperation::LogicalOr { .. } => return operation.unsupported(self), PairedBinaryOperation::Remainder { .. } => return operation.output_if_some(lhs.checked_rem(rhs), overflow_error), PairedBinaryOperation::BitXor { .. } => operation.output(lhs ^ rhs), PairedBinaryOperation::BitAnd { .. } => operation.output(lhs & rhs), @@ -580,7 +598,7 @@ macro_rules! impl_unsigned_unary_operations { Ok(match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - return operation.unsupported_for_value_type_err(stringify!($integer_type)) + return operation.unsupported(self) }, UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), @@ -614,7 +632,7 @@ macro_rules! impl_signed_unary_operations { Ok(match operation.operation { UnaryOperation::Neg { .. } => operation.output(-self), UnaryOperation::Not { .. } => { - return operation.unsupported_for_value_type_err(stringify!($integer_type)) + return operation.unsupported(self) }, UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => operation.output(UntypedInteger::from_fallback(self as FallbackInteger)), @@ -648,7 +666,7 @@ impl HandleUnaryOperation for u8 { ) -> ExecutionResult { Ok(match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - return operation.unsupported_for_value_type_err("u8") + return operation.unsupported(self) } UnaryOperation::Cast { target, .. } => match target { CastTarget::Integer(IntegerKind::Untyped) => { diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index f9751222..12d34890 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -23,3 +23,5 @@ use value::*; pub(crate) use expression::*; pub(crate) use value::*; +// For some reason Rust-analyzer didn't see it without this explicit export +pub(crate) use value::ExpressionValue; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 28603716..64f8e4c2 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -36,14 +36,11 @@ impl OutputSpanned<'_, T> { } } - pub(super) fn unsupported_for_value_type_err( - &self, - value_type: &'static str, - ) -> ExecutionResult { + pub(super) fn unsupported(&self, value: impl HasValueType) -> ExecutionResult { Err(self.operation.execution_error(format!( "The {} operator is not supported for {} values", self.operation.symbol(), - value_type, + value.value_type(), ))) } } diff --git a/src/expressions/string.rs b/src/expressions/string.rs index c78aaf5c..552e9d47 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -24,7 +24,7 @@ impl ExpressionString { match operation.operation { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } - | UnaryOperation::Cast { .. } => operation.unsupported_for_value_type_err("string"), + | UnaryOperation::Cast { .. } => operation.unsupported(self), } } @@ -33,7 +33,7 @@ impl ExpressionString { _right: ExpressionInteger, operation: OutputSpanned, ) -> ExecutionResult { - operation.unsupported_for_value_type_err("string") + operation.unsupported(self) } pub(super) fn handle_paired_binary_operation( @@ -53,9 +53,7 @@ impl ExpressionString { | PairedBinaryOperation::Remainder { .. } | PairedBinaryOperation::BitXor { .. } | PairedBinaryOperation::BitAnd { .. } - | PairedBinaryOperation::BitOr { .. } => { - return operation.unsupported_for_value_type_err("string") - } + | PairedBinaryOperation::BitOr { .. } => return operation.unsupported(lhs), PairedBinaryOperation::Equal { .. } => operation.output(lhs == rhs), PairedBinaryOperation::LessThan { .. } => operation.output(lhs < rhs), PairedBinaryOperation::LessThanOrEqual { .. } => operation.output(lhs <= rhs), @@ -70,6 +68,18 @@ impl ExpressionString { } } +impl HasValueType for ExpressionString { + fn value_type(&self) -> &'static str { + self.value.value_type() + } +} + +impl HasValueType for String { + fn value_type(&self) -> &'static str { + "string" + } +} + impl ToExpressionValue for String { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::String(ExpressionString { diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 7e98e3c5..8e013ddd 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -9,7 +9,7 @@ pub(crate) enum ExpressionValue { Char(ExpressionChar), } -pub(super) trait ToExpressionValue: Sized { +pub(crate) trait ToExpressionValue: Sized { fn to_value(self, span_range: SpanRange) -> ExpressionValue; } @@ -157,7 +157,7 @@ impl ExpressionValue { ExpressionIntegerValuePair::Isize(lhs, rhs) } (left_value, right_value) => { - return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.describe_type(), right_value.describe_type())); + return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.value_type(), right_value.value_type())); } }; EvaluationLiteralPair::Integer(integer_pair) @@ -196,7 +196,7 @@ impl ExpressionValue { ExpressionFloatValuePair::F64(lhs, rhs) } (left_value, right_value) => { - return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.describe_type(), right_value.describe_type())); + return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.value_type(), right_value.value_type())); } }; EvaluationLiteralPair::Float(float_pair) @@ -208,7 +208,7 @@ impl ExpressionValue { EvaluationLiteralPair::CharPair(left, right) } (left, right) => { - return operation.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left.describe_type(), right.describe_type())); + return operation.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left.value_type(), right.value_type())); } }) } @@ -246,10 +246,10 @@ impl ExpressionValue { } } - pub(super) fn describe_type(&self) -> &'static str { + pub(super) fn value_type(&self) -> &'static str { match self { - Self::Integer(int) => int.value.describe_type(), - Self::Float(float) => float.value.describe_type(), + Self::Integer(int) => int.value.value_type(), + Self::Float(float) => float.value.value_type(), Self::Boolean(_) => "bool", Self::String(_) => "string", Self::Char(_) => "char", @@ -329,6 +329,10 @@ impl HasSpanRange for ExpressionValue { } } +pub(super) trait HasValueType { + fn value_type(&self) -> &'static str; +} + impl ToTokens for ExpressionValue { fn to_tokens(&self, tokens: &mut TokenStream) { self.to_token_tree().to_tokens(tokens); From b1622539843c4f5552b3097e18289d27849efca8 Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 10 Feb 2025 21:40:26 +0000 Subject: [PATCH 081/126] refactor: Start expression rework --- CHANGELOG.md | 49 +++-- src/expressions/boolean.rs | 5 +- src/expressions/character.rs | 3 + src/expressions/expression.rs | 94 +++++----- src/expressions/float.rs | 10 +- src/expressions/integer.rs | 20 ++- src/expressions/mod.rs | 5 + src/expressions/operations.rs | 39 ++-- src/expressions/stream.rs | 93 ++++++++++ src/expressions/string.rs | 16 +- src/expressions/value.rs | 167 +++++++++++++----- src/extensions/tokens.rs | 16 -- src/interpretation/command.rs | 141 ++++++++++++--- .../commands/concat_commands.rs | 70 +++++--- .../commands/control_flow_commands.rs | 12 +- src/interpretation/commands/core_commands.rs | 76 +++++++- .../commands/expression_commands.rs | 62 +++---- src/interpretation/commands/token_commands.rs | 13 +- src/interpretation/interpreted_stream.rs | 11 ++ src/interpretation/source_code_block.rs | 2 +- src/interpretation/source_stream_input.rs | 3 +- src/interpretation/variable.rs | 34 ++++ src/misc/errors.rs | 2 + .../core/error_span_repeat.stderr | 2 +- .../expressions/add_float_and_int.stderr | 2 +- .../expressions/{brackets.rs => braces.rs} | 2 +- .../expressions/braces.stderr | 6 + .../expressions/brackets.stderr | 6 - .../code_blocks_are_not_reevaluated.stderr | 9 +- .../expressions/compare_int_and_float.stderr | 2 +- .../fix_me_negative_max_int_fails.stderr | 7 +- .../flattened_commands_in_expressions.stderr | 8 +- .../{inner_brackets.rs => inner_braces.rs} | 2 +- .../expressions/inner_braces.stderr | 6 + .../expressions/inner_brackets.stderr | 6 - .../no_output_commands_in_expressions.stderr | 8 +- tests/control_flow.rs | 2 +- tests/expressions.rs | 51 +++--- 38 files changed, 740 insertions(+), 322 deletions(-) create mode 100644 src/expressions/stream.rs rename tests/compilation_failures/expressions/{brackets.rs => braces.rs} (69%) create mode 100644 tests/compilation_failures/expressions/braces.stderr delete mode 100644 tests/compilation_failures/expressions/brackets.stderr rename tests/compilation_failures/expressions/{inner_brackets.rs => inner_braces.rs} (68%) create mode 100644 tests/compilation_failures/expressions/inner_braces.stderr delete mode 100644 tests/compilation_failures/expressions/inner_brackets.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f21f445..4264bb66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * `[!set! _ = ...]` interprets its arguments but then ignores any outputs. * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. * `[!output! ...]` can be used to just output its interpreted contents. Normally it's a no-op, but it can be useful inside a transformer. + * `[!reinterpret! ...]` is like an `eval` command in scripting languages. It takes a stream, and parses/interprets it. * `[!settings! { ... }]` can be used to adjust the iteration limit. * Expression commands: * `[!evaluate! ]` @@ -43,16 +44,21 @@ Expressions can be evaluated with `[!evaluate! ...]` and are also used in the `if`, `for` and `while` loops. They operate on literals as values. Currently supported are: -* Integer, Float, Bool, String and Char literals -* The operators: `+ - * / % & | ^` +* Values Model: + * Integer literals + * Float literals + * Boolean literals + * String literals + * Char literals + * Token streams which look like `[...]` and can be appended with the `+` operator. +* The numeric operators: `+ - * / % & | ^` * The lazy boolean operators: `|| &&` * The comparison operators: `== != < > <= >=` * The shift operators: `>> <<` -* Casting with `as` including to untyped integers/floats with `as int` and `as float` +* Casting with `as` including to untyped integers/floats with `as int` and `as float` and to a stream with `as stream`. * () and none-delimited groups for precedence -* Embedded `#x` grouped variables, whose contents are parsed as an expression - and evaluated. -* `{ ... }` for creating sub-expressions, which are parsed from the resultant token stream. +* Variables `#x` and flattened variables `#..x` +* Commands `[!xxx! ...]` Expressions behave intuitively as you'd expect from writing regular rust code, except they happen at compile time. @@ -86,6 +92,11 @@ Inside a transform stream, the following grammar is supported: ### To come +* Add basic `ExpressionBlock` support: + * `#(xxx)` which can e.g. output a variable as expression + * Support multiple statements (where only the last is output, and the rest have to be None): + * `#([!set! #x = 2]; 1 + 1)` => evaluate + * Change `boolean_operators_short_circuit` test back to use `#()` * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Separate `&mut Output` and `StreamOutput`, `ValueOutput = ExpressionOutput`, `ObjectOutput` * Objects: @@ -97,10 +108,9 @@ Inside a transform stream, the following grammar is supported: (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default stream for the object, or error and suggest fields the user should use instead. * Values: - * Can output to an internal stream as `[!group! ]` so it can be read by other transformers. * When we parse a `#x` (`@(#x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. * Output to final output as unwrapped content - * New expression & definition syntax (replaces `[!set!]`??) + * New ExpressionBlock syntax (replaces `[!set!]`??) * `#(#x = [... stream ...])` * `#(#x += [... stream ...])` // += acts like "extend" with a stream LHS * `#(#x += [!group! ...])` @@ -144,7 +154,7 @@ Inside a transform stream, the following grammar is supported: * Adding `!define_transformer!` * `[!is_set! #x]` * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) -* Add `[!reinterpret! ...]` command for an `eval` style command, and maybe a `@[REINTERPRET ..]` transformer. +* Maybe add a `@[REINTERPRET ..]` transformer. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed @@ -194,16 +204,15 @@ Inside a transform stream, the following grammar is supported: // * ...and only if it's redirected inside a @[x = $(...)] // * [!command! ...] // * Can output but does not parse. -// * Every transformer or transport stream has a typed output, which is either: -// * EITHER just its token tree / token stream (for simple matchers) -// * OR an OBJECT with at least two properties: +// * Every transformer has a typed output, which is either: +// * EITHER its token stream (for simple matchers) +// * OR an #output OBJECT with at least two properties: // * input => all matched characters (a slice reference which can be dropped...) // (it might only be possible to performantly capture this after the syn fork) // * stream => A lazy function, used to handle the output when #x is in the final output... // likely `input` or an error depending on the case. // * ... other properties, depending on the TRANSFORMER: -// * ITEM might have quite a few -// * STREAM has an `output` property (discussed below) +// * e.g. a Rust ITEM might have quite a few (mostly lazy) // * Drop @XXX syntax. Require: @[ ... ] instead, one of: // * @[XXX] or equivalently @[let _ = XXX ...] // * @[let x = XXX] or @[let x = XXX { ... }] @@ -314,9 +323,15 @@ Inside a transform stream, the following grammar is supported: ``` * Pushed to 0.4: - * Get rid of needless cloning of commands/variables etc - * Iterator variable type - * Trial making `for` lazily read its input somehow? Some callback on `OutputStream` I guess... + * Performance: + * Get rid of needless cloning of commands/variables etc + * Support `+=` inside expressions to allow appending of token streams + * Variable reference would need to be a sub-type of stream + * Then `#x += [] + []` could resolve to two variable reference appends and then a return null + * Iterators: + * Iterator value type (with an inbuilt iteration count / limit check) + * Allow `for` lazily reading from iterators + * Support unbounded iterators `[!range! xx..]` * Fork of syn to: * Fix issues in Rust Analyzer * Add support for a more general `TokenBuffer`, and ensure that Cursor can work in a backwards-compatible way with that buffer. Support: diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index acbf7328..ee973058 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -2,7 +2,7 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionBoolean { - pub(super) value: bool, + pub(crate) value: bool, /// The span range that generated this value. /// For a complex expression, the start span is the most left part /// of the expression, and the end span is the most right part. @@ -45,6 +45,9 @@ impl ExpressionBoolean { return operation.execution_err("This cast is not supported") } CastTarget::Boolean => operation.output(self.value), + CastTarget::Stream => { + operation.output(operation.output(input).into_new_output_stream()) + } }, }) } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 51e8d42f..fd05bbd7 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -44,6 +44,9 @@ impl ExpressionChar { CastTarget::Integer(IntegerKind::Usize) => operation.output(char as usize), CastTarget::Char => operation.output(char), CastTarget::Boolean | CastTarget::Float(_) => return operation.unsupported(self), + CastTarget::Stream => { + operation.output(operation.output(char).into_new_output_stream()) + } }, }) } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 091b0c01..d6c2d7e1 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -11,7 +11,7 @@ pub(crate) struct SourceExpression { impl Parse for SourceExpression { fn parse(input: ParseStream) -> ParseResult { Ok(Self { - inner: ExpressionParser::parse(input)?, + inner: input.parse()?, }) } } @@ -28,7 +28,8 @@ impl SourceExpression { pub(super) enum SourceExpressionLeaf { Command(Command), GroupedVariable(GroupedVariable), - CodeBlock(SourceCodeBlock), + FlattenedVariable(FlattenedVariable), + ExplicitStream(SourceGroup), Value(ExpressionValue), } @@ -38,38 +39,43 @@ impl Expressionable for Source { fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { Ok(match input.peek_grammar() { - SourcePeekMatch::Command(Some(output_kind)) => { - match output_kind.expression_support() { - Ok(()) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), - Err(error_message) => return input.parse_err(error_message), - } - } - SourcePeekMatch::Command(None) => return input.parse_err("Invalid command"), - SourcePeekMatch::GroupedVariable => UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)), - SourcePeekMatch::FlattenedVariable => return input.parse_err("Flattened variables cannot be used directly in expressions. Consider removing the .. or wrapping it inside a command such as [!group! ..] which returns an expression"), - SourcePeekMatch::AppendVariableBinding => return input.parse_err("Append variable operations are not supported in an expression"), - SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => return input.parse_err("Destructurings are not supported in an expression"), + SourcePeekMatch::Command(_) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), + SourcePeekMatch::GroupedVariable => { + UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)) + } + SourcePeekMatch::FlattenedVariable => { + UnaryAtom::Leaf(Self::Leaf::FlattenedVariable(input.parse()?)) + } + SourcePeekMatch::AppendVariableBinding => { + return input + .parse_err("Append variable operations are not supported in an expression") + } + SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => { + return input.parse_err("Destructurings are not supported in an expression") + } SourcePeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { let (_, delim_span) = input.parse_and_enter_group()?; UnaryAtom::Group(delim_span) - }, + } SourcePeekMatch::Group(Delimiter::Brace) => { - let leaf = SourceExpressionLeaf::CodeBlock(input.parse()?); - UnaryAtom::Leaf(leaf) + return input.parse_err("Braces { ... } are not supported in an expression") } - SourcePeekMatch::Group(Delimiter::Bracket) => return input.parse_err("Square brackets [ .. ] are not supported in an expression"), - SourcePeekMatch::Punct(_) => { - UnaryAtom::PrefixUnaryOperation(input.parse()?) - }, + SourcePeekMatch::Group(Delimiter::Bracket) => { + UnaryAtom::Leaf(Self::Leaf::ExplicitStream(input.parse()?)) + } + SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), SourcePeekMatch::Ident(_) => { - let value = ExpressionValue::Boolean(ExpressionBoolean::for_litbool(input.parse()?)); + let value = + ExpressionValue::Boolean(ExpressionBoolean::for_litbool(input.parse()?)); UnaryAtom::Leaf(Self::Leaf::Value(value)) - }, + } SourcePeekMatch::Literal(_) => { - let value = ExpressionValue::for_literal(input.parse()?)?; + let value = ExpressionValue::for_syn_lit(input.parse()?); UnaryAtom::Leaf(Self::Leaf::Value(value)) - }, - SourcePeekMatch::End => return input.parse_err("The expression ended in an incomplete state"), + } + SourcePeekMatch::End => { + return input.parse_err("The expression ended in an incomplete state") + } }) } @@ -92,23 +98,23 @@ impl Expressionable for Source { leaf: &Self::Leaf, interpreter: &mut Self::EvaluationContext, ) -> ExecutionResult { - let interpreted = match leaf { + Ok(match leaf { SourceExpressionLeaf::Command(command) => { - command.clone().interpret_to_new_stream(interpreter)? + command.clone().interpret_to_value(interpreter)? } SourceExpressionLeaf::GroupedVariable(grouped_variable) => { - grouped_variable.interpret_to_new_stream(interpreter)? - } - SourceExpressionLeaf::CodeBlock(code_block) => { - code_block.clone().interpret_to_new_stream(interpreter)? - } - SourceExpressionLeaf::Value(value) => return Ok(value.clone()), - }; - let parsed_expression = unsafe { - // RUST-ANALYZER SAFETY: This isn't very safe, as it could have a none-delimited group in it - interpreted.parse_as::()? - }; - parsed_expression.evaluate() + grouped_variable.read_as_expression_value(interpreter)? + } + SourceExpressionLeaf::FlattenedVariable(flattened_variable) => { + flattened_variable.read_as_expression_value(interpreter)? + } + SourceExpressionLeaf::ExplicitStream(source_group) => source_group + .clone() + .into_content() + .interpret_to_new_stream(interpreter)? + .to_value(source_group.span_range()), + SourceExpressionLeaf::Value(value) => value.clone(), + }) } } @@ -116,6 +122,7 @@ impl Expressionable for Source { // =========== #[derive(Clone)] +#[allow(unused)] pub(crate) struct OutputExpression { inner: Expression, } @@ -128,6 +135,7 @@ impl Parse for OutputExpression { } } +#[allow(unused)] impl OutputExpression { pub(crate) fn evaluate(&self) -> ExecutionResult { Output::evaluate(&self.inner, &mut ()) @@ -163,7 +171,7 @@ impl Expressionable for Output { )), OutputPeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), OutputPeekMatch::Literal(_) => { - UnaryAtom::Leaf(ExpressionValue::for_literal(input.parse()?)?) + UnaryAtom::Leaf(ExpressionValue::for_syn_lit(input.parse()?)) } OutputPeekMatch::End => { return input.parse_err("The expression ended in an incomplete state") @@ -204,6 +212,12 @@ impl Clone for Expression { } } +impl Parse for Expression { + fn parse(input: ParseStream) -> ParseResult { + ExpressionParser::parse(input) + } +} + #[derive(Clone, Copy)] pub(super) struct ExpressionNodeId(pub(super) usize); diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 72825735..f83fb993 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -11,7 +11,7 @@ pub(crate) struct ExpressionFloat { } impl ExpressionFloat { - pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { + pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ParseResult { let span_range = lit.span().span_range(); Ok(Self { value: ExpressionFloatValue::for_litfloat(lit)?, @@ -88,9 +88,9 @@ pub(super) enum ExpressionFloatValue { } impl ExpressionFloatValue { - pub(super) fn for_litfloat(lit: syn::LitFloat) -> ParseResult { + pub(super) fn for_litfloat(lit: &syn::LitFloat) -> ParseResult { Ok(match lit.suffix() { - "" => Self::Untyped(UntypedFloat::new_from_lit_float(lit)), + "" => Self::Untyped(UntypedFloat::new_from_lit_float(lit.clone())), "f32" => Self::F32(lit.base10_parse()?), "f64" => Self::F64(lit.base10_parse()?), suffix => { @@ -175,6 +175,9 @@ impl UntypedFloat { CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } + CastTarget::Stream => { + operation.output(operation.output(self).into_new_output_stream()) + } }, }) } @@ -318,6 +321,7 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream()), } }) } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 5c53e5a6..1bc3ff71 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -10,7 +10,7 @@ pub(crate) struct ExpressionInteger { } impl ExpressionInteger { - pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { + pub(super) fn for_litint(lit: &syn::LitInt) -> ParseResult { Ok(Self { span_range: lit.span().span_range(), value: ExpressionIntegerValue::for_litint(lit)?, @@ -194,9 +194,9 @@ pub(super) enum ExpressionIntegerValue { } impl ExpressionIntegerValue { - pub(super) fn for_litint(lit: syn::LitInt) -> ParseResult { + pub(super) fn for_litint(lit: &syn::LitInt) -> ParseResult { Ok(match lit.suffix() { - "" => Self::Untyped(UntypedInteger::new_from_lit_int(lit)), + "" => Self::Untyped(UntypedInteger::new_from_lit_int(lit.clone())), "u8" => Self::U8(lit.base10_parse()?), "u16" => Self::U16(lit.base10_parse()?), "u32" => Self::U32(lit.base10_parse()?), @@ -263,11 +263,11 @@ impl HasValueType for UntypedInteger { } #[derive(Clone)] -pub(super) struct UntypedInteger( +pub(crate) struct UntypedInteger( /// The span of the literal is ignored, and will be set when converted to an output. syn::LitInt, ); -pub(super) type FallbackInteger = i128; +pub(crate) type FallbackInteger = i128; impl UntypedInteger { pub(super) fn new_from_lit_int(lit_int: LitInt) -> Self { @@ -310,6 +310,9 @@ impl UntypedInteger { CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } + CastTarget::Stream => { + operation.output(operation.output(self).into_new_output_stream()) + } }, }) } @@ -440,7 +443,7 @@ impl UntypedInteger { }) } - pub(super) fn from_fallback(value: FallbackInteger) -> Self { + pub(crate) fn from_fallback(value: FallbackInteger) -> Self { Self::new_from_literal(Literal::i128_unsuffixed(value)) } @@ -618,6 +621,7 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream()), } }) } @@ -652,6 +656,7 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream()), } }) } @@ -693,6 +698,9 @@ impl HandleUnaryOperation for u8 { CastTarget::Boolean => { return operation.execution_err("This cast is not supported") } + CastTarget::Stream => { + operation.output(operation.output(self).into_new_output_stream()) + } }, }) } diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 12d34890..f0617363 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -6,6 +6,7 @@ mod expression_parsing; mod float; mod integer; mod operations; +mod stream; mod string; mod value; @@ -18,10 +19,14 @@ use expression_parsing::*; use float::*; use integer::*; use operations::*; +use stream::*; use string::*; use value::*; pub(crate) use expression::*; +pub(crate) use integer::*; +pub(crate) use operations::*; pub(crate) use value::*; // For some reason Rust-analyzer didn't see it without this explicit export +pub(crate) use operations::BinaryOperation; pub(crate) use value::ExpressionValue; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 64f8e4c2..955b66c2 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -127,6 +127,7 @@ impl UnaryOperation { "f64" => CastTarget::Float(FloatKind::F64), "bool" => CastTarget::Boolean, "char" => CastTarget::Char, + "stream" => CastTarget::Stream, _ => { return target_ident .parse_err("This type is not supported in preinterpret cast expressions") @@ -178,7 +179,7 @@ pub(super) trait HandleUnaryOperation: Sized { } #[derive(Clone)] -pub(super) enum BinaryOperation { +pub(crate) enum BinaryOperation { Paired(PairedBinaryOperation), Integer(IntegerBinaryOperation), } @@ -187,7 +188,7 @@ impl SynParse for BinaryOperation { fn parse(input: SynParseStream) -> SynResult { // In line with Syn's BinOp, we use peek instead of lookahead // ...I assume for slightly increased performance - // ...Or becuase 30 alternative options in the error message is too many + // ...Or because 30 alternative options in the error message is too many if input.peek(Token![+]) { Ok(Self::Paired(PairedBinaryOperation::Addition( input.parse()?, @@ -265,34 +266,26 @@ impl BinaryOperation { ) -> ExecutionResult> { match self { BinaryOperation::Paired(PairedBinaryOperation::LogicalAnd { .. }) => { - match left.clone().into_bool() { - Some(bool) => { - if !bool.value { - Ok(Some(ExpressionValue::Boolean(bool))) - } else { - Ok(None) - } - } - None => self.execution_err("The left operand was not a boolean"), + let bool = left.clone().expect_bool("The left operand to &&")?; + if !bool.value { + Ok(Some(ExpressionValue::Boolean(bool))) + } else { + Ok(None) } } BinaryOperation::Paired(PairedBinaryOperation::LogicalOr { .. }) => { - match left.clone().into_bool() { - Some(bool) => { - if bool.value { - Ok(Some(ExpressionValue::Boolean(bool))) - } else { - Ok(None) - } - } - None => self.execution_err("The left operand was not a boolean"), + let bool = left.clone().expect_bool("The left operand to ||")?; + if bool.value { + Ok(Some(ExpressionValue::Boolean(bool))) + } else { + Ok(None) } } _ => Ok(None), } } - pub(super) fn evaluate( + pub(crate) fn evaluate( &self, left: ExpressionValue, right: ExpressionValue, @@ -337,7 +330,7 @@ impl Operation for BinaryOperation { } #[derive(Copy, Clone)] -pub(super) enum PairedBinaryOperation { +pub(crate) enum PairedBinaryOperation { Addition(Token![+]), Subtraction(Token![-]), Multiplication(Token![*]), @@ -403,7 +396,7 @@ impl HasSpanRange for PairedBinaryOperation { } #[derive(Copy, Clone)] -pub(super) enum IntegerBinaryOperation { +pub(crate) enum IntegerBinaryOperation { ShiftLeft(Token![<<]), ShiftRight(Token![>>]), } diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs new file mode 100644 index 00000000..053cc440 --- /dev/null +++ b/src/expressions/stream.rs @@ -0,0 +1,93 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionStream { + pub(super) value: OutputStream, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(super) span_range: SpanRange, +} + +impl ExpressionStream { + pub(super) fn handle_unary_operation( + self, + operation: OutputSpanned, + ) -> ExecutionResult { + Ok(match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { + return operation.unsupported(self) + } + UnaryOperation::Cast { target, .. } => match target { + CastTarget::Stream => operation.output(self.value), + _ => { + let coerced = self.value.coerce_into_value(self.span_range); + if let ExpressionValue::Stream(_) = &coerced { + return operation.unsupported(coerced); + } + coerced.handle_unary_operation(operation)? + } + }, + }) + } + + pub(super) fn handle_integer_binary_operation( + self, + _right: ExpressionInteger, + operation: OutputSpanned, + ) -> ExecutionResult { + operation.unsupported(self) + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: OutputSpanned, + ) -> ExecutionResult { + let lhs = self.value; + let rhs = rhs.value; + Ok(match operation.operation { + PairedBinaryOperation::Addition { .. } => operation.output({ + let mut stream = lhs; + rhs.append_cloned_into(&mut stream); + stream + }), + PairedBinaryOperation::Subtraction { .. } + | PairedBinaryOperation::Multiplication { .. } + | PairedBinaryOperation::Division { .. } + | PairedBinaryOperation::LogicalAnd { .. } + | PairedBinaryOperation::LogicalOr { .. } + | PairedBinaryOperation::Remainder { .. } + | PairedBinaryOperation::BitXor { .. } + | PairedBinaryOperation::BitAnd { .. } + | PairedBinaryOperation::BitOr { .. } + | PairedBinaryOperation::Equal { .. } + | PairedBinaryOperation::LessThan { .. } + | PairedBinaryOperation::LessThanOrEqual { .. } + | PairedBinaryOperation::NotEqual { .. } + | PairedBinaryOperation::GreaterThanOrEqual { .. } + | PairedBinaryOperation::GreaterThan { .. } => return operation.unsupported(lhs), + }) + } +} + +impl HasValueType for ExpressionStream { + fn value_type(&self) -> &'static str { + self.value.value_type() + } +} + +impl HasValueType for OutputStream { + fn value_type(&self) -> &'static str { + "stream" + } +} + +impl ToExpressionValue for OutputStream { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Stream(ExpressionStream { + value: self, + span_range, + }) + } +} diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 552e9d47..f1beb333 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -21,11 +21,17 @@ impl ExpressionString { self, operation: OutputSpanned, ) -> ExecutionResult { - match operation.operation { - UnaryOperation::Neg { .. } - | UnaryOperation::Not { .. } - | UnaryOperation::Cast { .. } => operation.unsupported(self), - } + Ok(match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { + return operation.unsupported(self) + } + UnaryOperation::Cast { target, .. } => match target { + CastTarget::Stream => { + operation.output(operation.output(self.value).into_new_output_stream()) + } + _ => return operation.unsupported(self), + }, + }) } pub(super) fn handle_integer_binary_operation( diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 8e013ddd..5a9ae9ed 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -2,11 +2,16 @@ use super::*; #[derive(Clone)] pub(crate) enum ExpressionValue { + None(SpanRange), Integer(ExpressionInteger), Float(ExpressionFloat), Boolean(ExpressionBoolean), String(ExpressionString), Char(ExpressionChar), + // Unsupported literal is a type here so that we can parse such a token + // as a value rather than a stream, and give it better error messages + UnsupportedLiteral(UnsupportedLiteral), + Stream(ExpressionStream), } pub(crate) trait ToExpressionValue: Sized { @@ -14,20 +19,37 @@ pub(crate) trait ToExpressionValue: Sized { } impl ExpressionValue { - pub(super) fn for_literal(lit: syn::Lit) -> ParseResult { + pub(crate) fn for_literal(literal: Literal) -> Self { + // The unwrap should be safe because all Literal should be parsable + // as syn::Lit; falling back to syn::Lit::Verbatim if necessary. + Self::for_syn_lit(literal.to_token_stream().source_parse_as().unwrap()) + } + + pub(crate) fn for_syn_lit(lit: syn::Lit) -> Self { // https://docs.rs/syn/latest/syn/enum.Lit.html - Ok(match lit { - Lit::Int(lit) => Self::Integer(ExpressionInteger::for_litint(lit)?), - Lit::Float(lit) => Self::Float(ExpressionFloat::for_litfloat(lit)?), + match lit { + Lit::Int(lit) => match ExpressionInteger::for_litint(&lit) { + Ok(int) => Self::Integer(int), + Err(_) => Self::UnsupportedLiteral(UnsupportedLiteral { + span_range: lit.span().span_range(), + lit: Lit::Int(lit), + }), + }, + Lit::Float(lit) => match ExpressionFloat::for_litfloat(&lit) { + Ok(float) => Self::Float(float), + Err(_) => Self::UnsupportedLiteral(UnsupportedLiteral { + span_range: lit.span().span_range(), + lit: Lit::Float(lit), + }), + }, Lit::Bool(lit) => Self::Boolean(ExpressionBoolean::for_litbool(lit)), Lit::Str(lit) => Self::String(ExpressionString::for_litstr(lit)), Lit::Char(lit) => Self::Char(ExpressionChar::for_litchar(lit)), - other_literal => { - return other_literal - .span() - .parse_err("This literal is not supported in preinterpret expressions"); - } - }) + other => Self::UnsupportedLiteral(UnsupportedLiteral { + span_range: other.span().span_range(), + lit: other, + }), + } } pub(super) fn expect_value_pair( @@ -207,8 +229,11 @@ impl ExpressionValue { (ExpressionValue::Char(left), ExpressionValue::Char(right)) => { EvaluationLiteralPair::CharPair(left, right) } + (ExpressionValue::Stream(left), ExpressionValue::Stream(right)) => { + EvaluationLiteralPair::StreamPair(left, right) + } (left, right) => { - return operation.execution_err(format!("The {} operator cannot infer a common operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left.value_type(), right.value_type())); + return operation.execution_err(format!("Cannot infer common type from {} {} {}. Consider using `as` to cast the operands to matching types.", left.value_type(), operation.symbol(), right.value_type())); } }) } @@ -220,39 +245,21 @@ impl ExpressionValue { } } - pub(crate) fn into_bool(self) -> Option { + pub(crate) fn into_bool(self) -> Result { match self { - ExpressionValue::Boolean(value) => Some(value), - _ => None, + ExpressionValue::Boolean(value) => Ok(value), + other => Err(other.value_type()), } } - pub(crate) fn expect_bool(self, error_message: &str) -> ExecutionResult { + pub(crate) fn expect_bool(self, place_descriptor: &str) -> ExecutionResult { let error_span = self.span_range(); match self.into_bool() { - Some(boolean) => Ok(boolean.value), - None => error_span.execution_err(error_message), - } - } - - /// The span is used if there isn't already a span available - pub(crate) fn to_token_tree(&self) -> TokenTree { - match self { - Self::Integer(int) => int.to_literal().into(), - Self::Float(float) => float.to_literal().into(), - Self::Boolean(bool) => bool.to_ident().into(), - Self::String(string) => string.to_literal().into(), - Self::Char(char) => char.to_literal().into(), - } - } - - pub(super) fn value_type(&self) -> &'static str { - match self { - Self::Integer(int) => int.value.value_type(), - Self::Float(float) => float.value.value_type(), - Self::Boolean(_) => "bool", - Self::String(_) => "string", - Self::Char(_) => "char", + Ok(boolean) => Ok(boolean), + Err(value_type) => error_span.execution_err(format!( + "{} must be a boolean, but it is a {}", + place_descriptor, value_type, + )), } } @@ -261,11 +268,14 @@ impl ExpressionValue { operation: OutputSpanned, ) -> ExecutionResult { match self { + ExpressionValue::None(_) => operation.unsupported(self), ExpressionValue::Integer(value) => value.handle_unary_operation(operation), ExpressionValue::Float(value) => value.handle_unary_operation(operation), ExpressionValue::Boolean(value) => value.handle_unary_operation(operation), ExpressionValue::String(value) => value.handle_unary_operation(operation), ExpressionValue::Char(value) => value.handle_unary_operation(operation), + ExpressionValue::Stream(value) => value.handle_unary_operation(operation), + ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), } } @@ -275,6 +285,7 @@ impl ExpressionValue { operation: OutputSpanned, ) -> ExecutionResult { match self { + ExpressionValue::None(_) => operation.unsupported(self), ExpressionValue::Integer(value) => { value.handle_integer_binary_operation(right, operation) } @@ -288,6 +299,10 @@ impl ExpressionValue { value.handle_integer_binary_operation(right, operation) } ExpressionValue::Char(value) => value.handle_integer_binary_operation(right, operation), + ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), + ExpressionValue::Stream(value) => { + value.handle_integer_binary_operation(right, operation) + } } } @@ -303,11 +318,14 @@ impl ExpressionValue { fn span_range_mut(&mut self) -> &mut SpanRange { match self { + Self::None(span_range) => span_range, Self::Integer(value) => &mut value.span_range, Self::Float(value) => &mut value.span_range, Self::Boolean(value) => &mut value.span_range, Self::String(value) => &mut value.span_range, Self::Char(value) => &mut value.span_range, + Self::UnsupportedLiteral(value) => &mut value.span_range, + Self::Stream(value) => &mut value.span_range, } } @@ -315,16 +333,74 @@ impl ExpressionValue { *self.span_range_mut() = source_span.span_range(); self } + + pub(crate) fn into_new_output_stream(self) -> OutputStream { + match self { + Self::Stream(value) => value.value, + other => { + let mut output = OutputStream::new(); + other.output_to(&mut output); + output + } + } + } + + pub(crate) fn output_to(self, output: &mut OutputStream) { + match self { + Self::None { .. } => {} + Self::Integer(value) => { + // Grouped so that -1 is interpreted as a single thing, not a punct then a number + output.push_grouped( + |inner| Ok(inner.push_literal(value.to_literal())), + Delimiter::None, + value.span_range.join_into_span_else_start(), + ).unwrap() + }, + Self::Float(value) => { + // Grouped so that -1.0 is interpreted as a single thing, not a punct then a number + output.push_grouped( + |inner| Ok(inner.push_literal(value.to_literal())), + Delimiter::None, + value.span_range.join_into_span_else_start(), + ).unwrap() + } + Self::Boolean(value) => output.push_ident(value.to_ident()), + Self::String(value) => output.push_literal(value.to_literal()), + Self::Char(value) => output.push_literal(value.to_literal()), + Self::UnsupportedLiteral(literal) => { + output.extend_raw_tokens(literal.lit.into_token_stream()) + } + Self::Stream(value) => value.value.append_into(output), + } + } +} + +impl HasValueType for ExpressionValue { + fn value_type(&self) -> &'static str { + match self { + Self::None { .. } => "none", + Self::Integer(value) => value.value_type(), + Self::Float(value) => value.value_type(), + Self::Boolean(value) => value.value_type(), + Self::String(value) => value.value_type(), + Self::Char(value) => value.value_type(), + Self::UnsupportedLiteral(value) => value.value_type(), + Self::Stream(value) => value.value_type(), + } + } } impl HasSpanRange for ExpressionValue { fn span_range(&self) -> SpanRange { match self { + Self::None(span_range) => *span_range, Self::Integer(int) => int.span_range, Self::Float(float) => float.span_range, Self::Boolean(bool) => bool.span_range, Self::String(str) => str.span_range, Self::Char(char) => char.span_range, + Self::UnsupportedLiteral(lit) => lit.span_range, + Self::Stream(stream) => stream.span_range, } } } @@ -333,9 +409,15 @@ pub(super) trait HasValueType { fn value_type(&self) -> &'static str; } -impl ToTokens for ExpressionValue { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.to_token_tree().to_tokens(tokens); +#[derive(Clone)] +pub(crate) struct UnsupportedLiteral { + lit: syn::Lit, + span_range: SpanRange, +} + +impl HasValueType for UnsupportedLiteral { + fn value_type(&self) -> &'static str { + "unsupported literal" } } @@ -345,6 +427,7 @@ pub(super) enum CastTarget { Float(FloatKind), Boolean, Char, + Stream, } pub(super) enum EvaluationLiteralPair { @@ -353,6 +436,7 @@ pub(super) enum EvaluationLiteralPair { BooleanPair(ExpressionBoolean, ExpressionBoolean), StringPair(ExpressionString, ExpressionString), CharPair(ExpressionChar, ExpressionChar), + StreamPair(ExpressionStream, ExpressionStream), } impl EvaluationLiteralPair { @@ -366,6 +450,7 @@ impl EvaluationLiteralPair { Self::BooleanPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::StringPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::StreamPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } diff --git a/src/extensions/tokens.rs b/src/extensions/tokens.rs index 997a0358..b29e7f0c 100644 --- a/src/extensions/tokens.rs +++ b/src/extensions/tokens.rs @@ -25,22 +25,6 @@ impl TokenStreamExt for TokenStream { } } -pub(crate) trait TokenTreeExt: Sized { - fn group(tokens: TokenStream, delimeter: Delimiter, span: Span) -> Self; - fn into_singleton_group(self, delimiter: Delimiter) -> Self; -} - -impl TokenTreeExt for TokenTree { - fn group(inner_tokens: TokenStream, delimiter: Delimiter, span: Span) -> Self { - TokenTree::Group(Group::new(delimiter, inner_tokens).with_span(span)) - } - - fn into_singleton_group(self, delimiter: Delimiter) -> Self { - let span = self.span(); - Self::group(self.into_token_stream(), delimiter, span) - } -} - pub(crate) trait IdentExt: Sized { fn new_bool(value: bool, span: Span) -> Self; fn with_span(self, span: Span) -> Self; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index a84a27e1..ba08d3f5 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -7,23 +7,12 @@ pub(crate) enum CommandOutputKind { None, Value, Ident, + Literal, FlattenedStream, GroupedStream, Stream, } -impl CommandOutputKind { - pub(crate) fn expression_support(&self) -> Result<(), &'static str> { - match self { - CommandOutputKind::Value | CommandOutputKind::GroupedStream => Ok(()), - CommandOutputKind::None => Err("A command which returns nothing cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), - CommandOutputKind::Ident => Err("A command which returns idents cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), - CommandOutputKind::FlattenedStream => Err("A command which returns a flattened stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), - CommandOutputKind::Stream => Err("A control flow command which returns a code stream cannot be used directly in expressions.\nConsider wrapping it inside a { } block which returns an expression"), - } - } -} - pub(crate) trait CommandType { type OutputKind: OutputKind; } @@ -46,22 +35,8 @@ trait CommandInvocation { context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()>; -} -trait ClonableCommandInvocation: CommandInvocation { - fn clone_box(&self) -> Box; -} - -impl ClonableCommandInvocation for C { - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } -} - -impl Clone for Box { - fn clone(&self) -> Self { - self.clone_box() - } + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult; } // Using the trick for permitting multiple non-overlapping blanket @@ -72,6 +47,8 @@ trait CommandInvocationAs { context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()>; + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult; } impl> CommandInvocation for C { @@ -82,6 +59,10 @@ impl> CommandInvocation for ) -> ExecutionResult<()> { >::execute_into(self, context, output) } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + >::execute_to_value(self, context) + } } //=============== @@ -114,6 +95,11 @@ impl CommandInvocationAs for C { self.execute(context.interpreter)?; Ok(()) } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + self.execute(context.interpreter)?; + Ok(ExpressionValue::None(context.delim_span.span_range())) + } } //================ @@ -139,7 +125,7 @@ pub(crate) trait ValueCommandDefinition: { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult; } impl CommandInvocationAs for C { @@ -148,9 +134,13 @@ impl CommandInvocationAs for C { context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { - output.push_raw_token_tree(self.execute(context.interpreter)?); + self.execute(context.interpreter)?.output_to(output); Ok(()) } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + self.execute(context.interpreter) + } } //================ @@ -188,6 +178,55 @@ impl CommandInvocationAs for C { output.push_ident(self.execute(context.interpreter)?); Ok(()) } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + let span_range = context.delim_span.span_range(); + let mut output = OutputStream::new(); + output.push_ident(self.execute(context.interpreter)?); + Ok(output.to_value(span_range)) + } +} + +//================== +// OutputKindLiteral +//================== + +pub(crate) struct OutputKindLiteral; +impl OutputKind for OutputKindLiteral { + type Output = Literal; + + fn resolve_standard() -> CommandOutputKind { + CommandOutputKind::Literal + } + + fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { + error_span_range + .parse_err("This command outputs a single literal, so cannot be flattened with ..") + } +} + +pub(crate) trait LiteralCommandDefinition: + Sized + CommandType +{ + const COMMAND_NAME: &'static str; + fn parse(arguments: CommandArguments) -> ParseResult; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult; +} + +impl CommandInvocationAs for C { + fn execute_into( + self, + context: ExecutionContext, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + output.push_literal(self.execute(context.interpreter)?); + Ok(()) + } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + let literal = self.execute(context.interpreter)?; + Ok(ExpressionValue::for_literal(literal)) + } } //================= @@ -235,6 +274,17 @@ impl CommandInvocationAs unreachable!(), } } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + let span_range = context.delim_span.span_range(); + let mut output = OutputStream::new(); + >::execute_into( + self, + context, + &mut output, + )?; + Ok(output.to_value(span_range)) + } } //====================== @@ -275,6 +325,17 @@ impl CommandInvocationAs for ) -> ExecutionResult<()> { self.execute(context.interpreter, output) } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + let span_range = context.delim_span.span_range(); + let mut output = OutputStream::new(); + >::execute_into( + self, + context, + &mut output, + )?; + Ok(output.to_value(span_range)) + } } //========================= @@ -357,6 +418,14 @@ macro_rules! define_command_enums { )* } } + + fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { + match self { + $( + Self::$command(command) => <$command as CommandInvocation>::execute_to_value(command, context), + )* + } + } } }; } @@ -364,9 +433,11 @@ macro_rules! define_command_enums { define_command_enums! { // Core Commands SetCommand, + TypedSetCommand, RawCommand, OutputCommand, IgnoreCommand, + ReinterpretCommand, SettingsCommand, ErrorCommand, DebugCommand, @@ -477,6 +548,18 @@ impl Command { pub(crate) unsafe fn set_output_kind(&mut self, output_kind: CommandOutputKind) { self.output_kind = output_kind; } + + pub(crate) fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let context = ExecutionContext { + interpreter, + output_kind: self.output_kind, + delim_span: self.source_group_span, + }; + self.typed.execute_to_value(context) + } } impl HasSpan for Command { diff --git a/src/interpretation/commands/concat_commands.rs b/src/interpretation/commands/concat_commands.rs index fc7e2021..f31a22c1 100644 --- a/src/interpretation/commands/concat_commands.rs +++ b/src/interpretation/commands/concat_commands.rs @@ -8,13 +8,12 @@ fn concat_into_string( input: SourceStream, interpreter: &mut Interpreter, conversion_fn: impl Fn(&str) -> String, -) -> ExecutionResult { +) -> ExecutionResult { let output_span = input.span(); let concatenated = input .interpret_to_new_stream(interpreter)? .concat_recursive(&ConcatBehaviour::standard()); - let value = conversion_fn(&concatenated); - Ok(Literal::string(&value).with_span(output_span)) + Ok(conversion_fn(&concatenated).to_value(output_span.span_range())) } fn concat_into_ident( @@ -51,7 +50,7 @@ fn concat_into_literal( Ok(literal) } -macro_rules! define_literal_concat_command { +macro_rules! define_string_concat_command { ( $command_name:literal => $command:ident: $output_fn:ident($conversion_fn:expr) ) => { @@ -73,8 +72,8 @@ macro_rules! define_literal_concat_command { }) } - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - Ok($output_fn(self.arguments, interpreter, $conversion_fn)?.into()) + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + $output_fn(self.arguments, interpreter, $conversion_fn) } } }; @@ -103,7 +102,36 @@ macro_rules! define_ident_concat_command { } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - $output_fn(self.arguments, interpreter, $conversion_fn).into() + $output_fn(self.arguments, interpreter, $conversion_fn) + } + } + }; +} + +macro_rules! define_literal_concat_command { + ( + $command_name:literal => $command:ident: $output_fn:ident($conversion_fn:expr) + ) => { + #[derive(Clone)] + pub(crate) struct $command { + arguments: SourceStream, + } + + impl CommandType for $command { + type OutputKind = OutputKindLiteral; + } + + impl LiteralCommandDefinition for $command { + const COMMAND_NAME: &'static str = $command_name; + + fn parse(arguments: CommandArguments) -> ParseResult { + Ok(Self { + arguments: arguments.parse_all_as_source()?, + }) + } + + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + $output_fn(self.arguments, interpreter, $conversion_fn) } } }; @@ -113,7 +141,7 @@ macro_rules! define_ident_concat_command { // Concatenating type-conversion commands //======================================= -define_literal_concat_command!("string" => StringCommand: concat_into_string(|s| s.to_string())); +define_string_concat_command!("string" => StringCommand: concat_into_string(|s| s.to_string())); define_ident_concat_command!("ident" => IdentCommand: concat_into_ident(|s| s.to_string())); define_ident_concat_command!("ident_camel" => IdentCamelCommand: concat_into_ident(to_upper_camel_case)); define_ident_concat_command!("ident_snake" => IdentSnakeCommand: concat_into_ident(to_lower_snake_case)); @@ -124,20 +152,20 @@ define_literal_concat_command!("literal" => LiteralCommand: concat_into_literal( // String conversion commands //=========================== -define_literal_concat_command!("upper" => UpperCommand: concat_into_string(to_uppercase)); -define_literal_concat_command!("lower" => LowerCommand: concat_into_string(to_lowercase)); +define_string_concat_command!("upper" => UpperCommand: concat_into_string(to_uppercase)); +define_string_concat_command!("lower" => LowerCommand: concat_into_string(to_lowercase)); // Snake case is typically lower snake case in Rust, so default to that -define_literal_concat_command!("snake" => SnakeCommand: concat_into_string(to_lower_snake_case)); -define_literal_concat_command!("lower_snake" => LowerSnakeCommand: concat_into_string(to_lower_snake_case)); -define_literal_concat_command!("upper_snake" => UpperSnakeCommand: concat_into_string(to_upper_snake_case)); +define_string_concat_command!("snake" => SnakeCommand: concat_into_string(to_lower_snake_case)); +define_string_concat_command!("lower_snake" => LowerSnakeCommand: concat_into_string(to_lower_snake_case)); +define_string_concat_command!("upper_snake" => UpperSnakeCommand: concat_into_string(to_upper_snake_case)); // Kebab case is normally lower case (including in Rust where it's used - e.g. crate names) // It can always be combined with other casing to get other versions -define_literal_concat_command!("kebab" => KebabCommand: concat_into_string(to_lower_kebab_case)); +define_string_concat_command!("kebab" => KebabCommand: concat_into_string(to_lower_kebab_case)); // Upper camel case is the more common casing in Rust, so default to that -define_literal_concat_command!("camel" => CamelCommand: concat_into_string(to_upper_camel_case)); -define_literal_concat_command!("lower_camel" => LowerCamelCommand: concat_into_string(to_lower_camel_case)); -define_literal_concat_command!("upper_camel" => UpperCamelCommand: concat_into_string(to_upper_camel_case)); -define_literal_concat_command!("capitalize" => CapitalizeCommand: concat_into_string(capitalize)); -define_literal_concat_command!("decapitalize" => DecapitalizeCommand: concat_into_string(decapitalize)); -define_literal_concat_command!("title" => TitleCommand: concat_into_string(title_case)); -define_literal_concat_command!("insert_spaces" => InsertSpacesCommand: concat_into_string(insert_spaces_between_words)); +define_string_concat_command!("camel" => CamelCommand: concat_into_string(to_upper_camel_case)); +define_string_concat_command!("lower_camel" => LowerCamelCommand: concat_into_string(to_lower_camel_case)); +define_string_concat_command!("upper_camel" => UpperCamelCommand: concat_into_string(to_upper_camel_case)); +define_string_concat_command!("capitalize" => CapitalizeCommand: concat_into_string(capitalize)); +define_string_concat_command!("decapitalize" => DecapitalizeCommand: concat_into_string(decapitalize)); +define_string_concat_command!("title" => TitleCommand: concat_into_string(title_case)); +define_string_concat_command!("insert_spaces" => InsertSpacesCommand: concat_into_string(insert_spaces_between_words)); diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 567950b6..5cdb7127 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -54,18 +54,18 @@ impl StreamingCommandDefinition for IfCommand { let evaluated_condition = self .condition .evaluate(interpreter)? - .expect_bool("An if condition must evaluate to a boolean")?; + .expect_bool("An if condition")?; - if evaluated_condition { + if evaluated_condition.value { return self.true_code.interpret_into(interpreter, output); } for (condition, code) in self.else_ifs { let evaluated_condition = condition .evaluate(interpreter)? - .expect_bool("An else if condition must evaluate to a boolean")?; + .expect_bool("An else if condition")?; - if evaluated_condition { + if evaluated_condition.value { return code.interpret_into(interpreter, output); } } @@ -116,9 +116,9 @@ impl StreamingCommandDefinition for WhileCommand { .condition .clone() .evaluate(interpreter)? - .expect_bool("An if condition must evaluate to a boolean")?; + .expect_bool("A while condition")?; - if !evaluated_condition { + if !evaluated_condition.value { break; } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index ecc5d1fc..8f8eac71 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -111,6 +111,42 @@ impl NoOutputCommandDefinition for SetCommand { } } +/// This is temporary until we have a proper implementation of #(...) +#[derive(Clone)] +pub(crate) struct TypedSetCommand { + variable: GroupedVariable, + #[allow(unused)] + equals: Token![=], + content: SourceExpression, +} + +impl CommandType for TypedSetCommand { + type OutputKind = OutputKindNone; +} + +impl NoOutputCommandDefinition for TypedSetCommand { + const COMMAND_NAME: &'static str = "typed_set"; + + fn parse(arguments: CommandArguments) -> ParseResult { + arguments.fully_parse_or_error( + |input| { + Ok(TypedSetCommand { + variable: input.parse()?, + equals: input.parse()?, + content: input.parse()?, + }) + }, + "Expected [!typed_set! #var1 = ]", + ) + } + + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { + let content = self.content.evaluate(interpreter)?; + self.variable.set_value(interpreter, content)?; + Ok(()) + } +} + #[derive(Clone)] pub(crate) struct RawCommand { token_stream: TokenStream, @@ -187,6 +223,42 @@ impl NoOutputCommandDefinition for IgnoreCommand { } } +/// This is temporary until we have a proper implementation of #(...) +#[derive(Clone)] +pub(crate) struct ReinterpretCommand { + content: SourceStream, +} + +impl CommandType for ReinterpretCommand { + type OutputKind = OutputKindStreaming; +} + +impl StreamingCommandDefinition for ReinterpretCommand { + const COMMAND_NAME: &'static str = "reinterpret"; + + fn parse(arguments: CommandArguments) -> ParseResult { + Ok(Self { + content: arguments.parse_all_as_source()?, + }) + } + + fn execute( + self, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + let command_span = self.content.span(); + let interpreted = self.content.interpret_to_new_stream(interpreter)?; + let source = unsafe { + // RUST-ANALYZER-SAFETY - Can't do much better than this + interpreted.into_token_stream() + }; + let reparsed_source_stream = + source.source_parse_with(|input| SourceStream::parse(input, command_span))?; + reparsed_source_stream.interpret_into(interpreter, output) + } +} + #[derive(Clone)] pub(crate) struct SettingsCommand { inputs: SettingsInputs, @@ -350,12 +422,12 @@ impl ValueCommandDefinition for DebugCommand { }) } - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { let span = self.inner.span(); let debug_string = self .inner .interpret_to_new_stream(interpreter)? .concat_recursive(&ConcatBehaviour::debug()); - Ok(Literal::string(&debug_string).with_span(span).into()) + Ok(debug_string.to_value(span.span_range())) } } diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index ba2c9140..bdac57cc 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -25,19 +25,19 @@ impl ValueCommandDefinition for EvaluateCommand { ) } - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - Ok(self + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let value = self .expression .evaluate(interpreter)? - .with_span(self.command_span) - .to_token_tree()) + .with_span(self.command_span); + Ok(value) } } #[derive(Clone)] pub(crate) struct AssignCommand { variable: GroupedVariable, - operator: Option, + operation: Option, #[allow(unused)] equals: Token![=], expression: SourceExpression, @@ -56,16 +56,19 @@ impl NoOutputCommandDefinition for AssignCommand { |input| { Ok(Self { variable: input.parse()?, - operator: { + operation: { if input.peek(Token![=]) { None } else { - let operator: Punct = input.parse()?; - match operator.as_char() { + let operator_char = match input.cursor().punct() { + Some((operator, _)) => operator.as_char(), + None => 'X', + }; + match operator_char { '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => return operator.parse_err("Expected one of + - * / % & | or ^"), + _ => return input.parse_err("Expected one of + - * / % & | or ^"), } - Some(operator) + Some(input.parse()?) } }, equals: input.parse()?, @@ -80,36 +83,21 @@ impl NoOutputCommandDefinition for AssignCommand { fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { let Self { variable, - operator, + operation, equals: _, expression, command_span, } = self; - let expression = if let Some(operator) = operator { - let mut calculation = TokenStream::new(); - unsafe { - // RUST-ANALYZER SAFETY: Hopefully it won't contain a none-delimited group - variable - .interpret_to_new_stream(interpreter)? - .parse_as::()? - .evaluate()? - .to_tokens(&mut calculation); - }; - operator.to_tokens(&mut calculation); - expression - .evaluate(interpreter)? - .to_tokens(&mut calculation); - calculation.source_parse_as()? + let value = if let Some(operation) = operation { + let left = variable.read_as_expression_value(interpreter)?; + let right = expression.evaluate(interpreter)?; + operation.evaluate(left, right)? } else { - expression + expression.evaluate(interpreter)? }; - let output = expression - .evaluate(interpreter)? - .with_span(command_span) - .to_token_tree(); - variable.set(interpreter, output.into())?; + variable.set_value(interpreter, value.with_span(command_span))?; Ok(()) } @@ -166,13 +154,9 @@ impl GroupedStreamCommandDefinition for RangeCommand { } } - output.extend_raw_tokens(range_iterator.map(|value| { - value - .to_token_tree() - // We wrap it in a singleton group to ensure that negative - // numbers are treated as single items in other stream commands - .into_singleton_group(Delimiter::None) - })); + for value in range_iterator { + value.output_to(output) + } Ok(()) } diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 3e8c8a65..50ac75c2 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -18,10 +18,10 @@ impl ValueCommandDefinition for IsEmptyCommand { }) } - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let output_span = self.arguments.span_range().join_into_span_else_start(); + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let output_span_range = self.arguments.span_range(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; - Ok(Ident::new_bool(interpreted.is_empty(), output_span).into()) + Ok(interpreted.is_empty().to_value(output_span_range)) } } @@ -43,11 +43,10 @@ impl ValueCommandDefinition for LengthCommand { }) } - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let output_span = self.arguments.span_range().join_into_span_else_start(); + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let output_span_range = self.arguments.span_range(); let interpreted = self.arguments.interpret_to_new_stream(interpreter)?; - let length_literal = Literal::usize_unsuffixed(interpreted.len()).with_span(output_span); - Ok(length_literal.into()) + Ok(interpreted.len().to_value(output_span_range)) } } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 45a09fd8..b11824a3 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -123,6 +123,17 @@ impl OutputStream { self.token_length == 0 } + pub(crate) fn coerce_into_value(self, stream_span_range: SpanRange) -> ExpressionValue { + let parse_result = unsafe { + // RUST-ANALYZER SAFETY: This is actually safe. + self.clone().parse_as::() + }; + match parse_result { + Ok(syn_lit) => ExpressionValue::for_syn_lit(syn_lit), + Err(_) => self.to_value(stream_span_range), + } + } + /// WARNING: With rust-analyzer, this loses transparent groups which have been inserted. /// Use only where that doesn't matter: https://github.com/rust-lang/rust-analyzer/issues/18211#issuecomment-2604547032 /// diff --git a/src/interpretation/source_code_block.rs b/src/interpretation/source_code_block.rs index b7399a14..12a33327 100644 --- a/src/interpretation/source_code_block.rs +++ b/src/interpretation/source_code_block.rs @@ -1,6 +1,6 @@ use crate::internal_prelude::*; -/// A group { .. } representing code which can be interpreted +/// A group `{ ... }` representing code which can be interpreted #[derive(Clone)] pub(crate) struct SourceCodeBlock { delim_span: DelimSpan, diff --git a/src/interpretation/source_stream_input.rs b/src/interpretation/source_stream_input.rs index f499369d..9a24ec8c 100644 --- a/src/interpretation/source_stream_input.rs +++ b/src/interpretation/source_stream_input.rs @@ -58,7 +58,8 @@ impl Interpret for SourceStreamInput { match command.output_kind() { CommandOutputKind::None | CommandOutputKind::Value - | CommandOutputKind::Ident => { + | CommandOutputKind::Ident + | CommandOutputKind::Literal => { command.execution_err("The command does not output a stream") } CommandOutputKind::FlattenedStream => parse_as_stream_input( diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index d8199a69..6d72067b 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -33,6 +33,19 @@ impl GroupedVariable { interpreter.set_variable(self, value) } + pub(crate) fn set_value( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + let value = { + let mut output = OutputStream::new(); + value.output_to(&mut output); + output + }; + interpreter.set_variable(self, value) + } + pub(crate) fn get_existing_for_mutation( &self, interpreter: &Interpreter, @@ -44,6 +57,18 @@ impl GroupedVariable { .cheap_clone()) } + pub(crate) fn read_as_expression_value( + &self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let value = self + .read_existing(interpreter)? + .get(self)? + .clone() + .coerce_into_value(self.span_range()); + Ok(value) + } + pub(crate) fn substitute_ungrouped_contents_into( &self, interpreter: &mut Interpreter, @@ -149,6 +174,15 @@ impl FlattenedVariable { Ok(()) } + pub(crate) fn read_as_expression_value( + &self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut output_stream = OutputStream::new(); + self.substitute_into(interpreter, &mut output_stream)?; + Ok(output_stream.to_value(self.span_range())) + } + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { interpreter.get_existing_variable_data( self, diff --git a/src/misc/errors.rs b/src/misc/errors.rs index 94e0143b..ada214de 100644 --- a/src/misc/errors.rs +++ b/src/misc/errors.rs @@ -84,12 +84,14 @@ impl ExecutionResultExt for ExecutionResult { } } +#[derive(Debug)] pub(crate) enum ExecutionInterrupt { Error(syn::Error), DestructureError(ParseError), ControlFlow(ControlFlowInterrupt, Span), } +#[derive(Debug)] pub(crate) enum ControlFlowInterrupt { Break, Continue, diff --git a/tests/compilation_failures/core/error_span_repeat.stderr b/tests/compilation_failures/core/error_span_repeat.stderr index 3f27bea3..56694847 100644 --- a/tests/compilation_failures/core/error_span_repeat.stderr +++ b/tests/compilation_failures/core/error_span_repeat.stderr @@ -1,4 +1,4 @@ -error: Expected 3 inputs, got 4 +error: Expected 3 inputs, got 4usize --> tests/compilation_failures/core/error_span_repeat.rs:16:31 | 16 | assert_input_length_of_3!(42 101 666 1024); diff --git a/tests/compilation_failures/expressions/add_float_and_int.stderr b/tests/compilation_failures/expressions/add_float_and_int.stderr index 7cd3f383..bd756751 100644 --- a/tests/compilation_failures/expressions/add_float_and_int.stderr +++ b/tests/compilation_failures/expressions/add_float_and_int.stderr @@ -1,4 +1,4 @@ -error: The + operator cannot infer a common operand type from untyped float and untyped integer. Consider using `as` to cast to matching types. +error: Cannot infer common type from untyped float + untyped integer. Consider using `as` to cast the operands to matching types. --> tests/compilation_failures/expressions/add_float_and_int.rs:5:25 | 5 | [!evaluate! 1.2 + 1] diff --git a/tests/compilation_failures/expressions/brackets.rs b/tests/compilation_failures/expressions/braces.rs similarity index 69% rename from tests/compilation_failures/expressions/brackets.rs rename to tests/compilation_failures/expressions/braces.rs index a85e809a..bcd94df6 100644 --- a/tests/compilation_failures/expressions/brackets.rs +++ b/tests/compilation_failures/expressions/braces.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! [true]] + [!evaluate! { true }] }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr new file mode 100644 index 00000000..a02f3849 --- /dev/null +++ b/tests/compilation_failures/expressions/braces.stderr @@ -0,0 +1,6 @@ +error: Braces { ... } are not supported in an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/braces.rs:5:21 + | +5 | [!evaluate! { true }] + | ^ diff --git a/tests/compilation_failures/expressions/brackets.stderr b/tests/compilation_failures/expressions/brackets.stderr deleted file mode 100644 index 72795d65..00000000 --- a/tests/compilation_failures/expressions/brackets.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Square brackets [ .. ] are not supported in an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/brackets.rs:5:21 - | -5 | [!evaluate! [true]] - | ^ diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr index 19b4b2c0..13c95e37 100644 --- a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr @@ -1,5 +1,6 @@ -error: Expected ! or - - --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:6:35 +error: Braces { ... } are not supported in an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:7:21 | -6 | [!set! #indirect = [!raw! #value]] - | ^ +7 | [!evaluate! { #indirect }] + | ^ diff --git a/tests/compilation_failures/expressions/compare_int_and_float.stderr b/tests/compilation_failures/expressions/compare_int_and_float.stderr index 4208a21a..131ae9a0 100644 --- a/tests/compilation_failures/expressions/compare_int_and_float.stderr +++ b/tests/compilation_failures/expressions/compare_int_and_float.stderr @@ -1,4 +1,4 @@ -error: The < operator cannot infer a common operand type from untyped integer and untyped float. Consider using `as` to cast to matching types. +error: Cannot infer common type from untyped integer < untyped float. Consider using `as` to cast the operands to matching types. --> tests/compilation_failures/expressions/compare_int_and_float.rs:5:23 | 5 | [!evaluate! 5 < 6.4] diff --git a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr index d442be70..ab93316a 100644 --- a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr +++ b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr @@ -1,6 +1,5 @@ -error: number too large to fit in target type - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs:8:22 +error: The - operator is not supported for unsupported literal values + --> tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs:8:21 | 8 | [!evaluate! -128i8] - | ^^^^^ + | ^ diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr index 9d64994e..f8fb7f79 100644 --- a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr @@ -1,7 +1,5 @@ -error: A command which returns a flattened stream cannot be used directly in expressions. - Consider wrapping it inside a { } block which returns an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:25 +error: Cannot infer common type from untyped integer + stream. Consider using `as` to cast the operands to matching types. + --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:23 | 5 | [!evaluate! 5 + [!..range! 1..2]] - | ^ + | ^ diff --git a/tests/compilation_failures/expressions/inner_brackets.rs b/tests/compilation_failures/expressions/inner_braces.rs similarity index 68% rename from tests/compilation_failures/expressions/inner_brackets.rs rename to tests/compilation_failures/expressions/inner_braces.rs index 0607a729..29e32db2 100644 --- a/tests/compilation_failures/expressions/inner_brackets.rs +++ b/tests/compilation_failures/expressions/inner_braces.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! ([true])] + [!evaluate! ({ true })] }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/inner_braces.stderr b/tests/compilation_failures/expressions/inner_braces.stderr new file mode 100644 index 00000000..45f5a12a --- /dev/null +++ b/tests/compilation_failures/expressions/inner_braces.stderr @@ -0,0 +1,6 @@ +error: Braces { ... } are not supported in an expression + Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/inner_braces.rs:5:22 + | +5 | [!evaluate! ({ true })] + | ^ diff --git a/tests/compilation_failures/expressions/inner_brackets.stderr b/tests/compilation_failures/expressions/inner_brackets.stderr deleted file mode 100644 index 947d2f82..00000000 --- a/tests/compilation_failures/expressions/inner_brackets.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Square brackets [ .. ] are not supported in an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/inner_brackets.rs:5:22 - | -5 | [!evaluate! ([true])] - | ^ diff --git a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr index c906afbb..93b3c5a2 100644 --- a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr +++ b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr @@ -1,7 +1,5 @@ -error: A command which returns nothing cannot be used directly in expressions. - Consider wrapping it inside a { } block which returns an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/no_output_commands_in_expressions.rs:5:25 +error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression + --> tests/compilation_failures/expressions/no_output_commands_in_expressions.rs:5:40 | 5 | [!evaluate! 5 + [!set! #x = 2] 2] - | ^ + | ^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index e30ab4b2..0264459e 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -19,7 +19,7 @@ fn test_control_flow_compilation_failures() { fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); assert_preinterpret_eq!({ - [!set! #x = 1 == 2] + [!set! #x = [!evaluate! 1 == 2]] [!if! #x { "YES" } !else! { "NO" }] }, "NO"); assert_preinterpret_eq!({ diff --git a/tests/expressions.rs b/tests/expressions.rs index 74dbb014..c47f4b21 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -42,7 +42,7 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! 123 > 456], false); assert_preinterpret_eq!( { - [!set! #six_as_sum = 3 + 3] // The token stream '3 + 3'. They're not evaluated to 6 (yet). + [!typed_set! #six_as_sum = 3 + 3] [!evaluate! #six_as_sum * #six_as_sum] }, 36 @@ -50,26 +50,17 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!( { [!set! #partial_sum = + 2] - // The [!group! ...] constructs an expression from tokens, - // which is then interpreted / executed. - [!evaluate! [!group! 5 #..partial_sum]] + [!debug! + // [...] is a token stream, which is not evaluated. + [!evaluate! [5 #..partial_sum]] + = + // !reinterpret! can be used to force an evaluation + [!reinterpret! [[!raw! !evaluate!] 5 #..partial_sum]] + ] }, - 7 - ); - assert_preinterpret_eq!( - { - [!set! #partial_sum = + 2] - // A { ... } block is evaluated as an expression after interpretation - [!evaluate! { 1 #..partial_sum }] - }, - 3 - ); - assert_preinterpret_eq!( - { - [!evaluate! 1 + [!range! 1..2]] - }, - 2 + "5 + 2 = [!group! 7]" ); + assert_preinterpret_eq!([!evaluate! 1 + [!range! 1..2] as int], 2); assert_preinterpret_eq!([!evaluate! "hello" == "world"], false); assert_preinterpret_eq!([!evaluate! "hello" == "hello"], true); assert_preinterpret_eq!([!evaluate! 'A' as u8 == 65], true); @@ -78,6 +69,10 @@ fn test_basic_evaluate_works() { assert_preinterpret_eq!([!evaluate! 'A' == 'B'], false); assert_preinterpret_eq!([!evaluate! 'A' < 'B'], true); assert_preinterpret_eq!([!evaluate! "Zoo" > "Aardvark"], true); + assert_preinterpret_eq!( + [!debug! [!evaluate! "Hello" as stream + "World" as stream + (1 + 1) as stream]], + r#""Hello" "World" [!group! 2]"# + ); } #[test] @@ -102,9 +97,9 @@ fn test_very_long_expression_works() { { [!settings! { iteration_limit: 100000, - }][!evaluate! { - 0 [!for! #i in [!range! 0..100000] { + 1 }] }] + [!set! #expression = 0 [!for! #i in [!range! 0..100000] { + 1 }]] + [!reinterpret! [[!raw! !evaluate!] #expression]] }, 100000 ); @@ -116,7 +111,7 @@ fn boolean_operators_short_circuit() { assert_preinterpret_eq!( { [!set! #is_lazy = true] - [!set! _ = [!evaluate! false && { [!set! #is_lazy = false] true }]] + [!set! _ = [!evaluate! false && [!set! #is_lazy = false]]] #is_lazy }, true @@ -125,7 +120,7 @@ fn boolean_operators_short_circuit() { assert_preinterpret_eq!( { [!set! #is_lazy = true] - [!set! _ = [!evaluate! true || { [!set! #is_lazy = false] true }]] + [!set! _ = [!evaluate! true || [!set! #is_lazy = false]]] #is_lazy }, true @@ -139,13 +134,13 @@ fn assign_works() { [!assign! #x = 5 + 5] [!debug! #..x] }, - "10" + "[!group! 10]" ); assert_preinterpret_eq!( { - [!set! #x = 8 + 2] // 8 + 2 (not evaluated) - [!assign! #x /= 1 + 1] // ((8 + 2) / (1 + 1)) => 5 - [!assign! #x += 2 + #x] // ((10) + 2) => 12 + [!set! #x = 10] + [!assign! #x /= 1 + 1] // (10 / (1 + 1)) => 5 + [!assign! #x += 2 + #x] // (5 + (2 + 5)) => 12 #x }, 12 @@ -200,6 +195,6 @@ fn test_range() { assert_preinterpret_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); assert_preinterpret_eq!( { [!debug! [!..range! -1i8..3i8]] }, - "[!group! - 1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]" + "[!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]" ); } From 9f951918800e4561cafe5a5bc79305ddaaf0ec3f Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 11 Feb 2025 14:05:03 +0000 Subject: [PATCH 082/126] feature: Support expression block `#(..)` --- CHANGELOG.md | 87 +++---- Cargo.toml | 8 +- README.md | 2 +- src/expressions/boolean.rs | 13 +- src/expressions/character.rs | 13 +- src/expressions/expression.rs | 40 ++- src/expressions/expression_block.rs | 245 ++++++++++++++++++ src/expressions/float.rs | 16 +- src/expressions/integer.rs | 32 ++- src/expressions/mod.rs | 7 +- src/expressions/operations.rs | 1 + src/expressions/stream.rs | 5 + src/expressions/string.rs | 13 +- src/expressions/value.rs | 61 +++-- src/interpretation/command.rs | 42 +-- .../commands/control_flow_commands.rs | 7 +- src/interpretation/commands/core_commands.rs | 2 +- .../commands/expression_commands.rs | 110 +------- src/interpretation/interpret_traits.rs | 4 +- src/interpretation/source_stream.rs | 14 +- src/interpretation/source_stream_input.rs | 9 +- src/interpretation/source_value.rs | 21 +- src/interpretation/variable.rs | 126 +++++++-- src/misc/parse_traits.rs | 14 +- src/transformation/exact_stream.rs | 14 +- src/transformation/parse_utilities.rs | 6 +- src/transformation/transform_stream.rs | 9 +- .../control_flow/error_after_continue.rs | 6 +- .../control_flow/error_after_continue.stderr | 4 +- .../expressions/add_float_and_int.rs | 2 +- .../expressions/add_float_and_int.stderr | 6 +- .../expressions/braces.rs | 2 +- .../expressions/braces.stderr | 7 +- .../expressions/cast_int_to_bool.rs | 2 +- .../expressions/cast_int_to_bool.stderr | 6 +- .../code_blocks_are_not_reevaluated.rs | 7 +- .../code_blocks_are_not_reevaluated.stderr | 9 +- .../expressions/compare_int_and_float.rs | 2 +- .../expressions/compare_int_and_float.stderr | 6 +- ...micolon_expressions_cannot_return_value.rs | 10 + ...lon_expressions_cannot_return_value.stderr | 5 + .../fix_me_negative_max_int_fails.rs | 4 +- .../fix_me_negative_max_int_fails.stderr | 6 +- .../flattened_commands_in_expressions.rs | 2 +- .../flattened_commands_in_expressions.stderr | 6 +- .../flattened_variables_in_expressions.rs | 3 +- .../flattened_variables_in_expressions.stderr | 8 +- ...ped_variable_with_incomplete_expression.rs | 3 +- ...variable_with_incomplete_expression.stderr | 8 +- .../expressions/inner_braces.rs | 2 +- .../expressions/inner_braces.stderr | 7 +- .../expressions/invalid_binary_operator.rs | 6 +- .../invalid_binary_operator.stderr | 8 +- .../expressions/invalid_unary_operator.rs | 2 +- .../expressions/invalid_unary_operator.stderr | 7 +- .../no_output_commands_in_expressions.rs | 2 +- .../no_output_commands_in_expressions.stderr | 8 +- tests/control_flow.rs | 17 +- tests/core.rs | 2 +- tests/expressions.rs | 178 ++++++------- tests/tokens.rs | 4 +- 61 files changed, 808 insertions(+), 470 deletions(-) create mode 100644 src/expressions/expression_block.rs create mode 100644 tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.rs create mode 100644 tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 4264bb66..7010a987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,7 @@ * `[!reinterpret! ...]` is like an `eval` command in scripting languages. It takes a stream, and parses/interprets it. * `[!settings! { ... }]` can be used to adjust the iteration limit. * Expression commands: - * `[!evaluate! ]` - * `[!assign! #x += ]` for `+` and other supported operators + * The expression block `#(let x = 123; y /= x; y + 1)` which is discussed in more detail below. * `[!range! 0..5]` outputs `0 1 2 3 4` * Control flow commands: * `[!if! { ... }]` and `[!if! { ... } !elif! { ... } !else! { ... }]` @@ -41,26 +40,31 @@ ### Expressions -Expressions can be evaluated with `[!evaluate! ...]` and are also used in the `if`, `for` and `while` loops. They operate on literals as values. +Expressions can be evaluated with `#(...)` and are also used in the `!if!` and `!while!` loop conditions. -Currently supported are: -* Values Model: - * Integer literals - * Float literals - * Boolean literals - * String literals - * Char literals - * Token streams which look like `[...]` and can be appended with the `+` operator. +Expressions behave intuitively as you'd expect from writing regular rust code, except they are executed at compile time. + +The `#(...)` expression block behaves much like a `{ .. }` block in rust. It supports multiple statements ending with `;` and optionally a final statement. +Statements are either expressions `EXPR` or `let x = EXPR`, `x = EXPR`, `x += EXPR` for some operator such as `+`. + +The following are recognized values: +* Integer literals, with or without a suffix +* Float literals, with or without a suffix +* Boolean literals +* String literals +* Char literals +* Other literals +* Token streams which are defined as `[...]` and can be appended with the `+` operator. + +The following operators are supported: * The numeric operators: `+ - * / % & | ^` * The lazy boolean operators: `|| &&` * The comparison operators: `== != < > <= >=` * The shift operators: `>> <<` -* Casting with `as` including to untyped integers/floats with `as int` and `as float` and to a stream with `as stream`. +* Casting with `as` including to untyped integers/floats with `as int` and `as float`, to a grouped stream with `as group` and to a flattened stream with `as stream`. * () and none-delimited groups for precedence -* Variables `#x` and flattened variables `#..x` -* Commands `[!xxx! ...]` -Expressions behave intuitively as you'd expect from writing regular rust code, except they happen at compile time. +An expression also supports embedding commands `[!xxx! ...]`, other expression blocks, variables and flattened variables. The value type outputted by a command depends on the command. ### Transforming @@ -73,10 +77,10 @@ Inside a transform stream, the following grammar is supported: * `@(...)`, `@(#x = ...)`, `@(#x += ...)` and `@(_ = ...)` - Explicit transform (sub)streams which either output, set, append or discard its output. * Explicit punctuation, idents, literals and groups. These aren't output by default, except directly inside a `@[EXACT ...]` transformer. * Variable bindings: - * `#x` - Reads a token tree, writes its content (opposite of `#x`). Equivalent to `@(#x = @TOKEN_OR_GROUP_CONTENT)` + * `#x` - Reads a token tree, writes its content (opposite of `#x`). Equivalent to `@(x = @TOKEN_OR_GROUP_CONTENT)` * `#..x` - Reads a stream, writes a stream (opposite of `#..x`). - * If it's at the end of the transformer stream, it's equivalent to `@(#x = @REST)`. - * If it's followed by a token `T` in the transformer stream, it's equivalent to `@(#x = @[UNTIL T])` + * If it's at the end of the transformer stream, it's equivalent to `@(x = @REST)`. + * If it's followed by a token `T` in the transformer stream, it's equivalent to `@(x = @[UNTIL T])` * `#>>x` - Reads a token tree, appends a token tree (can be read back with `!for! #y in #x { ... }`) * `#>>..x` - Reads a token tree, appends a stream (i.e. flatten it if it's a group) * `#..>>x` - Reads a stream, appends a group (can be read back with `!for! #y in #x { ... }`) @@ -88,48 +92,39 @@ Inside a transform stream, the following grammar is supported: * `@[GROUP ...]` - Consumes a none-delimited group. Its arguments are used to transform the group's contents. * `@[EXACT ...]` - Interprets its arguments (i.e. variables are substituted, not bound; and command output is gathered) into an "exact match stream". And then expects to consume exactly the same stream from the input. It outputs the parsed stream. * Commands: Their output is appended to the transform's output. Useful patterns include: - * `@(#inner = ...) [!output! #inner]` - wraps the output in a transparent group + * `@(inner = ...) [!output! #inner]` - wraps the output in a transparent group ### To come -* Add basic `ExpressionBlock` support: - * `#(xxx)` which can e.g. output a variable as expression - * Support multiple statements (where only the last is output, and the rest have to be None): - * `#([!set! #x = 2]; 1 + 1)` => evaluate - * Change `boolean_operators_short_circuit` test back to use `#()` +* Complete `ExpressionBlock` support: + * Change variables to store expression values + * Support `#(x[..])` syntax for indexing streams at read time + * Via a post-fix `[...]` operator + * `#(x[0])` + * `#(x[0..3])` returns a TokenStream + * `#(x[0..=3])` returns a TokenStream + * Add `+` support for concatenating strings + * Create an `enum MarkedVariable { Grouped(GroupedMarkedVariable), Flattened(FlattenedMarkedVariable) }` + * Revisit the `SourceStreamInput` abstraction * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Separate `&mut Output` and `StreamOutput`, `ValueOutput = ExpressionOutput`, `ObjectOutput` * Objects: + * Can be created with `#({ a: x, ... })` * Can be destructured with `@{ #hello, world: _, ... }` or read with `#(x.hello)` or `#(x["hello"])` - * Debug impl is `[!object! { hello: [!group! BLAH], ["world"]: Hi, }]` - * Fields can be lazy (for e.g. exposing functions on syn objects). + * Debug impl is `#({ hello: [!group! BLAH], ["world"]: Hi, })` * They have an input object + * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` * (Until we get custom type support into a syn fork), can be embedded into an output stream as a single token - e.g. `PREINTERPRET_OBJECT_2313` (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default stream for the object, or error and suggest fields the user should use instead. + * Have `!zip!` support `{ objects }` * Values: * When we parse a `#x` (`@(#x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. * Output to final output as unwrapped content - * New ExpressionBlock syntax (replaces `[!set!]`??) - * `#(#x = [... stream ...])` - * `#(#x += [... stream ...])` // += acts like "extend" with a stream LHS - * `#(#x += [!group! ...])` - * `#(#x = 1 + 2 + 3)` // (Expression) Value - * `#(#x += 1)` // += acts as plus with a value LHS - * `#(#x = {})` // Object - * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` - * `#(#x = xxx {})` // For some other custom type `xxx`. - * `#([ ... stream ... ])` - * `#({ a: x, ... })` - * `#([ ... stream ... ].length)` ?? - * `#(let #x = 123; let #y = 123; let #z = {})`... - * Support `#(x)` syntax for indexing streams: - * `#(x[0])` - * `#(x[0..3])` - * `#..(x[0..3])` and other things like `[ ..=3]` - ... basically - this becomes one big expression. - * Equivalent to a `syn::Block` which is a `Vec` - * ...where a `Stmt` is either a local `let` binding or an `expression` + * Method calls + * Also add support for methods (for e.g. exposing functions on syn objects). + * `.len()` on stream + * Consider `.map(|| {})` * Destructurers => Transformers cont * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. diff --git a/Cargo.toml b/Cargo.toml index 339a560f..7e811f53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,9 @@ rust-version = "1.61" proc-macro = true [dependencies] -proc-macro2 = { version = "1.0" } -syn = { version = "2.0", default-features = false, features = ["parsing", "derive", "printing", "clone-impls", "full"] } -quote = { version = "1.0", default-features = false } +proc-macro2 = { version = "1.0.93" } +syn = { version = "2.0.98", default-features = false, features = ["parsing", "derive", "printing", "clone-impls", "full"] } +quote = { version = "1.0.38", default-features = false } [dev-dependencies] -trybuild = { version = "1.0.66", features = ["diff"] } +trybuild = { version = "1.0.103", features = ["diff"] } diff --git a/README.md b/README.md index 304d087f..12c09f23 100644 --- a/README.md +++ b/README.md @@ -447,7 +447,7 @@ Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example: * `[!tokens_eq! (3 4) (3 4)]` outputs `true` because the token stream ignores spacing. * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. - * This can be effectively done already with `[!evaluate! [!debug! #x] == [!debug! #y]]` + * This can be effectively done already with `#([!debug! #x] == [!debug! #y])` * `[!str_split! { input: Value, separator: Value, }]` * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index ee973058..b53c2747 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -45,9 +45,16 @@ impl ExpressionBoolean { return operation.execution_err("This cast is not supported") } CastTarget::Boolean => operation.output(self.value), - CastTarget::Stream => { - operation.output(operation.output(input).into_new_output_stream()) - } + CastTarget::Stream => operation.output( + operation + .output(input) + .into_new_output_stream(Grouping::Flattened), + ), + CastTarget::Group => operation.output( + operation + .output(input) + .into_new_output_stream(Grouping::Grouped), + ), }, }) } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index fd05bbd7..36c00447 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -44,9 +44,16 @@ impl ExpressionChar { CastTarget::Integer(IntegerKind::Usize) => operation.output(char as usize), CastTarget::Char => operation.output(char), CastTarget::Boolean | CastTarget::Float(_) => return operation.unsupported(self), - CastTarget::Stream => { - operation.output(operation.output(char).into_new_output_stream()) - } + CastTarget::Stream => operation.output( + operation + .output(char) + .into_new_output_stream(Grouping::Flattened), + ), + CastTarget::Group => operation.output( + operation + .output(char) + .into_new_output_stream(Grouping::Grouped), + ), }, }) } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index d6c2d7e1..c0cd37bf 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -16,11 +16,13 @@ impl Parse for SourceExpression { } } -impl SourceExpression { - pub(crate) fn evaluate( - &self, +impl InterpretToValue for &SourceExpression { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { Source::evaluate(&self.inner, interpreter) } } @@ -29,6 +31,8 @@ pub(super) enum SourceExpressionLeaf { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), + VariablePath(VariablePath), + ExpressionBlock(ExpressionBlock), ExplicitStream(SourceGroup), Value(ExpressionValue), } @@ -40,12 +44,15 @@ impl Expressionable for Source { fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), - SourcePeekMatch::GroupedVariable => { + SourcePeekMatch::Variable(Grouping::Grouped) => { UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)) } - SourcePeekMatch::FlattenedVariable => { + SourcePeekMatch::Variable(Grouping::Flattened) => { UnaryAtom::Leaf(Self::Leaf::FlattenedVariable(input.parse()?)) } + SourcePeekMatch::ExpressionBlock(_) => { + UnaryAtom::Leaf(Self::Leaf::ExpressionBlock(input.parse()?)) + } SourcePeekMatch::AppendVariableBinding => { return input .parse_err("Append variable operations are not supported in an expression") @@ -64,11 +71,12 @@ impl Expressionable for Source { UnaryAtom::Leaf(Self::Leaf::ExplicitStream(input.parse()?)) } SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), - SourcePeekMatch::Ident(_) => { - let value = - ExpressionValue::Boolean(ExpressionBoolean::for_litbool(input.parse()?)); - UnaryAtom::Leaf(Self::Leaf::Value(value)) - } + SourcePeekMatch::Ident(_) => match input.try_parse_or_revert() { + Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( + ExpressionBoolean::for_litbool(bool), + ))), + Err(_) => UnaryAtom::Leaf(Self::Leaf::VariablePath(input.parse()?)), + }, SourcePeekMatch::Literal(_) => { let value = ExpressionValue::for_syn_lit(input.parse()?); UnaryAtom::Leaf(Self::Leaf::Value(value)) @@ -103,10 +111,16 @@ impl Expressionable for Source { command.clone().interpret_to_value(interpreter)? } SourceExpressionLeaf::GroupedVariable(grouped_variable) => { - grouped_variable.read_as_expression_value(interpreter)? + grouped_variable.interpret_to_value(interpreter)? } SourceExpressionLeaf::FlattenedVariable(flattened_variable) => { - flattened_variable.read_as_expression_value(interpreter)? + flattened_variable.interpret_to_value(interpreter)? + } + SourceExpressionLeaf::VariablePath(variable_path) => { + variable_path.interpret_to_value(interpreter)? + } + SourceExpressionLeaf::ExpressionBlock(block) => { + block.interpret_to_value(interpreter)? } SourceExpressionLeaf::ExplicitStream(source_group) => source_group .clone() diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs new file mode 100644 index 00000000..a989bf2d --- /dev/null +++ b/src/expressions/expression_block.rs @@ -0,0 +1,245 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionBlock { + marker: Token![#], + flattening: Option, + delim_span: DelimSpan, + standard_statements: Vec<(Statement, Token![;])>, + return_statement: Option, +} + +impl Parse for ExpressionBlock { + fn parse(input: ParseStream) -> ParseResult { + let marker = input.parse()?; + let flattening = if input.peek(Token![..]) { + Some(input.parse()?) + } else { + None + }; + let (delim_span, inner) = input.parse_specific_group(Delimiter::Parenthesis)?; + let mut standard_statements = Vec::new(); + let return_statement = loop { + if inner.is_empty() { + break None; + } + let statement = inner.parse()?; + if inner.is_empty() { + break Some(statement); + } else if inner.peek(Token![;]) { + standard_statements.push((statement, inner.parse()?)); + } else { + return inner.parse_err("Expected an operator to continue the expression, or ; to mark the end of the expression statement"); + } + }; + Ok(Self { + marker, + flattening, + delim_span, + standard_statements, + return_statement, + }) + } +} + +impl HasSpanRange for ExpressionBlock { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.marker.span, self.delim_span.close()) + } +} + +impl ExpressionBlock { + pub(crate) fn evaluate( + &self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let output_span_range = self.span_range(); + for (statement, ..) in &self.standard_statements { + let value = statement.interpret_to_value(interpreter)?; + match value { + ExpressionValue::None { .. } => {}, + other_value => return other_value.execution_err("A statement ending with ; must not return a value. If you wish to explicitly discard the expression's result, use `let _ = ...;`"), + } + } + if let Some(return_statement) = &self.return_statement { + return_statement.interpret_to_value(interpreter) + } else { + Ok(ExpressionValue::None(output_span_range)) + } + } +} + +impl Interpret for &ExpressionBlock { + fn interpret_into( + self, + interpreter: &mut Interpreter, + output: &mut OutputStream, + ) -> ExecutionResult<()> { + let grouping = match self.flattening { + Some(_) => Grouping::Flattened, + None => Grouping::Grouped, + }; + self.evaluate(interpreter)?.output_to(grouping, output); + Ok(()) + } +} + +impl InterpretToValue for &ExpressionBlock { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + if let Some(flattening) = &self.flattening { + return flattening + .execution_err("Flattening is not supported when outputting as a value"); + } + self.evaluate(interpreter) + } +} + +#[derive(Clone)] +pub(crate) enum Statement { + Assignment(AssignmentStatement), + Expression(SourceExpression), +} + +impl Parse for Statement { + fn parse(input: ParseStream) -> ParseResult { + let forked = input.fork(); + match forked.parse() { + Ok(assignment) => { + input.advance_to(&forked); + Ok(Statement::Assignment(assignment)) + } + Err(_) => Ok(Statement::Expression(input.parse()?)), + } + } +} + +impl InterpretToValue for &Statement { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + match self { + Statement::Assignment(assignment) => assignment.interpret_to_value(interpreter), + Statement::Expression(expression) => expression.interpret_to_value(interpreter), + } + } +} + +#[derive(Clone)] +pub(crate) struct AssignmentStatement { + destination: Destination, + #[allow(unused)] + equals: Token![=], + expression: SourceExpression, +} + +impl Parse for AssignmentStatement { + fn parse(input: ParseStream) -> ParseResult { + Ok(Self { + destination: input.parse()?, + equals: input.parse()?, + expression: input.parse()?, + }) + } +} + +impl InterpretToValue for &AssignmentStatement { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + match &self.destination { + Destination::NewVariable { let_token, path } => { + let value = self.expression.interpret_to_value(interpreter)?; + let mut span_range = value.span_range(); + path.set_value(interpreter, value)?; + span_range.set_start(let_token.span); + Ok(ExpressionValue::None(span_range)) + } + Destination::ExistingVariable { path, operation } => { + let value = if let Some(operation) = operation { + let left = path.interpret_to_value(interpreter)?; + let right = self.expression.interpret_to_value(interpreter)?; + operation.evaluate(left, right)? + } else { + self.expression.interpret_to_value(interpreter)? + }; + let mut span_range = value.span_range(); + path.set_value(interpreter, value)?; + span_range.set_start(path.span_range().start()); + Ok(ExpressionValue::None(span_range)) + } + Destination::Discarded { let_token, .. } => { + let value = self.expression.interpret_to_value(interpreter)?; + let mut span_range = value.span_range(); + span_range.set_start(let_token.span); + Ok(ExpressionValue::None(span_range)) + } + } + } +} + +#[derive(Clone)] +enum Destination { + NewVariable { + let_token: Token![let], + path: VariablePath, + }, + ExistingVariable { + path: VariablePath, + operation: Option, + }, + Discarded { + let_token: Token![let], + #[allow(unused)] + discarded_token: Token![_], + }, +} + +impl Parse for Destination { + fn parse(input: ParseStream) -> ParseResult { + if input.peek(Token![let]) { + let let_token = input.parse()?; + if input.peek(Token![_]) { + Ok(Self::Discarded { + let_token, + discarded_token: input.parse()?, + }) + } else { + Ok(Self::NewVariable { + let_token, + path: input.parse()?, + }) + } + } else { + Ok(Self::ExistingVariable { + path: input.parse()?, + operation: if input.peek(Token![=]) { + None + } else { + let operator_char = match input.cursor().punct() { + Some((operator, _)) => operator.as_char(), + None => 'X', + }; + match operator_char { + '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} + _ => { + return input + .parse_err("Expected = or one of += -= *= /= %= &= |= or ^=") + } + } + Some(input.parse()?) + }, + }) + } + } +} diff --git a/src/expressions/float.rs b/src/expressions/float.rs index f83fb993..6a3b983a 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -175,9 +175,16 @@ impl UntypedFloat { CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } - CastTarget::Stream => { - operation.output(operation.output(self).into_new_output_stream()) - } + CastTarget::Stream => operation.output( + operation + .output(self) + .into_new_output_stream(Grouping::Flattened), + ), + CastTarget::Group => operation.output( + operation + .output(self) + .into_new_output_stream(Grouping::Grouped), + ), }, }) } @@ -321,7 +328,8 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream()), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)), } }) } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 1bc3ff71..acabb7fe 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -310,9 +310,16 @@ impl UntypedInteger { CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } - CastTarget::Stream => { - operation.output(operation.output(self).into_new_output_stream()) - } + CastTarget::Stream => operation.output( + operation + .output(self) + .into_new_output_stream(Grouping::Flattened), + ), + CastTarget::Group => operation.output( + operation + .output(self) + .into_new_output_stream(Grouping::Grouped), + ), }, }) } @@ -621,7 +628,8 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream()), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)), } }) } @@ -656,7 +664,8 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream()), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)), } }) } @@ -698,9 +707,16 @@ impl HandleUnaryOperation for u8 { CastTarget::Boolean => { return operation.execution_err("This cast is not supported") } - CastTarget::Stream => { - operation.output(operation.output(self).into_new_output_stream()) - } + CastTarget::Stream => operation.output( + operation + .output(self) + .into_new_output_stream(Grouping::Flattened), + ), + CastTarget::Group => operation.output( + operation + .output(self) + .into_new_output_stream(Grouping::Grouped), + ), }, }) } diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index f0617363..98a6314f 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -2,6 +2,7 @@ mod boolean; mod character; mod evaluation; mod expression; +mod expression_block; mod expression_parsing; mod float; mod integer; @@ -24,9 +25,7 @@ use string::*; use value::*; pub(crate) use expression::*; -pub(crate) use integer::*; -pub(crate) use operations::*; +pub(crate) use expression_block::*; pub(crate) use value::*; -// For some reason Rust-analyzer didn't see it without this explicit export -pub(crate) use operations::BinaryOperation; +// For some mysterious reason Rust-analyzer can't resolve this without an explicit export pub(crate) use value::ExpressionValue; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 955b66c2..6619249a 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -128,6 +128,7 @@ impl UnaryOperation { "bool" => CastTarget::Boolean, "char" => CastTarget::Char, "stream" => CastTarget::Stream, + "group" => CastTarget::Group, _ => { return target_ident .parse_err("This type is not supported in preinterpret cast expressions") diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs index 053cc440..ea33d760 100644 --- a/src/expressions/stream.rs +++ b/src/expressions/stream.rs @@ -20,6 +20,11 @@ impl ExpressionStream { } UnaryOperation::Cast { target, .. } => match target { CastTarget::Stream => operation.output(self.value), + CastTarget::Group => operation.output( + operation + .output(self.value) + .into_new_output_stream(Grouping::Grouped), + ), _ => { let coerced = self.value.coerce_into_value(self.span_range); if let ExpressionValue::Stream(_) = &coerced { diff --git a/src/expressions/string.rs b/src/expressions/string.rs index f1beb333..624d9f26 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -26,9 +26,16 @@ impl ExpressionString { return operation.unsupported(self) } UnaryOperation::Cast { target, .. } => match target { - CastTarget::Stream => { - operation.output(operation.output(self.value).into_new_output_stream()) - } + CastTarget::Stream => operation.output( + operation + .output(self.value) + .into_new_output_stream(Grouping::Flattened), + ), + CastTarget::Group => operation.output( + operation + .output(self.value) + .into_new_output_stream(Grouping::Grouped), + ), _ => return operation.unsupported(self), }, }) diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 5a9ae9ed..c698cce9 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -334,36 +334,47 @@ impl ExpressionValue { self } - pub(crate) fn into_new_output_stream(self) -> OutputStream { - match self { - Self::Stream(value) => value.value, - other => { + pub(crate) fn into_new_output_stream(self, grouping: Grouping) -> OutputStream { + match (self, grouping) { + (Self::Stream(value), Grouping::Flattened) => value.value, + (other, grouping) => { let mut output = OutputStream::new(); - other.output_to(&mut output); + other.output_to(grouping, &mut output); output } } } - pub(crate) fn output_to(self, output: &mut OutputStream) { + pub(crate) fn output_to(self, grouping: Grouping, output: &mut OutputStream) { + match grouping { + Grouping::Grouped => { + // Grouping can be important for different values, to ensure they're read atomically + // when the output stream is viewed as an array/iterable, e.g. in a for loop. + // * Grouping means -1 is interpreted atomically, rather than as a punct then a number + // * Grouping means that a stream is interpreted atomically + let span = self.span_range().join_into_span_else_start(); + output + .push_grouped( + |inner| { + self.output_flattened_to(inner); + Ok(()) + }, + Delimiter::None, + span, + ) + .unwrap() + } + Grouping::Flattened => { + self.output_flattened_to(output); + } + } + } + + fn output_flattened_to(self, output: &mut OutputStream) { match self { Self::None { .. } => {} - Self::Integer(value) => { - // Grouped so that -1 is interpreted as a single thing, not a punct then a number - output.push_grouped( - |inner| Ok(inner.push_literal(value.to_literal())), - Delimiter::None, - value.span_range.join_into_span_else_start(), - ).unwrap() - }, - Self::Float(value) => { - // Grouped so that -1.0 is interpreted as a single thing, not a punct then a number - output.push_grouped( - |inner| Ok(inner.push_literal(value.to_literal())), - Delimiter::None, - value.span_range.join_into_span_else_start(), - ).unwrap() - } + Self::Integer(value) => output.push_literal(value.to_literal()), + Self::Float(value) => output.push_literal(value.to_literal()), Self::Boolean(value) => output.push_ident(value.to_ident()), Self::String(value) => output.push_literal(value.to_literal()), Self::Char(value) => output.push_literal(value.to_literal()), @@ -375,6 +386,11 @@ impl ExpressionValue { } } +pub(crate) enum Grouping { + Grouped, + Flattened, +} + impl HasValueType for ExpressionValue { fn value_type(&self) -> &'static str { match self { @@ -428,6 +444,7 @@ pub(super) enum CastTarget { Boolean, Char, Stream, + Group, } pub(super) enum EvaluationLiteralPair { diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index ba08d3f5..1a34a49b 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -5,7 +5,8 @@ use crate::internal_prelude::*; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum CommandOutputKind { None, - Value, + FlattenedValue, + GroupedValue, Ident, Literal, FlattenedStream, @@ -111,7 +112,7 @@ impl OutputKind for OutputKindValue { type Output = TokenTree; fn resolve_standard() -> CommandOutputKind { - CommandOutputKind::Value + CommandOutputKind::FlattenedValue } fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { @@ -134,7 +135,12 @@ impl CommandInvocationAs for C { context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.execute(context.interpreter)?.output_to(output); + let grouping = match context.output_kind { + CommandOutputKind::FlattenedValue => Grouping::Flattened, + _ => Grouping::Grouped, + }; + self.execute(context.interpreter)? + .output_to(grouping, output); Ok(()) } @@ -466,8 +472,6 @@ define_command_enums! { InsertSpacesCommand, // Expression Commands - EvaluateCommand, - AssignCommand, RangeCommand, // Control flow commands @@ -548,18 +552,6 @@ impl Command { pub(crate) unsafe fn set_output_kind(&mut self, output_kind: CommandOutputKind) { self.output_kind = output_kind; } - - pub(crate) fn interpret_to_value( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let context = ExecutionContext { - interpreter, - output_kind: self.output_kind, - delim_span: self.source_group_span, - }; - self.typed.execute_to_value(context) - } } impl HasSpan for Command { @@ -582,3 +574,19 @@ impl Interpret for Command { self.typed.execute_into(context, output) } } + +impl InterpretToValue for Command { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let context = ExecutionContext { + interpreter, + output_kind: self.output_kind, + delim_span: self.source_group_span, + }; + self.typed.execute_to_value(context) + } +} diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 5cdb7127..36c8e81d 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -53,7 +53,7 @@ impl StreamingCommandDefinition for IfCommand { ) -> ExecutionResult<()> { let evaluated_condition = self .condition - .evaluate(interpreter)? + .interpret_to_value(interpreter)? .expect_bool("An if condition")?; if evaluated_condition.value { @@ -62,7 +62,7 @@ impl StreamingCommandDefinition for IfCommand { for (condition, code) in self.else_ifs { let evaluated_condition = condition - .evaluate(interpreter)? + .interpret_to_value(interpreter)? .expect_bool("An else if condition")?; if evaluated_condition.value { @@ -114,8 +114,7 @@ impl StreamingCommandDefinition for WhileCommand { let evaluated_condition = self .condition - .clone() - .evaluate(interpreter)? + .interpret_to_value(interpreter)? .expect_bool("A while condition")?; if !evaluated_condition.value { diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 8f8eac71..411ae265 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -141,7 +141,7 @@ impl NoOutputCommandDefinition for TypedSetCommand { } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let content = self.content.evaluate(interpreter)?; + let content = self.content.interpret_to_value(interpreter)?; self.variable.set_value(interpreter, content)?; Ok(()) } diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index bdac57cc..73e69fe9 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -1,108 +1,5 @@ use crate::internal_prelude::*; -#[derive(Clone)] -pub(crate) struct EvaluateCommand { - expression: SourceExpression, - command_span: Span, -} - -impl CommandType for EvaluateCommand { - type OutputKind = OutputKindValue; -} - -impl ValueCommandDefinition for EvaluateCommand { - const COMMAND_NAME: &'static str = "evaluate"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - expression: input.parse()?, - command_span: arguments.command_span(), - }) - }, - "Expected [!evaluate! ...] containing a valid preinterpret expression", - ) - } - - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let value = self - .expression - .evaluate(interpreter)? - .with_span(self.command_span); - Ok(value) - } -} - -#[derive(Clone)] -pub(crate) struct AssignCommand { - variable: GroupedVariable, - operation: Option, - #[allow(unused)] - equals: Token![=], - expression: SourceExpression, - command_span: Span, -} - -impl CommandType for AssignCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for AssignCommand { - const COMMAND_NAME: &'static str = "assign"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - variable: input.parse()?, - operation: { - if input.peek(Token![=]) { - None - } else { - let operator_char = match input.cursor().punct() { - Some((operator, _)) => operator.as_char(), - None => 'X', - }; - match operator_char { - '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => return input.parse_err("Expected one of + - * / % & | or ^"), - } - Some(input.parse()?) - } - }, - equals: input.parse()?, - expression: input.parse()?, - command_span: arguments.command_span(), - }) - }, - "Expected [!assign! #variable = ] or [!assign! #variable X= ] for X one of + - * / % & | or ^", - ) - } - - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let Self { - variable, - operation, - equals: _, - expression, - command_span, - } = self; - - let value = if let Some(operation) = operation { - let left = variable.read_as_expression_value(interpreter)?; - let right = expression.evaluate(interpreter)?; - operation.evaluate(left, right)? - } else { - expression.evaluate(interpreter)? - }; - - variable.set_value(interpreter, value.with_span(command_span))?; - - Ok(()) - } -} - #[derive(Clone)] pub(crate) struct RangeCommand { left: SourceExpression, @@ -136,8 +33,8 @@ impl GroupedStreamCommandDefinition for RangeCommand { output: &mut OutputStream, ) -> ExecutionResult<()> { let range_limits = self.range_limits; - let left = self.left.evaluate(interpreter)?; - let right = self.right.evaluate(interpreter)?; + let left = self.left.interpret_to_value(interpreter)?; + let right = self.right.interpret_to_value(interpreter)?; let range_iterator = left.create_range(right, &range_limits)?; @@ -155,7 +52,8 @@ impl GroupedStreamCommandDefinition for RangeCommand { } for value in range_iterator { - value.output_to(output) + // It needs to be grouped so that e.g. -1 is interpreted as a single item, not two separate tokens. + value.output_to(Grouping::Grouped, output) } Ok(()) diff --git a/src/interpretation/interpret_traits.rs b/src/interpretation/interpret_traits.rs index 35e5457d..37202f90 100644 --- a/src/interpretation/interpret_traits.rs +++ b/src/interpretation/interpret_traits.rs @@ -17,7 +17,7 @@ pub(crate) trait Interpret: Sized { } } -pub(crate) trait InterpretValue: Sized { +pub(crate) trait InterpretToValue: Sized { type OutputValue; fn interpret_to_value( @@ -26,7 +26,7 @@ pub(crate) trait InterpretValue: Sized { ) -> ExecutionResult; } -impl InterpretValue for T { +impl InterpretToValue for T { type OutputValue = Self; fn interpret_to_value( diff --git a/src/interpretation/source_stream.rs b/src/interpretation/source_stream.rs index 51b20ea8..f662c482 100644 --- a/src/interpretation/source_stream.rs +++ b/src/interpretation/source_stream.rs @@ -43,6 +43,7 @@ pub(crate) enum SourceItem { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), + ExpressionBlock(ExpressionBlock), SourceGroup(SourceGroup), Punct(Punct), Ident(Ident), @@ -54,8 +55,13 @@ impl Parse for SourceItem { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => SourceItem::Command(input.parse()?), SourcePeekMatch::Group(_) => SourceItem::SourceGroup(input.parse()?), - SourcePeekMatch::GroupedVariable => SourceItem::GroupedVariable(input.parse()?), - SourcePeekMatch::FlattenedVariable => SourceItem::FlattenedVariable(input.parse()?), + SourcePeekMatch::Variable(Grouping::Grouped) => { + SourceItem::GroupedVariable(input.parse()?) + } + SourcePeekMatch::Variable(Grouping::Flattened) => { + SourceItem::FlattenedVariable(input.parse()?) + } + SourcePeekMatch::ExpressionBlock(_) => SourceItem::ExpressionBlock(input.parse()?), SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => { return input.parse_err("Destructurings are not supported here. If this wasn't intended to be a destructuring, replace @ with [!raw! @]"); } @@ -86,6 +92,9 @@ impl Interpret for SourceItem { SourceItem::FlattenedVariable(variable) => { variable.interpret_into(interpreter, output)?; } + SourceItem::ExpressionBlock(block) => { + block.interpret_into(interpreter, output)?; + } SourceItem::SourceGroup(group) => { group.interpret_into(interpreter, output)?; } @@ -103,6 +112,7 @@ impl HasSpanRange for SourceItem { SourceItem::Command(command_invocation) => command_invocation.span_range(), SourceItem::FlattenedVariable(variable) => variable.span_range(), SourceItem::GroupedVariable(variable) => variable.span_range(), + SourceItem::ExpressionBlock(block) => block.span_range(), SourceItem::SourceGroup(group) => group.span_range(), SourceItem::Punct(punct) => punct.span_range(), SourceItem::Ident(ident) => ident.span_range(), diff --git a/src/interpretation/source_stream_input.rs b/src/interpretation/source_stream_input.rs index 9a24ec8c..4b16b2f6 100644 --- a/src/interpretation/source_stream_input.rs +++ b/src/interpretation/source_stream_input.rs @@ -25,8 +25,8 @@ impl Parse for SourceStreamInput { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - SourcePeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + SourcePeekMatch::Variable(Grouping::Grouped) => Self::GroupedVariable(input.parse()?), + SourcePeekMatch::Variable(Grouping::Flattened) => Self::FlattenedVariable(input.parse()?), SourcePeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), _ => input.span() @@ -57,7 +57,8 @@ impl Interpret for SourceStreamInput { SourceStreamInput::Command(mut command) => { match command.output_kind() { CommandOutputKind::None - | CommandOutputKind::Value + | CommandOutputKind::GroupedValue + | CommandOutputKind::FlattenedValue | CommandOutputKind::Ident | CommandOutputKind::Literal => { command.execution_err("The command does not output a stream") @@ -136,7 +137,7 @@ fn parse_as_stream_input( Ok(()) } -impl InterpretValue for SourceStreamInput { +impl InterpretToValue for SourceStreamInput { type OutputValue = OutputCommandStream; fn interpret_to_value( diff --git a/src/interpretation/source_value.rs b/src/interpretation/source_value.rs index 292443c0..0d8ce2cb 100644 --- a/src/interpretation/source_value.rs +++ b/src/interpretation/source_value.rs @@ -9,6 +9,7 @@ pub(crate) enum SourceValue { Command(Command), GroupedVariable(GroupedVariable), FlattenedVariable(FlattenedVariable), + ExpressionBlock(ExpressionBlock), Code(SourceCodeBlock), Value(T), } @@ -17,8 +18,11 @@ impl> Parse for SourceValue { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::GroupedVariable => Self::GroupedVariable(input.parse()?), - SourcePeekMatch::FlattenedVariable => Self::FlattenedVariable(input.parse()?), + SourcePeekMatch::Variable(Grouping::Grouped) => Self::GroupedVariable(input.parse()?), + SourcePeekMatch::Variable(Grouping::Flattened) => { + Self::FlattenedVariable(input.parse()?) + } + SourcePeekMatch::ExpressionBlock(_) => Self::ExpressionBlock(input.parse()?), SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::AppendVariableBinding @@ -48,13 +52,14 @@ impl HasSpanRange for SourceValue { SourceValue::Command(command) => command.span_range(), SourceValue::GroupedVariable(variable) => variable.span_range(), SourceValue::FlattenedVariable(variable) => variable.span_range(), + SourceValue::ExpressionBlock(block) => block.span_range(), SourceValue::Code(code) => code.span_range(), SourceValue::Value(value) => value.span_range(), } } } -impl, I: Parse> InterpretValue for SourceValue { +impl, I: Parse> InterpretToValue for SourceValue { type OutputValue = I; fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { @@ -62,6 +67,7 @@ impl, I: Parse> InterpretValue for So SourceValue::Command(_) => "command output", SourceValue::GroupedVariable(_) => "grouped variable output", SourceValue::FlattenedVariable(_) => "flattened variable output", + SourceValue::ExpressionBlock(_) => "an #(...) expression block", SourceValue::Code(_) => "output from the { ... } block", SourceValue::Value(_) => "value", }; @@ -73,6 +79,7 @@ impl, I: Parse> InterpretValue for So SourceValue::FlattenedVariable(variable) => { variable.interpret_to_new_stream(interpreter)? } + SourceValue::ExpressionBlock(block) => block.interpret_to_new_stream(interpreter)?, SourceValue::Code(code) => code.interpret_to_new_stream(interpreter)?, SourceValue::Value(value) => return value.interpret_to_value(interpreter), }; @@ -122,9 +129,9 @@ impl> Parse for Grouped { } } -impl InterpretValue for Grouped +impl InterpretToValue for Grouped where - T: InterpretValue, + T: InterpretToValue, { type OutputValue = Grouped; @@ -165,9 +172,9 @@ impl> Parse for Repeated { } } -impl InterpretValue for Repeated +impl InterpretToValue for Repeated where - T: InterpretValue, + T: InterpretToValue, { type OutputValue = Repeated; diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 6d72067b..1c988959 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -38,12 +38,8 @@ impl GroupedVariable { interpreter: &mut Interpreter, value: ExpressionValue, ) -> ExecutionResult<()> { - let value = { - let mut output = OutputStream::new(); - value.output_to(&mut output); - output - }; - interpreter.set_variable(self, value) + // It will be grouped on the way out; not it. + interpreter.set_variable(self, value.into_new_output_stream(Grouping::Flattened)) } pub(crate) fn get_existing_for_mutation( @@ -57,18 +53,6 @@ impl GroupedVariable { .cheap_clone()) } - pub(crate) fn read_as_expression_value( - &self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let value = self - .read_existing(interpreter)? - .get(self)? - .clone() - .coerce_into_value(self.span_range()); - Ok(value) - } - pub(crate) fn substitute_ungrouped_contents_into( &self, interpreter: &mut Interpreter, @@ -121,6 +105,22 @@ impl Interpret for &GroupedVariable { } } +impl InterpretToValue for &GroupedVariable { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let value = self + .read_existing(interpreter)? + .get(self)? + .clone() + .coerce_into_value(self.span_range()); + Ok(value) + } +} + impl HasSpanRange for GroupedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) @@ -174,21 +174,12 @@ impl FlattenedVariable { Ok(()) } - pub(crate) fn read_as_expression_value( - &self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let mut output_stream = OutputStream::new(); - self.substitute_into(interpreter, &mut output_stream)?; - Ok(output_stream.to_value(self.span_range())) - } - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { interpreter.get_existing_variable_data( self, || self.error(format!( "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self, + self.variable_name, self, )), ) @@ -215,6 +206,19 @@ impl Interpret for &FlattenedVariable { } } +impl InterpretToValue for &FlattenedVariable { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut output_stream = OutputStream::new(); + self.substitute_into(interpreter, &mut output_stream)?; + Ok(output_stream.to_value(self.span_range())) + } +} + impl HasSpanRange for FlattenedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) @@ -232,3 +236,69 @@ impl core::fmt::Display for FlattenedVariable { write!(f, "#..{}", self.variable_name) } } + +// An identifier for a variable path in an expression +#[derive(Clone)] +pub(crate) struct VariablePath { + root: Ident, + fields: Vec<(Token![.], Ident)>, +} + +impl Parse for VariablePath { + fn parse(input: ParseStream) -> ParseResult { + Ok(Self { + root: input.parse()?, + fields: { + let mut fields = vec![]; + while input.peek(Token![.]) { + fields.push((input.parse()?, input.parse()?)); + } + fields + }, + }) + } +} + +impl VariablePath { + pub(crate) fn set_value( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + // It will be grouped on the way out, not in + interpreter.set_variable(self, value.into_new_output_stream(Grouping::Flattened)) + } +} + +impl IsVariable for VariablePath { + fn get_name(&self) -> String { + self.root.to_string() + } +} + +impl HasSpanRange for VariablePath { + fn span_range(&self) -> SpanRange { + match self.fields.last() { + Some((_, ident)) => SpanRange::new_between(self.root.span(), ident.span()), + None => self.root.span_range(), + } + } +} + +impl InterpretToValue for &VariablePath { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let value = interpreter + .get_existing_variable_data(self, || { + self.error(format!("The variable {} wasn't set.", &self.root,)) + })? + .get(self)? + .clone() + .coerce_into_value(self.span_range()); + Ok(value) + } +} diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 7e4fb1cc..716564a3 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -16,8 +16,8 @@ impl ParseBuffer<'_, Source> { #[allow(unused)] pub(crate) enum SourcePeekMatch { Command(Option), - GroupedVariable, - FlattenedVariable, + ExpressionBlock(Grouping), + Variable(Grouping), AppendVariableBinding, ExplicitTransformStream, Transformer(Option), @@ -71,12 +71,15 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> SourcePeekMatch { } if let Some((_, next)) = cursor.punct_matching('#') { if next.ident().is_some() { - return SourcePeekMatch::GroupedVariable; + return SourcePeekMatch::Variable(Grouping::Grouped); } if let Some((_, next)) = next.punct_matching('.') { if let Some((_, next)) = next.punct_matching('.') { if next.ident().is_some() { - return SourcePeekMatch::FlattenedVariable; + return SourcePeekMatch::Variable(Grouping::Flattened); + } + if next.group_matching(Delimiter::Parenthesis).is_some() { + return SourcePeekMatch::ExpressionBlock(Grouping::Flattened); } if let Some((_, next)) = next.punct_matching('>') { if next.punct_matching('>').is_some() { @@ -90,6 +93,9 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> SourcePeekMatch { return SourcePeekMatch::AppendVariableBinding; } } + if next.group_matching(Delimiter::Parenthesis).is_some() { + return SourcePeekMatch::ExpressionBlock(Grouping::Grouped); + } } if let Some((_, next)) = cursor.punct_matching('@') { diff --git a/src/transformation/exact_stream.rs b/src/transformation/exact_stream.rs index a470c3c8..987047ed 100644 --- a/src/transformation/exact_stream.rs +++ b/src/transformation/exact_stream.rs @@ -53,6 +53,7 @@ pub(crate) enum ExactItem { ExactCommandOutput(Command), ExactGroupedVariableOutput(GroupedVariable), ExactFlattenedVariableOutput(FlattenedVariable), + ExactExpressionBlock(ExpressionBlock), ExactPunct(Punct), ExactIdent(Ident), ExactLiteral(Literal), @@ -63,10 +64,13 @@ impl Parse for ExactItem { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::ExactCommandOutput(input.parse()?), - SourcePeekMatch::GroupedVariable => Self::ExactGroupedVariableOutput(input.parse()?), - SourcePeekMatch::FlattenedVariable => { + SourcePeekMatch::Variable(Grouping::Grouped) => { + Self::ExactGroupedVariableOutput(input.parse()?) + } + SourcePeekMatch::Variable(Grouping::Flattened) => { Self::ExactFlattenedVariableOutput(input.parse()?) } + SourcePeekMatch::ExpressionBlock(_) => Self::ExactExpressionBlock(input.parse()?), SourcePeekMatch::AppendVariableBinding => { return input .parse_err("Append variable bindings are not supported in an EXACT stream") @@ -127,6 +131,12 @@ impl HandleTransformation for ExactItem { .into_exact_stream()? .handle_transform(input, interpreter, output)?; } + ExactItem::ExactExpressionBlock(expression_block) => { + expression_block + .interpret_to_new_stream(interpreter)? + .into_exact_stream()? + .handle_transform(input, interpreter, output)?; + } ExactItem::ExactPunct(punct) => { output.push_punct(input.parse_punct_matching(punct.as_char())?); } diff --git a/src/transformation/parse_utilities.rs b/src/transformation/parse_utilities.rs index 0221583b..86fe984d 100644 --- a/src/transformation/parse_utilities.rs +++ b/src/transformation/parse_utilities.rs @@ -69,11 +69,11 @@ impl ParseUntil { } Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) - | SourcePeekMatch::GroupedVariable - | SourcePeekMatch::FlattenedVariable + | SourcePeekMatch::Variable(_) | SourcePeekMatch::Transformer(_) | SourcePeekMatch::ExplicitTransformStream - | SourcePeekMatch::AppendVariableBinding => { + | SourcePeekMatch::AppendVariableBinding + | SourcePeekMatch::ExpressionBlock(_) => { return input .span() .parse_err("This cannot follow a flattened variable binding"); diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index 76a94e24..ce711a5d 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -40,6 +40,7 @@ impl HandleTransformation for TransformSegment { pub(crate) enum TransformItem { Command(Command), Variable(VariableBinding), + ExpressionBlock(ExpressionBlock), Transformer(Transformer), TransformStreamInput(ExplicitTransformStream), ExactPunct(Punct), @@ -58,11 +59,10 @@ impl TransformItem { ) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::GroupedVariable - | SourcePeekMatch::FlattenedVariable - | SourcePeekMatch::AppendVariableBinding => { + SourcePeekMatch::Variable(_) | SourcePeekMatch::AppendVariableBinding => { Self::Variable(VariableBinding::parse_until::(input)?) } + SourcePeekMatch::ExpressionBlock(_) => Self::ExpressionBlock(input.parse()?), SourcePeekMatch::Group(_) => Self::ExactGroup(input.parse()?), SourcePeekMatch::ExplicitTransformStream => Self::TransformStreamInput(input.parse()?), SourcePeekMatch::Transformer(_) => Self::Transformer(input.parse()?), @@ -94,6 +94,9 @@ impl HandleTransformation for TransformItem { TransformItem::TransformStreamInput(stream) => { stream.handle_transform(input, interpreter, output)?; } + TransformItem::ExpressionBlock(block) => { + block.interpret_into(interpreter, output)?; + } TransformItem::ExactPunct(punct) => { input.parse_punct_matching(punct.as_char())?; } diff --git a/tests/compilation_failures/control_flow/error_after_continue.rs b/tests/compilation_failures/control_flow/error_after_continue.rs index 891942a4..4c1ae919 100644 --- a/tests/compilation_failures/control_flow/error_after_continue.rs +++ b/tests/compilation_failures/control_flow/error_after_continue.rs @@ -2,14 +2,14 @@ use preinterpret::*; fn main() { preinterpret!( - [!set! #x = 0] + #(let x = 0) [!while! true { - [!assign! #x += 1] + #(x += 1) [!if! #x == 3 { [!continue!] } !elif! #x >= 3 { // This checks that the "continue" flag is consumed, - // and future errors propogate correctly. + // and future errors propagate correctly. [!error! { message: "And now we error" }] diff --git a/tests/compilation_failures/control_flow/error_after_continue.stderr b/tests/compilation_failures/control_flow/error_after_continue.stderr index ea662970..b98f5442 100644 --- a/tests/compilation_failures/control_flow/error_after_continue.stderr +++ b/tests/compilation_failures/control_flow/error_after_continue.stderr @@ -2,9 +2,9 @@ error: And now we error --> tests/compilation_failures/control_flow/error_after_continue.rs:4:5 | 4 | / preinterpret!( -5 | | [!set! #x = 0] +5 | | #(let x = 0) 6 | | [!while! true { -7 | | [!assign! #x += 1] +7 | | #(x += 1) ... | 17 | | }] 18 | | ); diff --git a/tests/compilation_failures/expressions/add_float_and_int.rs b/tests/compilation_failures/expressions/add_float_and_int.rs index 9daca0b4..6532d39d 100644 --- a/tests/compilation_failures/expressions/add_float_and_int.rs +++ b/tests/compilation_failures/expressions/add_float_and_int.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! 1.2 + 1] + #(1.2 + 1) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/add_float_and_int.stderr b/tests/compilation_failures/expressions/add_float_and_int.stderr index bd756751..cf52f4cc 100644 --- a/tests/compilation_failures/expressions/add_float_and_int.stderr +++ b/tests/compilation_failures/expressions/add_float_and_int.stderr @@ -1,5 +1,5 @@ error: Cannot infer common type from untyped float + untyped integer. Consider using `as` to cast the operands to matching types. - --> tests/compilation_failures/expressions/add_float_and_int.rs:5:25 + --> tests/compilation_failures/expressions/add_float_and_int.rs:5:15 | -5 | [!evaluate! 1.2 + 1] - | ^ +5 | #(1.2 + 1) + | ^ diff --git a/tests/compilation_failures/expressions/braces.rs b/tests/compilation_failures/expressions/braces.rs index bcd94df6..a4712e16 100644 --- a/tests/compilation_failures/expressions/braces.rs +++ b/tests/compilation_failures/expressions/braces.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! { true }] + #({ true }) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr index a02f3849..e59b2e83 100644 --- a/tests/compilation_failures/expressions/braces.stderr +++ b/tests/compilation_failures/expressions/braces.stderr @@ -1,6 +1,5 @@ error: Braces { ... } are not supported in an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/braces.rs:5:21 + --> tests/compilation_failures/expressions/braces.rs:5:11 | -5 | [!evaluate! { true }] - | ^ +5 | #({ true }) + | ^ diff --git a/tests/compilation_failures/expressions/cast_int_to_bool.rs b/tests/compilation_failures/expressions/cast_int_to_bool.rs index 891ef418..744fc192 100644 --- a/tests/compilation_failures/expressions/cast_int_to_bool.rs +++ b/tests/compilation_failures/expressions/cast_int_to_bool.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! 1 as bool] + #(1 as bool) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/cast_int_to_bool.stderr b/tests/compilation_failures/expressions/cast_int_to_bool.stderr index a8c67dc7..1a036393 100644 --- a/tests/compilation_failures/expressions/cast_int_to_bool.stderr +++ b/tests/compilation_failures/expressions/cast_int_to_bool.stderr @@ -1,5 +1,5 @@ error: This cast is not supported - --> tests/compilation_failures/expressions/cast_int_to_bool.rs:5:23 + --> tests/compilation_failures/expressions/cast_int_to_bool.rs:5:13 | -5 | [!evaluate! 1 as bool] - | ^^ +5 | #(1 as bool) + | ^^ diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs index c5c1bc94..73c69dbc 100644 --- a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs @@ -2,8 +2,9 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!set! #value = 1] - [!set! #indirect = [!raw! #value]] - [!evaluate! { #indirect }] + // We don't get a re-evaluation. Instead, we get a parse error, because we end up + // with let _ = [!error! "This was a re-evaluation"]; which is a parse error in + // normal rust land. + #(indirect = [!raw! [!error! "This was a re-evaluation"]]; #(indirect)) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr index 13c95e37..e9924386 100644 --- a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr @@ -1,6 +1,5 @@ -error: Braces { ... } are not supported in an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:7:21 +error: expected one of `(`, `[`, or `{`, found `"This was a re-evaluation"` + --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:8:38 | -7 | [!evaluate! { #indirect }] - | ^ +8 | #(indirect = [!raw! [!error! "This was a re-evaluation"]]; #(indirect)) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected one of `(`, `[`, or `{` diff --git a/tests/compilation_failures/expressions/compare_int_and_float.rs b/tests/compilation_failures/expressions/compare_int_and_float.rs index e2f7b29b..d4e08b5c 100644 --- a/tests/compilation_failures/expressions/compare_int_and_float.rs +++ b/tests/compilation_failures/expressions/compare_int_and_float.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! 5 < 6.4] + #(5 < 6.4) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/compare_int_and_float.stderr b/tests/compilation_failures/expressions/compare_int_and_float.stderr index 131ae9a0..738a561c 100644 --- a/tests/compilation_failures/expressions/compare_int_and_float.stderr +++ b/tests/compilation_failures/expressions/compare_int_and_float.stderr @@ -1,5 +1,5 @@ error: Cannot infer common type from untyped integer < untyped float. Consider using `as` to cast the operands to matching types. - --> tests/compilation_failures/expressions/compare_int_and_float.rs:5:23 + --> tests/compilation_failures/expressions/compare_int_and_float.rs:5:13 | -5 | [!evaluate! 5 < 6.4] - | ^ +5 | #(5 < 6.4) + | ^ diff --git a/tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.rs b/tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.rs new file mode 100644 index 00000000..c6cb2193 --- /dev/null +++ b/tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.rs @@ -0,0 +1,10 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #( + 1 + 2 + 3 + 4; + "This gets returned" + ) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.stderr b/tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.stderr new file mode 100644 index 00000000..b2ef1fc9 --- /dev/null +++ b/tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.stderr @@ -0,0 +1,5 @@ +error: A statement ending with ; must not return a value. If you wish to explicitly discard the expression's result, use `let _ = ...;` + --> tests/compilation_failures/expressions/expression_block_semicolon_expressions_cannot_return_value.rs:6:13 + | +6 | 1 + 2 + 3 + 4; + | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs index a051e68c..6cc0b2ce 100644 --- a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs +++ b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs @@ -5,8 +5,8 @@ fn main() { // This should not fail, and should be fixed. // This test just records the fact it doesn't work as a known issue. // A fix of this should remove this test and move it to a working test. - [!evaluate! -128i8] + #(-128i8) // This should also not fail according to the rules of rustc. - [!evaluate! -(--128i8)] + #(-(--128i8)) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr index ab93316a..20765e62 100644 --- a/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr +++ b/tests/compilation_failures/expressions/fix_me_negative_max_int_fails.stderr @@ -1,5 +1,5 @@ error: The - operator is not supported for unsupported literal values - --> tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs:8:21 + --> tests/compilation_failures/expressions/fix_me_negative_max_int_fails.rs:8:11 | -8 | [!evaluate! -128i8] - | ^ +8 | #(-128i8) + | ^ diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs b/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs index ada3e9c3..b64f1552 100644 --- a/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs +++ b/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - [!evaluate! 5 + [!..range! 1..2]] + #(5 + [!..range! 1..2]) }; } diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr index f8fb7f79..20b58c43 100644 --- a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr @@ -1,5 +1,5 @@ error: Cannot infer common type from untyped integer + stream. Consider using `as` to cast the operands to matching types. - --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:23 + --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:13 | -5 | [!evaluate! 5 + [!..range! 1..2]] - | ^ +5 | #(5 + [!..range! 1..2]) + | ^ diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs index b1c7cbe4..f44c21ca 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs @@ -2,7 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - [!set! #partial_sum = + 2] - [!evaluate! 5 #..partial_sum] + #(partial_sum = [+ 2]; 5 #..partial_sum) }; } diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr index 70cced4f..6dca115f 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr @@ -1,5 +1,5 @@ -error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:6:23 +error: Expected an operator to continue the expression, or ; to mark the end of the expression statement + --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:5:34 | -6 | [!evaluate! 5 #..partial_sum] - | ^ +5 | #(partial_sum = [+ 2]; 5 #..partial_sum) + | ^ diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs index df4b2cc3..a48f9db4 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs @@ -2,7 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!set! #x = + 1] - [!evaluate! 1 #x] + #(x = [+ 1]; 1 x) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr index 5e2a1cec..4d774dad 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr @@ -1,5 +1,5 @@ -error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:6:23 +error: Expected an operator to continue the expression, or ; to mark the end of the expression statement + --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:24 | -6 | [!evaluate! 1 #x] - | ^ +5 | #(x = [+ 1]; 1 x) + | ^ diff --git a/tests/compilation_failures/expressions/inner_braces.rs b/tests/compilation_failures/expressions/inner_braces.rs index 29e32db2..58b6efa4 100644 --- a/tests/compilation_failures/expressions/inner_braces.rs +++ b/tests/compilation_failures/expressions/inner_braces.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!evaluate! ({ true })] + #(({ true })) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/inner_braces.stderr b/tests/compilation_failures/expressions/inner_braces.stderr index 45f5a12a..820014a6 100644 --- a/tests/compilation_failures/expressions/inner_braces.stderr +++ b/tests/compilation_failures/expressions/inner_braces.stderr @@ -1,6 +1,5 @@ error: Braces { ... } are not supported in an expression - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/inner_braces.rs:5:22 + --> tests/compilation_failures/expressions/inner_braces.rs:5:12 | -5 | [!evaluate! ({ true })] - | ^ +5 | #(({ true })) + | ^ diff --git a/tests/compilation_failures/expressions/invalid_binary_operator.rs b/tests/compilation_failures/expressions/invalid_binary_operator.rs index 181a6c4f..28dfe336 100644 --- a/tests/compilation_failures/expressions/invalid_binary_operator.rs +++ b/tests/compilation_failures/expressions/invalid_binary_operator.rs @@ -2,10 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - // The error message here isn't as good as it could be, - // because we end the Expression when we detect an invalid extension. - // This is useful to allow e.g. [!if! true == false { ... }] - // But in future, perhaps we can use different parse modes for these cases. - [!evaluate! 10 _ 10] + #(10 _ 10) }; } diff --git a/tests/compilation_failures/expressions/invalid_binary_operator.stderr b/tests/compilation_failures/expressions/invalid_binary_operator.stderr index 888b4c39..ac1b8398 100644 --- a/tests/compilation_failures/expressions/invalid_binary_operator.stderr +++ b/tests/compilation_failures/expressions/invalid_binary_operator.stderr @@ -1,5 +1,5 @@ -error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/invalid_binary_operator.rs:9:24 +error: Expected an operator to continue the expression, or ; to mark the end of the expression statement + --> tests/compilation_failures/expressions/invalid_binary_operator.rs:5:14 | -9 | [!evaluate! 10 _ 10] - | ^ +5 | #(10 _ 10) + | ^ diff --git a/tests/compilation_failures/expressions/invalid_unary_operator.rs b/tests/compilation_failures/expressions/invalid_unary_operator.rs index 7f771285..96f34295 100644 --- a/tests/compilation_failures/expressions/invalid_unary_operator.rs +++ b/tests/compilation_failures/expressions/invalid_unary_operator.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - [!evaluate! ^10] + #(^10) }; } diff --git a/tests/compilation_failures/expressions/invalid_unary_operator.stderr b/tests/compilation_failures/expressions/invalid_unary_operator.stderr index b8679c3f..c2abe00b 100644 --- a/tests/compilation_failures/expressions/invalid_unary_operator.stderr +++ b/tests/compilation_failures/expressions/invalid_unary_operator.stderr @@ -1,6 +1,5 @@ error: Expected ! or - - Occurred whilst parsing [!evaluate! ...] - Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/invalid_unary_operator.rs:5:21 + --> tests/compilation_failures/expressions/invalid_unary_operator.rs:5:11 | -5 | [!evaluate! ^10] - | ^ +5 | #(^10) + | ^ diff --git a/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs b/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs index 67fecaba..df4e28d6 100644 --- a/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs +++ b/tests/compilation_failures/expressions/no_output_commands_in_expressions.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - [!evaluate! 5 + [!set! #x = 2] 2] + #( 5 + [!set! #x = 2] 2) }; } diff --git a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr index 93b3c5a2..24d51ad0 100644 --- a/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr +++ b/tests/compilation_failures/expressions/no_output_commands_in_expressions.stderr @@ -1,5 +1,5 @@ -error: Unexpected extra tokens. Expected [!evaluate! ...] containing a valid preinterpret expression - --> tests/compilation_failures/expressions/no_output_commands_in_expressions.rs:5:40 +error: Expected an operator to continue the expression, or ; to mark the end of the expression statement + --> tests/compilation_failures/expressions/no_output_commands_in_expressions.rs:5:31 | -5 | [!evaluate! 5 + [!set! #x = 2] 2] - | ^ +5 | #( 5 + [!set! #x = 2] 2) + | ^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 0264459e..edf1b45d 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -19,12 +19,11 @@ fn test_control_flow_compilation_failures() { fn test_if() { assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); assert_preinterpret_eq!({ - [!set! #x = [!evaluate! 1 == 2]] + #(x = 1 == 2) [!if! #x { "YES" } !else! { "NO" }] }, "NO"); assert_preinterpret_eq!({ - [!set! #x = 1] - [!set! #y = 2] + #(x = 1; y = 2) [!if! #x == #y { "YES" } !else! { "NO" }] }, "NO"); assert_preinterpret_eq!({ @@ -51,8 +50,8 @@ fn test_if() { #[test] fn test_while() { assert_preinterpret_eq!({ - [!set! #x = 0] - [!while! #x < 5 { [!assign! #x += 1] }] + #(x = 0) + [!while! #x < 5 { #(x += 1) }] #x }, 5); } @@ -61,9 +60,9 @@ fn test_while() { fn test_loop_continue_and_break() { assert_preinterpret_eq!( { - [!set! #x = 0] + #(x = 0) [!loop! { - [!assign! #x += 1] + #(x += 1) [!if! #x >= 10 { [!break!] }] }] #x @@ -74,7 +73,7 @@ fn test_loop_continue_and_break() { { [!string! [!for! #x in [!range! 65..75] { [!if! #x % 2 == 0 { [!continue!] }] - [!evaluate! #x as u8 as char] + #(#x as u8 as char) }]] }, "ACEGI" @@ -86,7 +85,7 @@ fn test_for() { assert_preinterpret_eq!( { [!string! [!for! #x in [!range! 65..70] { - [!evaluate! #x as u8 as char] + #(#x as u8 as char) }]] }, "ABCDE" diff --git a/tests/core.rs b/tests/core.rs index 30149cb1..51b57f6c 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -55,7 +55,7 @@ fn test_extend() { [!if! (#i <= 3) { [!set! #output += ", "] }] - [!assign! #i += 1] + #(i += 1) }] [!string! #output] }, diff --git a/tests/expressions.rs b/tests/expressions.rs index c47f4b21..13ad28c5 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -6,6 +6,12 @@ macro_rules! assert_preinterpret_eq { }; } +macro_rules! assert_expression_eq { + (#($($input:tt)*), $($output:tt)*) => { + assert_eq!(preinterpret!(#($($input)*)), $($output)*); + }; +} + #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_expression_compilation_failures() { @@ -15,63 +21,48 @@ fn test_expression_compilation_failures() { #[test] fn test_basic_evaluate_works() { - assert_preinterpret_eq!([!evaluate! !!(!!(true))], true); - assert_preinterpret_eq!([!evaluate! 1 + 5], 6u8); - assert_preinterpret_eq!([!evaluate! 1 + 5], 6i128); - assert_preinterpret_eq!([!evaluate! 1 + 5u16], 6u16); - assert_preinterpret_eq!([!evaluate! 127i8 + (-127i8) + (-127i8)], -127i8); - assert_preinterpret_eq!([!evaluate! 3.0 + 3.2], 6.2); - assert_preinterpret_eq!([!evaluate! 3.6 + 3999999999999999992.0], 3.6 + 3999999999999999992.0); - assert_preinterpret_eq!([!evaluate! -3.2], -3.2); - assert_preinterpret_eq!([!evaluate! true && true || false], true); - assert_preinterpret_eq!([!evaluate! true || false && false], true); // The && has priority - assert_preinterpret_eq!([!evaluate! true | false & false], true); // The & has priority - assert_preinterpret_eq!([!evaluate! true as u32 + 2], 3); - assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u32); - assert_preinterpret_eq!([!evaluate! 3.57 as int + 1], 4u64); - assert_preinterpret_eq!([!evaluate! 0b1000 & 0b1101], 0b1000); - assert_preinterpret_eq!([!evaluate! 0b1000 | 0b1101], 0b1101); - assert_preinterpret_eq!([!evaluate! 0b1000 ^ 0b1101], 0b101); - assert_preinterpret_eq!([!evaluate! 5 << 2], 20); - assert_preinterpret_eq!([!evaluate! 5 >> 1], 2); - assert_preinterpret_eq!([!evaluate! 123 == 456], false); - assert_preinterpret_eq!([!evaluate! 123 < 456], true); - assert_preinterpret_eq!([!evaluate! 123 <= 456], true); - assert_preinterpret_eq!([!evaluate! 123 != 456], true); - assert_preinterpret_eq!([!evaluate! 123 >= 456], false); - assert_preinterpret_eq!([!evaluate! 123 > 456], false); - assert_preinterpret_eq!( - { - [!typed_set! #six_as_sum = 3 + 3] - [!evaluate! #six_as_sum * #six_as_sum] - }, - 36 - ); - assert_preinterpret_eq!( - { - [!set! #partial_sum = + 2] - [!debug! - // [...] is a token stream, which is not evaluated. - [!evaluate! [5 #..partial_sum]] - = - // !reinterpret! can be used to force an evaluation - [!reinterpret! [[!raw! !evaluate!] 5 #..partial_sum]] - ] - }, - "5 + 2 = [!group! 7]" - ); - assert_preinterpret_eq!([!evaluate! 1 + [!range! 1..2] as int], 2); - assert_preinterpret_eq!([!evaluate! "hello" == "world"], false); - assert_preinterpret_eq!([!evaluate! "hello" == "hello"], true); - assert_preinterpret_eq!([!evaluate! 'A' as u8 == 65], true); - assert_preinterpret_eq!([!evaluate! 65u8 as char == 'A'], true); - assert_preinterpret_eq!([!evaluate! 'A' == 'A'], true); - assert_preinterpret_eq!([!evaluate! 'A' == 'B'], false); - assert_preinterpret_eq!([!evaluate! 'A' < 'B'], true); - assert_preinterpret_eq!([!evaluate! "Zoo" > "Aardvark"], true); - assert_preinterpret_eq!( - [!debug! [!evaluate! "Hello" as stream + "World" as stream + (1 + 1) as stream]], - r#""Hello" "World" [!group! 2]"# + assert_expression_eq!(#(!!(!!(true))), true); + assert_expression_eq!(#(1 + 5), 6u8); + assert_expression_eq!(#(1 + 5), 6i128); + assert_expression_eq!(#(1 + 5u16), 6u16); + assert_expression_eq!(#(127i8 + (-127i8) + (-127i8)), -127i8); + assert_expression_eq!(#(3.0 + 3.2), 6.2); + assert_expression_eq!(#(3.6 + 3999999999999999992.0), 3.6 + 3999999999999999992.0); + assert_expression_eq!(#(-3.2), -3.2); + assert_expression_eq!(#(true && true || false), true); + assert_expression_eq!(#(true || false && false), true); // The && has priority + assert_expression_eq!(#(true | false & false), true); // The & has priority + assert_expression_eq!(#(true as u32 + 2), 3); + assert_expression_eq!(#(3.57 as int + 1), 4u32); + assert_expression_eq!(#(3.57 as int + 1), 4u64); + assert_expression_eq!(#(0b1000 & 0b1101), 0b1000); + assert_expression_eq!(#(0b1000 | 0b1101), 0b1101); + assert_expression_eq!(#(0b1000 ^ 0b1101), 0b101); + assert_expression_eq!(#(5 << 2), 20); + assert_expression_eq!(#(5 >> 1), 2); + assert_expression_eq!(#(123 == 456), false); + assert_expression_eq!(#(123 < 456), true); + assert_expression_eq!(#(123 <= 456), true); + assert_expression_eq!(#(123 != 456), true); + assert_expression_eq!(#(123 >= 456), false); + assert_expression_eq!(#(123 > 456), false); + assert_expression_eq!(#(six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); + assert_expression_eq!(#( + partial_sum = [+ 2]; + [!debug! #([5] + partial_sum) = [!reinterpret! [!raw! #](5 #..partial_sum)]] + ), "[!group! 5 + 2] = [!group! 7]"); + assert_expression_eq!(#(1 + [!range! 1..2] as int), 2); + assert_expression_eq!(#("hello" == "world"), false); + assert_expression_eq!(#("hello" == "hello"), true); + assert_expression_eq!(#('A' as u8 == 65), true); + assert_expression_eq!(#(65u8 as char == 'A'), true); + assert_expression_eq!(#('A' == 'A'), true); + assert_expression_eq!(#('A' == 'B'), false); + assert_expression_eq!(#('A' < 'B'), true); + assert_expression_eq!(#("Zoo" > "Aardvark"), true); + assert_expression_eq!( + #([!debug! #..("Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group)]), + r#""Hello" "World" 2 [!group! 2]"# ); } @@ -82,13 +73,13 @@ fn test_expression_precedence() { // * Operators at the same precedence should left-associate. // 1 + -1 + ((2 + 4) * 3) - 9 => 1 + -1 + 18 - 9 => 9 - assert_preinterpret_eq!([!evaluate! 1 + -(1) + (2 + 4) * 3 - 9], 9); + assert_expression_eq!(#(1 + -(1) + (2 + 4) * 3 - 9), 9); // (true > true) > true => false > true => false - assert_preinterpret_eq!([!evaluate! true > true > true], false); + assert_expression_eq!(#(true > true > true), false); // (5 - 2) - 1 => 3 - 1 => 2 - assert_preinterpret_eq!([!evaluate! 5 - 2 - 1], 2); + assert_expression_eq!(#(5 - 2 - 1), 2); // ((3 * 3 - 4) < (3 << 1)) && true => 5 < 6 => true - assert_preinterpret_eq!([!evaluate! 3 * 3 - 4 < 3 << 1 && true], true); + assert_expression_eq!(#(3 * 3 - 4 < 3 << 1 && true), true); } #[test] @@ -98,8 +89,8 @@ fn test_very_long_expression_works() { [!settings! { iteration_limit: 100000, }] - [!set! #expression = 0 [!for! #i in [!range! 0..100000] { + 1 }]] - [!reinterpret! [[!raw! !evaluate!] #expression]] + #(let expression = [0] + [!for! #i in [!range! 0..100000] { + 1 }]) + [!reinterpret! [!raw! #](#expression)] }, 100000 ); @@ -108,51 +99,60 @@ fn test_very_long_expression_works() { #[test] fn boolean_operators_short_circuit() { // && short-circuits if first operand is false - assert_preinterpret_eq!( - { - [!set! #is_lazy = true] - [!set! _ = [!evaluate! false && [!set! #is_lazy = false]]] + assert_expression_eq!( + #( + let is_lazy = true; + let _ = false && #(is_lazy = false; true); #is_lazy - }, + ), true ); // || short-circuits if first operand is true - assert_preinterpret_eq!( - { - [!set! #is_lazy = true] - [!set! _ = [!evaluate! true || [!set! #is_lazy = false]]] + assert_expression_eq!( + #( + let is_lazy = true; + let _ = true || #(is_lazy = false; true); #is_lazy - }, + ), true ); + // For comparison, the & operator does _not_ short-circuit + assert_expression_eq!( + #( + is_lazy = true; + let _ = false & #(is_lazy = false; true); + #is_lazy + ), + false + ); } #[test] fn assign_works() { - assert_preinterpret_eq!( - { - [!assign! #x = 5 + 5] + assert_expression_eq!( + #( + let x = 5 + 5; [!debug! #..x] - }, - "[!group! 10]" + ), + "10" ); - assert_preinterpret_eq!( - { - [!set! #x = 10] - [!assign! #x /= 1 + 1] // (10 / (1 + 1)) => 5 - [!assign! #x += 2 + #x] // (5 + (2 + 5)) => 12 + assert_expression_eq!( + #( + let x = 10; + x /= 1 + 1; // 10 / (1 + 1) + x += 2 + #x; // 5 + (2 + 5) #x - }, + ), 12 ); // Assign can reference itself in its expression, // because the expression result is buffered. - assert_preinterpret_eq!( - { - [!set! #x = 2] // 10 - [!assign! #x += #x] + assert_expression_eq!( + #( + let x = 2; + x += #x; #x - }, + ), 4 ); } diff --git a/tests/tokens.rs b/tests/tokens.rs index 1445bcba..7d36e075 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -191,12 +191,12 @@ fn complex_cases_for_intersperse_and_input_types() { ); // The separator is interpreted each time it is included assert_preinterpret_eq!({ - [!set! #i = 0] + #(let i = 0) [!string! [!intersperse! { items: [A B C D E F G], separator: [ (#i) - [!assign! #i += 1] + #(i += 1) ], add_trailing: true, }]] From 3b04d3667d346bef8814d73369e2485f4463fc56 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 11 Feb 2025 14:31:59 +0000 Subject: [PATCH 083/126] fix: Fix styling / raise clippy issue --- CHANGELOG.md | 4 ++-- tests/control_flow.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7010a987..ee0b9be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,7 +119,7 @@ Inside a transform stream, the following grammar is supported: stream for the object, or error and suggest fields the user should use instead. * Have `!zip!` support `{ objects }` * Values: - * When we parse a `#x` (`@(#x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. + * When we parse a `#x` (`@(x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. * Output to final output as unwrapped content * Method calls * Also add support for methods (for e.g. exposing functions on syn objects). @@ -144,7 +144,7 @@ Inside a transform stream, the following grammar is supported: * Scrap `#>>x` etc in favour of `@(#x += ...)` * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * If the `[!split!]` command should actually be a transformer? - * Adding `!define_macro!` + * Adding `preinterpret::macro` * Adding `!define_command!` * Adding `!define_transformer!` * `[!is_set! #x]` diff --git a/tests/control_flow.rs b/tests/control_flow.rs index edf1b45d..cb8ec038 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -1,4 +1,5 @@ #![allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 +#![allow(clippy::zero_prefixed_literal)] // https://github.com/rust-lang/rust-clippy/issues/14199 use preinterpret::preinterpret; From e4a9e5781958cee7997e9085b752e6db5fd3a60b Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 12 Feb 2025 09:34:40 +0000 Subject: [PATCH 084/126] refactor: Variables now store expression values --- CHANGELOG.md | 46 ++-- src/expressions/expression.rs | 17 +- src/expressions/mod.rs | 2 +- src/expressions/stream.rs | 4 +- src/expressions/string.rs | 4 +- src/expressions/value.rs | 13 +- src/interpretation/commands/core_commands.rs | 10 +- src/interpretation/commands/token_commands.rs | 16 +- src/interpretation/interpreter.rs | 38 ++- src/interpretation/source_stream.rs | 18 +- src/interpretation/source_stream_input.rs | 36 +-- src/interpretation/source_value.rs | 21 +- src/interpretation/variable.rs | 230 +++++++++--------- src/transformation/exact_stream.rs | 26 +- src/transformation/transform_stream.rs | 4 +- src/transformation/variable_binding.rs | 32 +-- .../core/error_span_repeat.stderr | 11 +- .../core/extend_non_existing_variable.stderr | 2 +- .../transforming/append_before_set.stderr | 2 +- tests/core.rs | 6 +- tests/expressions.rs | 3 +- tests/tokens.rs | 11 + 22 files changed, 288 insertions(+), 264 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee0b9be9..02d8e07d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,13 +54,14 @@ The following are recognized values: * String literals * Char literals * Other literals -* Token streams which are defined as `[...]` and can be appended with the `+` operator. +* Token streams which are defined as `[...]`. The following operators are supported: * The numeric operators: `+ - * / % & | ^` * The lazy boolean operators: `|| &&` * The comparison operators: `== != < > <= >=` * The shift operators: `>> <<` +* The concatenation operator: `+` can be used to concatenate strings and streams. * Casting with `as` including to untyped integers/floats with `as int` and `as float`, to a grouped stream with `as group` and to a flattened stream with `as stream`. * () and none-delimited groups for precedence @@ -96,18 +97,28 @@ Inside a transform stream, the following grammar is supported: ### To come -* Complete `ExpressionBlock` support: - * Change variables to store expression values - * Support `#(x[..])` syntax for indexing streams at read time - * Via a post-fix `[...]` operator - * `#(x[0])` - * `#(x[0..3])` returns a TokenStream - * `#(x[0..=3])` returns a TokenStream - * Add `+` support for concatenating strings - * Create an `enum MarkedVariable { Grouped(GroupedMarkedVariable), Flattened(FlattenedMarkedVariable) }` - * Revisit the `SourceStreamInput` abstraction +* Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model +```rust +#(x = [!stream! Hello World]) +#(x = %{Hello World}) +#(x = %[Hello World]) // Prefer this one so far +#(x = stream { Hello World }) +``` + * Replace `[!output! ...]` with `[!stream! ...]` + * Revisit the `SourceStreamInput` abstraction - maybe it's replaced with a `SourceExpression` which needs to output a stream? + * Split outputs an array + * Intersperse works with either an array or a stream (?) + * For works only with an array (in future, also an iterator) + * We can then consider dropping lots of the `group` wrappers I guess? + * Add `+` support for concatenating arrays + * Then destructuring and parsing become different: + * Destructuring works over the value model; parsing works over the token stream model. +* Support `#(x[..])` syntax for indexing arrays and streams at read time + * Via a post-fix `[...]` operation with high priority + * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) + * `#(x[0..3])` returns a TokenStream + * `#(x[0..=3])` returns a TokenStream * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: - * Separate `&mut Output` and `StreamOutput`, `ValueOutput = ExpressionOutput`, `ObjectOutput` * Objects: * Can be created with `#({ a: x, ... })` * Can be destructured with `@{ #hello, world: _, ... }` or read with `#(x.hello)` or `#(x["hello"])` @@ -121,11 +132,11 @@ Inside a transform stream, the following grammar is supported: * Values: * When we parse a `#x` (`@(x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. * Output to final output as unwrapped content - * Method calls - * Also add support for methods (for e.g. exposing functions on syn objects). - * `.len()` on stream - * Consider `.map(|| {})` -* Destructurers => Transformers cont +* Method calls + * Also add support for methods (for e.g. exposing functions on syn objects). + * `.len()` on stream + * Consider `.map(|| {})` +* TRANSFORMERS => PARSERS cont * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. * `@[ANY_GROUP ...]` @@ -151,6 +162,7 @@ Inside a transform stream, the following grammar is supported: * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) * Maybe add a `@[REINTERPRET ..]` transformer. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` +* Put `[!set! ...]` inside an opt-in feature. * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed * Work on book diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index c0cd37bf..d197a31f 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -29,9 +29,8 @@ impl InterpretToValue for &SourceExpression { pub(super) enum SourceExpressionLeaf { Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), VariablePath(VariablePath), + MarkedVariable(MarkedVariable), ExpressionBlock(ExpressionBlock), ExplicitStream(SourceGroup), Value(ExpressionValue), @@ -44,11 +43,8 @@ impl Expressionable for Source { fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), - SourcePeekMatch::Variable(Grouping::Grouped) => { - UnaryAtom::Leaf(Self::Leaf::GroupedVariable(input.parse()?)) - } - SourcePeekMatch::Variable(Grouping::Flattened) => { - UnaryAtom::Leaf(Self::Leaf::FlattenedVariable(input.parse()?)) + SourcePeekMatch::Variable(_) => { + UnaryAtom::Leaf(Self::Leaf::MarkedVariable(input.parse()?)) } SourcePeekMatch::ExpressionBlock(_) => { UnaryAtom::Leaf(Self::Leaf::ExpressionBlock(input.parse()?)) @@ -110,11 +106,8 @@ impl Expressionable for Source { SourceExpressionLeaf::Command(command) => { command.clone().interpret_to_value(interpreter)? } - SourceExpressionLeaf::GroupedVariable(grouped_variable) => { - grouped_variable.interpret_to_value(interpreter)? - } - SourceExpressionLeaf::FlattenedVariable(flattened_variable) => { - flattened_variable.interpret_to_value(interpreter)? + SourceExpressionLeaf::MarkedVariable(variable) => { + variable.interpret_to_value(interpreter)? } SourceExpressionLeaf::VariablePath(variable_path) => { variable_path.interpret_to_value(interpreter)? diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 98a6314f..5f0ad137 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -20,12 +20,12 @@ use expression_parsing::*; use float::*; use integer::*; use operations::*; -use stream::*; use string::*; use value::*; pub(crate) use expression::*; pub(crate) use expression_block::*; +pub(crate) use stream::*; pub(crate) use value::*; // For some mysterious reason Rust-analyzer can't resolve this without an explicit export pub(crate) use value::ExpressionValue; diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs index ea33d760..27e76a83 100644 --- a/src/expressions/stream.rs +++ b/src/expressions/stream.rs @@ -2,11 +2,11 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionStream { - pub(super) value: OutputStream, + pub(crate) value: OutputStream, /// The span range that generated this value. /// For a complex expression, the start span is the most left part /// of the expression, and the end span is the most right part. - pub(super) span_range: SpanRange, + pub(crate) span_range: SpanRange, } impl ExpressionStream { diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 624d9f26..131919a0 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -57,8 +57,8 @@ impl ExpressionString { let lhs = self.value; let rhs = rhs.value; Ok(match operation.operation { - PairedBinaryOperation::Addition { .. } - | PairedBinaryOperation::Subtraction { .. } + PairedBinaryOperation::Addition { .. } => operation.output(lhs + &rhs), + PairedBinaryOperation::Subtraction { .. } | PairedBinaryOperation::Multiplication { .. } | PairedBinaryOperation::Division { .. } | PairedBinaryOperation::LogicalAnd { .. } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index c698cce9..11785f1a 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -334,6 +334,11 @@ impl ExpressionValue { self } + pub(crate) fn with_span_range(mut self, source_span_range: SpanRange) -> ExpressionValue { + *self.span_range_mut() = source_span_range; + self + } + pub(crate) fn into_new_output_stream(self, grouping: Grouping) -> OutputStream { match (self, grouping) { (Self::Stream(value), Grouping::Flattened) => value.value, @@ -345,7 +350,7 @@ impl ExpressionValue { } } - pub(crate) fn output_to(self, grouping: Grouping, output: &mut OutputStream) { + pub(crate) fn output_to(&self, grouping: Grouping, output: &mut OutputStream) { match grouping { Grouping::Grouped => { // Grouping can be important for different values, to ensure they're read atomically @@ -370,7 +375,7 @@ impl ExpressionValue { } } - fn output_flattened_to(self, output: &mut OutputStream) { + fn output_flattened_to(&self, output: &mut OutputStream) { match self { Self::None { .. } => {} Self::Integer(value) => output.push_literal(value.to_literal()), @@ -379,9 +384,9 @@ impl ExpressionValue { Self::String(value) => output.push_literal(value.to_literal()), Self::Char(value) => output.push_literal(value.to_literal()), Self::UnsupportedLiteral(literal) => { - output.extend_raw_tokens(literal.lit.into_token_stream()) + output.extend_raw_tokens(literal.lit.to_token_stream()) } - Self::Stream(value) => value.value.append_into(output), + Self::Stream(value) => value.value.append_cloned_into(output), } } } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 411ae265..53d3ec7f 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -89,18 +89,20 @@ impl NoOutputCommandDefinition for SetCommand { variable, content, .. } => { let content = content.interpret_to_new_stream(interpreter)?; - variable.set(interpreter, content)?; + variable.set_stream(interpreter, content)?; } SetArguments::ExtendVariable { variable, content, .. } => { let variable_data = variable.get_existing_for_mutation(interpreter)?; - content - .interpret_into(interpreter, variable_data.get_mut(&variable)?.deref_mut())?; + content.interpret_into( + interpreter, + variable_data.get_mut_stream(&variable)?.deref_mut(), + )?; } SetArguments::SetVariablesEmpty { variables } => { for variable in variables { - variable.set(interpreter, OutputStream::new())?; + variable.set_stream(interpreter, OutputStream::new())?; } } SetArguments::Discard { content, .. } => { diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 50ac75c2..46af6708 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -299,6 +299,17 @@ fn handle_split( // Typically the separator won't contain none-delimited groups, so we're OK input.parse_with(move |input| { let mut current_item = OutputStream::new(); + + // Special case separator.len() == 0 to avoid an infinite loop + if separator.len() == 0 { + while !input.is_empty() { + current_item.push_raw_token_tree(input.parse()?); + let complete_item = core::mem::replace(&mut current_item, OutputStream::new()); + output.push_new_group(complete_item, Delimiter::None, output_span); + } + return Ok(()); + } + let mut drop_empty_next = drop_empty_start; while !input.is_empty() { let separator_fork = input.fork(); @@ -314,14 +325,15 @@ fn handle_split( current_item.push_raw_token_tree(input.parse()?); continue; } + // This is guaranteed to progress the parser because the separator is non-empty input.advance_to(&separator_fork); - if !(current_item.is_empty() && drop_empty_next) { + if !current_item.is_empty() || !drop_empty_next { let complete_item = core::mem::replace(&mut current_item, OutputStream::new()); output.push_new_group(complete_item, Delimiter::None, output_span); } drop_empty_next = drop_empty_middle; } - if !(current_item.is_empty() && drop_empty_end) { + if !current_item.is_empty() || !drop_empty_end { output.push_new_group(current_item, Delimiter::None, output_span); } Ok(()) diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 700657f8..481bc9e7 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -12,11 +12,11 @@ pub(crate) struct Interpreter { #[derive(Clone)] pub(crate) struct VariableData { - value: Rc>, + value: Rc>, } impl VariableData { - fn new(tokens: OutputStream) -> Self { + fn new(tokens: ExpressionValue) -> Self { Self { value: Rc::new(RefCell::new(tokens)), } @@ -24,8 +24,8 @@ impl VariableData { pub(crate) fn get<'d>( &'d self, - variable: &impl IsVariable, - ) -> ExecutionResult> { + variable: &(impl IsVariable + ?Sized), + ) -> ExecutionResult> { self.value.try_borrow().map_err(|_| { variable .error("The variable cannot be read if it is currently being modified") @@ -35,8 +35,8 @@ impl VariableData { pub(crate) fn get_mut<'d>( &'d self, - variable: &impl IsVariable, - ) -> ExecutionResult> { + variable: &(impl IsVariable + ?Sized), + ) -> ExecutionResult> { self.value.try_borrow_mut().map_err(|_| { variable.execution_error( "The variable cannot be modified if it is already currently being modified", @@ -44,10 +44,22 @@ impl VariableData { }) } + pub(crate) fn get_mut_stream<'d>( + &'d self, + variable: &(impl IsVariable + ?Sized), + ) -> ExecutionResult> { + let mut_guard = self.get_mut(variable)?; + RefMut::filter_map(mut_guard, |mut_guard| match mut_guard { + ExpressionValue::Stream(stream) => Some(&mut stream.value), + _ => None, + }) + .map_err(|_| variable.execution_error("The variable is not a stream")) + } + pub(crate) fn set( &self, - variable: &impl IsVariable, - content: OutputStream, + variable: &(impl IsVariable + ?Sized), + content: ExpressionValue, ) -> ExecutionResult<()> { *self.get_mut(variable)? = content; Ok(()) @@ -70,15 +82,15 @@ impl Interpreter { pub(crate) fn set_variable( &mut self, - variable: &impl IsVariable, - tokens: OutputStream, + variable: &(impl IsVariable + ?Sized), + value: ExpressionValue, ) -> ExecutionResult<()> { match self.variable_data.entry(variable.get_name()) { Entry::Occupied(mut entry) => { - entry.get_mut().set(variable, tokens)?; + entry.get_mut().set(variable, value)?; } Entry::Vacant(entry) => { - entry.insert(VariableData::new(tokens)); + entry.insert(VariableData::new(value)); } } Ok(()) @@ -86,7 +98,7 @@ impl Interpreter { pub(crate) fn get_existing_variable_data( &self, - variable: &impl IsVariable, + variable: &(impl IsVariable + ?Sized), make_error: impl FnOnce() -> SynError, ) -> ExecutionResult<&VariableData> { self.variable_data diff --git a/src/interpretation/source_stream.rs b/src/interpretation/source_stream.rs index f662c482..6e159ccd 100644 --- a/src/interpretation/source_stream.rs +++ b/src/interpretation/source_stream.rs @@ -41,8 +41,7 @@ impl HasSpan for SourceStream { #[derive(Clone)] pub(crate) enum SourceItem { Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), + Variable(MarkedVariable), ExpressionBlock(ExpressionBlock), SourceGroup(SourceGroup), Punct(Punct), @@ -55,12 +54,7 @@ impl Parse for SourceItem { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => SourceItem::Command(input.parse()?), SourcePeekMatch::Group(_) => SourceItem::SourceGroup(input.parse()?), - SourcePeekMatch::Variable(Grouping::Grouped) => { - SourceItem::GroupedVariable(input.parse()?) - } - SourcePeekMatch::Variable(Grouping::Flattened) => { - SourceItem::FlattenedVariable(input.parse()?) - } + SourcePeekMatch::Variable(_) => SourceItem::Variable(input.parse()?), SourcePeekMatch::ExpressionBlock(_) => SourceItem::ExpressionBlock(input.parse()?), SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => { return input.parse_err("Destructurings are not supported here. If this wasn't intended to be a destructuring, replace @ with [!raw! @]"); @@ -86,10 +80,7 @@ impl Interpret for SourceItem { SourceItem::Command(command_invocation) => { command_invocation.interpret_into(interpreter, output)?; } - SourceItem::GroupedVariable(variable) => { - variable.interpret_into(interpreter, output)?; - } - SourceItem::FlattenedVariable(variable) => { + SourceItem::Variable(variable) => { variable.interpret_into(interpreter, output)?; } SourceItem::ExpressionBlock(block) => { @@ -110,8 +101,7 @@ impl HasSpanRange for SourceItem { fn span_range(&self) -> SpanRange { match self { SourceItem::Command(command_invocation) => command_invocation.span_range(), - SourceItem::FlattenedVariable(variable) => variable.span_range(), - SourceItem::GroupedVariable(variable) => variable.span_range(), + SourceItem::Variable(variable) => variable.span_range(), SourceItem::ExpressionBlock(block) => block.span_range(), SourceItem::SourceGroup(group) => group.span_range(), SourceItem::Punct(punct) => punct.span_range(), diff --git a/src/interpretation/source_stream_input.rs b/src/interpretation/source_stream_input.rs index 4b16b2f6..38ad7791 100644 --- a/src/interpretation/source_stream_input.rs +++ b/src/interpretation/source_stream_input.rs @@ -15,8 +15,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum SourceStreamInput { Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), + Variable(MarkedVariable), Code(SourceCodeBlock), ExplicitStream(SourceGroup), } @@ -25,8 +24,7 @@ impl Parse for SourceStreamInput { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::Variable(Grouping::Grouped) => Self::GroupedVariable(input.parse()?), - SourcePeekMatch::Variable(Grouping::Flattened) => Self::FlattenedVariable(input.parse()?), + SourcePeekMatch::Variable(_) => Self::Variable(input.parse()?), SourcePeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), _ => input.span() @@ -39,8 +37,7 @@ impl HasSpanRange for SourceStreamInput { fn span_range(&self) -> SpanRange { match self { SourceStreamInput::Command(command) => command.span_range(), - SourceStreamInput::GroupedVariable(variable) => variable.span_range(), - SourceStreamInput::FlattenedVariable(variable) => variable.span_range(), + SourceStreamInput::Variable(variable) => variable.span_range(), SourceStreamInput::Code(code) => code.span_range(), SourceStreamInput::ExplicitStream(group) => group.span_range(), } @@ -88,19 +85,22 @@ impl Interpret for SourceStreamInput { ), } } - SourceStreamInput::FlattenedVariable(variable) => parse_as_stream_input( - &variable, - interpreter, - || { - format!( - "Expected variable to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to use {} instead, to use the content of the variable as the stream.", - variable.display_grouped_variable_token(), + SourceStreamInput::Variable(MarkedVariable::Flattened(variable)) => { + parse_as_stream_input( + &variable, + interpreter, + || { + format!( + "Expected variable to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to use #{} instead, to use the content of the variable as the stream.", + variable.get_name(), ) - }, - output, - ), - SourceStreamInput::GroupedVariable(variable) => { - variable.substitute_ungrouped_contents_into(interpreter, output) + }, + output, + ) + } + SourceStreamInput::Variable(MarkedVariable::Grouped(variable)) => { + // We extract its contents via explicitly using Grouping::Flattened here + variable.substitute_into(interpreter, Grouping::Flattened, output) } SourceStreamInput::Code(code) => parse_as_stream_input( code, diff --git a/src/interpretation/source_value.rs b/src/interpretation/source_value.rs index 0d8ce2cb..be035c8a 100644 --- a/src/interpretation/source_value.rs +++ b/src/interpretation/source_value.rs @@ -7,8 +7,7 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) enum SourceValue { Command(Command), - GroupedVariable(GroupedVariable), - FlattenedVariable(FlattenedVariable), + Variable(MarkedVariable), ExpressionBlock(ExpressionBlock), Code(SourceCodeBlock), Value(T), @@ -18,10 +17,7 @@ impl> Parse for SourceValue { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::Variable(Grouping::Grouped) => Self::GroupedVariable(input.parse()?), - SourcePeekMatch::Variable(Grouping::Flattened) => { - Self::FlattenedVariable(input.parse()?) - } + SourcePeekMatch::Variable(_) => Self::Variable(input.parse()?), SourcePeekMatch::ExpressionBlock(_) => Self::ExpressionBlock(input.parse()?), SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), SourcePeekMatch::ExplicitTransformStream @@ -50,8 +46,7 @@ impl HasSpanRange for SourceValue { fn span_range(&self) -> SpanRange { match self { SourceValue::Command(command) => command.span_range(), - SourceValue::GroupedVariable(variable) => variable.span_range(), - SourceValue::FlattenedVariable(variable) => variable.span_range(), + SourceValue::Variable(variable) => variable.span_range(), SourceValue::ExpressionBlock(block) => block.span_range(), SourceValue::Code(code) => code.span_range(), SourceValue::Value(value) => value.span_range(), @@ -65,20 +60,14 @@ impl, I: Parse> InterpretToValue fo fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { let descriptor = match self { SourceValue::Command(_) => "command output", - SourceValue::GroupedVariable(_) => "grouped variable output", - SourceValue::FlattenedVariable(_) => "flattened variable output", + SourceValue::Variable(_) => "variable output", SourceValue::ExpressionBlock(_) => "an #(...) expression block", SourceValue::Code(_) => "output from the { ... } block", SourceValue::Value(_) => "value", }; let interpreted_stream = match self { SourceValue::Command(command) => command.interpret_to_new_stream(interpreter)?, - SourceValue::GroupedVariable(variable) => { - variable.interpret_to_new_stream(interpreter)? - } - SourceValue::FlattenedVariable(variable) => { - variable.interpret_to_new_stream(interpreter)? - } + SourceValue::Variable(variable) => variable.interpret_to_new_stream(interpreter)?, SourceValue::ExpressionBlock(block) => block.interpret_to_new_stream(interpreter)?, SourceValue::Code(code) => code.interpret_to_new_stream(interpreter)?, SourceValue::Value(value) => return value.interpret_to_value(interpreter), diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 1c988959..b217c55c 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -2,89 +2,149 @@ use crate::internal_prelude::*; pub(crate) trait IsVariable: HasSpanRange { fn get_name(&self) -> String; -} - -#[derive(Clone)] -pub(crate) struct GroupedVariable { - marker: Token![#], - variable_name: Ident, -} -impl Parse for GroupedVariable { - fn parse(input: ParseStream) -> ParseResult { - input.try_parse_or_error( - |input| { - Ok(Self { - marker: input.parse()?, - variable_name: input.parse_any_ident()?, - }) - }, - "Expected #variable", - ) + fn set_stream( + &self, + interpreter: &mut Interpreter, + stream: OutputStream, + ) -> ExecutionResult<()> { + interpreter.set_variable(self, stream.to_value(self.span_range())) } -} -impl GroupedVariable { - pub(crate) fn set( + fn set_coerced_stream( &self, interpreter: &mut Interpreter, - value: OutputStream, + stream: OutputStream, ) -> ExecutionResult<()> { - interpreter.set_variable(self, value) + interpreter.set_variable(self, stream.coerce_into_value(self.span_range())) } - pub(crate) fn set_value( + fn set_value( &self, interpreter: &mut Interpreter, value: ExpressionValue, ) -> ExecutionResult<()> { - // It will be grouped on the way out; not it. - interpreter.set_variable(self, value.into_new_output_stream(Grouping::Flattened)) + interpreter.set_variable(self, value) } - pub(crate) fn get_existing_for_mutation( + fn get_existing_for_mutation( &self, interpreter: &Interpreter, ) -> ExecutionResult { - Ok(interpreter - .get_existing_variable_data(self, || { - self.error(format!("The variable {} wasn't already set", self)) - })? - .cheap_clone()) + Ok(self.read_existing(interpreter)?.cheap_clone()) + } + + fn get_value(&self, interpreter: &Interpreter) -> ExecutionResult { + let value = self + .read_existing(interpreter)? + .get(self)? + .clone() + .with_span_range(self.span_range()); + Ok(value) } - pub(crate) fn substitute_ungrouped_contents_into( + fn substitute_into( &self, interpreter: &mut Interpreter, + grouping: Grouping, output: &mut OutputStream, ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? - .append_cloned_into(output); + .output_to(grouping, output); Ok(()) } - pub(crate) fn substitute_grouped_into( - &self, + fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { + interpreter.get_existing_variable_data(self, || { + self.error("The variable does not already exist in the current scope") + }) + } +} + +#[derive(Clone)] +pub(crate) enum MarkedVariable { + Grouped(GroupedVariable), + Flattened(FlattenedVariable), +} + +impl Parse for MarkedVariable { + fn parse(input: ParseStream) -> ParseResult { + if input.peek2(Token![..]) { + Ok(MarkedVariable::Flattened(input.parse()?)) + } else { + Ok(MarkedVariable::Grouped(input.parse()?)) + } + } +} + +impl HasSpanRange for MarkedVariable { + fn span_range(&self) -> SpanRange { + match self { + MarkedVariable::Grouped(variable) => variable.span_range(), + MarkedVariable::Flattened(variable) => variable.span_range(), + } + } +} + +impl HasSpanRange for &MarkedVariable { + fn span_range(&self) -> SpanRange { + ::span_range(self) + } +} + +impl IsVariable for MarkedVariable { + fn get_name(&self) -> String { + match self { + MarkedVariable::Grouped(variable) => variable.get_name(), + MarkedVariable::Flattened(variable) => variable.get_name(), + } + } +} + +impl Interpret for &MarkedVariable { + fn interpret_into( + self, interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - output.push_new_group( - self.read_existing(interpreter)?.get(self)?.clone(), - Delimiter::None, - self.span_range().join_into_span_else_start(), - ); - Ok(()) + match self { + MarkedVariable::Grouped(variable) => variable.interpret_into(interpreter, output), + MarkedVariable::Flattened(variable) => variable.interpret_into(interpreter, output), + } } +} - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { - interpreter.get_existing_variable_data( - self, - || self.error(format!( - "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self, - self, - )), +impl InterpretToValue for &MarkedVariable { + type OutputValue = ExpressionValue; + + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + match self { + MarkedVariable::Grouped(variable) => variable.interpret_to_value(interpreter), + MarkedVariable::Flattened(variable) => variable.interpret_to_value(interpreter), + } + } +} + +#[derive(Clone)] +pub(crate) struct GroupedVariable { + marker: Token![#], + variable_name: Ident, +} + +impl Parse for GroupedVariable { + fn parse(input: ParseStream) -> ParseResult { + input.try_parse_or_error( + |input| { + Ok(Self { + marker: input.parse()?, + variable_name: input.parse_any_ident()?, + }) + }, + "Expected #variable", ) } } @@ -101,7 +161,7 @@ impl Interpret for &GroupedVariable { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.substitute_grouped_into(interpreter, output) + self.substitute_into(interpreter, Grouping::Grouped, output) } } @@ -112,12 +172,7 @@ impl InterpretToValue for &GroupedVariable { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - let value = self - .read_existing(interpreter)? - .get(self)? - .clone() - .coerce_into_value(self.span_range()); - Ok(value) + self.get_value(interpreter) } } @@ -129,7 +184,7 @@ impl HasSpanRange for GroupedVariable { impl HasSpanRange for &GroupedVariable { fn span_range(&self) -> SpanRange { - GroupedVariable::span_range(self) + ::span_range(self) } } @@ -162,34 +217,6 @@ impl Parse for FlattenedVariable { } } -impl FlattenedVariable { - pub(crate) fn substitute_into( - &self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - self.read_existing(interpreter)? - .get(self)? - .append_cloned_into(output); - Ok(()) - } - - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { - interpreter.get_existing_variable_data( - self, - || self.error(format!( - "The variable {} wasn't set.\nIf this wasn't intended to be a variable, work around this with [!raw! {}]", - self.variable_name, - self, - )), - ) - } - - pub(crate) fn display_grouped_variable_token(&self) -> String { - format!("#{}", self.variable_name) - } -} - impl IsVariable for FlattenedVariable { fn get_name(&self) -> String { self.variable_name.to_string() @@ -202,7 +229,7 @@ impl Interpret for &FlattenedVariable { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.substitute_into(interpreter, output) + self.substitute_into(interpreter, Grouping::Flattened, output) } } @@ -213,9 +240,10 @@ impl InterpretToValue for &FlattenedVariable { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - let mut output_stream = OutputStream::new(); - self.substitute_into(interpreter, &mut output_stream)?; - Ok(output_stream.to_value(self.span_range())) + // We always output a stream value for a flattened variable + Ok(self + .interpret_to_new_stream(interpreter)? + .to_value(self.span_range())) } } @@ -227,7 +255,7 @@ impl HasSpanRange for FlattenedVariable { impl HasSpanRange for &FlattenedVariable { fn span_range(&self) -> SpanRange { - FlattenedVariable::span_range(self) + ::span_range(self) } } @@ -259,17 +287,6 @@ impl Parse for VariablePath { } } -impl VariablePath { - pub(crate) fn set_value( - &self, - interpreter: &mut Interpreter, - value: ExpressionValue, - ) -> ExecutionResult<()> { - // It will be grouped on the way out, not in - interpreter.set_variable(self, value.into_new_output_stream(Grouping::Flattened)) - } -} - impl IsVariable for VariablePath { fn get_name(&self) -> String { self.root.to_string() @@ -292,13 +309,6 @@ impl InterpretToValue for &VariablePath { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - let value = interpreter - .get_existing_variable_data(self, || { - self.error(format!("The variable {} wasn't set.", &self.root,)) - })? - .get(self)? - .clone() - .coerce_into_value(self.span_range()); - Ok(value) + self.get_value(interpreter) } } diff --git a/src/transformation/exact_stream.rs b/src/transformation/exact_stream.rs index 987047ed..e575e454 100644 --- a/src/transformation/exact_stream.rs +++ b/src/transformation/exact_stream.rs @@ -31,6 +31,12 @@ where } } +impl ExactSegment { + pub(crate) fn len(&self) -> usize { + self.inner.len() + } +} + impl HandleTransformation for ExactSegment { fn handle_transform( &self, @@ -51,8 +57,7 @@ pub(crate) enum ExactItem { TransformStreamInput(ExplicitTransformStream), Transformer(Transformer), ExactCommandOutput(Command), - ExactGroupedVariableOutput(GroupedVariable), - ExactFlattenedVariableOutput(FlattenedVariable), + ExactVariableOutput(MarkedVariable), ExactExpressionBlock(ExpressionBlock), ExactPunct(Punct), ExactIdent(Ident), @@ -64,12 +69,7 @@ impl Parse for ExactItem { fn parse(input: ParseStream) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::ExactCommandOutput(input.parse()?), - SourcePeekMatch::Variable(Grouping::Grouped) => { - Self::ExactGroupedVariableOutput(input.parse()?) - } - SourcePeekMatch::Variable(Grouping::Flattened) => { - Self::ExactFlattenedVariableOutput(input.parse()?) - } + SourcePeekMatch::Variable(_) => Self::ExactVariableOutput(input.parse()?), SourcePeekMatch::ExpressionBlock(_) => Self::ExactExpressionBlock(input.parse()?), SourcePeekMatch::AppendVariableBinding => { return input @@ -119,14 +119,8 @@ impl HandleTransformation for ExactItem { .into_exact_stream()? .handle_transform(input, interpreter, output)?; } - ExactItem::ExactGroupedVariableOutput(grouped_variable) => { - grouped_variable - .interpret_to_new_stream(interpreter)? - .into_exact_stream()? - .handle_transform(input, interpreter, output)?; - } - ExactItem::ExactFlattenedVariableOutput(flattened_variable) => { - flattened_variable + ExactItem::ExactVariableOutput(variable) => { + variable .interpret_to_new_stream(interpreter)? .into_exact_stream()? .handle_transform(input, interpreter, output)?; diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index ce711a5d..18f3c604 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -240,7 +240,7 @@ impl HandleTransformation for ExplicitTransformStream { } => { let mut new_output = OutputStream::new(); content.handle_transform(input, interpreter, &mut new_output)?; - variable.set(interpreter, new_output)?; + variable.set_stream(interpreter, new_output)?; } ExplicitTransformStreamArguments::ExtendToVariable { variable, content, .. @@ -249,7 +249,7 @@ impl HandleTransformation for ExplicitTransformStream { content.handle_transform( input, interpreter, - variable_data.get_mut(variable)?.deref_mut(), + variable_data.get_mut_stream(variable)?.deref_mut(), )?; } ExplicitTransformStreamArguments::Discard { content, .. } => { diff --git a/src/transformation/variable_binding.rs b/src/transformation/variable_binding.rs index b1dd31c6..7db29363 100644 --- a/src/transformation/variable_binding.rs +++ b/src/transformation/variable_binding.rs @@ -166,18 +166,6 @@ impl VariableBinding { | VariableBinding::FlattenedAppendFlattened { .. } ) } - - fn get_variable_data(&self, interpreter: &mut Interpreter) -> ExecutionResult { - let variable_data = interpreter - .get_existing_variable_data(self, || { - self.error(format!( - "The variable #{} wasn't already set", - self.get_name() - )) - })? - .cheap_clone(); - Ok(variable_data) - } } impl HandleTransformation for VariableBinding { @@ -190,36 +178,36 @@ impl HandleTransformation for VariableBinding { match self { VariableBinding::Grouped { .. } => { let content = input.parse::()?.into_interpreted(); - interpreter.set_variable(self, content)?; + self.set_coerced_stream(interpreter, content)?; } VariableBinding::Flattened { until, .. } => { let mut content = OutputStream::new(); until.handle_parse_into(input, &mut content)?; - interpreter.set_variable(self, content)?; + self.set_coerced_stream(interpreter, content)?; } VariableBinding::GroupedAppendGrouped { .. } => { - let variable_data = self.get_variable_data(interpreter)?; + let variable_data = self.get_existing_for_mutation(interpreter)?; input .parse::()? - .push_as_token_tree(variable_data.get_mut(self)?.deref_mut()); + .push_as_token_tree(variable_data.get_mut_stream(self)?.deref_mut()); } VariableBinding::GroupedAppendFlattened { .. } => { - let variable_data = self.get_variable_data(interpreter)?; + let variable_data = self.get_existing_for_mutation(interpreter)?; input .parse::()? - .flatten_into(variable_data.get_mut(self)?.deref_mut()); + .flatten_into(variable_data.get_mut_stream(self)?.deref_mut()); } VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { - let variable_data = self.get_variable_data(interpreter)?; - variable_data.get_mut(self)?.push_grouped( + let variable_data = self.get_existing_for_mutation(interpreter)?; + variable_data.get_mut_stream(self)?.push_grouped( |inner| until.handle_parse_into(input, inner), Delimiter::None, marker.span, )?; } VariableBinding::FlattenedAppendFlattened { until, .. } => { - let variable_data = self.get_variable_data(interpreter)?; - until.handle_parse_into(input, variable_data.get_mut(self)?.deref_mut())?; + let variable_data = self.get_existing_for_mutation(interpreter)?; + until.handle_parse_into(input, variable_data.get_mut_stream(self)?.deref_mut())?; } } Ok(()) diff --git a/tests/compilation_failures/core/error_span_repeat.stderr b/tests/compilation_failures/core/error_span_repeat.stderr index 56694847..44b72dc4 100644 --- a/tests/compilation_failures/core/error_span_repeat.stderr +++ b/tests/compilation_failures/core/error_span_repeat.stderr @@ -1,5 +1,10 @@ -error: Expected 3 inputs, got 4usize - --> tests/compilation_failures/core/error_span_repeat.rs:16:31 +error: Cannot infer common type from stream != untyped integer. Consider using `as` to cast the operands to matching types. + --> tests/compilation_failures/core/error_span_repeat.rs:6:30 | +6 | [!if! (#input_length != 3) { + | ^^ +... 16 | assert_input_length_of_3!(42 101 666 1024); - | ^^^^^^^^^^^^^^^ + | ------------------------------------------ in this macro invocation + | + = note: this error originates in the macro `assert_input_length_of_3` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/compilation_failures/core/extend_non_existing_variable.stderr b/tests/compilation_failures/core/extend_non_existing_variable.stderr index a03f9546..50d3ba93 100644 --- a/tests/compilation_failures/core/extend_non_existing_variable.stderr +++ b/tests/compilation_failures/core/extend_non_existing_variable.stderr @@ -1,4 +1,4 @@ -error: The variable #variable wasn't already set +error: The variable does not already exist in the current scope --> tests/compilation_failures/core/extend_non_existing_variable.rs:5:16 | 5 | [!set! #variable += 2] diff --git a/tests/compilation_failures/transforming/append_before_set.stderr b/tests/compilation_failures/transforming/append_before_set.stderr index 30719a07..0a5734ae 100644 --- a/tests/compilation_failures/transforming/append_before_set.stderr +++ b/tests/compilation_failures/transforming/append_before_set.stderr @@ -1,4 +1,4 @@ -error: The variable #x wasn't already set +error: The variable does not already exist in the current scope --> tests/compilation_failures/transforming/append_before_set.rs:4:26 | 4 | preinterpret!([!let! #>>x = Hello]); diff --git a/tests/core.rs b/tests/core.rs index 51b57f6c..390f5719 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -48,11 +48,11 @@ fn test_extend() { ); assert_preinterpret_eq!( { - [!set! #i = 1] + #(i = 1) [!set! #output = [!..group!]] - [!while! (#i <= 4) { + [!while! i <= 4 { [!set! #output += #i] - [!if! (#i <= 3) { + [!if! i <= 3 { [!set! #output += ", "] }] #(i += 1) diff --git a/tests/expressions.rs b/tests/expressions.rs index 13ad28c5..63cdae38 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -24,6 +24,7 @@ fn test_basic_evaluate_works() { assert_expression_eq!(#(!!(!!(true))), true); assert_expression_eq!(#(1 + 5), 6u8); assert_expression_eq!(#(1 + 5), 6i128); + assert_expression_eq!(#("Hello" + " " + "World!"), "Hello World!"); assert_expression_eq!(#(1 + 5u16), 6u16); assert_expression_eq!(#(127i8 + (-127i8) + (-127i8)), -127i8); assert_expression_eq!(#(3.0 + 3.2), 6.2); @@ -175,7 +176,7 @@ fn test_range() { ); assert_preinterpret_eq!( { - [!set! #x = 2] + #(x = 2) [!string! [!intersperse! { items: [!range! (#x + #x)..=5], separator: [" "], diff --git a/tests/tokens.rs b/tests/tokens.rs index 7d36e075..9e8d9f7c 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -316,6 +316,17 @@ fn complex_cases_for_intersperse_and_input_types() { #[test] fn test_split() { + // Empty separators are allowed, and split on every token + // In this case, drop_empty_start / drop_empty_end are ignored + assert_preinterpret_eq!( + { + [!debug! [!..split! { + stream: [A::B], + separator: [], + }]] + }, + "[!group! A] [!group! :] [!group! :] [!group! B]" + ); // Double separators are allowed assert_preinterpret_eq!( { From 6d8583b24013cbe0ced7748d71af33f3abc89303 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 12 Feb 2025 09:45:08 +0000 Subject: [PATCH 085/126] tweak: Update MSRV to 1.63 for `RefMut::filter_map` --- .github/workflows/ci.yml | 4 ++-- CHANGELOG.md | 14 ++++++++------ Cargo.toml | 3 ++- local-check-msrv.sh | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f0760d7..b60bacb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,14 +41,14 @@ jobs: path: Cargo.lock msrv: - name: MSRV (1.61) Compiles + name: MSRV (1.63) Compiles runs-on: ubuntu-latest timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.61 + toolchain: 1.63 - run: cargo check style-check: diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d8e07d..1952225e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,13 +98,13 @@ Inside a transform stream, the following grammar is supported: ### To come * Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model + * Start with new stream syntax: `%[...]` ```rust #(x = [!stream! Hello World]) #(x = %{Hello World}) #(x = %[Hello World]) // Prefer this one so far #(x = stream { Hello World }) ``` - * Replace `[!output! ...]` with `[!stream! ...]` * Revisit the `SourceStreamInput` abstraction - maybe it's replaced with a `SourceExpression` which needs to output a stream? * Split outputs an array * Intersperse works with either an array or a stream (?) @@ -120,11 +120,13 @@ Inside a transform stream, the following grammar is supported: * `#(x[0..=3])` returns a TokenStream * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Objects: + * Backed by an indexmap * Can be created with `#({ a: x, ... })` - * Can be destructured with `@{ #hello, world: _, ... }` or read with `#(x.hello)` or `#(x["hello"])` - * Debug impl is `#({ hello: [!group! BLAH], ["world"]: Hi, })` + * Can be read with `#(x.hello)` or `#(x["hello"])` + * Debug impl is `#({ hello: [!group! BLAH], ["#world"]: Hi, })` * They have an input object * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` + * Can be destructured with `{ hello, world: _, ... }` * (Until we get custom type support into a syn fork), can be embedded into an output stream as a single token - e.g. `PREINTERPRET_OBJECT_2313` (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default stream for the object, or error and suggest fields the user should use instead. @@ -137,6 +139,8 @@ Inside a transform stream, the following grammar is supported: * `.len()` on stream * Consider `.map(|| {})` * TRANSFORMERS => PARSERS cont + * Transformers no longer output + * Scrap `#>>x` etc in favour of `@(#x += ...)` * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. * `@[ANY_GROUP ...]` @@ -152,9 +156,7 @@ Inside a transform stream, the following grammar is supported: * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` * Scrap `[!let!]` in favour of `[!parse! [...] as @(_ = ...)]` * Consider: - * Scrap `#>>x` etc in favour of `@(#x += ...)` - * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty - * If the `[!split!]` command should actually be a transformer? + * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` * Adding `!define_command!` * Adding `!define_transformer!` diff --git a/Cargo.toml b/Cargo.toml index 7e811f53..7cdd6213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,9 @@ keywords = ["macros", "declarative-macros", "toolkit", "interpreter", "preproces categories = ["development-tools", "compilers"] # MSRV 1.56.0 is the start of Edition 2021 # MSRV 1.61.0 is the current MSRV of syn +# MRSV 1.63.0 is needed to support RefMut::filter_map # If changing this, also update the local-check-msrv.sh script and ci.yml -rust-version = "1.61" +rust-version = "1.63" [lib] proc-macro = true diff --git a/local-check-msrv.sh b/local-check-msrv.sh index 3cce99fb..a307d955 100755 --- a/local-check-msrv.sh +++ b/local-check-msrv.sh @@ -4,5 +4,5 @@ set -e cd "$(dirname "$0")" -rustup install 1.61 -rm Cargo.lock && rustup run 1.61 cargo check +rustup install 1.63 +rm Cargo.lock && rustup run 1.63 cargo check From 7b1f5ce1cc3533b88de5c4bc848f5ac51976c0f3 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 12 Feb 2025 10:48:37 +0000 Subject: [PATCH 086/126] refactor: Trial new stream expression syntax --- src/expressions/expression.rs | 88 ++----------------- src/expressions/expression_block.rs | 31 +++++-- src/expressions/value.rs | 19 ++++ src/extensions/parsing.rs | 15 ++-- src/interpretation/commands/core_commands.rs | 22 +++-- src/misc/parse_traits.rs | 6 +- .../flattened_variables_in_expressions.rs | 2 +- .../flattened_variables_in_expressions.stderr | 6 +- ...ped_variable_with_incomplete_expression.rs | 2 +- ...variable_with_incomplete_expression.stderr | 6 +- tests/core.rs | 10 +-- tests/expressions.rs | 16 ++-- tests/tokens.rs | 30 +++---- tests/transforming.rs | 22 ++--- 14 files changed, 127 insertions(+), 148 deletions(-) diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index d197a31f..d95a1ab1 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -64,9 +64,16 @@ impl Expressionable for Source { return input.parse_err("Braces { ... } are not supported in an expression") } SourcePeekMatch::Group(Delimiter::Bracket) => { - UnaryAtom::Leaf(Self::Leaf::ExplicitStream(input.parse()?)) + return input.parse_err("Brackets [ ... ] are not supported in an expression") + } + SourcePeekMatch::Punct(punct) => { + if punct.as_char() == '%' && input.peek2(syn::token::Bracket) { + let _ = input.parse::()?; + UnaryAtom::Leaf(Self::Leaf::ExplicitStream(input.parse()?)) + } else { + UnaryAtom::PrefixUnaryOperation(input.parse()?) + } } - SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), SourcePeekMatch::Ident(_) => match input.try_parse_or_revert() { Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( ExpressionBoolean::for_litbool(bool), @@ -125,83 +132,6 @@ impl Expressionable for Source { } } -// Output -// =========== - -#[derive(Clone)] -#[allow(unused)] -pub(crate) struct OutputExpression { - inner: Expression, -} - -impl Parse for OutputExpression { - fn parse(input: ParseStream) -> ParseResult { - Ok(Self { - inner: ExpressionParser::parse(input)?, - }) - } -} - -#[allow(unused)] -impl OutputExpression { - pub(crate) fn evaluate(&self) -> ExecutionResult { - Output::evaluate(&self.inner, &mut ()) - } -} - -impl Expressionable for Output { - type Leaf = ExpressionValue; - type EvaluationContext = (); - - fn evaluate_leaf( - leaf: &Self::Leaf, - _: &mut Self::EvaluationContext, - ) -> ExecutionResult { - Ok(leaf.clone()) - } - - fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { - Ok(match input.peek_grammar() { - OutputPeekMatch::Group(Delimiter::None | Delimiter::Parenthesis) => { - let (_, delim_span) = input.parse_and_enter_group()?; - UnaryAtom::Group(delim_span) - } - OutputPeekMatch::Group(Delimiter::Brace) => { - return input - .parse_err("Curly braces are not supported in a re-interpreted expression") - } - OutputPeekMatch::Group(Delimiter::Bracket) => { - return input.parse_err("Square brackets [ .. ] are not supported in an expression") - } - OutputPeekMatch::Ident(_) => UnaryAtom::Leaf(ExpressionValue::Boolean( - ExpressionBoolean::for_litbool(input.parse()?), - )), - OutputPeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), - OutputPeekMatch::Literal(_) => { - UnaryAtom::Leaf(ExpressionValue::for_syn_lit(input.parse()?)) - } - OutputPeekMatch::End => { - return input.parse_err("The expression ended in an incomplete state") - } - }) - } - - fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { - Ok(match input.peek_grammar() { - OutputPeekMatch::Punct(_) => match input.try_parse_or_revert::() { - Ok(operation) => NodeExtension::BinaryOperation(operation), - Err(_) => NodeExtension::NoneMatched, - }, - OutputPeekMatch::Ident(ident) if ident == "as" => { - let cast_operation = - UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; - NodeExtension::PostfixOperation(cast_operation) - } - _ => NodeExtension::NoneMatched, - }) - } -} - // Generic // ======= diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index a989bf2d..97964a64 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -107,14 +107,31 @@ pub(crate) enum Statement { impl Parse for Statement { fn parse(input: ParseStream) -> ParseResult { - let forked = input.fork(); - match forked.parse() { - Ok(assignment) => { - input.advance_to(&forked); - Ok(Statement::Assignment(assignment)) + Ok(match input.cursor().ident() { + // let or some ident for a variable + // It may be the start of an assignment. + Some((ident, _)) if { let str = ident.to_string(); str != "true" && str != "false" } => { + let forked = input.fork(); + match forked.call(|input| { + let destination = input.parse()?; + let equals = input.parse()?; + Ok((destination, equals)) + }) { + Ok((destination, equals)) => { + input.advance_to(&forked); + // We commit to the fork after successfully parsing the destination and equals. + // This gives better error messages, if there is an error in the expression itself. + Statement::Assignment(AssignmentStatement { + destination, + equals, + expression: input.parse()?, + }) + }, + Err(_) => Statement::Expression(input.parse()?), + } } - Err(_) => Ok(Statement::Expression(input.parse()?)), - } + _ => Statement::Expression(input.parse()?), + }) } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 11785f1a..28a78919 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -389,6 +389,25 @@ impl ExpressionValue { Self::Stream(value) => value.value.append_cloned_into(output), } } + + pub(crate) fn debug(&self) -> String { + use std::fmt::Write; + let mut output = String::new(); + match self { + ExpressionValue::None { .. } => { + write!(output, "None").unwrap(); + }, + ExpressionValue::Stream(stream) => { + write!(output, "%[{}]", stream.value.clone().concat_recursive(&ConcatBehaviour::debug())).unwrap(); + }, + _ => { + let mut stream = OutputStream::new(); + self.output_flattened_to(&mut stream); + write!(output, "{}", stream.concat_recursive(&ConcatBehaviour::debug())).unwrap(); + } + } + output + } } pub(crate) enum Grouping { diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index fe02ebd6..0345ccf2 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -168,6 +168,15 @@ impl<'a, K> ParseStreamStack<'a, K> { self.current().parse() } + #[allow(unused)] + pub(crate) fn peek(&mut self, token: T) -> bool { + self.current().peek(token) + } + + pub(crate) fn peek2(&mut self, token: T) -> bool { + self.current().peek2(token) + } + pub(crate) fn parse_any_ident(&mut self) -> ParseResult { self.current().parse_any_ident() } @@ -224,12 +233,6 @@ impl ParseStreamStack<'_, Source> { } } -impl ParseStreamStack<'_, Output> { - pub(crate) fn peek_grammar(&mut self) -> OutputPeekMatch { - self.current().peek_grammar() - } -} - impl Drop for ParseStreamStack<'_, K> { fn drop(&mut self) { while !self.group_stack.is_empty() { diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 53d3ec7f..b6d3bed2 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -408,7 +408,8 @@ impl NoOutputCommandDefinition for ErrorCommand { #[derive(Clone)] pub(crate) struct DebugCommand { - inner: SourceStream, + span: Span, + inner: SourceExpression, } impl CommandType for DebugCommand { @@ -419,17 +420,22 @@ impl ValueCommandDefinition for DebugCommand { const COMMAND_NAME: &'static str = "debug"; fn parse(arguments: CommandArguments) -> ParseResult { - Ok(Self { - inner: arguments.parse_all_as_source()?, - }) + arguments.fully_parse_or_error( + |input| { + Ok(Self { + span: arguments.command_span(), + inner: input.parse()?, + }) + }, + "Expected [!debug! ]. To provide a stream, wrap in %[..]", + ) } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let span = self.inner.span(); let debug_string = self .inner - .interpret_to_new_stream(interpreter)? - .concat_recursive(&ConcatBehaviour::debug()); - Ok(debug_string.to_value(span.span_range())) + .interpret_to_value(interpreter)? + .debug(); + Ok(debug_string.to_value(self.span.span_range())) } } diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 716564a3..a949c3a3 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -224,6 +224,10 @@ impl<'a, K> ParseBuffer<'a, K> { T::parse(self, context) } + pub(crate) fn call) -> ParseResult>(&self, f: F) -> ParseResult { + f(self) + } + pub(crate) fn try_parse_or_error< T, F: FnOnce(&Self) -> ParseResult, @@ -246,7 +250,7 @@ impl<'a, K> ParseBuffer<'a, K> { } pub(crate) fn parse_any_ident(&self) -> ParseResult { - Ok(self.call(Ident::parse_any)?) + Ok(self.call(|stream| Ok(Ident::parse_any(&stream.inner)?))?) } pub(crate) fn peek_ident_matching(&self, content: &str) -> bool { diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs index f44c21ca..77e053ba 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - #(partial_sum = [+ 2]; 5 #..partial_sum) + #(partial_sum = %[+ 2]; 5 #..partial_sum) }; } diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr index 6dca115f..9b31d145 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr @@ -1,5 +1,5 @@ error: Expected an operator to continue the expression, or ; to mark the end of the expression statement - --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:5:34 + --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:5:35 | -5 | #(partial_sum = [+ 2]; 5 #..partial_sum) - | ^ +5 | #(partial_sum = %[+ 2]; 5 #..partial_sum) + | ^ diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs index a48f9db4..bd6eb39a 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - #(x = [+ 1]; 1 x) + #(x = %[+ 1]; 1 x) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr index 4d774dad..a69f7865 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr @@ -1,5 +1,5 @@ error: Expected an operator to continue the expression, or ; to mark the end of the expression statement - --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:24 + --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:25 | -5 | #(x = [+ 1]; 1 x) - | ^ +5 | #(x = %[+ 1]; 1 x) + | ^ diff --git a/tests/core.rs b/tests/core.rs index 390f5719..c01b6a39 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -107,12 +107,12 @@ fn test_debug() { // It keeps the semantic punctuation spacing intact // (e.g. it keeps 'a and >> together) assert_preinterpret_eq!( - [!debug! impl<'a, T> MyStruct<'a, T> { + [!debug! %[impl<'a, T> MyStruct<'a, T> { pub fn new() -> Self { !($crate::Test::CONSTANT >> 5 > 1) } - }], - "impl < 'a , T > MyStruct < 'a , T > { pub fn new () -> Self { ! ($ crate :: Test :: CONSTANT >> 5 > 1) } }" + }]], + "%[impl < 'a , T > MyStruct < 'a , T > { pub fn new () -> Self { ! ($ crate :: Test :: CONSTANT >> 5 > 1) } }]" ); // It shows transparent groups // NOTE: The output code can't be used directly as preinterpret input @@ -121,8 +121,8 @@ fn test_debug() { assert_preinterpret_eq!( { [!set! #x = Hello (World)] - [!debug! #x [!raw! #test] "and" [!raw! ##] #..x] + [!debug! %[#x [!raw! #test] "and" [!raw! ##] #..x]] }, - r###"[!group! Hello (World)] # test "and" ## Hello (World)"### + r###"%[[!group! Hello (World)] # test "and" ## Hello (World)]"### ); } diff --git a/tests/expressions.rs b/tests/expressions.rs index 63cdae38..5ae0e34e 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -49,9 +49,9 @@ fn test_basic_evaluate_works() { assert_expression_eq!(#(123 > 456), false); assert_expression_eq!(#(six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); assert_expression_eq!(#( - partial_sum = [+ 2]; - [!debug! #([5] + partial_sum) = [!reinterpret! [!raw! #](5 #..partial_sum)]] - ), "[!group! 5 + 2] = [!group! 7]"); + partial_sum = %[+ 2]; + [!debug! %[#(%[5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] + ), "%[[!group! 5 + 2] = [!group! 7]]"); assert_expression_eq!(#(1 + [!range! 1..2] as int), 2); assert_expression_eq!(#("hello" == "world"), false); assert_expression_eq!(#("hello" == "hello"), true); @@ -62,8 +62,8 @@ fn test_basic_evaluate_works() { assert_expression_eq!(#('A' < 'B'), true); assert_expression_eq!(#("Zoo" > "Aardvark"), true); assert_expression_eq!( - #([!debug! #..("Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group)]), - r#""Hello" "World" 2 [!group! 2]"# + #([!debug! "Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group]), + r#"%["Hello" "World" 2 [!group! 2]]"# ); } @@ -90,7 +90,7 @@ fn test_very_long_expression_works() { [!settings! { iteration_limit: 100000, }] - #(let expression = [0] + [!for! #i in [!range! 0..100000] { + 1 }]) + #(let expression = %[0] + [!for! #i in [!range! 0..100000] { + 1 }]) [!reinterpret! [!raw! #](#expression)] }, 100000 @@ -135,7 +135,7 @@ fn assign_works() { let x = 5 + 5; [!debug! #..x] ), - "10" + "%[10]" ); assert_expression_eq!( #( @@ -196,6 +196,6 @@ fn test_range() { assert_preinterpret_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); assert_preinterpret_eq!( { [!debug! [!..range! -1i8..3i8]] }, - "[!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]" + "%[[!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]]" ); } diff --git a/tests/tokens.rs b/tests/tokens.rs index 9e8d9f7c..440c8576 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -325,7 +325,7 @@ fn test_split() { separator: [], }]] }, - "[!group! A] [!group! :] [!group! :] [!group! B]" + "%[[!group! A] [!group! :] [!group! :] [!group! B]]" ); // Double separators are allowed assert_preinterpret_eq!( @@ -335,7 +335,7 @@ fn test_split() { separator: [::], }]] }, - "[!group! A] [!group! B] [!group! C]" + "%[[!group! A] [!group! B] [!group! C]]" ); // Trailing separator is ignored by default assert_preinterpret_eq!( @@ -345,7 +345,7 @@ fn test_split() { separator: [,], }]] }, - "[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]" + "%[[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" ); // By default, empty groups are included except at the end assert_preinterpret_eq!( @@ -355,7 +355,7 @@ fn test_split() { separator: [::], }]] }, - "[!group!] [!group! A] [!group! B] [!group!] [!group! C]" + "%[[!group!] [!group! A] [!group! B] [!group!] [!group! C]]" ); // Stream and separator are both interpreted assert_preinterpret_eq!({ @@ -367,7 +367,7 @@ fn test_split() { drop_empty_middle: true, drop_empty_end: true, }]] - }, "[!group! A] [!group! B] [!group! C] [!group! D] [!group! E]"); + }, "%[[!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); // Drop empty false works assert_preinterpret_eq!({ [!set! #x = ;] @@ -378,7 +378,7 @@ fn test_split() { drop_empty_middle: false, drop_empty_end: false, }]] - }, "[!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]"); + }, "%[[!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]"); // Drop empty middle works assert_preinterpret_eq!( { @@ -390,7 +390,7 @@ fn test_split() { drop_empty_end: false, }]] }, - "[!group!] [!group! A] [!group! B] [!group! E] [!group!]" + "%[[!group!] [!group! A] [!group! B] [!group! E] [!group!]]" ); // Code blocks are only evaluated once // (i.e. no "unknown variable B is output in the below") @@ -403,7 +403,7 @@ fn test_split() { separator: [#], }]] }, - "[!group! A] [!group! B] [!group! C]" + "%[[!group! A] [!group! B] [!group! C]]" ); } @@ -411,7 +411,7 @@ fn test_split() { fn test_comma_split() { assert_preinterpret_eq!( { [!debug! [!..comma_split! Pizza, Mac and Cheese, Hamburger,]] }, - "[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]" + "%[[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" ); } @@ -419,7 +419,7 @@ fn test_comma_split() { fn test_zip() { assert_preinterpret_eq!( { [!debug! [!..zip! ([Hello Goodbye] [World Friend])]] }, - "(Hello World) (Goodbye Friend)" + "%[(Hello World) (Goodbye Friend)]" ); assert_preinterpret_eq!( { @@ -430,7 +430,7 @@ fn test_zip() { streams: [#countries #flags #capitals], }]] }, - r#"[!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]"#, + r#"%[[!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]]"#, ); assert_preinterpret_eq!( { @@ -442,7 +442,7 @@ fn test_zip() { error_on_length_mismatch: false, }]] }, - r#"[!group! A 1] [!group! B 2] [!group! C 3]"#, + r#"%[[!group! A 1] [!group! B 2] [!group! C 3]]"#, ); assert_preinterpret_eq!( { @@ -455,7 +455,7 @@ fn test_zip() { }, }]] }, - r#"{ A 1 } { B 2 } { C 3 }"#, + r#"%[{ A 1 } { B 2 } { C 3 }]"#, ); assert_preinterpret_eq!( { @@ -466,7 +466,7 @@ fn test_zip() { streams: #..combined, }]] }, - r#"{ A 1 } { B 2 } { C 3 }"#, + r#"%[{ A 1 } { B 2 } { C 3 }]"#, ); assert_preinterpret_eq!( { @@ -475,7 +475,7 @@ fn test_zip() { [!set! #combined = [#letters #numbers]] [!debug! [!..zip! #..combined]] }, - r#"[A 1] [B 2] [C 3]"#, + r#"%[[A 1] [B 2] [C 3]]"#, ); } diff --git a/tests/transforming.rs b/tests/transforming.rs index 939c06cb..ab13d557 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -97,7 +97,7 @@ fn test_literal_transformer() { [!set! #x] [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] [!debug! #..x] - }, "'c' 0b1010 r#\"123\"#"); + }, "%['c' 0b1010 r#\"123\"#]"); } #[test] @@ -105,18 +105,18 @@ fn test_punct_transformer() { assert_preinterpret_eq!({ [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] [!debug! #..x] - }, "!"); + }, "%[!]"); // Test for ' which is treated weirdly by syn / rustc assert_preinterpret_eq!({ [!let! The "quick" fox isn 't brown and doesn @(#x = @PUNCT) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] [!debug! #..x] - }, "'"); + }, "%[']"); // Lots of punctuation, most of it ignored assert_preinterpret_eq!({ [!set! #x =] [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] [!debug! #..x] - }, "% |"); + }, "%[% |]"); } #[test] @@ -124,18 +124,18 @@ fn test_group_transformer() { assert_preinterpret_eq!({ [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] [!debug! #..x] - }, "fox"); + }, "%[fox]"); assert_preinterpret_eq!({ [!set! #x = "hello" "world"] [!let! I said @[GROUP #..y]! = I said #x!] [!debug! #..y] - }, "\"hello\" \"world\""); + }, "%[\"hello\" \"world\"]"); // ... which is equivalent to this: assert_preinterpret_eq!({ [!set! #x = "hello" "world"] [!let! I said #y! = I said #x!] [!debug! #..y] - }, "\"hello\" \"world\""); + }, "%[\"hello\" \"world\"]"); } #[test] @@ -143,7 +143,7 @@ fn test_none_output_commands_mid_parse() { assert_preinterpret_eq!({ [!let! The "quick" @(#x = @LITERAL) fox [!let! #y = #x] @(#x = @IDENT) = The "quick" "brown" fox jumps] [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] - }, "#x = jumps; #y = \"brown\""); + }, "#x = %[jumps]; #y = %[\"brown\"]"); } #[test] @@ -176,7 +176,7 @@ fn test_parse_command_and_exact_transformer() { // The output stream is additive assert_preinterpret_eq!( { [!debug! [!parse! [Hello World] as @(@IDENT @IDENT)]] }, - "Hello World" + "%[Hello World]" ); // Substreams redirected to a variable are not included in the output assert_preinterpret_eq!( @@ -185,7 +185,7 @@ fn test_parse_command_and_exact_transformer() { @[EXACT The] quick @IDENT @(#x = @IDENT) )]] }, - "The brown" + "%[The brown]" ); // This tests that: // * Can nest EXACT and transform streams @@ -197,5 +197,5 @@ fn test_parse_command_and_exact_transformer() { // The outputs are only from the EXACT transformer The quick @(_ = @IDENT) @[EXACT #x @(_ = @IDENT a) #..x - right?!] )]] - }, "fox fox - right ?!"); + }, "%[fox fox - right ?!]"); } From 1bf1db90be8f3529cc05a3ff31ab87f1599c836c Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 12 Feb 2025 11:10:19 +0000 Subject: [PATCH 087/126] refactor: Changed from experimental `%[...]` to `[!stream ...]` for stream values --- CHANGELOG.md | 14 +++------ src/expressions/expression.rs | 15 +--------- src/expressions/expression_block.rs | 9 ++++-- src/expressions/value.rs | 15 +++++++--- src/extensions/parsing.rs | 1 + src/interpretation/command.rs | 2 +- src/interpretation/commands/core_commands.rs | 15 ++++------ src/misc/parse_traits.rs | 7 +++-- .../flattened_variables_in_expressions.rs | 2 +- .../flattened_variables_in_expressions.stderr | 6 ++-- ...ped_variable_with_incomplete_expression.rs | 2 +- ...variable_with_incomplete_expression.stderr | 6 ++-- tests/core.rs | 8 ++--- tests/expressions.rs | 14 ++++----- tests/tokens.rs | 30 +++++++++---------- tests/transforming.rs | 22 +++++++------- 16 files changed, 81 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1952225e..d8d70465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ * `[!set! #x += ...]` to performantly add extra characters to the stream. * `[!set! _ = ...]` interprets its arguments but then ignores any outputs. * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. - * `[!output! ...]` can be used to just output its interpreted contents. Normally it's a no-op, but it can be useful inside a transformer. + * `[!stream! ...]` can be used to just output its interpreted contents. It's useful to create a stream value inside an expression. * `[!reinterpret! ...]` is like an `eval` command in scripting languages. It takes a stream, and parses/interprets it. * `[!settings! { ... }]` can be used to adjust the iteration limit. * Expression commands: @@ -93,24 +93,18 @@ Inside a transform stream, the following grammar is supported: * `@[GROUP ...]` - Consumes a none-delimited group. Its arguments are used to transform the group's contents. * `@[EXACT ...]` - Interprets its arguments (i.e. variables are substituted, not bound; and command output is gathered) into an "exact match stream". And then expects to consume exactly the same stream from the input. It outputs the parsed stream. * Commands: Their output is appended to the transform's output. Useful patterns include: - * `@(inner = ...) [!output! #inner]` - wraps the output in a transparent group + * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come * Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model - * Start with new stream syntax: `%[...]` -```rust -#(x = [!stream! Hello World]) -#(x = %{Hello World}) -#(x = %[Hello World]) // Prefer this one so far -#(x = stream { Hello World }) -``` + * Create array value + * Add `+` support for concatenating arrays * Revisit the `SourceStreamInput` abstraction - maybe it's replaced with a `SourceExpression` which needs to output a stream? * Split outputs an array * Intersperse works with either an array or a stream (?) * For works only with an array (in future, also an iterator) * We can then consider dropping lots of the `group` wrappers I guess? - * Add `+` support for concatenating arrays * Then destructuring and parsing become different: * Destructuring works over the value model; parsing works over the token stream model. * Support `#(x[..])` syntax for indexing arrays and streams at read time diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index d95a1ab1..18b42ef4 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -32,7 +32,6 @@ pub(super) enum SourceExpressionLeaf { VariablePath(VariablePath), MarkedVariable(MarkedVariable), ExpressionBlock(ExpressionBlock), - ExplicitStream(SourceGroup), Value(ExpressionValue), } @@ -66,14 +65,7 @@ impl Expressionable for Source { SourcePeekMatch::Group(Delimiter::Bracket) => { return input.parse_err("Brackets [ ... ] are not supported in an expression") } - SourcePeekMatch::Punct(punct) => { - if punct.as_char() == '%' && input.peek2(syn::token::Bracket) { - let _ = input.parse::()?; - UnaryAtom::Leaf(Self::Leaf::ExplicitStream(input.parse()?)) - } else { - UnaryAtom::PrefixUnaryOperation(input.parse()?) - } - } + SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), SourcePeekMatch::Ident(_) => match input.try_parse_or_revert() { Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( ExpressionBoolean::for_litbool(bool), @@ -122,11 +114,6 @@ impl Expressionable for Source { SourceExpressionLeaf::ExpressionBlock(block) => { block.interpret_to_value(interpreter)? } - SourceExpressionLeaf::ExplicitStream(source_group) => source_group - .clone() - .into_content() - .interpret_to_new_stream(interpreter)? - .to_value(source_group.span_range()), SourceExpressionLeaf::Value(value) => value.clone(), }) } diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index 97964a64..72f5a558 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -110,7 +110,12 @@ impl Parse for Statement { Ok(match input.cursor().ident() { // let or some ident for a variable // It may be the start of an assignment. - Some((ident, _)) if { let str = ident.to_string(); str != "true" && str != "false" } => { + Some((ident, _)) + if { + let str = ident.to_string(); + str != "true" && str != "false" + } => + { let forked = input.fork(); match forked.call(|input| { let destination = input.parse()?; @@ -126,7 +131,7 @@ impl Parse for Statement { equals, expression: input.parse()?, }) - }, + } Err(_) => Statement::Expression(input.parse()?), } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 28a78919..0f38dd3f 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -396,14 +396,21 @@ impl ExpressionValue { match self { ExpressionValue::None { .. } => { write!(output, "None").unwrap(); - }, + } ExpressionValue::Stream(stream) => { - write!(output, "%[{}]", stream.value.clone().concat_recursive(&ConcatBehaviour::debug())).unwrap(); - }, + if stream.value.is_empty() { + write!(output, "[!stream!]").unwrap(); + } else { + let stream = stream.value.clone(); + let string_rep = stream.concat_recursive(&ConcatBehaviour::debug()); + write!(output, "[!stream! {}]", string_rep).unwrap(); + } + } _ => { let mut stream = OutputStream::new(); self.output_flattened_to(&mut stream); - write!(output, "{}", stream.concat_recursive(&ConcatBehaviour::debug())).unwrap(); + let string_rep = stream.concat_recursive(&ConcatBehaviour::debug()); + write!(output, "{}", string_rep).unwrap(); } } output diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index 0345ccf2..bb06c7c3 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -173,6 +173,7 @@ impl<'a, K> ParseStreamStack<'a, K> { self.current().peek(token) } + #[allow(unused)] pub(crate) fn peek2(&mut self, token: T) -> bool { self.current().peek2(token) } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 1a34a49b..5766c8fb 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -441,7 +441,7 @@ define_command_enums! { SetCommand, TypedSetCommand, RawCommand, - OutputCommand, + StreamCommand, IgnoreCommand, ReinterpretCommand, SettingsCommand, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index b6d3bed2..c991e032 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -178,16 +178,16 @@ impl StreamingCommandDefinition for RawCommand { } #[derive(Clone)] -pub(crate) struct OutputCommand { +pub(crate) struct StreamCommand { inner: SourceStream, } -impl CommandType for OutputCommand { +impl CommandType for StreamCommand { type OutputKind = OutputKindStreaming; } -impl StreamingCommandDefinition for OutputCommand { - const COMMAND_NAME: &'static str = "output"; +impl StreamingCommandDefinition for StreamCommand { + const COMMAND_NAME: &'static str = "stream"; fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { @@ -427,15 +427,12 @@ impl ValueCommandDefinition for DebugCommand { inner: input.parse()?, }) }, - "Expected [!debug! ]. To provide a stream, wrap in %[..]", + "Expected [!debug! ]. To provide a stream, use [!debug! [!stream! ...]]", ) } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let debug_string = self - .inner - .interpret_to_value(interpreter)? - .debug(); + let debug_string = self.inner.interpret_to_value(interpreter)?.debug(); Ok(debug_string.to_value(self.span.span_range())) } } diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index a949c3a3..55ea7d2c 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -224,7 +224,10 @@ impl<'a, K> ParseBuffer<'a, K> { T::parse(self, context) } - pub(crate) fn call) -> ParseResult>(&self, f: F) -> ParseResult { + pub(crate) fn call) -> ParseResult>( + &self, + f: F, + ) -> ParseResult { f(self) } @@ -250,7 +253,7 @@ impl<'a, K> ParseBuffer<'a, K> { } pub(crate) fn parse_any_ident(&self) -> ParseResult { - Ok(self.call(|stream| Ok(Ident::parse_any(&stream.inner)?))?) + self.call(|stream| Ok(Ident::parse_any(&stream.inner)?)) } pub(crate) fn peek_ident_matching(&self, content: &str) -> bool { diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs index 77e053ba..8c14c97c 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret! { - #(partial_sum = %[+ 2]; 5 #..partial_sum) + #(partial_sum = [!stream! + 2]; 5 #..partial_sum) }; } diff --git a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr index 9b31d145..861019a5 100644 --- a/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr +++ b/tests/compilation_failures/expressions/flattened_variables_in_expressions.stderr @@ -1,5 +1,5 @@ error: Expected an operator to continue the expression, or ; to mark the end of the expression statement - --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:5:35 + --> tests/compilation_failures/expressions/flattened_variables_in_expressions.rs:5:43 | -5 | #(partial_sum = %[+ 2]; 5 #..partial_sum) - | ^ +5 | #(partial_sum = [!stream! + 2]; 5 #..partial_sum) + | ^ diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs index bd6eb39a..28cfa92d 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - #(x = %[+ 1]; 1 x) + #(x = [!stream! + 1]; 1 x) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr index a69f7865..5c70238a 100644 --- a/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr +++ b/tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.stderr @@ -1,5 +1,5 @@ error: Expected an operator to continue the expression, or ; to mark the end of the expression statement - --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:25 + --> tests/compilation_failures/expressions/grouped_variable_with_incomplete_expression.rs:5:33 | -5 | #(x = %[+ 1]; 1 x) - | ^ +5 | #(x = [!stream! + 1]; 1 x) + | ^ diff --git a/tests/core.rs b/tests/core.rs index c01b6a39..1fc0813c 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -107,12 +107,12 @@ fn test_debug() { // It keeps the semantic punctuation spacing intact // (e.g. it keeps 'a and >> together) assert_preinterpret_eq!( - [!debug! %[impl<'a, T> MyStruct<'a, T> { + [!debug! [!stream! impl<'a, T> MyStruct<'a, T> { pub fn new() -> Self { !($crate::Test::CONSTANT >> 5 > 1) } }]], - "%[impl < 'a , T > MyStruct < 'a , T > { pub fn new () -> Self { ! ($ crate :: Test :: CONSTANT >> 5 > 1) } }]" + "[!stream! impl < 'a , T > MyStruct < 'a , T > { pub fn new () -> Self { ! ($ crate :: Test :: CONSTANT >> 5 > 1) } }]" ); // It shows transparent groups // NOTE: The output code can't be used directly as preinterpret input @@ -121,8 +121,8 @@ fn test_debug() { assert_preinterpret_eq!( { [!set! #x = Hello (World)] - [!debug! %[#x [!raw! #test] "and" [!raw! ##] #..x]] + [!debug! [!stream! #x [!raw! #test] "and" [!raw! ##] #..x]] }, - r###"%[[!group! Hello (World)] # test "and" ## Hello (World)]"### + r###"[!stream! [!group! Hello (World)] # test "and" ## Hello (World)]"### ); } diff --git a/tests/expressions.rs b/tests/expressions.rs index 5ae0e34e..935c390c 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -49,9 +49,9 @@ fn test_basic_evaluate_works() { assert_expression_eq!(#(123 > 456), false); assert_expression_eq!(#(six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); assert_expression_eq!(#( - partial_sum = %[+ 2]; - [!debug! %[#(%[5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] - ), "%[[!group! 5 + 2] = [!group! 7]]"); + partial_sum = [!stream! + 2]; + [!debug! [!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] + ), "[!stream! [!group! 5 + 2] = [!group! 7]]"); assert_expression_eq!(#(1 + [!range! 1..2] as int), 2); assert_expression_eq!(#("hello" == "world"), false); assert_expression_eq!(#("hello" == "hello"), true); @@ -63,7 +63,7 @@ fn test_basic_evaluate_works() { assert_expression_eq!(#("Zoo" > "Aardvark"), true); assert_expression_eq!( #([!debug! "Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group]), - r#"%["Hello" "World" 2 [!group! 2]]"# + r#"[!stream! "Hello" "World" 2 [!group! 2]]"# ); } @@ -90,7 +90,7 @@ fn test_very_long_expression_works() { [!settings! { iteration_limit: 100000, }] - #(let expression = %[0] + [!for! #i in [!range! 0..100000] { + 1 }]) + #(let expression = [!stream! 0] + [!for! #i in [!range! 0..100000] { + 1 }]) [!reinterpret! [!raw! #](#expression)] }, 100000 @@ -135,7 +135,7 @@ fn assign_works() { let x = 5 + 5; [!debug! #..x] ), - "%[10]" + "[!stream! 10]" ); assert_expression_eq!( #( @@ -196,6 +196,6 @@ fn test_range() { assert_preinterpret_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); assert_preinterpret_eq!( { [!debug! [!..range! -1i8..3i8]] }, - "%[[!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]]" + "[!stream! [!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]]" ); } diff --git a/tests/tokens.rs b/tests/tokens.rs index 440c8576..d657df6f 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -325,7 +325,7 @@ fn test_split() { separator: [], }]] }, - "%[[!group! A] [!group! :] [!group! :] [!group! B]]" + "[!stream! [!group! A] [!group! :] [!group! :] [!group! B]]" ); // Double separators are allowed assert_preinterpret_eq!( @@ -335,7 +335,7 @@ fn test_split() { separator: [::], }]] }, - "%[[!group! A] [!group! B] [!group! C]]" + "[!stream! [!group! A] [!group! B] [!group! C]]" ); // Trailing separator is ignored by default assert_preinterpret_eq!( @@ -345,7 +345,7 @@ fn test_split() { separator: [,], }]] }, - "%[[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" + "[!stream! [!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" ); // By default, empty groups are included except at the end assert_preinterpret_eq!( @@ -355,7 +355,7 @@ fn test_split() { separator: [::], }]] }, - "%[[!group!] [!group! A] [!group! B] [!group!] [!group! C]]" + "[!stream! [!group!] [!group! A] [!group! B] [!group!] [!group! C]]" ); // Stream and separator are both interpreted assert_preinterpret_eq!({ @@ -367,7 +367,7 @@ fn test_split() { drop_empty_middle: true, drop_empty_end: true, }]] - }, "%[[!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); + }, "[!stream! [!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); // Drop empty false works assert_preinterpret_eq!({ [!set! #x = ;] @@ -378,7 +378,7 @@ fn test_split() { drop_empty_middle: false, drop_empty_end: false, }]] - }, "%[[!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]"); + }, "[!stream! [!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]"); // Drop empty middle works assert_preinterpret_eq!( { @@ -390,7 +390,7 @@ fn test_split() { drop_empty_end: false, }]] }, - "%[[!group!] [!group! A] [!group! B] [!group! E] [!group!]]" + "[!stream! [!group!] [!group! A] [!group! B] [!group! E] [!group!]]" ); // Code blocks are only evaluated once // (i.e. no "unknown variable B is output in the below") @@ -403,7 +403,7 @@ fn test_split() { separator: [#], }]] }, - "%[[!group! A] [!group! B] [!group! C]]" + "[!stream! [!group! A] [!group! B] [!group! C]]" ); } @@ -411,7 +411,7 @@ fn test_split() { fn test_comma_split() { assert_preinterpret_eq!( { [!debug! [!..comma_split! Pizza, Mac and Cheese, Hamburger,]] }, - "%[[!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" + "[!stream! [!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" ); } @@ -419,7 +419,7 @@ fn test_comma_split() { fn test_zip() { assert_preinterpret_eq!( { [!debug! [!..zip! ([Hello Goodbye] [World Friend])]] }, - "%[(Hello World) (Goodbye Friend)]" + "[!stream! (Hello World) (Goodbye Friend)]" ); assert_preinterpret_eq!( { @@ -430,7 +430,7 @@ fn test_zip() { streams: [#countries #flags #capitals], }]] }, - r#"%[[!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]]"#, + r#"[!stream! [!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]]"#, ); assert_preinterpret_eq!( { @@ -442,7 +442,7 @@ fn test_zip() { error_on_length_mismatch: false, }]] }, - r#"%[[!group! A 1] [!group! B 2] [!group! C 3]]"#, + r#"[!stream! [!group! A 1] [!group! B 2] [!group! C 3]]"#, ); assert_preinterpret_eq!( { @@ -455,7 +455,7 @@ fn test_zip() { }, }]] }, - r#"%[{ A 1 } { B 2 } { C 3 }]"#, + r#"[!stream! { A 1 } { B 2 } { C 3 }]"#, ); assert_preinterpret_eq!( { @@ -466,7 +466,7 @@ fn test_zip() { streams: #..combined, }]] }, - r#"%[{ A 1 } { B 2 } { C 3 }]"#, + r#"[!stream! { A 1 } { B 2 } { C 3 }]"#, ); assert_preinterpret_eq!( { @@ -475,7 +475,7 @@ fn test_zip() { [!set! #combined = [#letters #numbers]] [!debug! [!..zip! #..combined]] }, - r#"%[[A 1] [B 2] [C 3]]"#, + r#"[!stream! [A 1] [B 2] [C 3]]"#, ); } diff --git a/tests/transforming.rs b/tests/transforming.rs index ab13d557..077389a6 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -97,7 +97,7 @@ fn test_literal_transformer() { [!set! #x] [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] [!debug! #..x] - }, "%['c' 0b1010 r#\"123\"#]"); + }, "[!stream! 'c' 0b1010 r#\"123\"#]"); } #[test] @@ -105,18 +105,18 @@ fn test_punct_transformer() { assert_preinterpret_eq!({ [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] [!debug! #..x] - }, "%[!]"); + }, "[!stream! !]"); // Test for ' which is treated weirdly by syn / rustc assert_preinterpret_eq!({ [!let! The "quick" fox isn 't brown and doesn @(#x = @PUNCT) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] [!debug! #..x] - }, "%[']"); + }, "[!stream! ']"); // Lots of punctuation, most of it ignored assert_preinterpret_eq!({ [!set! #x =] [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] [!debug! #..x] - }, "%[% |]"); + }, "[!stream! % |]"); } #[test] @@ -124,18 +124,18 @@ fn test_group_transformer() { assert_preinterpret_eq!({ [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] [!debug! #..x] - }, "%[fox]"); + }, "[!stream! fox]"); assert_preinterpret_eq!({ [!set! #x = "hello" "world"] [!let! I said @[GROUP #..y]! = I said #x!] [!debug! #..y] - }, "%[\"hello\" \"world\"]"); + }, "[!stream! \"hello\" \"world\"]"); // ... which is equivalent to this: assert_preinterpret_eq!({ [!set! #x = "hello" "world"] [!let! I said #y! = I said #x!] [!debug! #..y] - }, "%[\"hello\" \"world\"]"); + }, "[!stream! \"hello\" \"world\"]"); } #[test] @@ -143,7 +143,7 @@ fn test_none_output_commands_mid_parse() { assert_preinterpret_eq!({ [!let! The "quick" @(#x = @LITERAL) fox [!let! #y = #x] @(#x = @IDENT) = The "quick" "brown" fox jumps] [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] - }, "#x = %[jumps]; #y = %[\"brown\"]"); + }, "#x = [!stream! jumps]; #y = [!stream! \"brown\"]"); } #[test] @@ -176,7 +176,7 @@ fn test_parse_command_and_exact_transformer() { // The output stream is additive assert_preinterpret_eq!( { [!debug! [!parse! [Hello World] as @(@IDENT @IDENT)]] }, - "%[Hello World]" + "[!stream! Hello World]" ); // Substreams redirected to a variable are not included in the output assert_preinterpret_eq!( @@ -185,7 +185,7 @@ fn test_parse_command_and_exact_transformer() { @[EXACT The] quick @IDENT @(#x = @IDENT) )]] }, - "%[The brown]" + "[!stream! The brown]" ); // This tests that: // * Can nest EXACT and transform streams @@ -197,5 +197,5 @@ fn test_parse_command_and_exact_transformer() { // The outputs are only from the EXACT transformer The quick @(_ = @IDENT) @[EXACT #x @(_ = @IDENT a) #..x - right?!] )]] - }, "%[fox fox - right ?!]"); + }, "[!stream! fox fox - right ?!]"); } From 994d9d40f66042eef954eb617368e7748e6482fd Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 12 Feb 2025 13:42:29 +0000 Subject: [PATCH 088/126] feature: Add array value --- CHANGELOG.md | 15 +-- src/expressions/array.rs | 110 ++++++++++++++++++ src/expressions/boolean.rs | 4 +- src/expressions/character.rs | 4 +- src/expressions/expression_block.rs | 2 +- src/expressions/float.rs | 8 +- src/expressions/integer.rs | 16 +-- src/expressions/mod.rs | 2 + src/expressions/stream.rs | 2 +- src/expressions/string.rs | 4 +- src/expressions/value.rs | 63 ++++++---- src/interpretation/command.rs | 6 +- src/interpretation/commands/core_commands.rs | 36 ------ .../commands/expression_commands.rs | 2 +- src/interpretation/variable.rs | 3 +- src/transformation/transformer.rs | 2 +- 16 files changed, 185 insertions(+), 94 deletions(-) create mode 100644 src/expressions/array.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d70465..3759800a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,7 +98,8 @@ Inside a transform stream, the following grammar is supported: ### To come * Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model - * Create array value + * Create parsable array value + * Add test for casting to stream, and compile error when outputting to stream * Add `+` support for concatenating arrays * Revisit the `SourceStreamInput` abstraction - maybe it's replaced with a `SourceExpression` which needs to output a stream? * Split outputs an array @@ -107,6 +108,7 @@ Inside a transform stream, the following grammar is supported: * We can then consider dropping lots of the `group` wrappers I guess? * Then destructuring and parsing become different: * Destructuring works over the value model; parsing works over the token stream model. + * Support a CastTarget of Array (only supported for array and stream and iterator) * Support `#(x[..])` syntax for indexing arrays and streams at read time * Via a post-fix `[...]` operation with high priority * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) @@ -236,20 +238,13 @@ Inside a transform stream, the following grammar is supported: // EXAMPLE #(parsed = []) [!parse! [...] as - // OPTION 1 @( #(let item = {}) impl @[item.trait = IDENT] for @[item.type = IDENT] - #(parsed += item) + #(parsed.push(item)) ),* - // OPTION 2 - @[COMMA_REPEATED { - before: #(let item = {}), - item: @(impl @[#(item.trait) = IDENT] for @[#(item.type) = IDENT]), - after: #(parsed += item), - }] ] -[!for! @[{ #trait, #type }] in #(parsed.output) { +[!stream_for! { trait, type } in parsed.output { impl #trait for #type {} }] diff --git a/src/expressions/array.rs b/src/expressions/array.rs new file mode 100644 index 00000000..dadf7512 --- /dev/null +++ b/src/expressions/array.rs @@ -0,0 +1,110 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionArray { + pub(crate) items: Vec, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(crate) span_range: SpanRange, +} + +impl ExpressionArray { + pub(super) fn handle_unary_operation( + mut self, + operation: OutputSpanned, + ) -> ExecutionResult { + Ok(match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { + return operation.unsupported(self) + } + UnaryOperation::Cast { target, target_ident, .. } => match target { + CastTarget::Stream => { + operation.output(self.to_stream_with_grouped_items()?) + }, + CastTarget::Group => { + operation.output(operation.output(self.to_stream_with_grouped_items()?).into_new_output_stream(Grouping::Grouped)?) + }, + _ => { + if self.items.len() == 1 { + self.items.pop().unwrap().handle_unary_operation(operation)? + } else { + return operation.execution_err(format!( + "Cannot only attempt to cast a singleton array to {} but the array has {} elements", + target_ident, + self.items.len(), + )); + } + } + }, + }) + } + + fn to_stream_with_grouped_items(self) -> ExecutionResult { + let mut stream = OutputStream::new(); + for item in self.items { + item.output_to(Grouping::Grouped, &mut stream)?; + } + Ok(stream) + } + + pub(super) fn handle_integer_binary_operation( + self, + _right: ExpressionInteger, + operation: OutputSpanned, + ) -> ExecutionResult { + operation.unsupported(self) + } + + pub(super) fn handle_paired_binary_operation( + self, + rhs: Self, + operation: OutputSpanned, + ) -> ExecutionResult { + let lhs = self.items; + let rhs = rhs.items; + Ok(match operation.operation { + PairedBinaryOperation::Addition { .. } => operation.output({ + let mut stream = lhs; + stream.extend(rhs); + stream + }), + PairedBinaryOperation::Subtraction { .. } + | PairedBinaryOperation::Multiplication { .. } + | PairedBinaryOperation::Division { .. } + | PairedBinaryOperation::LogicalAnd { .. } + | PairedBinaryOperation::LogicalOr { .. } + | PairedBinaryOperation::Remainder { .. } + | PairedBinaryOperation::BitXor { .. } + | PairedBinaryOperation::BitAnd { .. } + | PairedBinaryOperation::BitOr { .. } + | PairedBinaryOperation::Equal { .. } + | PairedBinaryOperation::LessThan { .. } + | PairedBinaryOperation::LessThanOrEqual { .. } + | PairedBinaryOperation::NotEqual { .. } + | PairedBinaryOperation::GreaterThanOrEqual { .. } + | PairedBinaryOperation::GreaterThan { .. } => return operation.unsupported(lhs), + }) + } +} + +impl HasValueType for ExpressionArray { + fn value_type(&self) -> &'static str { + self.items.value_type() + } +} + +impl HasValueType for Vec { + fn value_type(&self) -> &'static str { + "array" + } +} + +impl ToExpressionValue for Vec { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Array(ExpressionArray { + items: self, + span_range, + }) + } +} diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index b53c2747..2f256e55 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -48,12 +48,12 @@ impl ExpressionBoolean { CastTarget::Stream => operation.output( operation .output(input) - .into_new_output_stream(Grouping::Flattened), + .into_new_output_stream(Grouping::Flattened)?, ), CastTarget::Group => operation.output( operation .output(input) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), }, }) diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 36c00447..1cad7492 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -47,12 +47,12 @@ impl ExpressionChar { CastTarget::Stream => operation.output( operation .output(char) - .into_new_output_stream(Grouping::Flattened), + .into_new_output_stream(Grouping::Flattened)?, ), CastTarget::Group => operation.output( operation .output(char) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), }, }) diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index 72f5a558..0e49f4b0 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -79,7 +79,7 @@ impl Interpret for &ExpressionBlock { Some(_) => Grouping::Flattened, None => Grouping::Grouped, }; - self.evaluate(interpreter)?.output_to(grouping, output); + self.evaluate(interpreter)?.output_to(grouping, output)?; Ok(()) } } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 6a3b983a..32abfc9f 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -178,12 +178,12 @@ impl UntypedFloat { CastTarget::Stream => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Flattened), + .into_new_output_stream(Grouping::Flattened)?, ), CastTarget::Group => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), }, }) @@ -328,8 +328,8 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)?), } }) } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index acabb7fe..253c45fc 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -313,12 +313,12 @@ impl UntypedInteger { CastTarget::Stream => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Flattened), + .into_new_output_stream(Grouping::Flattened)?, ), CastTarget::Group => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), }, }) @@ -628,8 +628,8 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)?), } }) } @@ -664,8 +664,8 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)?), } }) } @@ -710,12 +710,12 @@ impl HandleUnaryOperation for u8 { CastTarget::Stream => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Flattened), + .into_new_output_stream(Grouping::Flattened)?, ), CastTarget::Group => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), }, }) diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 5f0ad137..bad4e0ee 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -1,3 +1,4 @@ +mod array; mod boolean; mod character; mod evaluation; @@ -13,6 +14,7 @@ mod value; // Marked as use for expression sub-modules to use with a `use super::*` statement use crate::internal_prelude::*; +use array::*; use boolean::*; use character::*; use evaluation::*; diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs index 27e76a83..81c29a60 100644 --- a/src/expressions/stream.rs +++ b/src/expressions/stream.rs @@ -23,7 +23,7 @@ impl ExpressionStream { CastTarget::Group => operation.output( operation .output(self.value) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), _ => { let coerced = self.value.coerce_into_value(self.span_range); diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 131919a0..5cd1836c 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -29,12 +29,12 @@ impl ExpressionString { CastTarget::Stream => operation.output( operation .output(self.value) - .into_new_output_stream(Grouping::Flattened), + .into_new_output_stream(Grouping::Flattened)?, ), CastTarget::Group => operation.output( operation .output(self.value) - .into_new_output_stream(Grouping::Grouped), + .into_new_output_stream(Grouping::Grouped)?, ), _ => return operation.unsupported(self), }, diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 0f38dd3f..602d98fd 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -11,6 +11,7 @@ pub(crate) enum ExpressionValue { // Unsupported literal is a type here so that we can parse such a token // as a value rather than a stream, and give it better error messages UnsupportedLiteral(UnsupportedLiteral), + Array(ExpressionArray), Stream(ExpressionStream), } @@ -229,6 +230,9 @@ impl ExpressionValue { (ExpressionValue::Char(left), ExpressionValue::Char(right)) => { EvaluationLiteralPair::CharPair(left, right) } + (ExpressionValue::Array(left), ExpressionValue::Array(right)) => { + EvaluationLiteralPair::ArrayPair(left, right) + } (ExpressionValue::Stream(left), ExpressionValue::Stream(right)) => { EvaluationLiteralPair::StreamPair(left, right) } @@ -275,6 +279,7 @@ impl ExpressionValue { ExpressionValue::String(value) => value.handle_unary_operation(operation), ExpressionValue::Char(value) => value.handle_unary_operation(operation), ExpressionValue::Stream(value) => value.handle_unary_operation(operation), + ExpressionValue::Array(value) => value.handle_unary_operation(operation), ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), } } @@ -300,6 +305,7 @@ impl ExpressionValue { } ExpressionValue::Char(value) => value.handle_integer_binary_operation(right, operation), ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), + ExpressionValue::Array(value) => value.handle_integer_binary_operation(right, operation), ExpressionValue::Stream(value) => { value.handle_integer_binary_operation(right, operation) } @@ -325,6 +331,7 @@ impl ExpressionValue { Self::String(value) => &mut value.span_range, Self::Char(value) => &mut value.span_range, Self::UnsupportedLiteral(value) => &mut value.span_range, + Self::Array(value) => &mut value.span_range, Self::Stream(value) => &mut value.span_range, } } @@ -339,44 +346,40 @@ impl ExpressionValue { self } - pub(crate) fn into_new_output_stream(self, grouping: Grouping) -> OutputStream { - match (self, grouping) { + pub(crate) fn into_new_output_stream(self, grouping: Grouping) -> ExecutionResult { + Ok(match (self, grouping) { (Self::Stream(value), Grouping::Flattened) => value.value, (other, grouping) => { let mut output = OutputStream::new(); - other.output_to(grouping, &mut output); + other.output_to(grouping, &mut output)?; output } - } + }) } - pub(crate) fn output_to(&self, grouping: Grouping, output: &mut OutputStream) { + pub(crate) fn output_to(&self, grouping: Grouping, output: &mut OutputStream) -> ExecutionResult<()> { match grouping { Grouping::Grouped => { // Grouping can be important for different values, to ensure they're read atomically // when the output stream is viewed as an array/iterable, e.g. in a for loop. // * Grouping means -1 is interpreted atomically, rather than as a punct then a number // * Grouping means that a stream is interpreted atomically - let span = self.span_range().join_into_span_else_start(); output .push_grouped( - |inner| { - self.output_flattened_to(inner); - Ok(()) - }, + |inner| self.output_flattened_to(inner), Delimiter::None, - span, - ) - .unwrap() + self.span_range().join_into_span_else_start(), + )?; } Grouping::Flattened => { - self.output_flattened_to(output); + self.output_flattened_to(output)?; } } + Ok(()) } - fn output_flattened_to(&self, output: &mut OutputStream) { - match self { + fn output_flattened_to(&self, output: &mut OutputStream) -> ExecutionResult<()> { + Ok(match self { Self::None { .. } => {} Self::Integer(value) => output.push_literal(value.to_literal()), Self::Float(value) => output.push_literal(value.to_literal()), @@ -386,13 +389,19 @@ impl ExpressionValue { Self::UnsupportedLiteral(literal) => { output.extend_raw_tokens(literal.lit.to_token_stream()) } + Self::Array { .. } => return self.execution_err("Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `as stream` to cast the array to a stream."), Self::Stream(value) => value.value.append_cloned_into(output), - } + }) } pub(crate) fn debug(&self) -> String { - use std::fmt::Write; let mut output = String::new(); + self.debug_to(&mut output); + output + } + + fn debug_to(&self, output: &mut String) { + use std::fmt::Write; match self { ExpressionValue::None { .. } => { write!(output, "None").unwrap(); @@ -406,14 +415,24 @@ impl ExpressionValue { write!(output, "[!stream! {}]", string_rep).unwrap(); } } + ExpressionValue::Array(array) => { + write!(output, "[").unwrap(); + for (i, item) in array.items.iter().enumerate() { + if i != 0 { + write!(output, ", ").unwrap(); + } + item.debug_to(output); + } + write!(output, "]").unwrap(); + } _ => { + // This isn't the most efficient, but it's less code and debug doesn't need to be super efficient. let mut stream = OutputStream::new(); - self.output_flattened_to(&mut stream); + self.output_flattened_to(&mut stream).expect("Non-composite values should all be able to be outputted to a stream"); let string_rep = stream.concat_recursive(&ConcatBehaviour::debug()); write!(output, "{}", string_rep).unwrap(); } } - output } } @@ -432,6 +451,7 @@ impl HasValueType for ExpressionValue { Self::String(value) => value.value_type(), Self::Char(value) => value.value_type(), Self::UnsupportedLiteral(value) => value.value_type(), + Self::Array(value) => value.value_type(), Self::Stream(value) => value.value_type(), } } @@ -447,6 +467,7 @@ impl HasSpanRange for ExpressionValue { Self::String(str) => str.span_range, Self::Char(char) => char.span_range, Self::UnsupportedLiteral(lit) => lit.span_range, + Self::Array(array) => array.span_range, Self::Stream(stream) => stream.span_range, } } @@ -484,6 +505,7 @@ pub(super) enum EvaluationLiteralPair { BooleanPair(ExpressionBoolean, ExpressionBoolean), StringPair(ExpressionString, ExpressionString), CharPair(ExpressionChar, ExpressionChar), + ArrayPair(ExpressionArray, ExpressionArray), StreamPair(ExpressionStream, ExpressionStream), } @@ -498,6 +520,7 @@ impl EvaluationLiteralPair { Self::BooleanPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::StringPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::ArrayPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::StreamPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 5766c8fb..3e48001a 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -140,8 +140,7 @@ impl CommandInvocationAs for C { _ => Grouping::Grouped, }; self.execute(context.interpreter)? - .output_to(grouping, output); - Ok(()) + .output_to(grouping, output) } fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { @@ -399,7 +398,7 @@ macro_rules! define_command_enums { const ALL_KIND_NAMES: &'static [&'static str] = &[$($command::COMMAND_NAME,)*]; pub(crate) fn list_all() -> String { - // TODO improve to add an "and" at the end + // TODO: Separate by group, and add "and" at the end Self::ALL_KIND_NAMES.join(", ") } } @@ -439,7 +438,6 @@ macro_rules! define_command_enums { define_command_enums! { // Core Commands SetCommand, - TypedSetCommand, RawCommand, StreamCommand, IgnoreCommand, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index c991e032..0f14c2d2 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -113,42 +113,6 @@ impl NoOutputCommandDefinition for SetCommand { } } -/// This is temporary until we have a proper implementation of #(...) -#[derive(Clone)] -pub(crate) struct TypedSetCommand { - variable: GroupedVariable, - #[allow(unused)] - equals: Token![=], - content: SourceExpression, -} - -impl CommandType for TypedSetCommand { - type OutputKind = OutputKindNone; -} - -impl NoOutputCommandDefinition for TypedSetCommand { - const COMMAND_NAME: &'static str = "typed_set"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(TypedSetCommand { - variable: input.parse()?, - equals: input.parse()?, - content: input.parse()?, - }) - }, - "Expected [!typed_set! #var1 = ]", - ) - } - - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { - let content = self.content.interpret_to_value(interpreter)?; - self.variable.set_value(interpreter, content)?; - Ok(()) - } -} - #[derive(Clone)] pub(crate) struct RawCommand { token_stream: TokenStream, diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 73e69fe9..80274658 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -53,7 +53,7 @@ impl GroupedStreamCommandDefinition for RangeCommand { for value in range_iterator { // It needs to be grouped so that e.g. -1 is interpreted as a single item, not two separate tokens. - value.output_to(Grouping::Grouped, output) + value.output_to(Grouping::Grouped, output)? } Ok(()) diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index b217c55c..f18e1dc3 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -51,8 +51,7 @@ pub(crate) trait IsVariable: HasSpanRange { ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? - .output_to(grouping, output); - Ok(()) + .output_to(grouping, output) } fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { diff --git a/src/transformation/transformer.rs b/src/transformation/transformer.rs index 0b2b4f97..40b69939 100644 --- a/src/transformation/transformer.rs +++ b/src/transformation/transformer.rs @@ -197,7 +197,7 @@ macro_rules! define_transformers { const ALL_KIND_NAMES: &'static [&'static str] = &[$($transformer::TRANSFORMER_NAME,)*]; pub(crate) fn list_all() -> String { - // TODO improve to add an "and" at the end + // TODO: Add "and" at the end Self::ALL_KIND_NAMES.join(", ") } } From 71507632b5709e203990bab32c2e7cb4801275e4 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 12 Feb 2025 15:19:56 +0000 Subject: [PATCH 089/126] feature: Add basic support for arrays --- CHANGELOG.md | 5 +- src/expressions/array.rs | 25 +- src/expressions/evaluation.rs | 36 ++ src/expressions/expression.rs | 21 +- src/expressions/expression_parsing.rs | 96 +++- src/expressions/value.rs | 34 +- src/extensions/errors_and_spans.rs | 1 + src/extensions/parsing.rs | 4 + .../cannot_output_array_to_stream.rs | 7 + .../cannot_output_array_to_stream.stderr | 5 + .../expressions/tuple_syntax_helpful_error.rs | 7 + .../tuple_syntax_helpful_error.stderr | 5 + tests/complex.rs | 4 +- tests/control_flow.rs | 32 +- tests/core.rs | 32 +- tests/expressions.rs | 138 ++--- tests/helpers/prelude.rs | 15 + tests/literal.rs | 32 +- tests/string.rs | 490 +++++++++--------- tests/tokens.rs | 116 ++--- tests/transforming.rs | 62 ++- 21 files changed, 677 insertions(+), 490 deletions(-) create mode 100644 tests/compilation_failures/expressions/cannot_output_array_to_stream.rs create mode 100644 tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr create mode 100644 tests/compilation_failures/expressions/tuple_syntax_helpful_error.rs create mode 100644 tests/compilation_failures/expressions/tuple_syntax_helpful_error.stderr create mode 100644 tests/helpers/prelude.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3759800a..817ac9e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,9 +98,6 @@ Inside a transform stream, the following grammar is supported: ### To come * Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model - * Create parsable array value - * Add test for casting to stream, and compile error when outputting to stream - * Add `+` support for concatenating arrays * Revisit the `SourceStreamInput` abstraction - maybe it's replaced with a `SourceExpression` which needs to output a stream? * Split outputs an array * Intersperse works with either an array or a stream (?) @@ -108,6 +105,7 @@ Inside a transform stream, the following grammar is supported: * We can then consider dropping lots of the `group` wrappers I guess? * Then destructuring and parsing become different: * Destructuring works over the value model; parsing works over the token stream model. + * Parsers can be embedded inside a destructuring * Support a CastTarget of Array (only supported for array and stream and iterator) * Support `#(x[..])` syntax for indexing arrays and streams at read time * Via a post-fix `[...]` operation with high priority @@ -163,6 +161,7 @@ Inside a transform stream, the following grammar is supported: * Put `[!set! ...]` inside an opt-in feature. * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed +* Add benches inspired by this: https://github.com/dtolnay/quote/tree/master/benches * Work on book * Input paradigms: * Streams diff --git a/src/expressions/array.rs b/src/expressions/array.rs index dadf7512..1a353450 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -18,16 +18,23 @@ impl ExpressionArray { UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { return operation.unsupported(self) } - UnaryOperation::Cast { target, target_ident, .. } => match target { - CastTarget::Stream => { - operation.output(self.to_stream_with_grouped_items()?) - }, - CastTarget::Group => { - operation.output(operation.output(self.to_stream_with_grouped_items()?).into_new_output_stream(Grouping::Grouped)?) - }, + UnaryOperation::Cast { + target, + target_ident, + .. + } => match target { + CastTarget::Stream => operation.output(self.into_stream_with_grouped_items()?), + CastTarget::Group => operation.output( + operation + .output(self.into_stream_with_grouped_items()?) + .into_new_output_stream(Grouping::Grouped)?, + ), _ => { if self.items.len() == 1 { - self.items.pop().unwrap().handle_unary_operation(operation)? + self.items + .pop() + .unwrap() + .handle_unary_operation(operation)? } else { return operation.execution_err(format!( "Cannot only attempt to cast a singleton array to {} but the array has {} elements", @@ -40,7 +47,7 @@ impl ExpressionArray { }) } - fn to_stream_with_grouped_items(self) -> ExecutionResult { + fn into_stream_with_grouped_items(self) -> ExecutionResult { let mut stream = OutputStream::new(); for item in self.items { item.output_to(Grouping::Grouped, &mut stream)?; diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 1dd51b09..001e20ee 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -51,6 +51,12 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { }); NextAction::EnterNode(*inner) } + ExpressionNode::Array { delim_span, items } => ArrayStackFrame { + span: delim_span.join(), + unevaluated_items: items.clone(), + evaluated_items: Vec::with_capacity(items.len()), + } + .next(&mut self.operation_stack), ExpressionNode::UnaryOperation { operation, input } => { self.operation_stack .push(EvaluationStackFrame::UnaryOperation { @@ -85,6 +91,10 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { EvaluationStackFrame::UnaryOperation { operation } => { NextAction::HandleValue(operation.evaluate(value)?) } + EvaluationStackFrame::Array(mut array) => { + array.evaluated_items.push(value); + array.next(&mut self.operation_stack) + } EvaluationStackFrame::BinaryOperation { operation, state } => match state { BinaryPath::OnLeftBranch { right } => { if let Some(result) = operation.lazy_evaluate(&value)? { @@ -116,6 +126,7 @@ enum EvaluationStackFrame { Group { span: Span, }, + Array(ArrayStackFrame), UnaryOperation { operation: UnaryOperation, }, @@ -125,6 +136,31 @@ enum EvaluationStackFrame { }, } +struct ArrayStackFrame { + span: Span, + unevaluated_items: Vec, + evaluated_items: Vec, +} + +impl ArrayStackFrame { + fn next(self, operation_stack: &mut Vec) -> NextAction { + match self + .unevaluated_items + .get(self.evaluated_items.len()) + .cloned() + { + Some(next) => { + operation_stack.push(EvaluationStackFrame::Array(self)); + NextAction::EnterNode(next) + } + None => NextAction::HandleValue(ExpressionValue::Array(ExpressionArray { + items: self.evaluated_items, + span_range: self.span.span_range(), + })), + } + } +} + enum BinaryPath { OnLeftBranch { right: ExpressionNodeId }, OnRightBranch { left: ExpressionValue }, diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 18b42ef4..f9b2d237 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -63,7 +63,14 @@ impl Expressionable for Source { return input.parse_err("Braces { ... } are not supported in an expression") } SourcePeekMatch::Group(Delimiter::Bracket) => { - return input.parse_err("Brackets [ ... ] are not supported in an expression") + // This could be handled as parsing a vector of SourceExpressions, + // but it's more efficient to handle nested vectors as a single expression + // in the expression parser + let (_, delim_span) = input.parse_and_enter_group()?; + UnaryAtom::Array { + delim_span, + is_empty: input.is_empty(), + } } SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), SourcePeekMatch::Ident(_) => match input.try_parse_or_revert() { @@ -84,6 +91,12 @@ impl Expressionable for Source { fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { Ok(match input.peek_grammar() { + SourcePeekMatch::Punct(punct) if punct.as_char() == ',' => { + NodeExtension::CommaOperator { + comma: input.parse()?, + is_end_of_stream: input.is_empty(), + } + } SourcePeekMatch::Punct(_) => match input.try_parse_or_revert::() { Ok(operation) => NodeExtension::BinaryOperation(operation), Err(_) => NodeExtension::NoneMatched, @@ -142,7 +155,7 @@ impl Parse for Expression { } } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq, Eq)] pub(super) struct ExpressionNodeId(pub(super) usize); pub(super) enum ExpressionNode { @@ -151,6 +164,10 @@ pub(super) enum ExpressionNode { delim_span: DelimSpan, inner: ExpressionNodeId, }, + Array { + delim_span: DelimSpan, + items: Vec, + }, UnaryOperation { operation: UnaryOperation, input: ExpressionNodeId, diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 04e5f077..a3330cac 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -46,6 +46,22 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { UnaryAtom::Group(delim_span) => { self.push_stack_frame(ExpressionStackFrame::Group { delim_span }) } + UnaryAtom::Array { + delim_span, + is_empty, + } => { + let array_node = self.nodes.add_node(ExpressionNode::Array { + delim_span, + items: Vec::new(), + }); + if is_empty { + self.streams.exit_group(); + WorkItem::TryParseAndApplyExtension { node: array_node } + } else { + self.push_stack_frame(ExpressionStackFrame::Array { array_node }); + WorkItem::RequireUnaryAtom + } + } UnaryAtom::PrefixUnaryOperation(operation) => { self.push_stack_frame(ExpressionStackFrame::IncompletePrefixOperation { operation }) } @@ -75,8 +91,33 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { operation, }) } - NodeExtension::NoneMatched => { - panic!("Not possible, as this has minimum precedence") + NodeExtension::CommaOperator { + comma, + is_end_of_stream: false, + } => { + let top_frame = self + .expression_stack + .last_mut() + .expect("There should always at least be a root"); + match top_frame { + ExpressionStackFrame::Array { array_node } => { + match self.nodes.node_mut(*array_node) { + ExpressionNode::Array { items, .. } => { + items.push(node); + }, + _ => unreachable!("Not possible, the array_node always corresponds to an array node"), + } + }, + _ => return comma.parse_err("Commas are only permitted inside preinterpret arrays []. Preinterpret arrays [a, b] can be used as a drop-in replacement for rust tuples (a, b)."), + } + WorkItem::RequireUnaryAtom + } + NodeExtension::CommaOperator { + is_end_of_stream: true, + .. + } + | NodeExtension::NoneMatched => { + unreachable!("Not possible, as these have minimum precedence") } }) } else { @@ -98,6 +139,26 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { }), } } + ExpressionStackFrame::Array { array_node } => { + assert!(matches!( + extension, + NodeExtension::NoneMatched + | NodeExtension::CommaOperator { + is_end_of_stream: true, + .. + } + )); + match self.nodes.node_mut(array_node) { + ExpressionNode::Array { items, .. } => { + items.push(node); + } + _ => unreachable!( + "Not possible, the array_node always corresponds to an array node" + ), + } + self.streams.exit_group(); + WorkItem::TryParseAndApplyExtension { node: array_node } + } ExpressionStackFrame::IncompletePrefixOperation { operation } => { WorkItem::TryApplyAlreadyParsedExtension { node: self.nodes.add_node(ExpressionNode::UnaryOperation { @@ -162,6 +223,14 @@ impl ExpressionNodes { node_id } + /// Panics if the node id isn't valid + pub(super) fn node_mut( + &mut self, + ExpressionNodeId(node_id): ExpressionNodeId, + ) -> &mut ExpressionNode { + self.nodes.get_mut(node_id).unwrap() + } + pub(super) fn complete(self, root: ExpressionNodeId) -> Expression { Expression { root, @@ -189,6 +258,8 @@ impl ExpressionNodes { enum OperatorPrecendence { // return, break, closures Jump, + // In arrays (this is a preinterpret addition) + NonTerminalComma, /// = += -= *= /= %= &= |= ^= <<= >>= Assign, // .. ..= @@ -370,6 +441,10 @@ enum ExpressionStackFrame { /// * When the group is opened, we add its inside to the parse stream stack /// * When the group is closed, we pop it from the parse stream stack Group { delim_span: DelimSpan }, + /// A marker for the bracketed array. + /// * When the array is opened, we add its inside to the parse stream stack + /// * When the array is closed, we pop it from the parse stream stack + Array { array_node: ExpressionNodeId }, /// An incomplete unary prefix operation /// NB: unary postfix operations such as `as` casting go straight to ExtendableNode IncompletePrefixOperation { operation: PrefixUnaryOperation }, @@ -385,6 +460,7 @@ impl ExpressionStackFrame { match self { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompletePrefixOperation { operation, .. } => { OperatorPrecendence::of_prefix_unary_operation(operation) } @@ -417,12 +493,20 @@ enum WorkItem { pub(super) enum UnaryAtom { Leaf(K::Leaf), Group(DelimSpan), + Array { + delim_span: DelimSpan, + is_empty: bool, + }, PrefixUnaryOperation(PrefixUnaryOperation), } pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), + CommaOperator { + comma: Token![,], + is_end_of_stream: bool, + }, NoneMatched, } @@ -431,6 +515,14 @@ impl NodeExtension { match self { NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), + NodeExtension::CommaOperator { + is_end_of_stream: false, + .. + } => OperatorPrecendence::NonTerminalComma, + NodeExtension::CommaOperator { + is_end_of_stream: true, + .. + } => OperatorPrecendence::MIN, NodeExtension::NoneMatched => OperatorPrecendence::MIN, } } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 602d98fd..f06748cd 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -305,7 +305,9 @@ impl ExpressionValue { } ExpressionValue::Char(value) => value.handle_integer_binary_operation(right, operation), ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), - ExpressionValue::Array(value) => value.handle_integer_binary_operation(right, operation), + ExpressionValue::Array(value) => { + value.handle_integer_binary_operation(right, operation) + } ExpressionValue::Stream(value) => { value.handle_integer_binary_operation(right, operation) } @@ -346,7 +348,10 @@ impl ExpressionValue { self } - pub(crate) fn into_new_output_stream(self, grouping: Grouping) -> ExecutionResult { + pub(crate) fn into_new_output_stream( + self, + grouping: Grouping, + ) -> ExecutionResult { Ok(match (self, grouping) { (Self::Stream(value), Grouping::Flattened) => value.value, (other, grouping) => { @@ -357,19 +362,22 @@ impl ExpressionValue { }) } - pub(crate) fn output_to(&self, grouping: Grouping, output: &mut OutputStream) -> ExecutionResult<()> { + pub(crate) fn output_to( + &self, + grouping: Grouping, + output: &mut OutputStream, + ) -> ExecutionResult<()> { match grouping { Grouping::Grouped => { // Grouping can be important for different values, to ensure they're read atomically // when the output stream is viewed as an array/iterable, e.g. in a for loop. // * Grouping means -1 is interpreted atomically, rather than as a punct then a number // * Grouping means that a stream is interpreted atomically - output - .push_grouped( - |inner| self.output_flattened_to(inner), - Delimiter::None, - self.span_range().join_into_span_else_start(), - )?; + output.push_grouped( + |inner| self.output_flattened_to(inner), + Delimiter::None, + self.span_range().join_into_span_else_start(), + )?; } Grouping::Flattened => { self.output_flattened_to(output)?; @@ -379,7 +387,7 @@ impl ExpressionValue { } fn output_flattened_to(&self, output: &mut OutputStream) -> ExecutionResult<()> { - Ok(match self { + match self { Self::None { .. } => {} Self::Integer(value) => output.push_literal(value.to_literal()), Self::Float(value) => output.push_literal(value.to_literal()), @@ -391,7 +399,8 @@ impl ExpressionValue { } Self::Array { .. } => return self.execution_err("Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `as stream` to cast the array to a stream."), Self::Stream(value) => value.value.append_cloned_into(output), - }) + }; + Ok(()) } pub(crate) fn debug(&self) -> String { @@ -428,7 +437,8 @@ impl ExpressionValue { _ => { // This isn't the most efficient, but it's less code and debug doesn't need to be super efficient. let mut stream = OutputStream::new(); - self.output_flattened_to(&mut stream).expect("Non-composite values should all be able to be outputted to a stream"); + self.output_flattened_to(&mut stream) + .expect("Non-composite values should all be able to be outputted to a stream"); let string_rep = stream.concat_recursive(&ConcatBehaviour::debug()); write!(output, "{}", string_rep).unwrap(); } diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index e76ea984..5e331499 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -227,6 +227,7 @@ impl_auto_span_range! { syn::token::Ne, syn::token::Ge, syn::token::Gt, + syn::token::Comma, } macro_rules! single_span_token { diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index bb06c7c3..149260c1 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -168,6 +168,10 @@ impl<'a, K> ParseStreamStack<'a, K> { self.current().parse() } + pub(crate) fn is_empty(&self) -> bool { + self.current().is_empty() + } + #[allow(unused)] pub(crate) fn peek(&mut self, token: T) -> bool { self.current().peek(token) diff --git a/tests/compilation_failures/expressions/cannot_output_array_to_stream.rs b/tests/compilation_failures/expressions/cannot_output_array_to_stream.rs new file mode 100644 index 00000000..a33b5c7e --- /dev/null +++ b/tests/compilation_failures/expressions/cannot_output_array_to_stream.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #([1, 2, 3]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr b/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr new file mode 100644 index 00000000..7d947dc5 --- /dev/null +++ b/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr @@ -0,0 +1,5 @@ +error: Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `as stream` to cast the array to a stream. + --> tests/compilation_failures/expressions/cannot_output_array_to_stream.rs:5:11 + | +5 | #([1, 2, 3]) + | ^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/tuple_syntax_helpful_error.rs b/tests/compilation_failures/expressions/tuple_syntax_helpful_error.rs new file mode 100644 index 00000000..1e1231fd --- /dev/null +++ b/tests/compilation_failures/expressions/tuple_syntax_helpful_error.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let x = (1, 2)) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/tuple_syntax_helpful_error.stderr b/tests/compilation_failures/expressions/tuple_syntax_helpful_error.stderr new file mode 100644 index 00000000..fe284cb9 --- /dev/null +++ b/tests/compilation_failures/expressions/tuple_syntax_helpful_error.stderr @@ -0,0 +1,5 @@ +error: Commas are only permitted inside preinterpret arrays []. Preinterpret arrays [a, b] can be used as a drop-in replacement for rust tuples (a, b). + --> tests/compilation_failures/expressions/tuple_syntax_helpful_error.rs:5:21 + | +5 | #(let x = (1, 2)) + | ^ diff --git a/tests/complex.rs b/tests/complex.rs index 1331a6fa..c968e17c 100644 --- a/tests/complex.rs +++ b/tests/complex.rs @@ -1,4 +1,6 @@ -use preinterpret::*; +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; preinterpret! { [!set! #bytes = 32] diff --git a/tests/control_flow.rs b/tests/control_flow.rs index cb8ec038..a7139072 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -1,13 +1,9 @@ #![allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 #![allow(clippy::zero_prefixed_literal)] // https://github.com/rust-lang/rust-clippy/issues/14199 -use preinterpret::preinterpret; - -macro_rules! assert_preinterpret_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] @@ -18,24 +14,24 @@ fn test_control_flow_compilation_failures() { #[test] fn test_if() { - assert_preinterpret_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); + preinterpret_assert_eq!({ #(x = 1 == 2) [!if! #x { "YES" } !else! { "NO" }] }, "NO"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ #(x = 1; y = 2) [!if! #x == #y { "YES" } !else! { "NO" }] }, "NO"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ 0 [!if! true { + 1 }] }, 1); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ 0 [!if! false { + 1 }] }, 0); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!if! false { 1 } !elif! false { @@ -50,7 +46,7 @@ fn test_if() { #[test] fn test_while() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ #(x = 0) [!while! #x < 5 { #(x += 1) }] #x @@ -59,7 +55,7 @@ fn test_while() { #[test] fn test_loop_continue_and_break() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { #(x = 0) [!loop! { @@ -70,7 +66,7 @@ fn test_loop_continue_and_break() { }, 10 ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string! [!for! #x in [!range! 65..75] { [!if! #x % 2 == 0 { [!continue!] }] @@ -83,7 +79,7 @@ fn test_loop_continue_and_break() { #[test] fn test_for() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string! [!for! #x in [!range! 65..70] { #(#x as u8 as char) @@ -91,7 +87,7 @@ fn test_for() { }, "ABCDE" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string! [!for! (#x,) in [(a,) (b,) (c,)] { #x diff --git a/tests/core.rs b/tests/core.rs index 1fc0813c..e364848f 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,10 +1,8 @@ use preinterpret::preinterpret; -macro_rules! assert_preinterpret_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] @@ -15,11 +13,11 @@ fn test_core_compilation_failures() { #[test] fn test_set() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #output = "Hello World!"] #output }, "Hello World!"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #hello = "Hello"] [!set! #world = "World"] [!set! #output = #hello " " #world "!"] @@ -30,7 +28,7 @@ fn test_set() { #[test] fn test_raw() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string! [!raw! #variable and [!command!] are not interpreted or error]] }, "#variableand[!command!]arenotinterpretedorerror" ); @@ -38,7 +36,7 @@ fn test_raw() { #[test] fn test_extend() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #variable = "Hello"] [!set! #variable += " World!"] @@ -46,7 +44,7 @@ fn test_extend() { }, "Hello World!" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { #(i = 1) [!set! #output = [!..group!]] @@ -65,7 +63,7 @@ fn test_extend() { #[test] fn test_ignore() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = false] [!ignore! [!set! #x = true] nothing is interpreted. Everything is ignored...] #x @@ -74,18 +72,18 @@ fn test_ignore() { #[test] fn test_empty_set() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x] [!set! #x += "hello"] #x }, "hello"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x, #y] [!set! #x += "hello"] [!set! #y += "world"] [!string! #x " " #y] }, "hello world"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x, #y, #z,] [!set! #x += "hello"] [!set! #y += "world"] @@ -95,7 +93,7 @@ fn test_empty_set() { #[test] fn test_discard_set() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = false] [!set! _ = [!set! #x = true] things _are_ interpreted, but the result is ignored...] #x @@ -106,7 +104,7 @@ fn test_discard_set() { fn test_debug() { // It keeps the semantic punctuation spacing intact // (e.g. it keeps 'a and >> together) - assert_preinterpret_eq!( + preinterpret_assert_eq!( [!debug! [!stream! impl<'a, T> MyStruct<'a, T> { pub fn new() -> Self { !($crate::Test::CONSTANT >> 5 > 1) @@ -118,7 +116,7 @@ fn test_debug() { // NOTE: The output code can't be used directly as preinterpret input // because it doesn't stick [!raw! ...] around things which could be confused // for the preinterpret grammar. Perhaps it could/should in future. - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #x = Hello (World)] [!debug! [!stream! #x [!raw! #test] "and" [!raw! ##] #..x]] diff --git a/tests/expressions.rs b/tests/expressions.rs index 935c390c..623e6366 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -1,16 +1,6 @@ -use preinterpret::preinterpret; - -macro_rules! assert_preinterpret_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} - -macro_rules! assert_expression_eq { - (#($($input:tt)*), $($output:tt)*) => { - assert_eq!(preinterpret!(#($($input)*)), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] @@ -21,47 +11,59 @@ fn test_expression_compilation_failures() { #[test] fn test_basic_evaluate_works() { - assert_expression_eq!(#(!!(!!(true))), true); - assert_expression_eq!(#(1 + 5), 6u8); - assert_expression_eq!(#(1 + 5), 6i128); - assert_expression_eq!(#("Hello" + " " + "World!"), "Hello World!"); - assert_expression_eq!(#(1 + 5u16), 6u16); - assert_expression_eq!(#(127i8 + (-127i8) + (-127i8)), -127i8); - assert_expression_eq!(#(3.0 + 3.2), 6.2); - assert_expression_eq!(#(3.6 + 3999999999999999992.0), 3.6 + 3999999999999999992.0); - assert_expression_eq!(#(-3.2), -3.2); - assert_expression_eq!(#(true && true || false), true); - assert_expression_eq!(#(true || false && false), true); // The && has priority - assert_expression_eq!(#(true | false & false), true); // The & has priority - assert_expression_eq!(#(true as u32 + 2), 3); - assert_expression_eq!(#(3.57 as int + 1), 4u32); - assert_expression_eq!(#(3.57 as int + 1), 4u64); - assert_expression_eq!(#(0b1000 & 0b1101), 0b1000); - assert_expression_eq!(#(0b1000 | 0b1101), 0b1101); - assert_expression_eq!(#(0b1000 ^ 0b1101), 0b101); - assert_expression_eq!(#(5 << 2), 20); - assert_expression_eq!(#(5 >> 1), 2); - assert_expression_eq!(#(123 == 456), false); - assert_expression_eq!(#(123 < 456), true); - assert_expression_eq!(#(123 <= 456), true); - assert_expression_eq!(#(123 != 456), true); - assert_expression_eq!(#(123 >= 456), false); - assert_expression_eq!(#(123 > 456), false); - assert_expression_eq!(#(six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); - assert_expression_eq!(#( + preinterpret_assert_eq!(#(!!(!!(true))), true); + preinterpret_assert_eq!(#(1 + 5), 6u8); + preinterpret_assert_eq!(#(1 + 5), 6i128); + preinterpret_assert_eq!(#("Hello" + " " + "World!"), "Hello World!"); + preinterpret_assert_eq!(#(1 + 5u16), 6u16); + preinterpret_assert_eq!(#(127i8 + (-127i8) + (-127i8)), -127i8); + preinterpret_assert_eq!(#(3.0 + 3.2), 6.2); + preinterpret_assert_eq!(#(3.6 + 3999999999999999992.0), 3.6 + 3999999999999999992.0); + preinterpret_assert_eq!(#(-3.2), -3.2); + preinterpret_assert_eq!(#(true && true || false), true); + preinterpret_assert_eq!(#(true || false && false), true); // The && has priority + preinterpret_assert_eq!(#(true | false & false), true); // The & has priority + preinterpret_assert_eq!(#(true as u32 + 2), 3); + preinterpret_assert_eq!(#(3.57 as int + 1), 4u32); + preinterpret_assert_eq!(#(3.57 as int + 1), 4u64); + preinterpret_assert_eq!(#(0b1000 & 0b1101), 0b1000); + preinterpret_assert_eq!(#(0b1000 | 0b1101), 0b1101); + preinterpret_assert_eq!(#(0b1000 ^ 0b1101), 0b101); + preinterpret_assert_eq!(#(5 << 2), 20); + preinterpret_assert_eq!(#(5 >> 1), 2); + preinterpret_assert_eq!(#(123 == 456), false); + preinterpret_assert_eq!(#(123 < 456), true); + preinterpret_assert_eq!(#(123 <= 456), true); + preinterpret_assert_eq!(#(123 != 456), true); + preinterpret_assert_eq!(#(123 >= 456), false); + preinterpret_assert_eq!(#(123 > 456), false); + preinterpret_assert_eq!(#(six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); + preinterpret_assert_eq!(#( partial_sum = [!stream! + 2]; [!debug! [!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] ), "[!stream! [!group! 5 + 2] = [!group! 7]]"); - assert_expression_eq!(#(1 + [!range! 1..2] as int), 2); - assert_expression_eq!(#("hello" == "world"), false); - assert_expression_eq!(#("hello" == "hello"), true); - assert_expression_eq!(#('A' as u8 == 65), true); - assert_expression_eq!(#(65u8 as char == 'A'), true); - assert_expression_eq!(#('A' == 'A'), true); - assert_expression_eq!(#('A' == 'B'), false); - assert_expression_eq!(#('A' < 'B'), true); - assert_expression_eq!(#("Zoo" > "Aardvark"), true); - assert_expression_eq!( + preinterpret_assert_eq!(#(1 + [!range! 1..2] as int), 2); + preinterpret_assert_eq!(#("hello" == "world"), false); + preinterpret_assert_eq!(#("hello" == "hello"), true); + preinterpret_assert_eq!(#('A' as u8 == 65), true); + preinterpret_assert_eq!(#(65u8 as char == 'A'), true); + preinterpret_assert_eq!(#('A' == 'A'), true); + preinterpret_assert_eq!(#('A' == 'B'), false); + preinterpret_assert_eq!(#('A' < 'B'), true); + preinterpret_assert_eq!(#("Zoo" > "Aardvark"), true); + preinterpret_assert_eq!( + #([!debug! "Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group]), + r#"[!stream! "Hello" "World" 2 [!group! 2]]"# + ); + preinterpret_assert_eq!( + [!debug! #([1, 2, 1 + 2, 4])], + "[1, 2, 3, 4]" + ); + preinterpret_assert_eq!( + [!debug! #([1 + (3 + 4), [5,] + [], [[6, 7],]] + [123])], + "[8, [5], [[6, 7]], 123]" + ); + preinterpret_assert_eq!( #([!debug! "Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group]), r#"[!stream! "Hello" "World" 2 [!group! 2]]"# ); @@ -74,18 +76,18 @@ fn test_expression_precedence() { // * Operators at the same precedence should left-associate. // 1 + -1 + ((2 + 4) * 3) - 9 => 1 + -1 + 18 - 9 => 9 - assert_expression_eq!(#(1 + -(1) + (2 + 4) * 3 - 9), 9); + preinterpret_assert_eq!(#(1 + -(1) + (2 + 4) * 3 - 9), 9); // (true > true) > true => false > true => false - assert_expression_eq!(#(true > true > true), false); + preinterpret_assert_eq!(#(true > true > true), false); // (5 - 2) - 1 => 3 - 1 => 2 - assert_expression_eq!(#(5 - 2 - 1), 2); + preinterpret_assert_eq!(#(5 - 2 - 1), 2); // ((3 * 3 - 4) < (3 << 1)) && true => 5 < 6 => true - assert_expression_eq!(#(3 * 3 - 4 < 3 << 1 && true), true); + preinterpret_assert_eq!(#(3 * 3 - 4 < 3 << 1 && true), true); } #[test] fn test_very_long_expression_works() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!settings! { iteration_limit: 100000, @@ -100,7 +102,7 @@ fn test_very_long_expression_works() { #[test] fn boolean_operators_short_circuit() { // && short-circuits if first operand is false - assert_expression_eq!( + preinterpret_assert_eq!( #( let is_lazy = true; let _ = false && #(is_lazy = false; true); @@ -109,7 +111,7 @@ fn boolean_operators_short_circuit() { true ); // || short-circuits if first operand is true - assert_expression_eq!( + preinterpret_assert_eq!( #( let is_lazy = true; let _ = true || #(is_lazy = false; true); @@ -118,7 +120,7 @@ fn boolean_operators_short_circuit() { true ); // For comparison, the & operator does _not_ short-circuit - assert_expression_eq!( + preinterpret_assert_eq!( #( is_lazy = true; let _ = false & #(is_lazy = false; true); @@ -130,14 +132,14 @@ fn boolean_operators_short_circuit() { #[test] fn assign_works() { - assert_expression_eq!( + preinterpret_assert_eq!( #( let x = 5 + 5; [!debug! #..x] ), "[!stream! 10]" ); - assert_expression_eq!( + preinterpret_assert_eq!( #( let x = 10; x /= 1 + 1; // 10 / (1 + 1) @@ -148,7 +150,7 @@ fn assign_works() { ); // Assign can reference itself in its expression, // because the expression result is buffered. - assert_expression_eq!( + preinterpret_assert_eq!( #( let x = 2; x += #x; @@ -160,21 +162,21 @@ fn assign_works() { #[test] fn test_range() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( [!string![!intersperse! { items: [!range! -2..5], separator: [" "], }]], "-2 -1 0 1 2 3 4" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( [!string![!intersperse! { items: [!range! -2..=5], separator: [" "], }]], "-2 -1 0 1 2 3 4 5" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { #(x = 2) [!string! [!intersperse! { @@ -184,7 +186,7 @@ fn test_range() { }, "4 5" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [!range! 8..=5], @@ -193,8 +195,8 @@ fn test_range() { }, "" ); - assert_preinterpret_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); - assert_preinterpret_eq!( + preinterpret_assert_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); + preinterpret_assert_eq!( { [!debug! [!..range! -1i8..3i8]] }, "[!stream! [!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]]" ); diff --git a/tests/helpers/prelude.rs b/tests/helpers/prelude.rs new file mode 100644 index 00000000..5c90d782 --- /dev/null +++ b/tests/helpers/prelude.rs @@ -0,0 +1,15 @@ +// This file is imported into lots of different integration test files, each of which is considered as a separate crates / compilation unit. +// Some of these exports aren't used by all integration test files, so we need to suppress the warnings. +#![allow(unused_imports, unused_macros)] +pub use preinterpret::*; + +macro_rules! preinterpret_assert_eq { + (#($($input:tt)*), $($output:tt)*) => { + assert_eq!(preinterpret!(#($($input)*)), $($output)*); + }; + ($input:tt, $($output:tt)*) => { + assert_eq!(preinterpret!($input), $($output)*); + }; +} + +pub(crate) use preinterpret_assert_eq; diff --git a/tests/literal.rs b/tests/literal.rs index 0f9ad07d..8acebd35 100644 --- a/tests/literal.rs +++ b/tests/literal.rs @@ -1,46 +1,42 @@ -use preinterpret::preinterpret; - -macro_rules! my_assert_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] fn test_string_literal() { - my_assert_eq!([!literal! '"' hello World! "\""], "helloWorld!"); + preinterpret_assert_eq!([!literal! '"' hello World! "\""], "helloWorld!"); } #[test] fn test_byte_string_literal() { - my_assert_eq!([!literal! b '"' hello World! "\""], b"helloWorld!"); + preinterpret_assert_eq!([!literal! b '"' hello World! "\""], b"helloWorld!"); } #[test] fn test_c_string_literal() { - my_assert_eq!([!literal! c '"' hello World! "\""], c"helloWorld!"); + preinterpret_assert_eq!([!literal! c '"' hello World! "\""], c"helloWorld!"); } #[test] fn test_integer_literal() { - my_assert_eq!([!literal! "123" 456], 123456); - my_assert_eq!([!literal! 456u "32"], 456); - my_assert_eq!([!literal! 000 u64], 0); + preinterpret_assert_eq!([!literal! "123" 456], 123456); + preinterpret_assert_eq!([!literal! 456u "32"], 456); + preinterpret_assert_eq!([!literal! 000 u64], 0); } #[test] fn test_float_literal() { - my_assert_eq!([!literal! 0 . 123], 0.123); - my_assert_eq!([!literal! 677f32], 677f32); - my_assert_eq!([!literal! "12" 9f64], 129f64); + preinterpret_assert_eq!([!literal! 0 . 123], 0.123); + preinterpret_assert_eq!([!literal! 677f32], 677f32); + preinterpret_assert_eq!([!literal! "12" 9f64], 129f64); } #[test] fn test_character() { - my_assert_eq!([!literal! "'" 7 "'"], '7'); + preinterpret_assert_eq!([!literal! "'" 7 "'"], '7'); } #[test] fn test_byte_character() { - my_assert_eq!([!literal! "b'a'"], b'a'); + preinterpret_assert_eq!([!literal! "b'a'"], b'a'); } diff --git a/tests/string.rs b/tests/string.rs index c9102b59..c3ebecfa 100644 --- a/tests/string.rs +++ b/tests/string.rs @@ -1,295 +1,291 @@ -use preinterpret::preinterpret; - -macro_rules! my_assert_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] fn test_string() { - my_assert_eq!([!string! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my_MixedCaseSTRINGWhichis #awesome -whatdo youthink?"); - my_assert_eq!([!string! UPPER], "UPPER"); - my_assert_eq!([!string! lower], "lower"); - my_assert_eq!([!string! lower_snake_case], "lower_snake_case"); - my_assert_eq!([!string! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); - my_assert_eq!([!string! lowerCamelCase], "lowerCamelCase"); - my_assert_eq!([!string! UpperCamelCase], "UpperCamelCase"); - my_assert_eq!([!string! Capitalized], "Capitalized"); - my_assert_eq!([!string! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID: A quick brown fox jumps over the lazy dog."); - my_assert_eq!([!string! "hello_w🌎rld"], "hello_w🌎rld"); - my_assert_eq!([!string! "kebab-case"], "kebab-case"); - my_assert_eq!([!string! "~~h4xx0rZ <3 1337c0de"], "~~h4xx0rZ <3 1337c0de"); - my_assert_eq!([!string! PostgreSQLConnection], "PostgreSQLConnection"); - my_assert_eq!([!string! PostgreSqlConnection], "PostgreSqlConnection"); - my_assert_eq!([!string! "U+000A LINE FEED (LF)"], "U+000A LINE FEED (LF)"); - my_assert_eq!([!string! "\nThis\r\n is a\tmulti-line\nstring"], "\nThis\r\n is a\tmulti-line\nstring"); - my_assert_eq!([!string! " lots of _ space and _whacky |c$ara_cte>>rs|"], " lots of _ space and _whacky |c$ara_cte>>rs|"); - my_assert_eq!([!string! "über CöÖl"], "über CöÖl"); - my_assert_eq!([!string! "◌̈ubër Cöol"], "◌̈ubër Cöol"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!string! "真是难以置信！"], "真是难以置信！"); + preinterpret_assert_eq!([!string! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my_MixedCaseSTRINGWhichis #awesome -whatdo youthink?"); + preinterpret_assert_eq!([!string! UPPER], "UPPER"); + preinterpret_assert_eq!([!string! lower], "lower"); + preinterpret_assert_eq!([!string! lower_snake_case], "lower_snake_case"); + preinterpret_assert_eq!([!string! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); + preinterpret_assert_eq!([!string! lowerCamelCase], "lowerCamelCase"); + preinterpret_assert_eq!([!string! UpperCamelCase], "UpperCamelCase"); + preinterpret_assert_eq!([!string! Capitalized], "Capitalized"); + preinterpret_assert_eq!([!string! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID: A quick brown fox jumps over the lazy dog."); + preinterpret_assert_eq!([!string! "hello_w🌎rld"], "hello_w🌎rld"); + preinterpret_assert_eq!([!string! "kebab-case"], "kebab-case"); + preinterpret_assert_eq!([!string! "~~h4xx0rZ <3 1337c0de"], "~~h4xx0rZ <3 1337c0de"); + preinterpret_assert_eq!([!string! PostgreSQLConnection], "PostgreSQLConnection"); + preinterpret_assert_eq!([!string! PostgreSqlConnection], "PostgreSqlConnection"); + preinterpret_assert_eq!([!string! "U+000A LINE FEED (LF)"], "U+000A LINE FEED (LF)"); + preinterpret_assert_eq!([!string! "\nThis\r\n is a\tmulti-line\nstring"], "\nThis\r\n is a\tmulti-line\nstring"); + preinterpret_assert_eq!([!string! " lots of _ space and _whacky |c$ara_cte>>rs|"], " lots of _ space and _whacky |c$ara_cte>>rs|"); + preinterpret_assert_eq!([!string! "über CöÖl"], "über CöÖl"); + preinterpret_assert_eq!([!string! "◌̈ubër Cöol"], "◌̈ubër Cöol"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!string! "真是难以置信！"], "真是难以置信！"); } #[test] fn test_upper() { - my_assert_eq!([!upper! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "MY_MIXEDCASESTRINGWHICHIS #AWESOME -WHATDO YOUTHINK?"); - my_assert_eq!([!upper! UPPER], "UPPER"); - my_assert_eq!([!upper! lower], "LOWER"); - my_assert_eq!([!upper! lower_snake_case], "LOWER_SNAKE_CASE"); - my_assert_eq!([!upper! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); - my_assert_eq!([!upper! lowerCamelCase], "LOWERCAMELCASE"); - my_assert_eq!([!upper! UpperCamelCase], "UPPERCAMELCASE"); - my_assert_eq!([!upper! Capitalized], "CAPITALIZED"); - my_assert_eq!([!upper! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID: A QUICK BROWN FOX JUMPS OVER THE LAZY DOG."); - my_assert_eq!([!upper! "hello_w🌎rld"], "HELLO_W🌎RLD"); - my_assert_eq!([!upper! "kebab-case"], "KEBAB-CASE"); - my_assert_eq!([!upper! "~~h4xx0rZ <3 1337c0de"], "~~H4XX0RZ <3 1337C0DE"); - my_assert_eq!([!upper! PostgreSQLConnection], "POSTGRESQLCONNECTION"); - my_assert_eq!([!upper! PostgreSqlConnection], "POSTGRESQLCONNECTION"); - my_assert_eq!([!upper! "U+000A LINE FEED (LF)"], "U+000A LINE FEED (LF)"); - my_assert_eq!([!upper! "\nThis\r\n is a\tmulti-line\nstring"], "\nTHIS\r\n IS A\tMULTI-LINE\nSTRING"); - my_assert_eq!([!upper! " lots of _ space and _whacky |c$ara_cte>>rs|"], " LOTS OF _ SPACE AND _WHACKY |C$ARA_CTE>>RS|"); - my_assert_eq!([!upper! "über CöÖl"], "ÜBER CÖÖL"); - my_assert_eq!([!upper! "◌̈ubër Cöol"], "◌̈UBËR CÖOL"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!upper! "真是难以置信！"], "真是难以置信！"); + preinterpret_assert_eq!([!upper! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "MY_MIXEDCASESTRINGWHICHIS #AWESOME -WHATDO YOUTHINK?"); + preinterpret_assert_eq!([!upper! UPPER], "UPPER"); + preinterpret_assert_eq!([!upper! lower], "LOWER"); + preinterpret_assert_eq!([!upper! lower_snake_case], "LOWER_SNAKE_CASE"); + preinterpret_assert_eq!([!upper! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); + preinterpret_assert_eq!([!upper! lowerCamelCase], "LOWERCAMELCASE"); + preinterpret_assert_eq!([!upper! UpperCamelCase], "UPPERCAMELCASE"); + preinterpret_assert_eq!([!upper! Capitalized], "CAPITALIZED"); + preinterpret_assert_eq!([!upper! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID: A QUICK BROWN FOX JUMPS OVER THE LAZY DOG."); + preinterpret_assert_eq!([!upper! "hello_w🌎rld"], "HELLO_W🌎RLD"); + preinterpret_assert_eq!([!upper! "kebab-case"], "KEBAB-CASE"); + preinterpret_assert_eq!([!upper! "~~h4xx0rZ <3 1337c0de"], "~~H4XX0RZ <3 1337C0DE"); + preinterpret_assert_eq!([!upper! PostgreSQLConnection], "POSTGRESQLCONNECTION"); + preinterpret_assert_eq!([!upper! PostgreSqlConnection], "POSTGRESQLCONNECTION"); + preinterpret_assert_eq!([!upper! "U+000A LINE FEED (LF)"], "U+000A LINE FEED (LF)"); + preinterpret_assert_eq!([!upper! "\nThis\r\n is a\tmulti-line\nstring"], "\nTHIS\r\n IS A\tMULTI-LINE\nSTRING"); + preinterpret_assert_eq!([!upper! " lots of _ space and _whacky |c$ara_cte>>rs|"], " LOTS OF _ SPACE AND _WHACKY |C$ARA_CTE>>RS|"); + preinterpret_assert_eq!([!upper! "über CöÖl"], "ÜBER CÖÖL"); + preinterpret_assert_eq!([!upper! "◌̈ubër Cöol"], "◌̈UBËR CÖOL"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!upper! "真是难以置信！"], "真是难以置信！"); } #[test] fn test_lower() { - my_assert_eq!([!lower! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my_mixedcasestringwhichis #awesome -whatdo youthink?"); - my_assert_eq!([!lower! UPPER], "upper"); - my_assert_eq!([!lower! lower], "lower"); - my_assert_eq!([!lower! lower_snake_case], "lower_snake_case"); - my_assert_eq!([!lower! UPPER_SNAKE_CASE], "upper_snake_case"); - my_assert_eq!([!lower! lowerCamelCase], "lowercamelcase"); - my_assert_eq!([!lower! UpperCamelCase], "uppercamelcase"); - my_assert_eq!([!lower! Capitalized], "capitalized"); - my_assert_eq!([!lower! "THEY SAID: A quick brown fox jumps over the lazy dog."], "they said: a quick brown fox jumps over the lazy dog."); - my_assert_eq!([!lower! "hello_w🌎rld"], "hello_w🌎rld"); - my_assert_eq!([!lower! "kebab-case"], "kebab-case"); - my_assert_eq!([!lower! "~~h4xx0rZ <3 1337c0de"], "~~h4xx0rz <3 1337c0de"); - my_assert_eq!([!lower! PostgreSQLConnection], "postgresqlconnection"); - my_assert_eq!([!lower! PostgreSqlConnection], "postgresqlconnection"); - my_assert_eq!([!lower! "U+000A LINE FEED (LF)"], "u+000a line feed (lf)"); - my_assert_eq!([!lower! "\nThis\r\n is a\tmulti-line\nstring"], "\nthis\r\n is a\tmulti-line\nstring"); - my_assert_eq!([!lower! " lots of _ space and _whacky |c$ara_cte>>rs|"], " lots of _ space and _whacky |c$ara_cte>>rs|"); - my_assert_eq!([!lower! "über CöÖl"], "über cööl"); - my_assert_eq!([!lower! "◌̈ubër Cööl"], "◌̈ubër cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!lower! "真是难以置信！"], "真是难以置信！"); + preinterpret_assert_eq!([!lower! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my_mixedcasestringwhichis #awesome -whatdo youthink?"); + preinterpret_assert_eq!([!lower! UPPER], "upper"); + preinterpret_assert_eq!([!lower! lower], "lower"); + preinterpret_assert_eq!([!lower! lower_snake_case], "lower_snake_case"); + preinterpret_assert_eq!([!lower! UPPER_SNAKE_CASE], "upper_snake_case"); + preinterpret_assert_eq!([!lower! lowerCamelCase], "lowercamelcase"); + preinterpret_assert_eq!([!lower! UpperCamelCase], "uppercamelcase"); + preinterpret_assert_eq!([!lower! Capitalized], "capitalized"); + preinterpret_assert_eq!([!lower! "THEY SAID: A quick brown fox jumps over the lazy dog."], "they said: a quick brown fox jumps over the lazy dog."); + preinterpret_assert_eq!([!lower! "hello_w🌎rld"], "hello_w🌎rld"); + preinterpret_assert_eq!([!lower! "kebab-case"], "kebab-case"); + preinterpret_assert_eq!([!lower! "~~h4xx0rZ <3 1337c0de"], "~~h4xx0rz <3 1337c0de"); + preinterpret_assert_eq!([!lower! PostgreSQLConnection], "postgresqlconnection"); + preinterpret_assert_eq!([!lower! PostgreSqlConnection], "postgresqlconnection"); + preinterpret_assert_eq!([!lower! "U+000A LINE FEED (LF)"], "u+000a line feed (lf)"); + preinterpret_assert_eq!([!lower! "\nThis\r\n is a\tmulti-line\nstring"], "\nthis\r\n is a\tmulti-line\nstring"); + preinterpret_assert_eq!([!lower! " lots of _ space and _whacky |c$ara_cte>>rs|"], " lots of _ space and _whacky |c$ara_cte>>rs|"); + preinterpret_assert_eq!([!lower! "über CöÖl"], "über cööl"); + preinterpret_assert_eq!([!lower! "◌̈ubër Cööl"], "◌̈ubër cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!lower! "真是难以置信！"], "真是难以置信！"); } #[test] fn test_snake() { - my_assert_eq!([!snake! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my_mixed_case_string_whichis_awesome_whatdo_youthink"); - my_assert_eq!([!snake! UPPER], "upper"); - my_assert_eq!([!snake! lower], "lower"); - my_assert_eq!([!snake! lower_snake_case], "lower_snake_case"); - my_assert_eq!([!snake! UPPER_SNAKE_CASE], "upper_snake_case"); - my_assert_eq!([!snake! lowerCamelCase], "lower_camel_case"); - my_assert_eq!([!snake! UpperCamelCase], "upper_camel_case"); - my_assert_eq!([!snake! Capitalized], "capitalized"); - my_assert_eq!([!snake! "THEY SAID: A quick brown fox jumps over the lazy dog."], "they_said_a_quick_brown_fox_jumps_over_the_lazy_dog"); - my_assert_eq!([!snake! "hello_w🌎rld"], "hello_w_rld"); - my_assert_eq!([!snake! "kebab-case"], "kebab_case"); - my_assert_eq!([!snake! "~~h4xx0rZ <3 1337c0de"], "h4xx0r_z_3_1337c0de"); - my_assert_eq!([!snake! PostgreSQLConnection], "postgre_sql_connection"); - my_assert_eq!([!snake! PostgreSqlConnection], "postgre_sql_connection"); - my_assert_eq!([!snake! "U+000A LINE FEED (LF)"], "u_000a_line_feed_lf"); - my_assert_eq!([!snake! "\nThis\r\n is a\tmulti-line\nstring"], "this_is_a_multi_line_string"); - my_assert_eq!([!snake! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lots_of_space_and_whacky_c_ara_cte_rs"); - my_assert_eq!([!snake! "über CöÖl"], "über_cö_öl"); - my_assert_eq!([!snake! "◌̈ubër Cööl"], "ube_r_cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!snake! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!snake! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my_mixed_case_string_whichis_awesome_whatdo_youthink"); + preinterpret_assert_eq!([!snake! UPPER], "upper"); + preinterpret_assert_eq!([!snake! lower], "lower"); + preinterpret_assert_eq!([!snake! lower_snake_case], "lower_snake_case"); + preinterpret_assert_eq!([!snake! UPPER_SNAKE_CASE], "upper_snake_case"); + preinterpret_assert_eq!([!snake! lowerCamelCase], "lower_camel_case"); + preinterpret_assert_eq!([!snake! UpperCamelCase], "upper_camel_case"); + preinterpret_assert_eq!([!snake! Capitalized], "capitalized"); + preinterpret_assert_eq!([!snake! "THEY SAID: A quick brown fox jumps over the lazy dog."], "they_said_a_quick_brown_fox_jumps_over_the_lazy_dog"); + preinterpret_assert_eq!([!snake! "hello_w🌎rld"], "hello_w_rld"); + preinterpret_assert_eq!([!snake! "kebab-case"], "kebab_case"); + preinterpret_assert_eq!([!snake! "~~h4xx0rZ <3 1337c0de"], "h4xx0r_z_3_1337c0de"); + preinterpret_assert_eq!([!snake! PostgreSQLConnection], "postgre_sql_connection"); + preinterpret_assert_eq!([!snake! PostgreSqlConnection], "postgre_sql_connection"); + preinterpret_assert_eq!([!snake! "U+000A LINE FEED (LF)"], "u_000a_line_feed_lf"); + preinterpret_assert_eq!([!snake! "\nThis\r\n is a\tmulti-line\nstring"], "this_is_a_multi_line_string"); + preinterpret_assert_eq!([!snake! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lots_of_space_and_whacky_c_ara_cte_rs"); + preinterpret_assert_eq!([!snake! "über CöÖl"], "über_cö_öl"); + preinterpret_assert_eq!([!snake! "◌̈ubër Cööl"], "ube_r_cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!snake! "真是难以置信！"], "真是难以置信"); } #[test] fn test_upper_snake() { - my_assert_eq!([!upper_snake! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "MY_MIXED_CASE_STRING_WHICHIS_AWESOME_WHATDO_YOUTHINK"); - my_assert_eq!([!upper_snake! UPPER], "UPPER"); - my_assert_eq!([!upper_snake! lower], "LOWER"); - my_assert_eq!([!upper_snake! lower_snake_case], "LOWER_SNAKE_CASE"); - my_assert_eq!([!upper_snake! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); - my_assert_eq!([!upper_snake! lowerCamelCase], "LOWER_CAMEL_CASE"); - my_assert_eq!([!upper_snake! UpperCamelCase], "UPPER_CAMEL_CASE"); - my_assert_eq!([!upper_snake! Capitalized], "CAPITALIZED"); - my_assert_eq!([!upper_snake! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY_SAID_A_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG"); - my_assert_eq!([!upper_snake! "hello_w🌎rld"], "HELLO_W_RLD"); - my_assert_eq!([!upper_snake! "kebab-case"], "KEBAB_CASE"); - my_assert_eq!([!upper_snake! "~~h4xx0rZ <3 1337c0de"], "H4XX0R_Z_3_1337C0DE"); - my_assert_eq!([!upper_snake! PostgreSQLConnection], "POSTGRE_SQL_CONNECTION"); - my_assert_eq!([!upper_snake! PostgreSqlConnection], "POSTGRE_SQL_CONNECTION"); - my_assert_eq!([!upper_snake! "U+000A LINE FEED (LF)"], "U_000A_LINE_FEED_LF"); - my_assert_eq!([!upper_snake! "\nThis\r\n is a\tmulti-line\nstring"], "THIS_IS_A_MULTI_LINE_STRING"); - my_assert_eq!([!upper_snake! " lots of _ space and _whacky |c$ara_cte>>rs|"], "LOTS_OF_SPACE_AND_WHACKY_C_ARA_CTE_RS"); - my_assert_eq!([!upper_snake! "über CöÖl"], "ÜBER_CÖ_ÖL"); - my_assert_eq!([!upper_snake! "◌̈ubër Cöol"], "UBE_R_CÖOL"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!upper_snake! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!upper_snake! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "MY_MIXED_CASE_STRING_WHICHIS_AWESOME_WHATDO_YOUTHINK"); + preinterpret_assert_eq!([!upper_snake! UPPER], "UPPER"); + preinterpret_assert_eq!([!upper_snake! lower], "LOWER"); + preinterpret_assert_eq!([!upper_snake! lower_snake_case], "LOWER_SNAKE_CASE"); + preinterpret_assert_eq!([!upper_snake! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); + preinterpret_assert_eq!([!upper_snake! lowerCamelCase], "LOWER_CAMEL_CASE"); + preinterpret_assert_eq!([!upper_snake! UpperCamelCase], "UPPER_CAMEL_CASE"); + preinterpret_assert_eq!([!upper_snake! Capitalized], "CAPITALIZED"); + preinterpret_assert_eq!([!upper_snake! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY_SAID_A_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG"); + preinterpret_assert_eq!([!upper_snake! "hello_w🌎rld"], "HELLO_W_RLD"); + preinterpret_assert_eq!([!upper_snake! "kebab-case"], "KEBAB_CASE"); + preinterpret_assert_eq!([!upper_snake! "~~h4xx0rZ <3 1337c0de"], "H4XX0R_Z_3_1337C0DE"); + preinterpret_assert_eq!([!upper_snake! PostgreSQLConnection], "POSTGRE_SQL_CONNECTION"); + preinterpret_assert_eq!([!upper_snake! PostgreSqlConnection], "POSTGRE_SQL_CONNECTION"); + preinterpret_assert_eq!([!upper_snake! "U+000A LINE FEED (LF)"], "U_000A_LINE_FEED_LF"); + preinterpret_assert_eq!([!upper_snake! "\nThis\r\n is a\tmulti-line\nstring"], "THIS_IS_A_MULTI_LINE_STRING"); + preinterpret_assert_eq!([!upper_snake! " lots of _ space and _whacky |c$ara_cte>>rs|"], "LOTS_OF_SPACE_AND_WHACKY_C_ARA_CTE_RS"); + preinterpret_assert_eq!([!upper_snake! "über CöÖl"], "ÜBER_CÖ_ÖL"); + preinterpret_assert_eq!([!upper_snake! "◌̈ubër Cöol"], "UBE_R_CÖOL"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!upper_snake! "真是难以置信！"], "真是难以置信"); } #[test] fn test_to_lower_kebab_case() { - my_assert_eq!([!kebab! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my-mixed-case-string-whichis-awesome-whatdo-youthink"); - my_assert_eq!([!kebab! UPPER], "upper"); - my_assert_eq!([!kebab! lower], "lower"); - my_assert_eq!([!kebab! lower_snake_case], "lower-snake-case"); - my_assert_eq!([!kebab! UPPER_SNAKE_CASE], "upper-snake-case"); - my_assert_eq!([!kebab! lowerCamelCase], "lower-camel-case"); - my_assert_eq!([!kebab! UpperCamelCase], "upper-camel-case"); - my_assert_eq!([!kebab! Capitalized], "capitalized"); - my_assert_eq!([!kebab! "THEY SAID: A quick brown fox jumps over the lazy dog."], "they-said-a-quick-brown-fox-jumps-over-the-lazy-dog"); - my_assert_eq!([!kebab! "hello_w🌎rld"], "hello-w-rld"); - my_assert_eq!([!kebab! "kebab-case"], "kebab-case"); - my_assert_eq!([!kebab! "~~h4xx0rZ <3 1337c0de"], "h4xx0r-z-3-1337c0de"); - my_assert_eq!([!kebab! PostgreSQLConnection], "postgre-sql-connection"); - my_assert_eq!([!kebab! PostgreSqlConnection], "postgre-sql-connection"); - my_assert_eq!([!kebab! "U+000A LINE FEED (LF)"], "u-000a-line-feed-lf"); - my_assert_eq!([!kebab! "\nThis\r\n is a\tmulti-line\nstring"], "this-is-a-multi-line-string"); - my_assert_eq!([!kebab! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lots-of-space-and-whacky-c-ara-cte-rs"); - my_assert_eq!([!kebab! "über CöÖl"], "über-cö-öl"); - my_assert_eq!([!kebab! "◌̈ubër Cööl"], "ube-r-cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!kebab! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!kebab! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "my-mixed-case-string-whichis-awesome-whatdo-youthink"); + preinterpret_assert_eq!([!kebab! UPPER], "upper"); + preinterpret_assert_eq!([!kebab! lower], "lower"); + preinterpret_assert_eq!([!kebab! lower_snake_case], "lower-snake-case"); + preinterpret_assert_eq!([!kebab! UPPER_SNAKE_CASE], "upper-snake-case"); + preinterpret_assert_eq!([!kebab! lowerCamelCase], "lower-camel-case"); + preinterpret_assert_eq!([!kebab! UpperCamelCase], "upper-camel-case"); + preinterpret_assert_eq!([!kebab! Capitalized], "capitalized"); + preinterpret_assert_eq!([!kebab! "THEY SAID: A quick brown fox jumps over the lazy dog."], "they-said-a-quick-brown-fox-jumps-over-the-lazy-dog"); + preinterpret_assert_eq!([!kebab! "hello_w🌎rld"], "hello-w-rld"); + preinterpret_assert_eq!([!kebab! "kebab-case"], "kebab-case"); + preinterpret_assert_eq!([!kebab! "~~h4xx0rZ <3 1337c0de"], "h4xx0r-z-3-1337c0de"); + preinterpret_assert_eq!([!kebab! PostgreSQLConnection], "postgre-sql-connection"); + preinterpret_assert_eq!([!kebab! PostgreSqlConnection], "postgre-sql-connection"); + preinterpret_assert_eq!([!kebab! "U+000A LINE FEED (LF)"], "u-000a-line-feed-lf"); + preinterpret_assert_eq!([!kebab! "\nThis\r\n is a\tmulti-line\nstring"], "this-is-a-multi-line-string"); + preinterpret_assert_eq!([!kebab! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lots-of-space-and-whacky-c-ara-cte-rs"); + preinterpret_assert_eq!([!kebab! "über CöÖl"], "über-cö-öl"); + preinterpret_assert_eq!([!kebab! "◌̈ubër Cööl"], "ube-r-cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!kebab! "真是难以置信！"], "真是难以置信"); } #[test] fn test_lower_camel() { - my_assert_eq!([!lower_camel! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "myMixedCaseStringWhichisAwesomeWhatdoYouthink"); - my_assert_eq!([!lower_camel! UPPER], "upper"); - my_assert_eq!([!lower_camel! lower], "lower"); - my_assert_eq!([!lower_camel! lower_snake_case], "lowerSnakeCase"); - my_assert_eq!([!lower_camel! UPPER_SNAKE_CASE], "upperSnakeCase"); - my_assert_eq!([!lower_camel! lowerCamelCase], "lowerCamelCase"); - my_assert_eq!([!lower_camel! UpperCamelCase], "upperCamelCase"); - my_assert_eq!([!lower_camel! Capitalized], "capitalized"); - my_assert_eq!([!lower_camel! "THEY SAID: A quick brown fox jumps over the lazy dog."], "theySaidAQuickBrownFoxJumpsOverTheLazyDog"); - my_assert_eq!([!lower_camel! "hello_w🌎rld"], "helloWRld"); - my_assert_eq!([!lower_camel! "kebab-case"], "kebabCase"); - my_assert_eq!([!lower_camel! "~~h4xx0rZ <3 1337c0de"], "h4xx0rZ31337c0de"); - my_assert_eq!([!lower_camel! PostgreSQLConnection], "postgreSqlConnection"); - my_assert_eq!([!lower_camel! PostgreSqlConnection], "postgreSqlConnection"); - my_assert_eq!([!lower_camel! "U+000A LINE FEED (LF)"], "u000aLineFeedLf"); - my_assert_eq!([!lower_camel! "\nThis\r\n is a\tmulti-line\nstring"], "thisIsAMultiLineString"); - my_assert_eq!([!lower_camel! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lotsOfSpaceAndWhackyCAraCteRs"); - my_assert_eq!([!lower_camel! "über CöÖl"], "überCöÖl"); - my_assert_eq!([!lower_camel! "◌̈ubër Cööl"], "ubeRCööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!lower_camel! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!lower_camel! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "myMixedCaseStringWhichisAwesomeWhatdoYouthink"); + preinterpret_assert_eq!([!lower_camel! UPPER], "upper"); + preinterpret_assert_eq!([!lower_camel! lower], "lower"); + preinterpret_assert_eq!([!lower_camel! lower_snake_case], "lowerSnakeCase"); + preinterpret_assert_eq!([!lower_camel! UPPER_SNAKE_CASE], "upperSnakeCase"); + preinterpret_assert_eq!([!lower_camel! lowerCamelCase], "lowerCamelCase"); + preinterpret_assert_eq!([!lower_camel! UpperCamelCase], "upperCamelCase"); + preinterpret_assert_eq!([!lower_camel! Capitalized], "capitalized"); + preinterpret_assert_eq!([!lower_camel! "THEY SAID: A quick brown fox jumps over the lazy dog."], "theySaidAQuickBrownFoxJumpsOverTheLazyDog"); + preinterpret_assert_eq!([!lower_camel! "hello_w🌎rld"], "helloWRld"); + preinterpret_assert_eq!([!lower_camel! "kebab-case"], "kebabCase"); + preinterpret_assert_eq!([!lower_camel! "~~h4xx0rZ <3 1337c0de"], "h4xx0rZ31337c0de"); + preinterpret_assert_eq!([!lower_camel! PostgreSQLConnection], "postgreSqlConnection"); + preinterpret_assert_eq!([!lower_camel! PostgreSqlConnection], "postgreSqlConnection"); + preinterpret_assert_eq!([!lower_camel! "U+000A LINE FEED (LF)"], "u000aLineFeedLf"); + preinterpret_assert_eq!([!lower_camel! "\nThis\r\n is a\tmulti-line\nstring"], "thisIsAMultiLineString"); + preinterpret_assert_eq!([!lower_camel! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lotsOfSpaceAndWhackyCAraCteRs"); + preinterpret_assert_eq!([!lower_camel! "über CöÖl"], "überCöÖl"); + preinterpret_assert_eq!([!lower_camel! "◌̈ubër Cööl"], "ubeRCööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!lower_camel! "真是难以置信！"], "真是难以置信"); } #[test] fn test_camel() { - my_assert_eq!([!camel! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "MyMixedCaseStringWhichisAwesomeWhatdoYouthink"); - my_assert_eq!([!camel! UPPER], "Upper"); - my_assert_eq!([!camel! lower], "Lower"); - my_assert_eq!([!camel! lower_snake_case], "LowerSnakeCase"); - my_assert_eq!([!camel! UPPER_SNAKE_CASE], "UpperSnakeCase"); - my_assert_eq!([!camel! lowerCamelCase], "LowerCamelCase"); - my_assert_eq!([!camel! UpperCamelCase], "UpperCamelCase"); - my_assert_eq!([!camel! Capitalized], "Capitalized"); - my_assert_eq!([!camel! "THEY SAID: A quick brown fox jumps over the lazy dog."], "TheySaidAQuickBrownFoxJumpsOverTheLazyDog"); - my_assert_eq!([!camel! "hello_w🌎rld"], "HelloWRld"); - my_assert_eq!([!camel! "kebab-case"], "KebabCase"); - my_assert_eq!([!camel! "~~h4xx0rZ <3 1337c0de"], "H4xx0rZ31337c0de"); - my_assert_eq!([!camel! PostgreSQLConnection], "PostgreSqlConnection"); - my_assert_eq!([!camel! PostgreSqlConnection], "PostgreSqlConnection"); - my_assert_eq!([!camel! "U+000A LINE FEED (LF)"], "U000aLineFeedLf"); - my_assert_eq!([!camel! "\nThis\r\n is a\tmulti-line\nstring"], "ThisIsAMultiLineString"); - my_assert_eq!([!camel! " lots of _ space and _whacky |c$ara_cte>>rs|"], "LotsOfSpaceAndWhackyCAraCteRs"); - my_assert_eq!([!camel! "über CöÖl"], "ÜberCöÖl"); - my_assert_eq!([!camel! "◌̈ubër Cööl"], "UbeRCööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!camel! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!camel! my_ MixedCase STRING Which is " #awesome " - what "do you" think?], "MyMixedCaseStringWhichisAwesomeWhatdoYouthink"); + preinterpret_assert_eq!([!camel! UPPER], "Upper"); + preinterpret_assert_eq!([!camel! lower], "Lower"); + preinterpret_assert_eq!([!camel! lower_snake_case], "LowerSnakeCase"); + preinterpret_assert_eq!([!camel! UPPER_SNAKE_CASE], "UpperSnakeCase"); + preinterpret_assert_eq!([!camel! lowerCamelCase], "LowerCamelCase"); + preinterpret_assert_eq!([!camel! UpperCamelCase], "UpperCamelCase"); + preinterpret_assert_eq!([!camel! Capitalized], "Capitalized"); + preinterpret_assert_eq!([!camel! "THEY SAID: A quick brown fox jumps over the lazy dog."], "TheySaidAQuickBrownFoxJumpsOverTheLazyDog"); + preinterpret_assert_eq!([!camel! "hello_w🌎rld"], "HelloWRld"); + preinterpret_assert_eq!([!camel! "kebab-case"], "KebabCase"); + preinterpret_assert_eq!([!camel! "~~h4xx0rZ <3 1337c0de"], "H4xx0rZ31337c0de"); + preinterpret_assert_eq!([!camel! PostgreSQLConnection], "PostgreSqlConnection"); + preinterpret_assert_eq!([!camel! PostgreSqlConnection], "PostgreSqlConnection"); + preinterpret_assert_eq!([!camel! "U+000A LINE FEED (LF)"], "U000aLineFeedLf"); + preinterpret_assert_eq!([!camel! "\nThis\r\n is a\tmulti-line\nstring"], "ThisIsAMultiLineString"); + preinterpret_assert_eq!([!camel! " lots of _ space and _whacky |c$ara_cte>>rs|"], "LotsOfSpaceAndWhackyCAraCteRs"); + preinterpret_assert_eq!([!camel! "über CöÖl"], "ÜberCöÖl"); + preinterpret_assert_eq!([!camel! "◌̈ubër Cööl"], "UbeRCööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!camel! "真是难以置信！"], "真是难以置信"); } #[test] fn test_capitalize() { - my_assert_eq!([!capitalize! my_ MixedCase STRING Which is " #awesome " - what do you think?], "My_MixedCaseSTRINGWhichis #awesome -whatdoyouthink?"); - my_assert_eq!([!capitalize! UPPER], "UPPER"); - my_assert_eq!([!capitalize! lower], "Lower"); - my_assert_eq!([!capitalize! lower_snake_case], "Lower_snake_case"); - my_assert_eq!([!capitalize! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); - my_assert_eq!([!capitalize! lowerCamelCase], "LowerCamelCase"); - my_assert_eq!([!capitalize! UpperCamelCase], "UpperCamelCase"); - my_assert_eq!([!capitalize! Capitalized], "Capitalized"); - my_assert_eq!([!capitalize! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID: A quick brown fox jumps over the lazy dog."); - my_assert_eq!([!capitalize! "hello_w🌎rld"], "Hello_w🌎rld"); - my_assert_eq!([!capitalize! "kebab-case"], "Kebab-case"); - my_assert_eq!([!capitalize! "~~h4xx0rZ <3 1337c0de"], "~~H4xx0rZ <3 1337c0de"); - my_assert_eq!([!capitalize! PostgreSQLConnection], "PostgreSQLConnection"); - my_assert_eq!([!capitalize! PostgreSqlConnection], "PostgreSqlConnection"); - my_assert_eq!([!capitalize! "U+000A LINE FEED (LF)"], "U+000A LINE FEED (LF)"); - my_assert_eq!([!capitalize! "\nThis\r\n is a\tmulti-line\nstring"], "\nThis\r\n is a\tmulti-line\nstring"); - my_assert_eq!([!capitalize! " lots of _ space and _whacky |c$ara_cte>>rs|"], " Lots of _ space and _whacky |c$ara_cte>>rs|"); - my_assert_eq!([!capitalize! "über CöÖl"], "Über CöÖl"); - my_assert_eq!([!capitalize! "◌̈ubër Cööl"], "◌̈Ubër Cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!capitalize! "真是难以置信！"], "真是难以置信！"); + preinterpret_assert_eq!([!capitalize! my_ MixedCase STRING Which is " #awesome " - what do you think?], "My_MixedCaseSTRINGWhichis #awesome -whatdoyouthink?"); + preinterpret_assert_eq!([!capitalize! UPPER], "UPPER"); + preinterpret_assert_eq!([!capitalize! lower], "Lower"); + preinterpret_assert_eq!([!capitalize! lower_snake_case], "Lower_snake_case"); + preinterpret_assert_eq!([!capitalize! UPPER_SNAKE_CASE], "UPPER_SNAKE_CASE"); + preinterpret_assert_eq!([!capitalize! lowerCamelCase], "LowerCamelCase"); + preinterpret_assert_eq!([!capitalize! UpperCamelCase], "UpperCamelCase"); + preinterpret_assert_eq!([!capitalize! Capitalized], "Capitalized"); + preinterpret_assert_eq!([!capitalize! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID: A quick brown fox jumps over the lazy dog."); + preinterpret_assert_eq!([!capitalize! "hello_w🌎rld"], "Hello_w🌎rld"); + preinterpret_assert_eq!([!capitalize! "kebab-case"], "Kebab-case"); + preinterpret_assert_eq!([!capitalize! "~~h4xx0rZ <3 1337c0de"], "~~H4xx0rZ <3 1337c0de"); + preinterpret_assert_eq!([!capitalize! PostgreSQLConnection], "PostgreSQLConnection"); + preinterpret_assert_eq!([!capitalize! PostgreSqlConnection], "PostgreSqlConnection"); + preinterpret_assert_eq!([!capitalize! "U+000A LINE FEED (LF)"], "U+000A LINE FEED (LF)"); + preinterpret_assert_eq!([!capitalize! "\nThis\r\n is a\tmulti-line\nstring"], "\nThis\r\n is a\tmulti-line\nstring"); + preinterpret_assert_eq!([!capitalize! " lots of _ space and _whacky |c$ara_cte>>rs|"], " Lots of _ space and _whacky |c$ara_cte>>rs|"); + preinterpret_assert_eq!([!capitalize! "über CöÖl"], "Über CöÖl"); + preinterpret_assert_eq!([!capitalize! "◌̈ubër Cööl"], "◌̈Ubër Cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!capitalize! "真是难以置信！"], "真是难以置信！"); } #[test] fn test_decapitalize() { - my_assert_eq!([!decapitalize! my_ MixedCase STRING Which is " #awesome " - what do you think?], "my_MixedCaseSTRINGWhichis #awesome -whatdoyouthink?"); - my_assert_eq!([!decapitalize! UPPER], "uPPER"); - my_assert_eq!([!decapitalize! lower], "lower"); - my_assert_eq!([!decapitalize! lower_snake_case], "lower_snake_case"); - my_assert_eq!([!decapitalize! UPPER_SNAKE_CASE], "uPPER_SNAKE_CASE"); - my_assert_eq!([!decapitalize! lowerCamelCase], "lowerCamelCase"); - my_assert_eq!([!decapitalize! UpperCamelCase], "upperCamelCase"); - my_assert_eq!([!decapitalize! Capitalized], "capitalized"); - my_assert_eq!([!decapitalize! "THEY SAID: A quick brown fox jumps over the lazy dog."], "tHEY SAID: A quick brown fox jumps over the lazy dog."); - my_assert_eq!([!decapitalize! "hello_w🌎rld"], "hello_w🌎rld"); - my_assert_eq!([!decapitalize! "kebab-case"], "kebab-case"); - my_assert_eq!([!decapitalize! "~~h4xx0rZ <3 1337c0de"], "~~h4xx0rZ <3 1337c0de"); - my_assert_eq!([!decapitalize! PostgreSQLConnection], "postgreSQLConnection"); - my_assert_eq!([!decapitalize! PostgreSqlConnection], "postgreSqlConnection"); - my_assert_eq!([!decapitalize! "U+000A LINE FEED (LF)"], "u+000A LINE FEED (LF)"); - my_assert_eq!([!decapitalize! "\nThis\r\n is a\tmulti-line\nstring"], "\nthis\r\n is a\tmulti-line\nstring"); - my_assert_eq!([!decapitalize! " lots of _ space and _whacky |c$ara_cte>>rs|"], " lots of _ space and _whacky |c$ara_cte>>rs|"); - my_assert_eq!([!decapitalize! "über CöÖl"], "über CöÖl"); - my_assert_eq!([!decapitalize! "◌̈ubër Cööl"], "◌̈ubër Cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!decapitalize! "真是难以置信！"], "真是难以置信！"); + preinterpret_assert_eq!([!decapitalize! my_ MixedCase STRING Which is " #awesome " - what do you think?], "my_MixedCaseSTRINGWhichis #awesome -whatdoyouthink?"); + preinterpret_assert_eq!([!decapitalize! UPPER], "uPPER"); + preinterpret_assert_eq!([!decapitalize! lower], "lower"); + preinterpret_assert_eq!([!decapitalize! lower_snake_case], "lower_snake_case"); + preinterpret_assert_eq!([!decapitalize! UPPER_SNAKE_CASE], "uPPER_SNAKE_CASE"); + preinterpret_assert_eq!([!decapitalize! lowerCamelCase], "lowerCamelCase"); + preinterpret_assert_eq!([!decapitalize! UpperCamelCase], "upperCamelCase"); + preinterpret_assert_eq!([!decapitalize! Capitalized], "capitalized"); + preinterpret_assert_eq!([!decapitalize! "THEY SAID: A quick brown fox jumps over the lazy dog."], "tHEY SAID: A quick brown fox jumps over the lazy dog."); + preinterpret_assert_eq!([!decapitalize! "hello_w🌎rld"], "hello_w🌎rld"); + preinterpret_assert_eq!([!decapitalize! "kebab-case"], "kebab-case"); + preinterpret_assert_eq!([!decapitalize! "~~h4xx0rZ <3 1337c0de"], "~~h4xx0rZ <3 1337c0de"); + preinterpret_assert_eq!([!decapitalize! PostgreSQLConnection], "postgreSQLConnection"); + preinterpret_assert_eq!([!decapitalize! PostgreSqlConnection], "postgreSqlConnection"); + preinterpret_assert_eq!([!decapitalize! "U+000A LINE FEED (LF)"], "u+000A LINE FEED (LF)"); + preinterpret_assert_eq!([!decapitalize! "\nThis\r\n is a\tmulti-line\nstring"], "\nthis\r\n is a\tmulti-line\nstring"); + preinterpret_assert_eq!([!decapitalize! " lots of _ space and _whacky |c$ara_cte>>rs|"], " lots of _ space and _whacky |c$ara_cte>>rs|"); + preinterpret_assert_eq!([!decapitalize! "über CöÖl"], "über CöÖl"); + preinterpret_assert_eq!([!decapitalize! "◌̈ubër Cööl"], "◌̈ubër Cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!decapitalize! "真是难以置信！"], "真是难以置信！"); } #[test] fn test_title() { - my_assert_eq!([!title! my_ MixedCase STRING Which is " #awesome " - what do you think?], "My Mixed Case String Whichis Awesome Whatdoyouthink"); - my_assert_eq!([!title! UPPER], "Upper"); - my_assert_eq!([!title! lower], "Lower"); - my_assert_eq!([!title! lower_snake_case], "Lower Snake Case"); - my_assert_eq!([!title! UPPER_SNAKE_CASE], "Upper Snake Case"); - my_assert_eq!([!title! lowerCamelCase], "Lower Camel Case"); - my_assert_eq!([!title! UpperCamelCase], "Upper Camel Case"); - my_assert_eq!([!title! Capitalized], "Capitalized"); - my_assert_eq!([!title! "THEY SAID: A quick brown fox jumps over the lazy dog."], "They Said A Quick Brown Fox Jumps Over The Lazy Dog"); - my_assert_eq!([!title! "hello_w🌎rld"], "Hello W Rld"); - my_assert_eq!([!title! "kebab-case"], "Kebab Case"); - my_assert_eq!([!title! "~~h4xx0rZ <3 1337c0de"], "H4xx0r Z 3 1337c0de"); - my_assert_eq!([!title! PostgreSQLConnection], "Postgre Sql Connection"); - my_assert_eq!([!title! PostgreSqlConnection], "Postgre Sql Connection"); - my_assert_eq!([!title! "U+000A LINE FEED (LF)"], "U 000a Line Feed Lf"); - my_assert_eq!([!title! "\nThis\r\n is a\tmulti-line\nstring"], "This Is A Multi Line String"); - my_assert_eq!([!title! " lots of _ space and _whacky |c$ara_cte>>rs|"], "Lots Of Space And Whacky C Ara Cte Rs"); - my_assert_eq!([!title! "über CöÖl"], "Über Cö Öl"); - my_assert_eq!([!title! "◌̈ubër Cööl"], "Ube R Cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!title! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!title! my_ MixedCase STRING Which is " #awesome " - what do you think?], "My Mixed Case String Whichis Awesome Whatdoyouthink"); + preinterpret_assert_eq!([!title! UPPER], "Upper"); + preinterpret_assert_eq!([!title! lower], "Lower"); + preinterpret_assert_eq!([!title! lower_snake_case], "Lower Snake Case"); + preinterpret_assert_eq!([!title! UPPER_SNAKE_CASE], "Upper Snake Case"); + preinterpret_assert_eq!([!title! lowerCamelCase], "Lower Camel Case"); + preinterpret_assert_eq!([!title! UpperCamelCase], "Upper Camel Case"); + preinterpret_assert_eq!([!title! Capitalized], "Capitalized"); + preinterpret_assert_eq!([!title! "THEY SAID: A quick brown fox jumps over the lazy dog."], "They Said A Quick Brown Fox Jumps Over The Lazy Dog"); + preinterpret_assert_eq!([!title! "hello_w🌎rld"], "Hello W Rld"); + preinterpret_assert_eq!([!title! "kebab-case"], "Kebab Case"); + preinterpret_assert_eq!([!title! "~~h4xx0rZ <3 1337c0de"], "H4xx0r Z 3 1337c0de"); + preinterpret_assert_eq!([!title! PostgreSQLConnection], "Postgre Sql Connection"); + preinterpret_assert_eq!([!title! PostgreSqlConnection], "Postgre Sql Connection"); + preinterpret_assert_eq!([!title! "U+000A LINE FEED (LF)"], "U 000a Line Feed Lf"); + preinterpret_assert_eq!([!title! "\nThis\r\n is a\tmulti-line\nstring"], "This Is A Multi Line String"); + preinterpret_assert_eq!([!title! " lots of _ space and _whacky |c$ara_cte>>rs|"], "Lots Of Space And Whacky C Ara Cte Rs"); + preinterpret_assert_eq!([!title! "über CöÖl"], "Über Cö Öl"); + preinterpret_assert_eq!([!title! "◌̈ubër Cööl"], "Ube R Cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!title! "真是难以置信！"], "真是难以置信"); } #[test] fn test_insert_spaces() { - my_assert_eq!([!insert_spaces! my_ MixedCase STRING Which is " #awesome " - what do you think?], "my Mixed Case STRING Whichis awesome whatdoyouthink"); - my_assert_eq!([!insert_spaces! UPPER], "UPPER"); - my_assert_eq!([!insert_spaces! lower], "lower"); - my_assert_eq!([!insert_spaces! lower_snake_case], "lower snake case"); - my_assert_eq!([!insert_spaces! UPPER_SNAKE_CASE], "UPPER SNAKE CASE"); - my_assert_eq!([!insert_spaces! lowerCamelCase], "lower Camel Case"); - my_assert_eq!([!insert_spaces! UpperCamelCase], "Upper Camel Case"); - my_assert_eq!([!insert_spaces! Capitalized], "Capitalized"); - my_assert_eq!([!insert_spaces! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID A quick brown fox jumps over the lazy dog"); - my_assert_eq!([!insert_spaces! "hello_w🌎rld"], "hello w rld"); - my_assert_eq!([!insert_spaces! "kebab-case"], "kebab case"); - my_assert_eq!([!insert_spaces! "~~h4xx0rZ <3 1337c0de"], "h4xx0r Z 3 1337c0de"); - my_assert_eq!([!insert_spaces! PostgreSQLConnection], "Postgre SQL Connection"); - my_assert_eq!([!insert_spaces! PostgreSqlConnection], "Postgre Sql Connection"); - my_assert_eq!([!insert_spaces! "U+000A LINE FEED (LF)"], "U 000A LINE FEED LF"); - my_assert_eq!([!insert_spaces! "\nThis\r\n is a\tmulti-line\nstring"], "This is a multi line string"); - my_assert_eq!([!insert_spaces! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lots of space and whacky c ara cte rs"); - my_assert_eq!([!insert_spaces! "über CöÖl"], "über Cö Öl"); - my_assert_eq!([!insert_spaces! "◌̈ubër Cööl"], "ube r Cööl"); // The ë (and only the e) uses a post-fix combining character - my_assert_eq!([!insert_spaces! "真是难以置信！"], "真是难以置信"); + preinterpret_assert_eq!([!insert_spaces! my_ MixedCase STRING Which is " #awesome " - what do you think?], "my Mixed Case STRING Whichis awesome whatdoyouthink"); + preinterpret_assert_eq!([!insert_spaces! UPPER], "UPPER"); + preinterpret_assert_eq!([!insert_spaces! lower], "lower"); + preinterpret_assert_eq!([!insert_spaces! lower_snake_case], "lower snake case"); + preinterpret_assert_eq!([!insert_spaces! UPPER_SNAKE_CASE], "UPPER SNAKE CASE"); + preinterpret_assert_eq!([!insert_spaces! lowerCamelCase], "lower Camel Case"); + preinterpret_assert_eq!([!insert_spaces! UpperCamelCase], "Upper Camel Case"); + preinterpret_assert_eq!([!insert_spaces! Capitalized], "Capitalized"); + preinterpret_assert_eq!([!insert_spaces! "THEY SAID: A quick brown fox jumps over the lazy dog."], "THEY SAID A quick brown fox jumps over the lazy dog"); + preinterpret_assert_eq!([!insert_spaces! "hello_w🌎rld"], "hello w rld"); + preinterpret_assert_eq!([!insert_spaces! "kebab-case"], "kebab case"); + preinterpret_assert_eq!([!insert_spaces! "~~h4xx0rZ <3 1337c0de"], "h4xx0r Z 3 1337c0de"); + preinterpret_assert_eq!([!insert_spaces! PostgreSQLConnection], "Postgre SQL Connection"); + preinterpret_assert_eq!([!insert_spaces! PostgreSqlConnection], "Postgre Sql Connection"); + preinterpret_assert_eq!([!insert_spaces! "U+000A LINE FEED (LF)"], "U 000A LINE FEED LF"); + preinterpret_assert_eq!([!insert_spaces! "\nThis\r\n is a\tmulti-line\nstring"], "This is a multi line string"); + preinterpret_assert_eq!([!insert_spaces! " lots of _ space and _whacky |c$ara_cte>>rs|"], "lots of space and whacky c ara cte rs"); + preinterpret_assert_eq!([!insert_spaces! "über CöÖl"], "über Cö Öl"); + preinterpret_assert_eq!([!insert_spaces! "◌̈ubër Cööl"], "ube r Cööl"); // The ë (and only the e) uses a post-fix combining character + preinterpret_assert_eq!([!insert_spaces! "真是难以置信！"], "真是难以置信"); } diff --git a/tests/tokens.rs b/tests/tokens.rs index d657df6f..3b87038a 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -1,10 +1,6 @@ -use preinterpret::preinterpret; - -macro_rules! assert_preinterpret_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] @@ -15,18 +11,18 @@ fn test_tokens_compilation_failures() { #[test] fn test_flattened_group_and_is_empty() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!..group!] "hello" [!..group!] [!..group!] }, "hello"); - assert_preinterpret_eq!([!is_empty!], true); - assert_preinterpret_eq!([!is_empty! [!..group!]], true); - assert_preinterpret_eq!([!is_empty! [!..group!] [!..group!]], true); - assert_preinterpret_eq!([!is_empty! Not Empty], false); - assert_preinterpret_eq!({ + preinterpret_assert_eq!([!is_empty!], true); + preinterpret_assert_eq!([!is_empty! [!..group!]], true); + preinterpret_assert_eq!([!is_empty! [!..group!] [!..group!]], true); + preinterpret_assert_eq!([!is_empty! Not Empty], false); + preinterpret_assert_eq!({ [!set! #x =] [!is_empty! #..x] }, true); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x =] [!set! #x = #x is no longer empty] [!is_empty! #x] @@ -35,16 +31,16 @@ fn test_flattened_group_and_is_empty() { #[test] fn test_length_and_group() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!length! "hello" World] }, 2); - assert_preinterpret_eq!({ [!length! ("hello" World)] }, 1); - assert_preinterpret_eq!({ [!length! [!group! "hello" World]] }, 1); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!length! ("hello" World)] }, 1); + preinterpret_assert_eq!({ [!length! [!group! "hello" World]] }, 1); + preinterpret_assert_eq!({ [!set! #x = Hello "World" (1 2 3 4 5)] [!length! #..x] }, 3); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = Hello "World" (1 2 3 4 5)] [!length! [!group! #..x]] }, 1); @@ -52,7 +48,7 @@ fn test_length_and_group() { #[test] fn test_intersperse() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [Hello World], @@ -61,7 +57,7 @@ fn test_intersperse() { }, "Hello, World" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [Hello World], @@ -70,7 +66,7 @@ fn test_intersperse() { }, "Hello_and_World" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [Hello World], @@ -80,7 +76,7 @@ fn test_intersperse() { }, "Hello_and_World_and_" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [The Quick Brown Fox], @@ -89,7 +85,7 @@ fn test_intersperse() { }, "TheQuickBrownFox" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [The Quick Brown Fox], @@ -99,7 +95,7 @@ fn test_intersperse() { }, "The,Quick,Brown,Fox," ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [Red Green Blue], @@ -109,7 +105,7 @@ fn test_intersperse() { }, "Red, Green and Blue" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [Red Green Blue], @@ -120,7 +116,7 @@ fn test_intersperse() { }, "Red, Green, Blue and " ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [], @@ -131,7 +127,7 @@ fn test_intersperse() { }, "" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [SingleItem], @@ -141,7 +137,7 @@ fn test_intersperse() { }, "SingleItem" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [SingleItem], @@ -152,7 +148,7 @@ fn test_intersperse() { }, "SingleItem!" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [SingleItem], @@ -167,7 +163,7 @@ fn test_intersperse() { #[test] fn complex_cases_for_intersperse_and_input_types() { // Normal separator is not interpreted if it is unneeded - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [], @@ -178,7 +174,7 @@ fn complex_cases_for_intersperse_and_input_types() { "" ); // Final separator is not interpreted if it is unneeded - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [], @@ -190,7 +186,7 @@ fn complex_cases_for_intersperse_and_input_types() { "" ); // The separator is interpreted each time it is included - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ #(let i = 0) [!string! [!intersperse! { items: [A B C D E F G], @@ -202,7 +198,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "A(0)B(1)C(2)D(3)E(4)F(5)G(6)"); // Command can be used for items - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [!range! 0..4], @@ -212,7 +208,7 @@ fn complex_cases_for_intersperse_and_input_types() { "0_1_2_3" ); // Grouped Variable can be used for items - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] [!string! [!intersperse! { items: #items, @@ -220,7 +216,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "0_1_2_3"); // Grouped variable containing flattened command can be used for items - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #items = [!..range! 0..4]] [!string! [!intersperse! { items: #items, @@ -228,7 +224,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "0_1_2_3"); // Flattened variable containing [ ... ] group - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #items = [0 1 2 3]] [!string! [!intersperse! { items: #..items, @@ -236,7 +232,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "0_1_2_3"); // Flattened variable containing transparent group - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] [!set! #wrapped_items = #items] // #items is "grouped variable" so outputs [!group! 0 1 2 3] [!string! [!intersperse! { @@ -245,7 +241,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "0_1_2_3"); // { ... } block returning transparent group (from variable) - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] [!string! [!intersperse! { items: { @@ -255,7 +251,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "0_1_2_3"); // { ... } block returning [ ... ] group - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: { @@ -267,7 +263,7 @@ fn complex_cases_for_intersperse_and_input_types() { "0_1_2_3" ); // Grouped variable containing two groups - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #items = 0 1] [!set! #item_groups = #items #items] // [!group! 0 1] [!group! 0 1] [!string! [!intersperse! { @@ -276,7 +272,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "01_01"); // Control stream commands can be used, if they return a valid stream grouping - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!string![!intersperse! { items: [!if! false { [0 1] } !else! { [2 3] }], @@ -287,7 +283,7 @@ fn complex_cases_for_intersperse_and_input_types() { ); // All inputs can be variables // Inputs can be in any order - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #people = Anna Barbara Charlie] [!set! #separator = ", "] [!set! #final_separator = " and "] @@ -300,7 +296,7 @@ fn complex_cases_for_intersperse_and_input_types() { }]] }, "Anna, Barbara and Charlie"); // Add trailing is executed even if it's irrelevant because there are no items - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = "NOT_EXECUTED"] [!intersperse! { items: [], @@ -318,7 +314,7 @@ fn complex_cases_for_intersperse_and_input_types() { fn test_split() { // Empty separators are allowed, and split on every token // In this case, drop_empty_start / drop_empty_end are ignored - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..split! { stream: [A::B], @@ -328,7 +324,7 @@ fn test_split() { "[!stream! [!group! A] [!group! :] [!group! :] [!group! B]]" ); // Double separators are allowed - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..split! { stream: [A::B::C], @@ -338,7 +334,7 @@ fn test_split() { "[!stream! [!group! A] [!group! B] [!group! C]]" ); // Trailing separator is ignored by default - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..split! { stream: [Pizza, Mac and Cheese, Hamburger,], @@ -348,7 +344,7 @@ fn test_split() { "[!stream! [!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" ); // By default, empty groups are included except at the end - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..split! { stream: [::A::B::::C::], @@ -358,7 +354,7 @@ fn test_split() { "[!stream! [!group!] [!group! A] [!group! B] [!group!] [!group! C]]" ); // Stream and separator are both interpreted - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = ;] [!debug! [!..split! { stream: [;A;;B;C;D #..x E;], @@ -369,7 +365,7 @@ fn test_split() { }]] }, "[!stream! [!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); // Drop empty false works - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = ;] [!debug! [!..split! { stream: [;A;;B;C;D #..x E;], @@ -380,7 +376,7 @@ fn test_split() { }]] }, "[!stream! [!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]"); // Drop empty middle works - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..split! { stream: [;A;;B;;;;E;], @@ -394,7 +390,7 @@ fn test_split() { ); // Code blocks are only evaluated once // (i.e. no "unknown variable B is output in the below") - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..split! { stream: { @@ -409,7 +405,7 @@ fn test_split() { #[test] fn test_comma_split() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..comma_split! Pizza, Mac and Cheese, Hamburger,]] }, "[!stream! [!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" ); @@ -417,11 +413,11 @@ fn test_comma_split() { #[test] fn test_zip() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!..zip! ([Hello Goodbye] [World Friend])]] }, "[!stream! (Hello World) (Goodbye Friend)]" ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #countries = France Germany Italy] [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] @@ -432,7 +428,7 @@ fn test_zip() { }, r#"[!stream! [!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]]"#, ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #longer = A B C D] [!set! #shorter = 1 2 3] @@ -444,7 +440,7 @@ fn test_zip() { }, r#"[!stream! [!group! A 1] [!group! B 2] [!group! C 3]]"#, ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #letters = A B C] [!set! #numbers = 1 2 3] @@ -457,7 +453,7 @@ fn test_zip() { }, r#"[!stream! { A 1 } { B 2 } { C 3 }]"#, ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #letters = A B C] [!set! #numbers = 1 2 3] @@ -468,7 +464,7 @@ fn test_zip() { }, r#"[!stream! { A 1 } { B 2 } { C 3 }]"#, ); - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #letters = A B C] [!set! #numbers = 1 2 3] @@ -481,7 +477,7 @@ fn test_zip() { #[test] fn test_zip_with_for() { - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!set! #countries = France Germany Italy] [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] diff --git a/tests/transforming.rs b/tests/transforming.rs index 077389a6..6332c73d 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -1,10 +1,6 @@ -use preinterpret::preinterpret; - -macro_rules! assert_preinterpret_eq { - ($input:tt, $($output:tt)*) => { - assert_eq!(preinterpret!($input), $($output)*); - }; -} +#[path = "helpers/prelude.rs"] +mod prelude; +use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] @@ -19,39 +15,39 @@ fn test_transfoming_compilation_failures() { #[test] fn test_variable_parsing() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! = ] [!string! #inner] }, "Beautiful"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! #..inner = ] [!string! #inner] }, ""); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! #..x = Hello => World] [!string! #x] }, "Hello=>World"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! Hello #..x!! = Hello => World!!] [!string! #x] }, "=>World"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! Hello #..x World = Hello => World] [!string! #x] }, "=>"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! Hello #..x World = Hello And Welcome To The Wonderful World] [!string! #x] }, "AndWelcomeToTheWonderful"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! Hello #..x "World"! = Hello World And Welcome To The Wonderful "World"!] [!string! #x] }, "WorldAndWelcomeToTheWonderful"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! #..x (#..y) = Why Hello (World)] [!string! "#x = " #x "; #y = " #y] }, "#x = WhyHello; #y = World"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x =] [!let! #>>x // Matches one tt and appends it: Why @@ -75,11 +71,11 @@ fn test_explicit_transform_stream() { #[test] fn test_ident_transformer() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! The "quick" @(#x = @IDENT) fox "jumps" = The "quick" brown fox "jumps"] [!string! #x] }, "brown"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x =] [!let! The quick @(#x += @IDENT) fox jumps @(#x += @IDENT) the lazy dog = The quick brown fox jumps over the lazy dog] [!string! [!intersperse! { items: #x, separator: ["_"] }]] @@ -88,12 +84,12 @@ fn test_ident_transformer() { #[test] fn test_literal_transformer() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! The "quick" @(#x = @LITERAL) fox "jumps" = The "quick" "brown" fox "jumps"] #x }, "brown"); // Lots of literals - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x] [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] [!debug! #..x] @@ -102,17 +98,17 @@ fn test_literal_transformer() { #[test] fn test_punct_transformer() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] [!debug! #..x] }, "[!stream! !]"); // Test for ' which is treated weirdly by syn / rustc - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! The "quick" fox isn 't brown and doesn @(#x = @PUNCT) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] [!debug! #..x] }, "[!stream! ']"); // Lots of punctuation, most of it ignored - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x =] [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] [!debug! #..x] @@ -121,17 +117,17 @@ fn test_punct_transformer() { #[test] fn test_group_transformer() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] [!debug! #..x] }, "[!stream! fox]"); - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said @[GROUP #..y]! = I said #x!] [!debug! #..y] }, "[!stream! \"hello\" \"world\"]"); // ... which is equivalent to this: - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said #y! = I said #x!] [!debug! #..y] @@ -140,7 +136,7 @@ fn test_group_transformer() { #[test] fn test_none_output_commands_mid_parse() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!let! The "quick" @(#x = @LITERAL) fox [!let! #y = #x] @(#x = @IDENT) = The "quick" "brown" fox jumps] [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] }, "#x = [!stream! jumps]; #y = [!stream! \"brown\"]"); @@ -148,7 +144,7 @@ fn test_none_output_commands_mid_parse() { #[test] fn test_raw_content_in_exact_transformer() { - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = true] [!let! The @[EXACT [!raw! #x]] = The [!raw! #] x] #x @@ -158,13 +154,13 @@ fn test_raw_content_in_exact_transformer() { #[test] fn test_exact_transformer() { // EXACT works - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = true] [!let! The @[EXACT #..x] = The true] #x }, true); // EXACT is evaluated at execution time - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x =] [!let! The #>>..x fox is #>>..x. It 's super @[EXACT #..x]. = The brown fox is brown. It 's super brown brown.] true @@ -174,12 +170,12 @@ fn test_exact_transformer() { #[test] fn test_parse_command_and_exact_transformer() { // The output stream is additive - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!parse! [Hello World] as @(@IDENT @IDENT)]] }, "[!stream! Hello World]" ); // Substreams redirected to a variable are not included in the output - assert_preinterpret_eq!( + preinterpret_assert_eq!( { [!debug! [!parse! [The quick brown fox] as @( @[EXACT The] quick @IDENT @(#x = @IDENT) @@ -191,7 +187,7 @@ fn test_parse_command_and_exact_transformer() { // * Can nest EXACT and transform streams // * Can discard output with @(_ = ...) // * That EXACT ignores none-delimited groups, to make it more intuitive - assert_preinterpret_eq!({ + preinterpret_assert_eq!({ [!set! #x = [!group! fox]] [!debug! [!parse! [The quick brown fox is a fox - right?!] as @( // The outputs are only from the EXACT transformer From 89f9ba3dc1ca0f679bc42f07b658b9b5c23aaa6f Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 13 Feb 2025 02:10:27 +0000 Subject: [PATCH 090/126] feature: Full support for array expressions --- CHANGELOG.md | 17 +- src/expressions/array.rs | 45 +- src/expressions/boolean.rs | 8 +- src/expressions/character.rs | 6 +- src/expressions/expression.rs | 77 ++- src/expressions/expression_block.rs | 40 +- src/expressions/expression_parsing.rs | 195 ++++---- src/expressions/float.rs | 12 +- src/expressions/integer.rs | 40 +- src/expressions/operations.rs | 111 +++-- src/expressions/stream.rs | 27 +- src/expressions/string.rs | 13 +- src/expressions/value.rs | 216 ++++++-- src/extensions/parsing.rs | 2 +- src/internal_prelude.rs | 3 +- src/interpretation/command.rs | 27 +- .../commands/control_flow_commands.rs | 32 +- src/interpretation/commands/core_commands.rs | 38 +- .../commands/expression_commands.rs | 21 +- src/interpretation/commands/token_commands.rs | 296 ++++++----- .../commands/transforming_commands.rs | 16 +- src/interpretation/interpreted_stream.rs | 63 ++- src/interpretation/mod.rs | 4 - src/interpretation/source_stream.rs | 6 - src/interpretation/source_stream_input.rs | 168 ------- src/interpretation/source_value.rs | 180 ------- src/interpretation/variable.rs | 65 +-- src/misc/errors.rs | 1 + src/misc/parse_traits.rs | 10 + src/transformation/destructuring.rs | 90 ++++ src/transformation/mod.rs | 2 + src/transformation/transform_stream.rs | 13 + .../complex/nested.stderr | 2 +- .../core/error_invalid_structure.stderr | 2 +- .../core/error_span_multiple.rs | 2 +- .../core/error_span_repeat.rs | 2 +- .../core/error_span_single.rs | 2 +- .../expressions/array_missing_comma.rs | 7 + .../expressions/array_missing_comma.stderr | 5 + .../cannot_output_array_to_stream.stderr | 2 +- .../flattened_commands_in_expressions.rs | 7 - .../flattened_commands_in_expressions.stderr | 5 - .../intersperse_bool_input_braces_issue.rs | 13 - ...intersperse_bool_input_braces_issue.stderr | 6 - .../intersperse_stream_input_braces_issue.rs | 10 - ...tersperse_stream_input_braces_issue.stderr | 5 - ...rsperse_stream_input_variable_issue.stderr | 14 +- .../tokens/zip_different_length_streams.rs | 2 +- .../zip_different_length_streams.stderr | 4 +- .../{zip_no_streams.rs => zip_no_input.rs} | 2 +- .../tokens/zip_no_input.stderr | 11 + .../tokens/zip_no_streams.stderr | 5 - tests/control_flow.rs | 12 +- tests/expressions.rs | 35 +- tests/tokens.rs | 470 ++++++++---------- tests/transforming.rs | 32 +- 56 files changed, 1244 insertions(+), 1257 deletions(-) delete mode 100644 src/interpretation/source_stream_input.rs delete mode 100644 src/interpretation/source_value.rs create mode 100644 src/transformation/destructuring.rs create mode 100644 tests/compilation_failures/expressions/array_missing_comma.rs create mode 100644 tests/compilation_failures/expressions/array_missing_comma.stderr delete mode 100644 tests/compilation_failures/expressions/flattened_commands_in_expressions.rs delete mode 100644 tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr delete mode 100644 tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs delete mode 100644 tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr delete mode 100644 tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs delete mode 100644 tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr rename tests/compilation_failures/tokens/{zip_no_streams.rs => zip_no_input.rs} (76%) create mode 100644 tests/compilation_failures/tokens/zip_no_input.stderr delete mode 100644 tests/compilation_failures/tokens/zip_no_streams.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 817ac9e7..d33f71a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,14 +98,18 @@ Inside a transform stream, the following grammar is supported: ### To come * Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model - * Revisit the `SourceStreamInput` abstraction - maybe it's replaced with a `SourceExpression` which needs to output a stream? - * Split outputs an array - * Intersperse works with either an array or a stream (?) - * For works only with an array (in future, also an iterator) - * We can then consider dropping lots of the `group` wrappers I guess? + * Add `as ident` casting and support it for array and stream using concat recursive. + * Support an iterator value as: + * (Not a real iterator - takes an `interpreter` for its `next(interpreter)` method) + * Is an input for for loops + * Is an output for zip and intersperse + * Supports casting to/from arrays and streams; with an iteration limit + * Get rid of `OutputKindGroupedStream` and the `[!..command]` thing + * Consider dropping lots of the `group` wrappers? * Then destructuring and parsing become different: * Destructuring works over the value model; parsing works over the token stream model. * Parsers can be embedded inside a destructuring + * Add `..` and `..x` support to the array destructurer * Support a CastTarget of Array (only supported for array and stream and iterator) * Support `#(x[..])` syntax for indexing arrays and streams at read time * Via a post-fix `[...]` operation with high priority @@ -148,7 +152,7 @@ Inside a transform stream, the following grammar is supported: * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` - * Scrap `[!let!]` in favour of `[!parse! [...] as @(_ = ...)]` + * Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` * Consider: * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` @@ -321,6 +325,7 @@ Inside a transform stream, the following grammar is supported: * Pushed to 0.4: * Performance: + * Use a small-vec optimization in some places * Get rid of needless cloning of commands/variables etc * Support `+=` inside expressions to allow appending of token streams * Variable reference would need to be a sub-type of stream diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 1a353450..d7c4a362 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -27,9 +27,18 @@ impl ExpressionArray { CastTarget::Group => operation.output( operation .output(self.into_stream_with_grouped_items()?) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), - _ => { + CastTarget::String => operation.output({ + let mut output = String::new(); + self.concat_recursive_into(&mut output, &ConcatBehaviour::standard()); + output + }), + CastTarget::DebugString => operation.output(self.items).into_debug_string_value(), + CastTarget::Boolean + | CastTarget::Char + | CastTarget::Integer(_) + | CastTarget::Float(_) => { if self.items.len() == 1 { self.items .pop() @@ -37,7 +46,7 @@ impl ExpressionArray { .handle_unary_operation(operation)? } else { return operation.execution_err(format!( - "Cannot only attempt to cast a singleton array to {} but the array has {} elements", + "Only a singleton array can be cast to {} but the array has {} elements", target_ident, self.items.len(), )); @@ -47,10 +56,10 @@ impl ExpressionArray { }) } - fn into_stream_with_grouped_items(self) -> ExecutionResult { + pub(crate) fn into_stream_with_grouped_items(self) -> ExecutionResult { let mut stream = OutputStream::new(); for item in self.items { - item.output_to(Grouping::Grouped, &mut stream)?; + item.output_to(Grouping::Grouped, &mut stream, true)?; } Ok(stream) } @@ -93,6 +102,32 @@ impl ExpressionArray { | PairedBinaryOperation::GreaterThan { .. } => return operation.unsupported(lhs), }) } + + pub(crate) fn concat_recursive_into(self, output: &mut String, behaviour: &ConcatBehaviour) { + if behaviour.output_array_structure { + output.push('['); + } + let mut is_first = true; + for item in self.items { + if !is_first && behaviour.output_array_structure { + output.push(','); + } + if !is_first && behaviour.add_space_between_token_trees { + output.push(' '); + } + item.concat_recursive_into(output, behaviour); + is_first = false; + } + if behaviour.output_array_structure { + output.push(']'); + } + } +} + +impl HasSpanRange for ExpressionArray { + fn span_range(&self) -> SpanRange { + self.span_range + } } impl HasValueType for ExpressionArray { diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index 2f256e55..e0743e5a 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -44,16 +44,18 @@ impl ExpressionBoolean { CastTarget::Float(_) | CastTarget::Char => { return operation.execution_err("This cast is not supported") } - CastTarget::Boolean => operation.output(self.value), + CastTarget::Boolean => operation.output(input), + CastTarget::String => operation.output(input.to_string()), + CastTarget::DebugString => operation.output(input.to_string()), CastTarget::Stream => operation.output( operation .output(input) - .into_new_output_stream(Grouping::Flattened)?, + .into_new_output_stream(Grouping::Flattened, false)?, ), CastTarget::Group => operation.output( operation .output(input) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), }, }) diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 1cad7492..2bb90606 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -44,15 +44,17 @@ impl ExpressionChar { CastTarget::Integer(IntegerKind::Usize) => operation.output(char as usize), CastTarget::Char => operation.output(char), CastTarget::Boolean | CastTarget::Float(_) => return operation.unsupported(self), + CastTarget::String => operation.output(char.to_string()), + CastTarget::DebugString => operation.output(format!("{:?}", char)), CastTarget::Stream => operation.output( operation .output(char) - .into_new_output_stream(Grouping::Flattened)?, + .into_new_output_stream(Grouping::Flattened, false)?, ), CastTarget::Group => operation.output( operation .output(char) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), }, }) diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index f9b2d237..b1024745 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -30,7 +30,7 @@ impl InterpretToValue for &SourceExpression { pub(super) enum SourceExpressionLeaf { Command(Command), VariablePath(VariablePath), - MarkedVariable(MarkedVariable), + Variable(GroupedVariable), ExpressionBlock(ExpressionBlock), Value(ExpressionValue), } @@ -42,8 +42,13 @@ impl Expressionable for Source { fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult> { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), - SourcePeekMatch::Variable(_) => { - UnaryAtom::Leaf(Self::Leaf::MarkedVariable(input.parse()?)) + SourcePeekMatch::Variable(Grouping::Grouped) => { + UnaryAtom::Leaf(Self::Leaf::Variable(input.parse()?)) + } + SourcePeekMatch::Variable(Grouping::Flattened) => { + return input.parse_err( + "Remove the .. prefix. Flattened variables are not supported in an expression.", + ) } SourcePeekMatch::ExpressionBlock(_) => { UnaryAtom::Leaf(Self::Leaf::ExpressionBlock(input.parse()?)) @@ -69,7 +74,7 @@ impl Expressionable for Source { let (_, delim_span) = input.parse_and_enter_group()?; UnaryAtom::Array { delim_span, - is_empty: input.is_empty(), + is_empty: input.is_current_empty(), } } SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), @@ -83,31 +88,56 @@ impl Expressionable for Source { let value = ExpressionValue::for_syn_lit(input.parse()?); UnaryAtom::Leaf(Self::Leaf::Value(value)) } - SourcePeekMatch::End => { - return input.parse_err("The expression ended in an incomplete state") - } + SourcePeekMatch::End => return input.parse_err("Expected an expression"), }) } - fn parse_extension(input: &mut ParseStreamStack) -> ParseResult { - Ok(match input.peek_grammar() { + fn parse_extension( + input: &mut ParseStreamStack, + parent_stack_frame: &ExpressionStackFrame, + ) -> ParseResult { + // We fall through if we have no match + match input.peek_grammar() { SourcePeekMatch::Punct(punct) if punct.as_char() == ',' => { - NodeExtension::CommaOperator { - comma: input.parse()?, - is_end_of_stream: input.is_empty(), + match parent_stack_frame { + ExpressionStackFrame::Array { .. } => { + input.parse::()?; + if input.is_current_empty() { + return Ok(NodeExtension::EndOfStream); + } else { + return Ok(NodeExtension::NonTerminalArrayComma); + } + } + ExpressionStackFrame::Group { .. } => { + return input.parse_err("Commas are only permitted inside preinterpret arrays []. Preinterpret arrays [a, b] can be used as a drop-in replacement for rust tuples (a, b).") + } + // + _ => {} } } - SourcePeekMatch::Punct(_) => match input.try_parse_or_revert::() { - Ok(operation) => NodeExtension::BinaryOperation(operation), - Err(_) => NodeExtension::NoneMatched, - }, + SourcePeekMatch::Punct(_) => if let Ok(operation) = input.try_parse_or_revert::() { return Ok(NodeExtension::BinaryOperation(operation)) }, SourcePeekMatch::Ident(ident) if ident == "as" => { let cast_operation = UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; - NodeExtension::PostfixOperation(cast_operation) + return Ok(NodeExtension::PostfixOperation(cast_operation)); } - _ => NodeExtension::NoneMatched, - }) + SourcePeekMatch::End => return Ok(NodeExtension::EndOfStream), + _ => {} + }; + // We are not at the end of the stream, but the tokens which follow are + // not a valid extension... + match parent_stack_frame { + ExpressionStackFrame::Root => Ok(NodeExtension::NoValidExtensionForCurrentParent), + ExpressionStackFrame::Group { .. } => input.parse_err("Expected ) or operator"), + ExpressionStackFrame::Array { .. } => input.parse_err("Expected comma, ], or operator"), + // e.g. I've just matched the true in !true or false || true, + // and I want to see if there's an extension (e.g. a cast). + // There's nothing matching, so we fall through to an EndOfFrame + ExpressionStackFrame::IncompleteUnaryPrefixOperation { .. } + | ExpressionStackFrame::IncompleteBinaryOperation { .. } => { + Ok(NodeExtension::NoValidExtensionForCurrentParent) + } + } } fn evaluate_leaf( @@ -118,9 +148,7 @@ impl Expressionable for Source { SourceExpressionLeaf::Command(command) => { command.clone().interpret_to_value(interpreter)? } - SourceExpressionLeaf::MarkedVariable(variable) => { - variable.interpret_to_value(interpreter)? - } + SourceExpressionLeaf::Variable(variable) => variable.interpret_to_value(interpreter)?, SourceExpressionLeaf::VariablePath(variable_path) => { variable_path.interpret_to_value(interpreter)? } @@ -184,7 +212,10 @@ pub(super) trait Expressionable: Sized { type EvaluationContext; fn parse_unary_atom(input: &mut ParseStreamStack) -> ParseResult>; - fn parse_extension(input: &mut ParseStreamStack) -> ParseResult; + fn parse_extension( + input: &mut ParseStreamStack, + parent_stack_frame: &ExpressionStackFrame, + ) -> ParseResult; fn evaluate_leaf( leaf: &Self::Leaf, diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index 0e49f4b0..4548eed8 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -79,7 +79,8 @@ impl Interpret for &ExpressionBlock { Some(_) => Grouping::Flattened, None => Grouping::Grouped, }; - self.evaluate(interpreter)?.output_to(grouping, output)?; + self.evaluate(interpreter)? + .output_to(grouping, output, false)?; Ok(()) } } @@ -180,10 +181,13 @@ impl InterpretToValue for &AssignmentStatement { interpreter: &mut Interpreter, ) -> ExecutionResult { match &self.destination { - Destination::NewVariable { let_token, path } => { + Destination::LetBinding { + let_token, + destructuring, + } => { let value = self.expression.interpret_to_value(interpreter)?; let mut span_range = value.span_range(); - path.set_value(interpreter, value)?; + destructuring.handle_destructure(interpreter, value)?; span_range.set_start(let_token.span); Ok(ExpressionValue::None(span_range)) } @@ -200,48 +204,30 @@ impl InterpretToValue for &AssignmentStatement { span_range.set_start(path.span_range().start()); Ok(ExpressionValue::None(span_range)) } - Destination::Discarded { let_token, .. } => { - let value = self.expression.interpret_to_value(interpreter)?; - let mut span_range = value.span_range(); - span_range.set_start(let_token.span); - Ok(ExpressionValue::None(span_range)) - } } } } #[derive(Clone)] enum Destination { - NewVariable { + LetBinding { let_token: Token![let], - path: VariablePath, + destructuring: Destructuring, }, ExistingVariable { path: VariablePath, operation: Option, }, - Discarded { - let_token: Token![let], - #[allow(unused)] - discarded_token: Token![_], - }, } impl Parse for Destination { fn parse(input: ParseStream) -> ParseResult { if input.peek(Token![let]) { let let_token = input.parse()?; - if input.peek(Token![_]) { - Ok(Self::Discarded { - let_token, - discarded_token: input.parse()?, - }) - } else { - Ok(Self::NewVariable { - let_token, - path: input.parse()?, - }) - } + Ok(Self::LetBinding { + let_token, + destructuring: input.parse()?, + }) } else { Ok(Self::ExistingVariable { path: input.parse()?, diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index a3330cac..b551922a 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -27,7 +27,10 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { self.extend_with_unary_atom(unary_atom)? } WorkItem::TryParseAndApplyExtension { node } => { - let extension = K::parse_extension(&mut self.streams)?; + let extension = K::parse_extension( + &mut self.streams, + self.expression_stack.last().unwrap(), + )?; self.attempt_extension(node, extension)? } WorkItem::TryApplyAlreadyParsedExtension { node, extension } => { @@ -48,22 +51,27 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } UnaryAtom::Array { delim_span, - is_empty, + is_empty: true, } => { - let array_node = self.nodes.add_node(ExpressionNode::Array { - delim_span, - items: Vec::new(), - }); - if is_empty { - self.streams.exit_group(); - WorkItem::TryParseAndApplyExtension { node: array_node } - } else { - self.push_stack_frame(ExpressionStackFrame::Array { array_node }); - WorkItem::RequireUnaryAtom + self.streams.exit_group(); + WorkItem::TryParseAndApplyExtension { + node: self.nodes.add_node(ExpressionNode::Array { + delim_span, + items: Vec::new(), + }), } } + UnaryAtom::Array { + delim_span, + is_empty: false, + } => self.push_stack_frame(ExpressionStackFrame::Array { + delim_span, + items: Vec::new(), + }), UnaryAtom::PrefixUnaryOperation(operation) => { - self.push_stack_frame(ExpressionStackFrame::IncompletePrefixOperation { operation }) + self.push_stack_frame(ExpressionStackFrame::IncompleteUnaryPrefixOperation { + operation, + }) } }) } @@ -91,32 +99,16 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { operation, }) } - NodeExtension::CommaOperator { - comma, - is_end_of_stream: false, - } => { - let top_frame = self - .expression_stack - .last_mut() - .expect("There should always at least be a root"); - match top_frame { - ExpressionStackFrame::Array { array_node } => { - match self.nodes.node_mut(*array_node) { - ExpressionNode::Array { items, .. } => { - items.push(node); - }, - _ => unreachable!("Not possible, the array_node always corresponds to an array node"), - } - }, - _ => return comma.parse_err("Commas are only permitted inside preinterpret arrays []. Preinterpret arrays [a, b] can be used as a drop-in replacement for rust tuples (a, b)."), + NodeExtension::NonTerminalArrayComma => { + match self.expression_stack.last_mut().unwrap() { + ExpressionStackFrame::Array { items, .. } => { + items.push(node); + } + _ => unreachable!("CommaOperator is only returned under an Array parent."), } WorkItem::RequireUnaryAtom } - NodeExtension::CommaOperator { - is_end_of_stream: true, - .. - } - | NodeExtension::NoneMatched => { + NodeExtension::EndOfStream | NodeExtension::NoValidExtensionForCurrentParent => { unreachable!("Not possible, as these have minimum precedence") } }) @@ -126,11 +118,15 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { let parent_stack_frame = self.pop_stack_frame(); Ok(match parent_stack_frame { ExpressionStackFrame::Root => { - assert!(matches!(extension, NodeExtension::NoneMatched)); + assert!(matches!( + extension, + NodeExtension::EndOfStream + | NodeExtension::NoValidExtensionForCurrentParent + )); WorkItem::Finished { root: node } } ExpressionStackFrame::Group { delim_span } => { - assert!(matches!(extension, NodeExtension::NoneMatched)); + assert!(matches!(extension, NodeExtension::EndOfStream)); self.streams.exit_group(); WorkItem::TryParseAndApplyExtension { node: self.nodes.add_node(ExpressionNode::Grouped { @@ -139,44 +135,33 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { }), } } - ExpressionStackFrame::Array { array_node } => { - assert!(matches!( - extension, - NodeExtension::NoneMatched - | NodeExtension::CommaOperator { - is_end_of_stream: true, - .. - } - )); - match self.nodes.node_mut(array_node) { - ExpressionNode::Array { items, .. } => { - items.push(node); - } - _ => unreachable!( - "Not possible, the array_node always corresponds to an array node" - ), - } + ExpressionStackFrame::Array { + mut items, + delim_span, + } => { + assert!(matches!(extension, NodeExtension::EndOfStream)); + items.push(node); self.streams.exit_group(); - WorkItem::TryParseAndApplyExtension { node: array_node } - } - ExpressionStackFrame::IncompletePrefixOperation { operation } => { - WorkItem::TryApplyAlreadyParsedExtension { - node: self.nodes.add_node(ExpressionNode::UnaryOperation { - operation: operation.into(), - input: node, - }), - extension, + WorkItem::TryParseAndApplyExtension { + node: self + .nodes + .add_node(ExpressionNode::Array { delim_span, items }), } } + ExpressionStackFrame::IncompleteUnaryPrefixOperation { operation } => { + let node = self.nodes.add_node(ExpressionNode::UnaryOperation { + operation: operation.into(), + input: node, + }); + extension.into_post_operation_completion_work_item(node) + } ExpressionStackFrame::IncompleteBinaryOperation { lhs, operation } => { - WorkItem::TryApplyAlreadyParsedExtension { - node: self.nodes.add_node(ExpressionNode::BinaryOperation { - operation, - left_input: lhs, - right_input: node, - }), - extension, - } + let node = self.nodes.add_node(ExpressionNode::BinaryOperation { + operation, + left_input: lhs, + right_input: node, + }); + extension.into_post_operation_completion_work_item(node) } }) } @@ -223,14 +208,6 @@ impl ExpressionNodes { node_id } - /// Panics if the node id isn't valid - pub(super) fn node_mut( - &mut self, - ExpressionNodeId(node_id): ExpressionNodeId, - ) -> &mut ExpressionNode { - self.nodes.get_mut(node_id).unwrap() - } - pub(super) fn complete(self, root: ExpressionNodeId) -> Expression { Expression { root, @@ -434,7 +411,7 @@ impl OperatorPrecendence { /// ===> Stack: [] /// ===> WorkItem::Finished(E) /// ``` -enum ExpressionStackFrame { +pub(super) enum ExpressionStackFrame { /// A marker for the root of the expression Root, /// A marker for the parenthesized or transparent group. @@ -444,10 +421,13 @@ enum ExpressionStackFrame { /// A marker for the bracketed array. /// * When the array is opened, we add its inside to the parse stream stack /// * When the array is closed, we pop it from the parse stream stack - Array { array_node: ExpressionNodeId }, + Array { + delim_span: DelimSpan, + items: Vec, + }, /// An incomplete unary prefix operation /// NB: unary postfix operations such as `as` casting go straight to ExtendableNode - IncompletePrefixOperation { operation: PrefixUnaryOperation }, + IncompleteUnaryPrefixOperation { operation: PrefixUnaryOperation }, /// An incomplete binary operation IncompleteBinaryOperation { lhs: ExpressionNodeId, @@ -461,7 +441,7 @@ impl ExpressionStackFrame { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, - ExpressionStackFrame::IncompletePrefixOperation { operation, .. } => { + ExpressionStackFrame::IncompleteUnaryPrefixOperation { operation, .. } => { OperatorPrecendence::of_prefix_unary_operation(operation) } ExpressionStackFrame::IncompleteBinaryOperation { operation, .. } => { @@ -481,6 +461,12 @@ enum WorkItem { TryParseAndApplyExtension { node: ExpressionNodeId, }, + /// The same as [`WorkItem::TryParseAndApplyExtension`], except that we have already + /// parsed the extension, and we are attempting to apply it again. + /// This happens if an extension is of lower precedence than an existing unary/binary + /// operation, and so we complete the existing operation and try to re-apply the extension + /// on the result. For example, the + in `2 * 3 + 4` fails to apply to 3, but can + /// then apply to (2 * 3). TryApplyAlreadyParsedExtension { node: ExpressionNodeId, extension: NodeExtension, @@ -503,11 +489,9 @@ pub(super) enum UnaryAtom { pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), - CommaOperator { - comma: Token![,], - is_end_of_stream: bool, - }, - NoneMatched, + NonTerminalArrayComma, + EndOfStream, + NoValidExtensionForCurrentParent, } impl NodeExtension { @@ -515,15 +499,30 @@ impl NodeExtension { match self { NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), - NodeExtension::CommaOperator { - is_end_of_stream: false, - .. - } => OperatorPrecendence::NonTerminalComma, - NodeExtension::CommaOperator { - is_end_of_stream: true, - .. - } => OperatorPrecendence::MIN, - NodeExtension::NoneMatched => OperatorPrecendence::MIN, + NodeExtension::NonTerminalArrayComma => OperatorPrecendence::NonTerminalComma, + NodeExtension::EndOfStream => OperatorPrecendence::MIN, + NodeExtension::NoValidExtensionForCurrentParent => OperatorPrecendence::MIN, + } + } + + fn into_post_operation_completion_work_item(self, node: ExpressionNodeId) -> WorkItem { + match self { + extension @ NodeExtension::PostfixOperation { .. } + | extension @ NodeExtension::BinaryOperation { .. } + | extension @ NodeExtension::EndOfStream => { + WorkItem::TryApplyAlreadyParsedExtension { node, extension } + } + NodeExtension::NonTerminalArrayComma => { + unreachable!("Array comma is only possible on array parent") + } + NodeExtension::NoValidExtensionForCurrentParent => { + // We have to reparse in case the extension is valid for the new parent. + // e.g. consider [2 + 3, 4] - initially the peeked comma has a parent of an + // incomplete binary operation 2 + 3 which is invalid. + // When the binary operation is completed, the comma now gets parsed against + // the parent array, which can succeed. + WorkItem::TryParseAndApplyExtension { node } + } } } } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 32abfc9f..4ac0f1af 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -172,18 +172,20 @@ impl UntypedFloat { } CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input), + CastTarget::String => operation.output(input.to_string()), + CastTarget::DebugString => operation.output(self).into_debug_string_value(), CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } CastTarget::Stream => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Flattened)?, + .into_new_output_stream(Grouping::Flattened, false)?, ), CastTarget::Group => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), }, }) @@ -327,9 +329,11 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::Untyped) => operation.output(UntypedFloat::from_fallback(self as FallbackFloat)), CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), + CastTarget::String => operation.output(self.to_string()), + CastTarget::DebugString => operation.output(self).into_debug_string_value(), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)?), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)?), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, false)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, false)?), } }) } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 253c45fc..ec7eeb10 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -86,6 +86,18 @@ impl ExpressionInteger { } } + pub(crate) fn expect_usize(self) -> ExecutionResult { + Ok(match self.value { + ExpressionIntegerValue::Untyped(input) => input.parse_as()?, + ExpressionIntegerValue::Usize(input) => input, + _ => { + return self + .span_range + .execution_err("Expected a usize or untyped integer") + } + }) + } + pub(super) fn to_literal(&self) -> Literal { self.value .to_unspanned_literal() @@ -310,15 +322,17 @@ impl UntypedInteger { CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } + CastTarget::String => operation.output(input.to_string()), + CastTarget::DebugString => operation.output(self).into_debug_string_value(), CastTarget::Stream => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Flattened)?, + .into_new_output_stream(Grouping::Flattened, false)?, ), CastTarget::Group => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), }, }) @@ -379,7 +393,7 @@ impl UntypedInteger { format!( "The untyped integer operation {:?} {} {:?} overflowed in i128 space", lhs, - operation.symbol(), + operation.symbolic_description(), rhs ) }; @@ -515,7 +529,7 @@ macro_rules! impl_int_operations_except_unary { impl HandleBinaryOperation for $integer_type { fn handle_paired_binary_operation(self, rhs: Self, operation: OutputSpanned) -> ExecutionResult { let lhs = self; - let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.symbol(), rhs); + let overflow_error = || format!("The {} operation {:?} {} {:?} overflowed", stringify!($integer_type), lhs, operation.symbolic_description(), rhs); Ok(match operation.operation { PairedBinaryOperation::Addition { .. } => return operation.output_if_some(lhs.checked_add(rhs), overflow_error), PairedBinaryOperation::Subtraction { .. } => return operation.output_if_some(lhs.checked_sub(rhs), overflow_error), @@ -628,8 +642,10 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)?), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)?), + CastTarget::String => operation.output(self.to_string()), + CastTarget::DebugString => operation.output(self).into_debug_string_value(), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, false)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, false)?), } }) } @@ -664,8 +680,10 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened)?), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped)?), + CastTarget::String => operation.output(self.to_string()), + CastTarget::DebugString => operation.output(self).into_debug_string_value(), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, false)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, false)?), } }) } @@ -707,15 +725,17 @@ impl HandleUnaryOperation for u8 { CastTarget::Boolean => { return operation.execution_err("This cast is not supported") } + CastTarget::String => operation.output(self.to_string()), + CastTarget::DebugString => operation.output(self).into_debug_string_value(), CastTarget::Stream => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Flattened)?, + .into_new_output_stream(Grouping::Flattened, false)?, ), CastTarget::Group => operation.output( operation .output(self) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), }, }) diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 6619249a..f96b564e 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -8,7 +8,7 @@ pub(super) trait Operation: HasSpanRange { } } - fn symbol(&self) -> &'static str; + fn symbolic_description(&self) -> &'static str; } pub(super) struct OutputSpanned<'a, T: Operation + ?Sized> { @@ -17,8 +17,8 @@ pub(super) struct OutputSpanned<'a, T: Operation + ?Sized> { } impl OutputSpanned<'_, T> { - pub(super) fn symbol(&self) -> &'static str { - self.operation.symbol() + pub(super) fn symbolic_description(&self) -> &'static str { + self.operation.symbolic_description() } pub(super) fn output(&self, output_value: impl ToExpressionValue) -> ExpressionValue { @@ -39,7 +39,7 @@ impl OutputSpanned<'_, T> { pub(super) fn unsupported(&self, value: impl HasValueType) -> ExecutionResult { Err(self.operation.execution_error(format!( "The {} operator is not supported for {} values", - self.operation.symbol(), + self.operation.symbolic_description(), value.value_type(), ))) } @@ -108,7 +108,45 @@ impl UnaryOperation { as_token: Token![as], target_ident: Ident, ) -> ParseResult { - let target = match target_ident.to_string().as_str() { + let target = CastTarget::from_str(target_ident.to_string().as_str()).map_err(|()| { + target_ident.parse_error("This type is not supported in cast expressions") + })?; + + Ok(Self::Cast { + as_token, + target, + target_ident, + }) + } + + pub(super) fn evaluate(self, input: ExpressionValue) -> ExecutionResult { + let mut span_range = input.span_range(); + match &self { + UnaryOperation::Neg { token } => span_range.set_start(token.span), + UnaryOperation::Not { token } => span_range.set_start(token.span), + UnaryOperation::Cast { target_ident, .. } => span_range.set_end(target_ident.span()), + }; + input.handle_unary_operation(self.with_output_span_range(span_range)) + } +} + +#[derive(Copy, Clone)] +pub(super) enum CastTarget { + Integer(IntegerKind), + Float(FloatKind), + Boolean, + String, + DebugString, + Char, + Stream, + Group, +} + +impl FromStr for CastTarget { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(match s { "int" | "integer" => CastTarget::Integer(IntegerKind::Untyped), "u8" => CastTarget::Integer(IntegerKind::U8), "u16" => CastTarget::Integer(IntegerKind::U16), @@ -129,35 +167,48 @@ impl UnaryOperation { "char" => CastTarget::Char, "stream" => CastTarget::Stream, "group" => CastTarget::Group, - _ => { - return target_ident - .parse_err("This type is not supported in preinterpret cast expressions") - } - }; - Ok(Self::Cast { - as_token, - target, - target_ident, + "string" => CastTarget::String, + "debug" => CastTarget::DebugString, + _ => return Err(()), }) } +} - pub(super) fn evaluate(self, input: ExpressionValue) -> ExecutionResult { - let mut span_range = input.span_range(); - match &self { - UnaryOperation::Neg { token } => span_range.set_start(token.span), - UnaryOperation::Not { token } => span_range.set_start(token.span), - UnaryOperation::Cast { target_ident, .. } => span_range.set_end(target_ident.span()), - }; - input.handle_unary_operation(self.with_output_span_range(span_range)) +impl CastTarget { + fn symbolic_description(&self) -> &'static str { + match self { + CastTarget::Integer(IntegerKind::Untyped) => "as int", + CastTarget::Integer(IntegerKind::U8) => "as u8", + CastTarget::Integer(IntegerKind::U16) => "as u16", + CastTarget::Integer(IntegerKind::U32) => "as u32", + CastTarget::Integer(IntegerKind::U64) => "as u64", + CastTarget::Integer(IntegerKind::U128) => "as u128", + CastTarget::Integer(IntegerKind::Usize) => "as usize", + CastTarget::Integer(IntegerKind::I8) => "as i8", + CastTarget::Integer(IntegerKind::I16) => "as i16", + CastTarget::Integer(IntegerKind::I32) => "as i32", + CastTarget::Integer(IntegerKind::I64) => "as i64", + CastTarget::Integer(IntegerKind::I128) => "as i128", + CastTarget::Integer(IntegerKind::Isize) => "as isize", + CastTarget::Float(FloatKind::Untyped) => "as float", + CastTarget::Float(FloatKind::F32) => "as f32", + CastTarget::Float(FloatKind::F64) => "as f64", + CastTarget::Boolean => "as bool", + CastTarget::String => "as string", + CastTarget::DebugString => "as debug", + CastTarget::Char => "as char", + CastTarget::Stream => "as stream", + CastTarget::Group => "as group", + } } } impl Operation for UnaryOperation { - fn symbol(&self) -> &'static str { + fn symbolic_description(&self) -> &'static str { match self { UnaryOperation::Neg { .. } => "-", UnaryOperation::Not { .. } => "!", - UnaryOperation::Cast { .. } => "as", + UnaryOperation::Cast { target, .. } => target.symbolic_description(), } } } @@ -322,10 +373,10 @@ impl HasSpanRange for BinaryOperation { } impl Operation for BinaryOperation { - fn symbol(&self) -> &'static str { + fn symbolic_description(&self) -> &'static str { match self { - BinaryOperation::Paired(paired) => paired.symbol(), - BinaryOperation::Integer(integer) => integer.symbol(), + BinaryOperation::Paired(paired) => paired.symbolic_description(), + BinaryOperation::Integer(integer) => integer.symbolic_description(), } } } @@ -351,7 +402,7 @@ pub(crate) enum PairedBinaryOperation { } impl Operation for PairedBinaryOperation { - fn symbol(&self) -> &'static str { + fn symbolic_description(&self) -> &'static str { match self { PairedBinaryOperation::Addition { .. } => "+", PairedBinaryOperation::Subtraction { .. } => "-", @@ -403,7 +454,7 @@ pub(crate) enum IntegerBinaryOperation { } impl Operation for IntegerBinaryOperation { - fn symbol(&self) -> &'static str { + fn symbolic_description(&self) -> &'static str { match self { IntegerBinaryOperation::ShiftLeft { .. } => "<<", IntegerBinaryOperation::ShiftRight { .. } => ">>", @@ -443,7 +494,7 @@ pub(super) trait HandleCreateRange: Sized { } impl Operation for syn::RangeLimits { - fn symbol(&self) -> &'static str { + fn symbolic_description(&self) -> &'static str { match self { syn::RangeLimits::HalfOpen(_) => "..", syn::RangeLimits::Closed(_) => "..=", diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs index 81c29a60..2f81ab30 100644 --- a/src/expressions/stream.rs +++ b/src/expressions/stream.rs @@ -19,13 +19,22 @@ impl ExpressionStream { return operation.unsupported(self) } UnaryOperation::Cast { target, .. } => match target { + CastTarget::String => operation.output({ + let mut output = String::new(); + self.concat_recursive_into(&mut output, &ConcatBehaviour::standard()); + output + }), + CastTarget::DebugString => operation.output(self.value).into_debug_string_value(), CastTarget::Stream => operation.output(self.value), CastTarget::Group => operation.output( operation .output(self.value) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), - _ => { + CastTarget::Boolean + | CastTarget::Char + | CastTarget::Integer(_) + | CastTarget::Float(_) => { let coerced = self.value.coerce_into_value(self.span_range); if let ExpressionValue::Stream(_) = &coerced { return operation.unsupported(coerced); @@ -74,6 +83,20 @@ impl ExpressionStream { | PairedBinaryOperation::GreaterThan { .. } => return operation.unsupported(lhs), }) } + + pub(crate) fn concat_recursive_into(self, output: &mut String, behaviour: &ConcatBehaviour) { + if behaviour.output_types_as_commands { + if self.value.is_empty() { + output.push_str("[!stream!]"); + } else { + output.push_str("[!stream! "); + self.value.concat_recursive_into(output, behaviour); + output.push(']'); + } + } else { + self.value.concat_recursive_into(output, behaviour); + } + } } impl HasValueType for ExpressionStream { diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 5cd1836c..b6a6f644 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -2,7 +2,7 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionString { - pub(super) value: String, + pub(crate) value: String, /// The span range that generated this value. /// For a complex expression, the start span is the most left part /// of the expression, and the end span is the most right part. @@ -29,14 +29,19 @@ impl ExpressionString { CastTarget::Stream => operation.output( operation .output(self.value) - .into_new_output_stream(Grouping::Flattened)?, + .into_new_output_stream(Grouping::Flattened, false)?, ), CastTarget::Group => operation.output( operation .output(self.value) - .into_new_output_stream(Grouping::Grouped)?, + .into_new_output_stream(Grouping::Grouped, false)?, ), - _ => return operation.unsupported(self), + CastTarget::String => operation.output(self.value), + CastTarget::DebugString => operation.output(format!("{:?}", self.value)), + CastTarget::Boolean + | CastTarget::Char + | CastTarget::Integer(_) + | CastTarget::Float(_) => return operation.unsupported(self), }, }) } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index f06748cd..b8370843 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -180,7 +180,7 @@ impl ExpressionValue { ExpressionIntegerValuePair::Isize(lhs, rhs) } (left_value, right_value) => { - return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.value_type(), right_value.value_type())); + return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbolic_description(), left_value.value_type(), right_value.value_type())); } }; EvaluationLiteralPair::Integer(integer_pair) @@ -219,7 +219,7 @@ impl ExpressionValue { ExpressionFloatValuePair::F64(lhs, rhs) } (left_value, right_value) => { - return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbol(), left_value.value_type(), right_value.value_type())); + return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbolic_description(), left_value.value_type(), right_value.value_type())); } }; EvaluationLiteralPair::Float(float_pair) @@ -237,7 +237,7 @@ impl ExpressionValue { EvaluationLiteralPair::StreamPair(left, right) } (left, right) => { - return operation.execution_err(format!("Cannot infer common type from {} {} {}. Consider using `as` to cast the operands to matching types.", left.value_type(), operation.symbol(), right.value_type())); + return operation.execution_err(format!("Cannot infer common type from {} {} {}. Consider using `as` to cast the operands to matching types.", left.value_type(), operation.symbolic_description(), right.value_type())); } }) } @@ -249,20 +249,75 @@ impl ExpressionValue { } } - pub(crate) fn into_bool(self) -> Result { + pub(crate) fn expect_bool(self, place_descriptor: &str) -> ExecutionResult { match self { ExpressionValue::Boolean(value) => Ok(value), - other => Err(other.value_type()), + other => other.execution_err(format!( + "{} must be a boolean, but it is a {}", + place_descriptor, + other.value_type(), + )), } } - pub(crate) fn expect_bool(self, place_descriptor: &str) -> ExecutionResult { - let error_span = self.span_range(); - match self.into_bool() { - Ok(boolean) => Ok(boolean), - Err(value_type) => error_span.execution_err(format!( - "{} must be a boolean, but it is a {}", - place_descriptor, value_type, + pub(crate) fn expect_integer( + self, + place_descriptor: &str, + ) -> ExecutionResult { + match self { + ExpressionValue::Integer(value) => Ok(value), + other => other.execution_err(format!( + "{} must be an integer, but it is a {}", + place_descriptor, + other.value_type(), + )), + } + } + + pub(crate) fn expect_string(self, place_descriptor: &str) -> ExecutionResult { + match self { + ExpressionValue::String(value) => Ok(value), + other => other.execution_err(format!( + "{} must be a string, but it is a {}", + place_descriptor, + other.value_type(), + )), + } + } + + pub(crate) fn expect_array(self, place_descriptor: &str) -> ExecutionResult { + match self { + ExpressionValue::Array(value) => Ok(value), + other => other.execution_err(format!( + "{} must be an array, but it is a {}", + place_descriptor, + other.value_type(), + )), + } + } + + pub(crate) fn expect_stream(self, place_descriptor: &str) -> ExecutionResult { + match self { + ExpressionValue::Stream(value) => Ok(value), + other => other.execution_err(format!( + "{} must be a stream, but it is a {}", + place_descriptor, + other.value_type(), + )), + } + } + + pub(crate) fn expect_any_iterator( + self, + place_descriptor: &str, + ) -> ExecutionResult { + match self { + ExpressionValue::Array(value) => Ok(ExpressionIterator::new_for_array(value)), + ExpressionValue::Stream(value) => Ok(ExpressionIterator::new_for_stream(value)), + other => other.execution_err(format!( + "{} must be iterable (an array or stream), but it is a {}", + place_descriptor, + other.value_type(), )), } } @@ -351,12 +406,13 @@ impl ExpressionValue { pub(crate) fn into_new_output_stream( self, grouping: Grouping, + output_arrays: bool, ) -> ExecutionResult { Ok(match (self, grouping) { (Self::Stream(value), Grouping::Flattened) => value.value, (other, grouping) => { let mut output = OutputStream::new(); - other.output_to(grouping, &mut output)?; + other.output_to(grouping, &mut output, output_arrays)?; output } }) @@ -366,6 +422,7 @@ impl ExpressionValue { &self, grouping: Grouping, output: &mut OutputStream, + output_arrays: bool, ) -> ExecutionResult<()> { match grouping { Grouping::Grouped => { @@ -374,19 +431,23 @@ impl ExpressionValue { // * Grouping means -1 is interpreted atomically, rather than as a punct then a number // * Grouping means that a stream is interpreted atomically output.push_grouped( - |inner| self.output_flattened_to(inner), + |inner| self.output_flattened_to(inner, output_arrays), Delimiter::None, self.span_range().join_into_span_else_start(), )?; } Grouping::Flattened => { - self.output_flattened_to(output)?; + self.output_flattened_to(output, output_arrays)?; } } Ok(()) } - fn output_flattened_to(&self, output: &mut OutputStream) -> ExecutionResult<()> { + fn output_flattened_to( + &self, + output: &mut OutputStream, + output_arrays: bool, + ) -> ExecutionResult<()> { match self { Self::None { .. } => {} Self::Integer(value) => output.push_literal(value.to_literal()), @@ -397,50 +458,51 @@ impl ExpressionValue { Self::UnsupportedLiteral(literal) => { output.extend_raw_tokens(literal.lit.to_token_stream()) } - Self::Array { .. } => return self.execution_err("Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `as stream` to cast the array to a stream."), + Self::Array(array) => { + if output_arrays { + for item in &array.items { + item.output_to(Grouping::Grouped, output, output_arrays)?; + } + } else { + return self.execution_err("Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `#(XXX as stream)` to cast the array to a stream."); + } + } Self::Stream(value) => value.value.append_cloned_into(output), }; Ok(()) } - pub(crate) fn debug(&self) -> String { + pub(crate) fn into_debug_string_value(self) -> ExpressionValue { + let span_range = self.span_range(); + self.concat_recursive(&ConcatBehaviour::debug()) + .to_value(span_range) + } + + pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> String { let mut output = String::new(); - self.debug_to(&mut output); + self.concat_recursive_into(&mut output, behaviour); output } - fn debug_to(&self, output: &mut String) { - use std::fmt::Write; + pub(crate) fn concat_recursive_into(self, output: &mut String, behaviour: &ConcatBehaviour) { match self { ExpressionValue::None { .. } => { - write!(output, "None").unwrap(); + if behaviour.show_none_values { + output.push_str("None"); + } } ExpressionValue::Stream(stream) => { - if stream.value.is_empty() { - write!(output, "[!stream!]").unwrap(); - } else { - let stream = stream.value.clone(); - let string_rep = stream.concat_recursive(&ConcatBehaviour::debug()); - write!(output, "[!stream! {}]", string_rep).unwrap(); - } + stream.concat_recursive_into(output, behaviour); } ExpressionValue::Array(array) => { - write!(output, "[").unwrap(); - for (i, item) in array.items.iter().enumerate() { - if i != 0 { - write!(output, ", ").unwrap(); - } - item.debug_to(output); - } - write!(output, "]").unwrap(); + array.concat_recursive_into(output, behaviour); } _ => { // This isn't the most efficient, but it's less code and debug doesn't need to be super efficient. let mut stream = OutputStream::new(); - self.output_flattened_to(&mut stream) + self.output_flattened_to(&mut stream, false) .expect("Non-composite values should all be able to be outputted to a stream"); - let string_rep = stream.concat_recursive(&ConcatBehaviour::debug()); - write!(output, "{}", string_rep).unwrap(); + stream.concat_recursive_into(output, behaviour); } } } @@ -499,16 +561,6 @@ impl HasValueType for UnsupportedLiteral { } } -#[derive(Copy, Clone)] -pub(super) enum CastTarget { - Integer(IntegerKind), - Float(FloatKind), - Boolean, - Char, - Stream, - Group, -} - pub(super) enum EvaluationLiteralPair { Integer(ExpressionIntegerValuePair), Float(ExpressionFloatValuePair), @@ -549,3 +601,67 @@ impl EvaluationLiteralPair { }) } } + +pub(crate) struct ExpressionIterator { + iterator: ExpressionIteratorInner, + #[allow(unused)] + pub(crate) span_range: SpanRange, +} + +impl ExpressionIterator { + pub(crate) fn new_for_array(array: ExpressionArray) -> Self { + Self { + iterator: ExpressionIteratorInner::Array(array.items.into_iter()), + span_range: array.span_range, + } + } + + pub(crate) fn new_for_stream(stream: ExpressionStream) -> Self { + Self { + iterator: ExpressionIteratorInner::Stream(stream.value.into_iter()), + span_range: stream.span_range, + } + } + + #[allow(unused)] + pub(crate) fn new_custom( + iterator: Box>, + span_range: SpanRange, + ) -> Self { + Self { + iterator: ExpressionIteratorInner::Other(iterator), + span_range, + } + } +} + +enum ExpressionIteratorInner { + Array( as IntoIterator>::IntoIter), + Stream(::IntoIter), + Other(Box>), +} + +impl Iterator for ExpressionIterator { + type Item = ExpressionValue; + + fn next(&mut self) -> Option { + match &mut self.iterator { + ExpressionIteratorInner::Array(iter) => iter.next(), + ExpressionIteratorInner::Stream(iter) => { + let item = iter.next()?; + let span = item.span(); + let stream: OutputStream = item.into(); + Some(stream.coerce_into_value(span.span_range())) + } + ExpressionIteratorInner::Other(iter) => iter.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + match &self.iterator { + ExpressionIteratorInner::Array(iter) => iter.size_hint(), + ExpressionIteratorInner::Stream(iter) => iter.size_hint(), + ExpressionIteratorInner::Other(iter) => iter.size_hint(), + } + } +} diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index 149260c1..1e56edc1 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -168,7 +168,7 @@ impl<'a, K> ParseStreamStack<'a, K> { self.current().parse() } - pub(crate) fn is_empty(&self) -> bool { + pub(crate) fn is_current_empty(&self) -> bool { self.current().is_empty() } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 8b45c829..e5348362 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -11,7 +11,8 @@ pub(crate) use syn::parse::{ discouraged::Speculative, Parse as SynParse, ParseBuffer as SynParseBuffer, ParseStream as SynParseStream, Parser as SynParser, }; -pub(crate) use syn::{parse_str, Lit, LitBool, LitFloat, LitInt, Token}; +pub(crate) use syn::punctuated::Punctuated; +pub(crate) use syn::{parse_str, Lit, LitFloat, LitInt, Token}; pub(crate) use syn::{Error as SynError, Result as SynResult}; pub(crate) use crate::expressions::*; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 3e48001a..90d7948b 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -140,7 +140,7 @@ impl CommandInvocationAs for C { _ => Grouping::Grouped, }; self.execute(context.interpreter)? - .output_to(grouping, output) + .output_to(grouping, output, false) } fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { @@ -296,8 +296,8 @@ impl CommandInvocationAs CommandOutputKind { @@ -311,7 +311,7 @@ impl OutputKind for OutputKindStreaming { // Control Flow or a command which is unlikely to want grouped output pub(crate) trait StreamingCommandDefinition: - Sized + CommandType + Sized + CommandType { const COMMAND_NAME: &'static str; fn parse(arguments: CommandArguments) -> ParseResult; @@ -322,7 +322,7 @@ pub(crate) trait StreamingCommandDefinition: ) -> ExecutionResult<()>; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self, context: ExecutionContext, @@ -334,11 +334,7 @@ impl CommandInvocationAs for fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { let span_range = context.delim_span.span_range(); let mut output = OutputStream::new(); - >::execute_into( - self, - context, - &mut output, - )?; + >::execute_into(self, context, &mut output)?; Ok(output.to_value(span_range)) } } @@ -541,17 +537,6 @@ impl Parse for Command { } } -impl Command { - pub(crate) fn output_kind(&self) -> CommandOutputKind { - self.output_kind - } - - /// Should only be used to swap valid kinds - pub(crate) unsafe fn set_output_kind(&mut self, output_kind: CommandOutputKind) { - self.output_kind = output_kind; - } -} - impl HasSpan for Command { fn span(&self) -> Span { self.source_group_span.join() diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 36c8e81d..45137350 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -9,7 +9,7 @@ pub(crate) struct IfCommand { } impl CommandType for IfCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for IfCommand { @@ -85,7 +85,7 @@ pub(crate) struct WhileCommand { } impl CommandType for WhileCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for WhileCommand { @@ -142,7 +142,7 @@ pub(crate) struct LoopCommand { } impl CommandType for LoopCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for LoopCommand { @@ -184,15 +184,15 @@ impl StreamingCommandDefinition for LoopCommand { #[derive(Clone)] pub(crate) struct ForCommand { - parse_place: TransformStreamUntilToken, + destructuring: Destructuring, #[allow(unused)] in_token: Token![in], - input: SourceStreamInput, + input: SourceExpression, loop_code: SourceCodeBlock, } impl CommandType for ForCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for ForCommand { @@ -202,13 +202,13 @@ impl StreamingCommandDefinition for ForCommand { arguments.fully_parse_or_error( |input| { Ok(Self { - parse_place: input.parse()?, + destructuring: input.parse()?, in_token: input.parse()?, input: input.parse()?, loop_code: input.parse()?, }) }, - "Expected [!for! #x in [ ... ] { code }]", + "Expected [!for! in { }]", ) } @@ -217,18 +217,18 @@ impl StreamingCommandDefinition for ForCommand { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - let stream = self.input.interpret_to_new_stream(interpreter)?; + let array = self + .input + .interpret_to_value(interpreter)? + .expect_any_iterator("The for loop input")?; let mut iteration_counter = interpreter.start_iteration_counter(&self.in_token); - for token in stream { + for item in array { iteration_counter.increment_and_check()?; - let mut ignored_transformer_output = OutputStream::new(); - self.parse_place.handle_transform_from_stream( - token.into(), - interpreter, - &mut ignored_transformer_output, - )?; + + self.destructuring.handle_destructure(interpreter, item)?; + match self .loop_code .clone() diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 0f14c2d2..a285c2dd 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -119,7 +119,7 @@ pub(crate) struct RawCommand { } impl CommandType for RawCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for RawCommand { @@ -147,7 +147,7 @@ pub(crate) struct StreamCommand { } impl CommandType for StreamCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for StreamCommand { @@ -196,7 +196,7 @@ pub(crate) struct ReinterpretCommand { } impl CommandType for ReinterpretCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for ReinterpretCommand { @@ -238,7 +238,7 @@ define_field_inputs! { SettingsInputs { required: {}, optional: { - iteration_limit: SourceValue = DEFAULT_ITERATION_LIMIT ("The new iteration limit"), + iteration_limit: SourceExpression = DEFAULT_ITERATION_LIMIT ("The new iteration limit"), } } } @@ -254,7 +254,10 @@ impl NoOutputCommandDefinition for SettingsCommand { fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { if let Some(limit) = self.inputs.iteration_limit { - let limit: usize = limit.interpret_to_value(interpreter)?.base10_parse()?; + let limit = limit + .interpret_to_value(interpreter)? + .expect_integer("The iteration limit")? + .expect_usize()?; interpreter.set_iteration_limit(Some(limit)); } Ok(()) @@ -279,10 +282,10 @@ enum EitherErrorInput { define_field_inputs! { ErrorInputs { required: { - message: SourceValue = r#""...""# ("The error message to display"), + message: SourceExpression = r#""...""# ("The error message to display"), }, optional: { - spans: SourceStreamInput = "[$abc]" ("An optional [token stream], to determine where to show the error message"), + spans: SourceExpression = "[!stream! $abc]" ("An optional [token stream], to determine where to show the error message"), } } } @@ -323,11 +326,18 @@ impl NoOutputCommandDefinition for ErrorCommand { } }; - let message = fields.message.interpret_to_value(interpreter)?.value(); + let error_message = fields + .message + .interpret_to_value(interpreter)? + .expect_string("Error message")? + .value; let error_span = match fields.spans { Some(spans) => { - let error_span_stream = spans.interpret_to_new_stream(interpreter)?; + let error_span_stream = spans + .interpret_to_value(interpreter)? + .expect_stream("The error spans")? + .value; // Consider the case where preinterpret embeds in a declarative macro, and we have // an error like this: @@ -366,7 +376,7 @@ impl NoOutputCommandDefinition for ErrorCommand { None => Span::call_site().span_range(), }; - error_span.execution_err(message) + error_span.execution_err(error_message) } } @@ -396,7 +406,11 @@ impl ValueCommandDefinition for DebugCommand { } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let debug_string = self.inner.interpret_to_value(interpreter)?.debug(); - Ok(debug_string.to_value(self.span.span_range())) + let value = self + .inner + .interpret_to_value(interpreter)? + .with_span(self.span) + .into_debug_string_value(); + Ok(value) } } diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index 80274658..f290786b 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -2,22 +2,24 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct RangeCommand { + span: Span, left: SourceExpression, range_limits: syn::RangeLimits, right: SourceExpression, } impl CommandType for RangeCommand { - type OutputKind = OutputKindGroupedStream; + type OutputKind = OutputKindValue; } -impl GroupedStreamCommandDefinition for RangeCommand { +impl ValueCommandDefinition for RangeCommand { const COMMAND_NAME: &'static str = "range"; fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { Ok(Self { + span: arguments.command_span(), left: input.parse()?, range_limits: input.parse()?, right: input.parse()?, @@ -27,11 +29,7 @@ impl GroupedStreamCommandDefinition for RangeCommand { ) } - fn execute( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { let range_limits = self.range_limits; let left = self.left.interpret_to_value(interpreter)?; let right = self.right.interpret_to_value(interpreter)?; @@ -51,11 +49,8 @@ impl GroupedStreamCommandDefinition for RangeCommand { } } - for value in range_iterator { - // It needs to be grouped so that e.g. -1 is interpreted as a single item, not two separate tokens. - value.output_to(Grouping::Grouped, output)? - } - - Ok(()) + Ok(range_iterator + .collect::>() + .to_value(self.span.span_range())) } } diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 46af6708..589fddc7 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -81,60 +81,71 @@ impl GroupedStreamCommandDefinition for GroupCommand { #[derive(Clone)] pub(crate) struct IntersperseCommand { + span: Span, inputs: IntersperseInputs, } impl CommandType for IntersperseCommand { - type OutputKind = OutputKindGroupedStream; + type OutputKind = OutputKindValue; } define_field_inputs! { IntersperseInputs { required: { - items: SourceStreamInput = "[Hello World] or #var or [!cmd! ...]", - separator: SourceStreamInput = "[,]" ("The token/s to add between each item"), + items: SourceExpression = r#"["Hello", "World"]"# ("An array or stream (by coerced token-tree) to intersperse"), + separator: SourceExpression = "[!stream! ,]" ("The value to add between each item"), }, optional: { - add_trailing: SourceValue = "false" ("Whether to add the separator after the last item (default: false)"), - final_separator: SourceStreamInput = "[or]" ("Define a different final separator (default: same as normal separator)"), + add_trailing: SourceExpression = "false" ("Whether to add the separator after the last item (default: false)"), + final_separator: SourceExpression = "[!stream! or]" ("Define a different final separator (default: same as normal separator)"), } } } -impl GroupedStreamCommandDefinition for IntersperseCommand { +impl ValueCommandDefinition for IntersperseCommand { const COMMAND_NAME: &'static str = "intersperse"; fn parse(arguments: CommandArguments) -> ParseResult { Ok(Self { + span: arguments.command_span(), inputs: arguments.fully_parse_as()?, }) } - fn execute( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - let items = self.inputs.items.interpret_to_new_stream(interpreter)?; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let items = self + .inputs + .items + .interpret_to_value(interpreter)? + .expect_any_iterator("The items")?; + let output_span_range = self.span.span_range(); let add_trailing = match self.inputs.add_trailing { - Some(add_trailing) => add_trailing.interpret_to_value(interpreter)?.value(), + Some(add_trailing) => { + add_trailing + .interpret_to_value(interpreter)? + .expect_bool("This parameter")? + .value + } None => false, }; - if items.is_empty() { - return Ok(()); - } + let mut output = Vec::new(); + + let mut items = items.into_iter().peekable(); + + let mut this_item = match items.next() { + Some(next) => next, + None => return Ok(output.to_value(output_span_range)), + }; let mut appender = SeparatorAppender { - separator: self.inputs.separator, - final_separator: self.inputs.final_separator, + separator: &self.inputs.separator, + final_separator: self.inputs.final_separator.as_ref(), add_trailing, }; - let mut items = items.into_iter().peekable(); - let mut this_item = items.next().unwrap(); // Safe to unwrap as non-empty loop { - output.push_interpreted_item(this_item); + output.push(this_item); let next_item = items.next(); match next_item { Some(next_item) => { @@ -143,41 +154,46 @@ impl GroupedStreamCommandDefinition for IntersperseCommand { } else { RemainingItemCount::ExactlyOne }; - appender.add_separator(interpreter, remaining, output)?; + appender.add_separator(interpreter, remaining, &mut output)?; this_item = next_item; } None => { - appender.add_separator(interpreter, RemainingItemCount::None, output)?; + appender.add_separator(interpreter, RemainingItemCount::None, &mut output)?; break; } } } - Ok(()) + Ok(output.to_value(output_span_range)) } } -struct SeparatorAppender { - separator: SourceStreamInput, - final_separator: Option, +struct SeparatorAppender<'a> { + separator: &'a SourceExpression, + final_separator: Option<&'a SourceExpression>, add_trailing: bool, } -impl SeparatorAppender { +impl SeparatorAppender<'_> { fn add_separator( &mut self, interpreter: &mut Interpreter, remaining: RemainingItemCount, - output: &mut OutputStream, + output: &mut Vec, ) -> ExecutionResult<()> { match self.separator(remaining) { - TrailingSeparator::Normal => self.separator.clone().interpret_into(interpreter, output), + TrailingSeparator::Normal => { + output.push(self.separator.interpret_to_value(interpreter)?) + } TrailingSeparator::Final => match self.final_separator.take() { - Some(final_separator) => final_separator.interpret_into(interpreter, output), - None => self.separator.clone().interpret_into(interpreter, output), + Some(final_separator) => { + output.push(final_separator.interpret_to_value(interpreter)?) + } + None => output.push(self.separator.interpret_to_value(interpreter)?), }, - TrailingSeparator::None => Ok(()), + TrailingSeparator::None => {} } + Ok(()) } fn separator(&self, remaining_item_count: RemainingItemCount) -> TrailingSeparator { @@ -219,24 +235,24 @@ pub(crate) struct SplitCommand { } impl CommandType for SplitCommand { - type OutputKind = OutputKindGroupedStream; + type OutputKind = OutputKindValue; } define_field_inputs! { SplitInputs { required: { - stream: SourceStreamInput = "[...] or #var or [!cmd! ...]", - separator: SourceStreamInput = "[::]" ("The token/s to split if they match"), + stream: SourceExpression = "[!stream! ...] or #var" ("The stream-valued expression to split"), + separator: SourceExpression = "[!stream! ::]" ("The token/s to split if they match"), }, optional: { - drop_empty_start: SourceValue = "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), - drop_empty_middle: SourceValue = "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), - drop_empty_end: SourceValue = "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), + drop_empty_start: SourceExpression = "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), + drop_empty_middle: SourceExpression = "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), + drop_empty_end: SourceExpression = "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), } } } -impl GroupedStreamCommandDefinition for SplitCommand { +impl ValueCommandDefinition for SplitCommand { const COMMAND_NAME: &'static str = "split"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -245,37 +261,52 @@ impl GroupedStreamCommandDefinition for SplitCommand { }) } - fn execute( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - let output_span = self.inputs.stream.span_range().join_into_span_else_start(); - let stream = self.inputs.stream.interpret_to_new_stream(interpreter)?; - let separator = self.inputs.separator.interpret_to_new_stream(interpreter)?; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let stream = self + .inputs + .stream + .interpret_to_value(interpreter)? + .expect_stream("The stream input")?; + + let separator = self + .inputs + .separator + .interpret_to_value(interpreter)? + .expect_stream("The separator")? + .value; let drop_empty_start = match self.inputs.drop_empty_start { - Some(value) => value.interpret_to_value(interpreter)?.value(), + Some(value) => { + value + .interpret_to_value(interpreter)? + .expect_bool("This parameter")? + .value + } None => false, }; let drop_empty_middle = match self.inputs.drop_empty_middle { - Some(value) => value.interpret_to_value(interpreter)?.value(), + Some(value) => { + value + .interpret_to_value(interpreter)? + .expect_bool("This parameter")? + .value + } None => false, }; let drop_empty_end = match self.inputs.drop_empty_end { - Some(value) => value.interpret_to_value(interpreter)?.value(), + Some(value) => { + value + .interpret_to_value(interpreter)? + .expect_bool("This parameter")? + .value + } None => true, }; handle_split( interpreter, stream, - output, - output_span, - unsafe { - // RUST-ANALYZER SAFETY: This is as safe as we can get. - separator.parse_as()? - }, + separator.into_exact_stream()?, drop_empty_start, drop_empty_middle, drop_empty_end, @@ -286,18 +317,18 @@ impl GroupedStreamCommandDefinition for SplitCommand { #[allow(clippy::too_many_arguments)] fn handle_split( interpreter: &mut Interpreter, - input: OutputStream, - output: &mut OutputStream, - output_span: Span, + input: ExpressionStream, separator: ExactStream, drop_empty_start: bool, drop_empty_middle: bool, drop_empty_end: bool, -) -> ExecutionResult<()> { +) -> ExecutionResult { + let output_span_range = input.span_range; unsafe { // RUST-ANALYZER SAFETY: This is as safe as we can get. // Typically the separator won't contain none-delimited groups, so we're OK - input.parse_with(move |input| { + input.value.parse_with(move |input| { + let mut output = Vec::new(); let mut current_item = OutputStream::new(); // Special case separator.len() == 0 to avoid an infinite loop @@ -305,9 +336,9 @@ fn handle_split( while !input.is_empty() { current_item.push_raw_token_tree(input.parse()?); let complete_item = core::mem::replace(&mut current_item, OutputStream::new()); - output.push_new_group(complete_item, Delimiter::None, output_span); + output.push(complete_item.to_value(output_span_range)); } - return Ok(()); + return Ok(output.to_value(output_span_range)); } let mut drop_empty_next = drop_empty_start; @@ -329,14 +360,14 @@ fn handle_split( input.advance_to(&separator_fork); if !current_item.is_empty() || !drop_empty_next { let complete_item = core::mem::replace(&mut current_item, OutputStream::new()); - output.push_new_group(complete_item, Delimiter::None, output_span); + output.push(complete_item.to_value(output_span_range)); } drop_empty_next = drop_empty_middle; } if !current_item.is_empty() || !drop_empty_end { - output.push_new_group(current_item, Delimiter::None, output_span); + output.push(current_item.to_value(output_span_range)); } - Ok(()) + Ok(output.to_value(output_span_range)) }) } } @@ -347,10 +378,10 @@ pub(crate) struct CommaSplitCommand { } impl CommandType for CommaSplitCommand { - type OutputKind = OutputKindGroupedStream; + type OutputKind = OutputKindValue; } -impl GroupedStreamCommandDefinition for CommaSplitCommand { +impl ValueCommandDefinition for CommaSplitCommand { const COMMAND_NAME: &'static str = "comma_split"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -359,28 +390,18 @@ impl GroupedStreamCommandDefinition for CommaSplitCommand { }) } - fn execute( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - let output_span = self.input.span_range().join_into_span_else_start(); - let stream = self.input.interpret_to_new_stream(interpreter)?; + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let output_span_range = self.input.span_range(); + let stream = ExpressionStream { + span_range: output_span_range, + value: self.input.interpret_to_new_stream(interpreter)?, + }; let separator = Punct::new(',', Spacing::Alone) - .with_span(output_span) + .with_span(output_span_range.join_into_span_else_start()) .to_token_stream() .source_parse_as()?; - handle_split( - interpreter, - stream, - output, - output_span, - separator, - false, - false, - true, - ) + handle_split(interpreter, stream, separator, false, false, true) } } @@ -389,30 +410,28 @@ pub(crate) struct ZipCommand { inputs: EitherZipInput, } -type Streams = SourceValue>>; - #[derive(Clone)] enum EitherZipInput { Fields(ZipInputs), - JustStream(Streams), + JustStream(SourceExpression), } define_field_inputs! { ZipInputs { required: { - streams: Streams = r#"([Hello Goodbye] [World Friend])"# ("A group of one or more streams to zip together. The outer brackets are used for the group."), + streams: SourceExpression = r#"[[!stream! Hello Goodbye] ["World", "Friend"]]"# ("An array of arrays/iterators/streams to zip together."), }, optional: { - error_on_length_mismatch: SourceValue = "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), + error_on_length_mismatch: SourceExpression = "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), } } } impl CommandType for ZipCommand { - type OutputKind = OutputKindGroupedStream; + type OutputKind = OutputKindValue; } -impl GroupedStreamCommandDefinition for ZipCommand { +impl ValueCommandDefinition for ZipCommand { const COMMAND_NAME: &'static str = "zip"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -429,61 +448,78 @@ impl GroupedStreamCommandDefinition for ZipCommand { } }, format!( - "Expected [!zip! (#a #b #c)] or [!zip! {}]", + "Expected [!zip! [... An array of iterables ...]] or [!zip! {}]", ZipInputs::fields_description() ), ) } - fn execute( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - let (grouped_streams, error_on_length_mismatch) = match self.inputs { + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + let (streams, error_on_length_mismatch) = match self.inputs { EitherZipInput::Fields(inputs) => (inputs.streams, inputs.error_on_length_mismatch), EitherZipInput::JustStream(streams) => (streams, None), }; - let grouped_streams = grouped_streams.interpret_to_value(interpreter)?; + let streams = streams + .interpret_to_value(interpreter)? + .expect_array("The zip input")?; + let output_span_range = streams.span_range; + let mut output = Vec::new(); + let mut iterators = streams + .items + .into_iter() + .map(|x| x.expect_any_iterator("A zip input")) + .collect::, _>>()?; + let error_on_length_mismatch = match error_on_length_mismatch { - Some(value) => value.interpret_to_value(interpreter)?.value(), + Some(value) => { + value + .interpret_to_value(interpreter)? + .expect_bool("This parameter")? + .value + } None => true, }; - let Grouped { - delimiter, - delim_span, - inner: Repeated { inner: streams }, - } = grouped_streams; - if streams.is_empty() { - return delim_span - .join() - .execution_err("At least one stream is required to zip"); + + if iterators.is_empty() { + return Ok(output.to_value(output_span_range)); } - let stream_lengths = streams - .iter() - .map(|stream| stream.stream.len()) - .collect::>(); - let min_stream_length = *stream_lengths.iter().min().unwrap(); + + let min_stream_length = iterators.iter().map(|x| x.size_hint().0).min().unwrap(); + if error_on_length_mismatch { - let max_stream_length = *stream_lengths.iter().max().unwrap(); - if min_stream_length != max_stream_length { - return delim_span.join().execution_err(format!( + let max_stream_length = iterators + .iter() + .map(|x| x.size_hint().1) + .max_by(|a, b| match (a, b) { + (None, None) => core::cmp::Ordering::Equal, + (None, Some(_)) => core::cmp::Ordering::Greater, + (Some(_), None) => core::cmp::Ordering::Less, + (Some(a), Some(b)) => a.cmp(b), + }) + .unwrap(); + if Some(min_stream_length) != max_stream_length { + return output_span_range.execution_err(format!( "Streams have different lengths and zip's error_on_length_mismatch is true. The lengths vary from {} to {}", - min_stream_length, max_stream_length + min_stream_length, + match max_stream_length { + Some(max_stream_length) => max_stream_length.to_string(), + None => "unbounded".to_string(), + }, )); } } - let mut iters: Vec<_> = streams - .into_iter() - .map(|stream| stream.stream.into_iter()) - .collect(); + + let mut counter = interpreter.start_iteration_counter(&output_span_range); + for _ in 0..min_stream_length { - let mut inner = OutputStream::new(); - for iter in iters.iter_mut() { - inner.push_interpreted_item(iter.next().unwrap()); + counter.increment_and_check()?; + let mut inner = Vec::with_capacity(iterators.len()); + for iter in iterators.iter_mut() { + inner.push(iter.next().unwrap()); } - output.push_new_group(inner, delimiter, delim_span.span()); + output.push(inner.to_value(output_span_range)); } - Ok(()) + + Ok(output.to_value(output_span_range)) } } diff --git a/src/interpretation/commands/transforming_commands.rs b/src/interpretation/commands/transforming_commands.rs index eeaf9339..3b256788 100644 --- a/src/interpretation/commands/transforming_commands.rs +++ b/src/interpretation/commands/transforming_commands.rs @@ -2,14 +2,14 @@ use crate::internal_prelude::*; #[derive(Clone)] pub(crate) struct ParseCommand { - input: SourceStreamInput, + input: SourceExpression, #[allow(unused)] - as_token: Token![as], + with_token: Ident, transformer: ExplicitTransformStream, } impl CommandType for ParseCommand { - type OutputKind = OutputKindStreaming; + type OutputKind = OutputKindStream; } impl StreamingCommandDefinition for ParseCommand { @@ -20,11 +20,11 @@ impl StreamingCommandDefinition for ParseCommand { |input| { Ok(Self { input: input.parse()?, - as_token: input.parse()?, + with_token: input.parse_ident_matching("with")?, transformer: input.parse()?, }) }, - "Expected [!parse! [...] as @(...)] or [!parse! #x as @(...)] where the latter is a transform stream", + "Expected [!parse! with ] where:\n* The is some stream-valued expression, such as `#x` or `[!stream! ...]`\n* The is some parser such as @(...)", ) } @@ -33,7 +33,11 @@ impl StreamingCommandDefinition for ParseCommand { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - let input = self.input.interpret_to_new_stream(interpreter)?; + let input = self + .input + .interpret_to_value(interpreter)? + .expect_stream("Parse input")? + .value; self.transformer .handle_transform_from_stream(input, interpreter, output) } diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index b11824a3..2a1c8710 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -36,6 +36,15 @@ impl From for OutputStream { } } +impl HasSpan for OutputTokenTree { + fn span(&self) -> Span { + match self { + OutputTokenTree::TokenTree(token_tree) => token_tree.span(), + OutputTokenTree::OutputGroup(_, span, _) => *span, + } + } +} + impl OutputStream { pub(crate) fn new() -> Self { Self { @@ -223,30 +232,6 @@ impl OutputStream { } } - pub(crate) fn unwrap_singleton_group( - self, - check_group: impl FnOnce(Delimiter) -> bool, - create_error: impl FnOnce() -> SynError, - ) -> ParseResult { - let mut item_vec = self.into_item_vec(); - if item_vec.len() == 1 { - match item_vec.pop().unwrap() { - OutputTokenTree::OutputGroup(delimiter, _, inner) => { - if check_group(delimiter) { - return Ok(inner); - } - } - OutputTokenTree::TokenTree(TokenTree::Group(group)) => { - if check_group(group.delimiter()) { - return Ok(Self::raw(group.stream())); - } - } - _ => {} - } - } - Err(create_error().into()) - } - pub(crate) fn into_item_vec(self) -> Vec { let mut output = Vec::with_capacity(self.token_length); for segment in self.segments { @@ -267,6 +252,12 @@ impl OutputStream { } pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> String { + let mut output = String::new(); + self.concat_recursive_into(&mut output, behaviour); + output + } + + pub(crate) fn concat_recursive_into(self, output: &mut String, behaviour: &ConcatBehaviour) { fn concat_recursive_interpreted_stream( behaviour: &ConcatBehaviour, output: &mut String, @@ -344,9 +335,7 @@ impl OutputStream { spacing } - let mut output = String::new(); - concat_recursive_interpreted_stream(behaviour, &mut output, Spacing::Joint, self); - output + concat_recursive_interpreted_stream(behaviour, output, Spacing::Joint, self); } pub(crate) fn into_exact_stream(self) -> ParseResult { @@ -368,34 +357,40 @@ impl IntoIterator for OutputStream { pub(crate) struct ConcatBehaviour { pub(crate) add_space_between_token_trees: bool, - pub(crate) output_transparent_group_as_command: bool, + pub(crate) output_types_as_commands: bool, + pub(crate) output_array_structure: bool, pub(crate) unwrap_contents_of_string_like_literals: bool, + pub(crate) show_none_values: bool, } impl ConcatBehaviour { pub(crate) fn standard() -> Self { Self { add_space_between_token_trees: false, - output_transparent_group_as_command: false, + output_types_as_commands: false, + output_array_structure: false, unwrap_contents_of_string_like_literals: true, + show_none_values: false, } } pub(crate) fn debug() -> Self { Self { add_space_between_token_trees: true, - output_transparent_group_as_command: true, + output_types_as_commands: true, + output_array_structure: true, unwrap_contents_of_string_like_literals: false, + show_none_values: true, } } - fn before_token_tree(&self, output: &mut String, spacing: Spacing) { + pub(crate) fn before_token_tree(&self, output: &mut String, spacing: Spacing) { if self.add_space_between_token_trees && spacing == Spacing::Alone { output.push(' '); } } - fn handle_literal(&self, output: &mut String, literal: Literal) { + pub(crate) fn handle_literal(&self, output: &mut String, literal: Literal) { match literal.content_if_string_like() { Some(content) if self.unwrap_contents_of_string_like_literals => { output.push_str(&content) @@ -404,7 +399,7 @@ impl ConcatBehaviour { } } - fn wrap_delimiters( + pub(crate) fn wrap_delimiters( &self, output: &mut String, delimiter: Delimiter, @@ -434,7 +429,7 @@ impl ConcatBehaviour { output.push(']'); } Delimiter::None => { - if self.output_transparent_group_as_command { + if self.output_types_as_commands { if is_empty { output.push_str("[!group!"); } else { diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 2dccad98..1300bb4a 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -6,8 +6,6 @@ mod interpreted_stream; mod interpreter; mod source_code_block; mod source_stream; -mod source_stream_input; -mod source_value; mod variable; pub(crate) use command::*; @@ -17,6 +15,4 @@ pub(crate) use interpreted_stream::*; pub(crate) use interpreter::*; pub(crate) use source_code_block::*; pub(crate) use source_stream::*; -pub(crate) use source_stream_input::*; -pub(crate) use source_value::*; pub(crate) use variable::*; diff --git a/src/interpretation/source_stream.rs b/src/interpretation/source_stream.rs index 6e159ccd..b5bb006b 100644 --- a/src/interpretation/source_stream.rs +++ b/src/interpretation/source_stream.rs @@ -119,12 +119,6 @@ pub(crate) struct SourceGroup { content: SourceStream, } -impl SourceGroup { - pub(crate) fn into_content(self) -> SourceStream { - self.content - } -} - impl Parse for SourceGroup { fn parse(input: ParseStream) -> ParseResult { let (delimiter, delim_span, content) = input.parse_any_group()?; diff --git a/src/interpretation/source_stream_input.rs b/src/interpretation/source_stream_input.rs deleted file mode 100644 index 38ad7791..00000000 --- a/src/interpretation/source_stream_input.rs +++ /dev/null @@ -1,168 +0,0 @@ -use crate::internal_prelude::*; - -/// For use as a stream input to a command, when the whole command isn't the stream. -/// -/// It accepts any of the following: -/// * A `[..]` group - the input stream is the interpreted contents of the brackets -/// * A [!command! ...] - the input stream is the command's output -/// * A `#variable` - the input stream is the content of the variable -/// * A flattened `#..variable` - the variable must contain a single `[..]` or transparent group, the input stream are the group contents -/// -/// We don't support transparent groups, because they are not used consistently and it would expose this -/// inconsistency to the user, and be a potentially breaking change for other tooling to add/remove them. -/// For example, as of Jan 2025, in declarative macro substitutions a $literal gets a wrapping group, -/// but a $tt or $($tt)* does not. -#[derive(Clone)] -pub(crate) enum SourceStreamInput { - Command(Command), - Variable(MarkedVariable), - Code(SourceCodeBlock), - ExplicitStream(SourceGroup), -} - -impl Parse for SourceStreamInput { - fn parse(input: ParseStream) -> ParseResult { - Ok(match input.peek_grammar() { - SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::Variable(_) => Self::Variable(input.parse()?), - SourcePeekMatch::Group(Delimiter::Bracket) => Self::ExplicitStream(input.parse()?), - SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), - _ => input.span() - .parse_err("Expected [ ..input stream.. ], { [..input stream..] } or a [!command! ..], #variable or #..variable.\nMacro substitutions such as $x should be placed inside square brackets.")?, - }) - } -} - -impl HasSpanRange for SourceStreamInput { - fn span_range(&self) -> SpanRange { - match self { - SourceStreamInput::Command(command) => command.span_range(), - SourceStreamInput::Variable(variable) => variable.span_range(), - SourceStreamInput::Code(code) => code.span_range(), - SourceStreamInput::ExplicitStream(group) => group.span_range(), - } - } -} - -impl Interpret for SourceStreamInput { - fn interpret_into( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - match self { - SourceStreamInput::Command(mut command) => { - match command.output_kind() { - CommandOutputKind::None - | CommandOutputKind::GroupedValue - | CommandOutputKind::FlattenedValue - | CommandOutputKind::Ident - | CommandOutputKind::Literal => { - command.execution_err("The command does not output a stream") - } - CommandOutputKind::FlattenedStream => parse_as_stream_input( - command, - interpreter, - || { - "Expected output of flattened command to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to remove the .., to use the command output as-is.".to_string() - }, - output, - ), - CommandOutputKind::GroupedStream => { - unsafe { - // SAFETY: The kind change GroupedStream <=> FlattenedStream is valid - command.set_output_kind(CommandOutputKind::FlattenedStream); - } - command.interpret_into(interpreter, output) - } - CommandOutputKind::Stream => parse_as_stream_input( - command, - interpreter, - || { - "Expected output of control flow command to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...].".to_string() - }, - output, - ), - } - } - SourceStreamInput::Variable(MarkedVariable::Flattened(variable)) => { - parse_as_stream_input( - &variable, - interpreter, - || { - format!( - "Expected variable to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to use #{} instead, to use the content of the variable as the stream.", - variable.get_name(), - ) - }, - output, - ) - } - SourceStreamInput::Variable(MarkedVariable::Grouped(variable)) => { - // We extract its contents via explicitly using Grouping::Flattened here - variable.substitute_into(interpreter, Grouping::Flattened, output) - } - SourceStreamInput::Code(code) => parse_as_stream_input( - code, - interpreter, - || { - "Expected the { ... } block to output a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string() - }, - output, - ), - SourceStreamInput::ExplicitStream(group) => { - group.into_content().interpret_into(interpreter, output) - } - } - } -} - -/// From a code neatness perspective, this would be better as `OutputCommandStream`... -/// However this approach avoids a round-trip to TokenStream, which causes bugs in RustAnalyzer. -/// This can be removed when we fork syn and can parse the OutputStream directly. -fn parse_as_stream_input( - input: impl Interpret + HasSpanRange, - interpreter: &mut Interpreter, - error_message: impl FnOnce() -> String, - output: &mut OutputStream, -) -> ExecutionResult<()> { - let span = input.span_range(); - input - .interpret_to_new_stream(interpreter)? - .unwrap_singleton_group( - |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None), - || span.error(error_message()), - )? - .append_into(output); - Ok(()) -} - -impl InterpretToValue for SourceStreamInput { - type OutputValue = OutputCommandStream; - - fn interpret_to_value( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - Ok(OutputCommandStream { - stream: self.interpret_to_new_stream(interpreter)?, - }) - } -} - -pub(crate) struct OutputCommandStream { - pub(crate) stream: OutputStream, -} - -impl Parse for OutputCommandStream { - fn parse(input: ParseStream) -> ParseResult { - // We replicate parse_as_stream_input - let (_, inner) = input.parse_group_matching( - |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None), - || "Expected [...] or a transparent group from a #variable or stream-output command such as [!group! ...]".to_string(), - )?; - Ok(Self { - stream: OutputStream::raw(inner.parse()?), - }) - } -} diff --git a/src/interpretation/source_value.rs b/src/interpretation/source_value.rs deleted file mode 100644 index be035c8a..00000000 --- a/src/interpretation/source_value.rs +++ /dev/null @@ -1,180 +0,0 @@ -use crate::internal_prelude::*; - -/// This can be use to represent a value, or a source of that value at parse time. -/// -/// For example, `SourceValue` could be used to represent a literal, -/// or a variable/command which could convert to a literal after interpretation. -#[derive(Clone)] -pub(crate) enum SourceValue { - Command(Command), - Variable(MarkedVariable), - ExpressionBlock(ExpressionBlock), - Code(SourceCodeBlock), - Value(T), -} - -impl> Parse for SourceValue { - fn parse(input: ParseStream) -> ParseResult { - Ok(match input.peek_grammar() { - SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::Variable(_) => Self::Variable(input.parse()?), - SourcePeekMatch::ExpressionBlock(_) => Self::ExpressionBlock(input.parse()?), - SourcePeekMatch::Group(Delimiter::Brace) => Self::Code(input.parse()?), - SourcePeekMatch::ExplicitTransformStream - | SourcePeekMatch::AppendVariableBinding - | SourcePeekMatch::Transformer(_) => { - return input - .span() - .parse_err("Destructurings are not supported here") - } - SourcePeekMatch::Group(_) - | SourcePeekMatch::Punct(_) - | SourcePeekMatch::Literal(_) - | SourcePeekMatch::Ident(_) - | SourcePeekMatch::End => Self::Value(input.parse()?), - }) - } -} - -impl> Parse for SourceValue { - fn parse(input: ParseStream) -> ParseResult { - Ok(Self::Value(input.parse()?)) - } -} - -impl HasSpanRange for SourceValue { - fn span_range(&self) -> SpanRange { - match self { - SourceValue::Command(command) => command.span_range(), - SourceValue::Variable(variable) => variable.span_range(), - SourceValue::ExpressionBlock(block) => block.span_range(), - SourceValue::Code(code) => code.span_range(), - SourceValue::Value(value) => value.span_range(), - } - } -} - -impl, I: Parse> InterpretToValue for SourceValue { - type OutputValue = I; - - fn interpret_to_value(self, interpreter: &mut Interpreter) -> ExecutionResult { - let descriptor = match self { - SourceValue::Command(_) => "command output", - SourceValue::Variable(_) => "variable output", - SourceValue::ExpressionBlock(_) => "an #(...) expression block", - SourceValue::Code(_) => "output from the { ... } block", - SourceValue::Value(_) => "value", - }; - let interpreted_stream = match self { - SourceValue::Command(command) => command.interpret_to_new_stream(interpreter)?, - SourceValue::Variable(variable) => variable.interpret_to_new_stream(interpreter)?, - SourceValue::ExpressionBlock(block) => block.interpret_to_new_stream(interpreter)?, - SourceValue::Code(code) => code.interpret_to_new_stream(interpreter)?, - SourceValue::Value(value) => return value.interpret_to_value(interpreter), - }; - unsafe { - // RUST-ANALYZER SAFETY: If I is a very simple parse function, this is safe. - // Zip uses it with a parse function which does care about none-delimited groups however. - interpreted_stream - .parse_with(I::parse) - .add_context_if_error_and_no_context(|| { - format!( - "Occurred whilst parsing the {} to a {}.", - descriptor, - std::any::type_name::() - ) - }) - .into_execution_result() - } - } -} - -#[derive(Clone)] -pub(crate) struct Grouped { - pub(crate) delimiter: Delimiter, - pub(crate) delim_span: DelimSpan, - pub(crate) inner: T, -} - -impl> Parse for Grouped { - fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, inner) = input.parse_any_group()?; - Ok(Self { - delimiter, - delim_span, - inner: inner.parse()?, - }) - } -} - -impl> Parse for Grouped { - fn parse(input: ParseStream) -> ParseResult { - let (delimiter, delim_span, inner) = input.parse_any_group()?; - Ok(Self { - delimiter, - delim_span, - inner: inner.parse()?, - }) - } -} - -impl InterpretToValue for Grouped -where - T: InterpretToValue, -{ - type OutputValue = Grouped; - - fn interpret_to_value( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - Ok(Grouped { - delimiter: self.delimiter, - delim_span: self.delim_span, - inner: self.inner.interpret_to_value(interpreter)?, - }) - } -} - -#[derive(Clone)] -pub(crate) struct Repeated { - pub(crate) inner: Vec, -} - -impl> Parse for Repeated { - fn parse(input: ParseStream) -> ParseResult { - let mut inner = vec![]; - while !input.is_empty() { - inner.push(input.parse::()?); - } - Ok(Self { inner }) - } -} - -impl> Parse for Repeated { - fn parse(input: ParseStream) -> ParseResult { - let mut inner = vec![]; - while !input.is_empty() { - inner.push(input.parse::()?); - } - Ok(Self { inner }) - } -} - -impl InterpretToValue for Repeated -where - T: InterpretToValue, -{ - type OutputValue = Repeated; - - fn interpret_to_value( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let mut interpreted = Vec::with_capacity(self.inner.len()); - for item in self.inner.into_iter() { - interpreted.push(item.interpret_to_value(interpreter)?); - } - Ok(Repeated { inner: interpreted }) - } -} diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index f18e1dc3..552a3db6 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -51,7 +51,7 @@ pub(crate) trait IsVariable: HasSpanRange { ) -> ExecutionResult<()> { self.read_existing(interpreter)? .get(self)? - .output_to(grouping, output) + .output_to(grouping, output, false) } fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { @@ -114,20 +114,6 @@ impl Interpret for &MarkedVariable { } } -impl InterpretToValue for &MarkedVariable { - type OutputValue = ExpressionValue; - - fn interpret_to_value( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - match self { - MarkedVariable::Grouped(variable) => variable.interpret_to_value(interpreter), - MarkedVariable::Flattened(variable) => variable.interpret_to_value(interpreter), - } - } -} - #[derive(Clone)] pub(crate) struct GroupedVariable { marker: Token![#], @@ -232,20 +218,6 @@ impl Interpret for &FlattenedVariable { } } -impl InterpretToValue for &FlattenedVariable { - type OutputValue = ExpressionValue; - - fn interpret_to_value( - self, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - // We always output a stream value for a flattened variable - Ok(self - .interpret_to_new_stream(interpreter)? - .to_value(self.span_range())) - } -} - impl HasSpanRange for FlattenedVariable { fn span_range(&self) -> SpanRange { SpanRange::new_between(self.marker.span, self.variable_name.span()) @@ -311,3 +283,38 @@ impl InterpretToValue for &VariablePath { self.get_value(interpreter) } } + +#[derive(Clone)] +pub(crate) struct VariableDestructuring { + name: Ident, +} + +impl Parse for VariableDestructuring { + fn parse(input: ParseStream) -> ParseResult { + Ok(Self { + name: input.parse()?, + }) + } +} + +impl IsVariable for VariableDestructuring { + fn get_name(&self) -> String { + self.name.to_string() + } +} + +impl HasSpan for VariableDestructuring { + fn span(&self) -> Span { + self.name.span() + } +} + +impl HandleDestructure for VariableDestructuring { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + self.set_value(interpreter, value) + } +} diff --git a/src/misc/errors.rs b/src/misc/errors.rs index ada214de..effbc1e6 100644 --- a/src/misc/errors.rs +++ b/src/misc/errors.rs @@ -2,6 +2,7 @@ use crate::internal_prelude::*; pub(crate) type ParseResult = core::result::Result; +#[allow(unused)] pub(crate) trait ParseResultExt { /// This is not a `From` because it wants to be explicit fn convert_to_final_result(self) -> syn::Result; diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 55ea7d2c..adbcb277 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -224,6 +224,16 @@ impl<'a, K> ParseBuffer<'a, K> { T::parse(self, context) } + pub fn parse_terminated, P: syn::parse::Parse>( + &'a self, + ) -> ParseResult> { + Ok(Punctuated::parse_terminated_with(&self.inner, |inner| { + ParseStream::::from(inner) + .parse() + .map_err(|err| err.convert_to_final_error()) + })?) + } + pub(crate) fn call) -> ParseResult>( &self, f: F, diff --git a/src/transformation/destructuring.rs b/src/transformation/destructuring.rs new file mode 100644 index 00000000..0a156b2f --- /dev/null +++ b/src/transformation/destructuring.rs @@ -0,0 +1,90 @@ +use crate::internal_prelude::*; + +pub(crate) trait HandleDestructure { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()>; +} + +#[derive(Clone)] +pub(crate) enum Destructuring { + Variable(VariableDestructuring), + Array(ArrayDestructuring), + Stream(ExplicitTransformStream), + #[allow(unused)] + Discarded(Token![_]), +} + +impl Parse for Destructuring { + fn parse(input: ParseStream) -> ParseResult { + let lookahead = input.lookahead1(); + if lookahead.peek(syn::Ident) { + Ok(Destructuring::Variable(input.parse()?)) + } else if lookahead.peek(syn::token::Bracket) { + Ok(Destructuring::Array(input.parse()?)) + } else if lookahead.peek(Token![@]) { + Ok(Destructuring::Stream(input.parse()?)) + } else if lookahead.peek(Token![_]) { + Ok(Destructuring::Discarded(input.parse()?)) + } else if input.peek(Token![#]) { + return input.parse_err("Use `var` instead of `#var` in a destructuring"); + } else { + Err(lookahead.error().into()) + } + } +} + +impl HandleDestructure for Destructuring { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + match self { + Destructuring::Variable(variable) => variable.handle_destructure(interpreter, value), + Destructuring::Array(array) => array.handle_destructure(interpreter, value), + Destructuring::Stream(stream) => stream.handle_destructure(interpreter, value), + Destructuring::Discarded(_) => Ok(()), + } + } +} + +#[derive(Clone)] +pub struct ArrayDestructuring { + #[allow(unused)] + delim_span: DelimSpan, + items: Punctuated, +} + +impl Parse for ArrayDestructuring { + fn parse(input: ParseStream) -> ParseResult { + let (delim_span, inner) = input.parse_specific_group(Delimiter::Bracket)?; + Ok(Self { + delim_span, + items: inner.parse_terminated()?, + }) + } +} + +impl HandleDestructure for ArrayDestructuring { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + let array = value.expect_array("The destructure source")?; + if array.items.len() != self.items.len() { + return array.execution_err(format!( + "The array has {} items, but the destructuring expected {}", + array.items.len(), + self.items.len() + )); + } + for (value, destructuring) in array.items.into_iter().zip(&self.items) { + destructuring.handle_destructure(interpreter, value)?; + } + Ok(()) + } +} diff --git a/src/transformation/mod.rs b/src/transformation/mod.rs index ad5193ae..3dd9203d 100644 --- a/src/transformation/mod.rs +++ b/src/transformation/mod.rs @@ -1,3 +1,4 @@ +mod destructuring; mod exact_stream; mod fields; mod parse_utilities; @@ -7,6 +8,7 @@ mod transformer; mod transformers; mod variable_binding; +pub(crate) use destructuring::*; pub(crate) use exact_stream::*; #[allow(unused)] pub(crate) use fields::*; diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index 18f3c604..221c596b 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -260,3 +260,16 @@ impl HandleTransformation for ExplicitTransformStream { Ok(()) } } + +impl HandleDestructure for ExplicitTransformStream { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + let stream = value.expect_stream("The destructure source")?; + let mut discarded = OutputStream::new(); + self.handle_transform_from_stream(stream.value, interpreter, &mut discarded)?; + Ok(()) + } +} diff --git a/tests/compilation_failures/complex/nested.stderr b/tests/compilation_failures/complex/nested.stderr index c360b421..0c6b8cac 100644 --- a/tests/compilation_failures/complex/nested.stderr +++ b/tests/compilation_failures/complex/nested.stderr @@ -3,7 +3,7 @@ error: required fields are missing: message // The error message to display message: "...", // An optional [token stream], to determine where to show the error message - spans?: [$abc], + spans?: [!stream! $abc], }] --> tests/compilation_failures/complex/nested.rs:7:26 | diff --git a/tests/compilation_failures/core/error_invalid_structure.stderr b/tests/compilation_failures/core/error_invalid_structure.stderr index 6bb513cd..851a6f35 100644 --- a/tests/compilation_failures/core/error_invalid_structure.stderr +++ b/tests/compilation_failures/core/error_invalid_structure.stderr @@ -3,7 +3,7 @@ error: required fields are missing: message // The error message to display message: "...", // An optional [token stream], to determine where to show the error message - spans?: [$abc], + spans?: [!stream! $abc], }] --> tests/compilation_failures/core/error_invalid_structure.rs:4:28 | diff --git a/tests/compilation_failures/core/error_span_multiple.rs b/tests/compilation_failures/core/error_span_multiple.rs index f890d643..ac86a85b 100644 --- a/tests/compilation_failures/core/error_span_multiple.rs +++ b/tests/compilation_failures/core/error_span_multiple.rs @@ -5,7 +5,7 @@ macro_rules! assert_literals_eq { [!if! ($input1 != $input2) { [!error! { message: [!string! "Expected " $input1 " to equal " $input2], - spans: [$input1, $input2], + spans: [!stream! $input1, $input2], }] }] }}; diff --git a/tests/compilation_failures/core/error_span_repeat.rs b/tests/compilation_failures/core/error_span_repeat.rs index 48bc84ba..e8c991e4 100644 --- a/tests/compilation_failures/core/error_span_repeat.rs +++ b/tests/compilation_failures/core/error_span_repeat.rs @@ -6,7 +6,7 @@ macro_rules! assert_input_length_of_3 { [!if! (#input_length != 3) { [!error! { message: [!string! "Expected 3 inputs, got " #input_length], - spans: [$($input)+], + spans: [!stream! $($input)+], }] }] }}; diff --git a/tests/compilation_failures/core/error_span_single.rs b/tests/compilation_failures/core/error_span_single.rs index 7c2d1dd2..a4cfad38 100644 --- a/tests/compilation_failures/core/error_span_single.rs +++ b/tests/compilation_failures/core/error_span_single.rs @@ -5,7 +5,7 @@ macro_rules! assert_is_100 { [!if! ($input != 100) { [!error! { message: [!string! "Expected 100, got " $input], - spans: [$input], + spans: [!stream! $input], }] }] }}; diff --git a/tests/compilation_failures/expressions/array_missing_comma.rs b/tests/compilation_failures/expressions/array_missing_comma.rs new file mode 100644 index 00000000..62895890 --- /dev/null +++ b/tests/compilation_failures/expressions/array_missing_comma.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #([1 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_missing_comma.stderr b/tests/compilation_failures/expressions/array_missing_comma.stderr new file mode 100644 index 00000000..245725c5 --- /dev/null +++ b/tests/compilation_failures/expressions/array_missing_comma.stderr @@ -0,0 +1,5 @@ +error: Expected comma, ], or operator + --> tests/compilation_failures/expressions/array_missing_comma.rs:5:14 + | +5 | #([1 2]) + | ^ diff --git a/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr b/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr index 7d947dc5..58705eb1 100644 --- a/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr +++ b/tests/compilation_failures/expressions/cannot_output_array_to_stream.stderr @@ -1,4 +1,4 @@ -error: Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `as stream` to cast the array to a stream. +error: Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `#(XXX as stream)` to cast the array to a stream. --> tests/compilation_failures/expressions/cannot_output_array_to_stream.rs:5:11 | 5 | #([1, 2, 3]) diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs b/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs deleted file mode 100644 index b64f1552..00000000 --- a/tests/compilation_failures/expressions/flattened_commands_in_expressions.rs +++ /dev/null @@ -1,7 +0,0 @@ -use preinterpret::*; - -fn main() { - let _ = preinterpret! { - #(5 + [!..range! 1..2]) - }; -} diff --git a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr b/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr deleted file mode 100644 index 20b58c43..00000000 --- a/tests/compilation_failures/expressions/flattened_commands_in_expressions.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Cannot infer common type from untyped integer + stream. Consider using `as` to cast the operands to matching types. - --> tests/compilation_failures/expressions/flattened_commands_in_expressions.rs:5:13 - | -5 | #(5 + [!..range! 1..2]) - | ^ diff --git a/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs b/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs deleted file mode 100644 index 87ed1bb9..00000000 --- a/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs +++ /dev/null @@ -1,13 +0,0 @@ -use preinterpret::*; - -fn main() { - preinterpret! { - [!intersperse! { - items: [1 2], - separator: [], - add_trailing: { - true false - } - }] - } -} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr b/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr deleted file mode 100644 index afea4d6b..00000000 --- a/tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: unexpected token - Occurred whilst parsing the output from the { ... } block to a syn::lit::LitBool. - --> tests/compilation_failures/tokens/intersperse_bool_input_braces_issue.rs:9:22 - | -9 | true false - | ^^^^^ diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs deleted file mode 100644 index 3a81831a..00000000 --- a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs +++ /dev/null @@ -1,10 +0,0 @@ -use preinterpret::*; - -fn main() { - preinterpret! { - [!intersperse! { - items: { 1 2 3 }, - separator: [] - }] - } -} \ No newline at end of file diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr deleted file mode 100644 index 4c40c9aa..00000000 --- a/tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Expected the { ... } block to output a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream. - --> tests/compilation_failures/tokens/intersperse_stream_input_braces_issue.rs:6:20 - | -6 | items: { 1 2 3 }, - | ^^^^^^^^^ diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr index 930db528..bf3d688a 100644 --- a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr +++ b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr @@ -1,5 +1,15 @@ -error: Expected variable to contain a single [...] group or transparent group from a #variable or stream-output command such as [!group! ...]. Perhaps you want to use #x instead, to use the content of the variable as the stream. +error: Remove the .. prefix. Flattened variables are not supported in an expression. + Occurred whilst parsing [!intersperse! ...] - Expected: { + // An array or stream (by coerced token-tree) to intersperse + items: ["Hello", "World"], + // The value to add between each item + separator: [!stream! ,], + // Whether to add the separator after the last item (default: false) + add_trailing?: false, + // Define a different final separator (default: same as normal separator) + final_separator?: [!stream! or], + } --> tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.rs:7:20 | 7 | items: #..x, - | ^^^^ + | ^ diff --git a/tests/compilation_failures/tokens/zip_different_length_streams.rs b/tests/compilation_failures/tokens/zip_different_length_streams.rs index f6589845..8f1c809d 100644 --- a/tests/compilation_failures/tokens/zip_different_length_streams.rs +++ b/tests/compilation_failures/tokens/zip_different_length_streams.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { preinterpret! { - [!zip! ([A B C] [1 2 3 4])] + [!zip! [["A", "B", "C"], [1, 2, 3, 4]]] } } \ No newline at end of file diff --git a/tests/compilation_failures/tokens/zip_different_length_streams.stderr b/tests/compilation_failures/tokens/zip_different_length_streams.stderr index 0572cc5a..757cc110 100644 --- a/tests/compilation_failures/tokens/zip_different_length_streams.stderr +++ b/tests/compilation_failures/tokens/zip_different_length_streams.stderr @@ -1,5 +1,5 @@ error: Streams have different lengths and zip's error_on_length_mismatch is true. The lengths vary from 3 to 4 --> tests/compilation_failures/tokens/zip_different_length_streams.rs:5:16 | -5 | [!zip! ([A B C] [1 2 3 4])] - | ^^^^^^^^^^^^^^^^^^^ +5 | [!zip! [["A", "B", "C"], [1, 2, 3, 4]]] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/tokens/zip_no_streams.rs b/tests/compilation_failures/tokens/zip_no_input.rs similarity index 76% rename from tests/compilation_failures/tokens/zip_no_streams.rs rename to tests/compilation_failures/tokens/zip_no_input.rs index 0dd8fe05..739dd4ff 100644 --- a/tests/compilation_failures/tokens/zip_no_streams.rs +++ b/tests/compilation_failures/tokens/zip_no_input.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { preinterpret! { - [!zip! ()] + [!zip!] } } \ No newline at end of file diff --git a/tests/compilation_failures/tokens/zip_no_input.stderr b/tests/compilation_failures/tokens/zip_no_input.stderr new file mode 100644 index 00000000..9dd7214c --- /dev/null +++ b/tests/compilation_failures/tokens/zip_no_input.stderr @@ -0,0 +1,11 @@ +error: Expected an expression + Occurred whilst parsing [!zip! ...] - Expected [!zip! [... An array of iterables ...]] or [!zip! { + // An array of arrays/iterators/streams to zip together. + streams: [[!stream! Hello Goodbye] ["World", "Friend"]], + // If false, uses shortest stream length, if true, errors on unequal length. Defaults to true. + error_on_length_mismatch?: true, + }] + --> tests/compilation_failures/tokens/zip_no_input.rs:5:15 + | +5 | [!zip!] + | ^ diff --git a/tests/compilation_failures/tokens/zip_no_streams.stderr b/tests/compilation_failures/tokens/zip_no_streams.stderr deleted file mode 100644 index 788a385d..00000000 --- a/tests/compilation_failures/tokens/zip_no_streams.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: At least one stream is required to zip - --> tests/compilation_failures/tokens/zip_no_streams.rs:5:16 - | -5 | [!zip! ()] - | ^^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index a7139072..d2ae64bb 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -60,7 +60,7 @@ fn test_loop_continue_and_break() { #(x = 0) [!loop! { #(x += 1) - [!if! #x >= 10 { [!break!] }] + [!if! x >= 10 { [!break!] }] }] #x }, @@ -68,9 +68,9 @@ fn test_loop_continue_and_break() { ); preinterpret_assert_eq!( { - [!string! [!for! #x in [!range! 65..75] { - [!if! #x % 2 == 0 { [!continue!] }] - #(#x as u8 as char) + [!string! [!for! x in [!range! 65..75] { + [!if! x % 2 == 0 { [!continue!] }] + #(x as u8 as char) }]] }, "ACEGI" @@ -81,7 +81,7 @@ fn test_loop_continue_and_break() { fn test_for() { preinterpret_assert_eq!( { - [!string! [!for! #x in [!range! 65..70] { + [!string! [!for! x in [!range! 65..70] { #(#x as u8 as char) }]] }, @@ -89,7 +89,7 @@ fn test_for() { ); preinterpret_assert_eq!( { - [!string! [!for! (#x,) in [(a,) (b,) (c,)] { + [!string! [!for! @((#x,)) in [!stream! (a,) (b,) (c,)] { #x [!if! [!string! #x] == "b" { [!break!] }] }]] diff --git a/tests/expressions.rs b/tests/expressions.rs index 623e6366..4f895dd7 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -92,7 +92,7 @@ fn test_very_long_expression_works() { [!settings! { iteration_limit: 100000, }] - #(let expression = [!stream! 0] + [!for! #i in [!range! 0..100000] { + 1 }]) + #(let expression = [!stream! 0] + [!for! _ in [!range! 0..100000] { + 1 }]) [!reinterpret! [!raw! #](#expression)] }, 100000 @@ -135,9 +135,9 @@ fn assign_works() { preinterpret_assert_eq!( #( let x = 5 + 5; - [!debug! #..x] + [!debug! #x] ), - "[!stream! 10]" + "10" ); preinterpret_assert_eq!( #( @@ -163,41 +163,38 @@ fn assign_works() { #[test] fn test_range() { preinterpret_assert_eq!( - [!string![!intersperse! { + #([!intersperse! { items: [!range! -2..5], separator: [" "], - }]], + }] as string), "-2 -1 0 1 2 3 4" ); preinterpret_assert_eq!( - [!string![!intersperse! { + #([!intersperse! { items: [!range! -2..=5], - separator: [" "], - }]], + separator: " ", + }] as stream as string), "-2 -1 0 1 2 3 4 5" ); preinterpret_assert_eq!( { #(x = 2) - [!string! [!intersperse! { + #([!intersperse! { items: [!range! (#x + #x)..=5], - separator: [" "], - }]] + separator: " ", + }] as stream as string) }, "4 5" ); preinterpret_assert_eq!( { - [!string![!intersperse! { + #([!intersperse! { items: [!range! 8..=5], - separator: [" "], - }]] + separator: " ", + }] as stream as string) }, "" ); - preinterpret_assert_eq!({ [!string! [!range! 'a'..='f']] }, "abcdef"); - preinterpret_assert_eq!( - { [!debug! [!..range! -1i8..3i8]] }, - "[!stream! [!group! -1i8] [!group! 0i8] [!group! 1i8] [!group! 2i8]]" - ); + preinterpret_assert_eq!({ [!string! #([!range! 'a'..='f'] as stream)] }, "abcdef"); + preinterpret_assert_eq!({ [!debug! [!range! -1i8..3i8]] }, "[-1i8, 0i8, 1i8, 2i8]"); } diff --git a/tests/tokens.rs b/tests/tokens.rs index 3b87038a..46b5c250 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -49,113 +49,91 @@ fn test_length_and_group() { #[test] fn test_intersperse() { preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [Hello World], - separator: [", "], - }]] - }, + #([!intersperse! { + items: [!stream! Hello World], + separator: [", "], + }] as string), "Hello, World" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [Hello World], - separator: [_ "and" _], - }]] - }, + #([!intersperse! { + items: [!stream! Hello World], + separator: [!stream! _ "and" _], + }] as stream as string), "Hello_and_World" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [Hello World], - separator: [_ "and" _], - add_trailing: true, - }]] - }, + #([!intersperse! { + items: [!stream! Hello World], + separator: [!stream! _ "and" _], + add_trailing: true, + }] as stream as string), "Hello_and_World_and_" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [The Quick Brown Fox], - separator: [], - }]] - }, + #([!intersperse! { + items: [!stream! The Quick Brown Fox], + separator: [!stream!], + }] as stream as string), "TheQuickBrownFox" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [The Quick Brown Fox], - separator: [,], - add_trailing: true, - }]] - }, + #([!intersperse! { + items: [!stream! The Quick Brown Fox], + separator: [!stream! ,], + add_trailing: true, + }] as stream as string), "The,Quick,Brown,Fox," ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [Red Green Blue], - separator: [", "], - final_separator: [" and "], - }]] - }, + #([!intersperse! { + items: [!stream! Red Green Blue], + separator: [!stream! ", "], + final_separator: [!stream! " and "], + }] as stream as string), "Red, Green and Blue" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [Red Green Blue], - separator: [", "], + #([!intersperse! { + items: [!stream! Red Green Blue], + separator: [!stream! ", "], add_trailing: true, - final_separator: [" and "], - }]] - }, + final_separator: [!stream! " and "], + }] as stream as string), "Red, Green, Blue and " ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [], - separator: [", "], - add_trailing: true, - final_separator: [" and "], - }]] - }, + #([!intersperse! { + items: [!stream!], + separator: [!stream! ", "], + add_trailing: true, + final_separator: [!stream! " and "], + }] as stream as string), "" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [SingleItem], - separator: [","], - final_separator: ["!"], - }]] - }, + #([!intersperse! { + items: [!stream! SingleItem], + separator: [!stream! ","], + final_separator: [!stream! "!"], + }] as stream as string), "SingleItem" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [SingleItem], - separator: [","], - final_separator: ["!"], - add_trailing: true, - }]] - }, + #([!intersperse! { + items: [!stream! SingleItem], + separator: [!stream! ","], + final_separator: [!stream! "!"], + add_trailing: true, + }] as stream as string), "SingleItem!" ); preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [SingleItem], - separator: [","], - add_trailing: true, - }]] - }, + #([!intersperse! { + items: [!stream! SingleItem], + separator: [!stream! ","], + add_trailing: true, + }] as stream as string), "SingleItem," ); } @@ -164,150 +142,131 @@ fn test_intersperse() { fn complex_cases_for_intersperse_and_input_types() { // Normal separator is not interpreted if it is unneeded preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [], - separator: [[!error! { message: "FAIL" }]], - add_trailing: true, - }]] - }, + #([!intersperse! { + items: [!stream!], + separator: [!error! { message: "FAIL" }], + add_trailing: true, + }] as stream as string), "" ); // Final separator is not interpreted if it is unneeded preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [], - separator: [], - final_separator: [[!error! { message: "FAIL" }]], - add_trailing: true, - }]] - }, + #([!intersperse! { + items: [!stream!], + separator: [!stream!], + final_separator: [!error! { message: "FAIL" }], + add_trailing: true, + }] as stream as string), "" ); // The separator is interpreted each time it is included - preinterpret_assert_eq!({ - #(let i = 0) - [!string! [!intersperse! { - items: [A B C D E F G], - separator: [ - (#i) - #(i += 1) - ], - add_trailing: true, - }]] - }, "A(0)B(1)C(2)D(3)E(4)F(5)G(6)"); + preinterpret_assert_eq!( + #( + let i = 0; + [!intersperse! { + items: [!stream! A B C D E F G], + separator: #( + let output = [!stream! (#i)]; + i += 1; + output + ), + add_trailing: true, + }] as string + ), + "A(0)B(1)C(2)D(3)E(4)F(5)G(6)", + ); // Command can be used for items preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [!range! 0..4], - separator: [_], - }]] - }, + #([!intersperse! { + items: [!range! 0..4], + separator: [!stream! _], + }] as stream as string), "0_1_2_3" ); - // Grouped Variable can be used for items + // Variable containing stream be used for items preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] - [!string! [!intersperse! { + #([!intersperse! { items: #items, - separator: [_], - }]] + separator: [!stream! _], + }] as stream as string) }, "0_1_2_3"); - // Grouped variable containing flattened command can be used for items + // Variable containing iterable can be used for items preinterpret_assert_eq!({ - [!set! #items = [!..range! 0..4]] - [!string! [!intersperse! { + #(let items = [!range! 0..4]) + #([!intersperse! { items: #items, - separator: [_], - }]] + separator: [!stream! _], + }] as stream as string) }, "0_1_2_3"); - // Flattened variable containing [ ... ] group - preinterpret_assert_eq!({ - [!set! #items = [0 1 2 3]] - [!string! [!intersperse! { - items: #..items, - separator: [_], - }]] - }, "0_1_2_3"); - // Flattened variable containing transparent group + // #(...) block returning token stream (from variable) preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] - [!set! #wrapped_items = #items] // #items is "grouped variable" so outputs [!group! 0 1 2 3] - [!string! [!intersperse! { - items: #..wrapped_items, // #..wrapped_items returns its contents: [!group! 0 1 2 3] - separator: [_], - }]] + #([!intersperse! { + items: #(items), + separator: ["_"], + }] as string) }, "0_1_2_3"); - // { ... } block returning transparent group (from variable) - preinterpret_assert_eq!({ - [!set! #items = 0 1 2 3] - [!string! [!intersperse! { - items: { - #items // #items is "grouped variable syntax" so outputs [!group! 0 1 2 3] - }, - separator: [_], - }]] - }, "0_1_2_3"); - // { ... } block returning [ ... ] group + // #(...) block returning array preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: { - [0 1 2 3] - }, - separator: [_], - }]] - }, + #([!intersperse! { + items: #([0, 1, 2, 3]), + separator: ["_"], + }] as string), "0_1_2_3" ); - // Grouped variable containing two groups - preinterpret_assert_eq!({ - [!set! #items = 0 1] - [!set! #item_groups = #items #items] // [!group! 0 1] [!group! 0 1] - [!string! [!intersperse! { - items: #item_groups, // [!group! [!group! 0 1] [!group! 0 1]] - separator: [_], - }]] - }, "01_01"); - // Control stream commands can be used, if they return a valid stream grouping + // Stream containing two groups preinterpret_assert_eq!( - { - [!string![!intersperse! { - items: [!if! false { [0 1] } !else! { [2 3] }], - separator: [_], - }]] - }, + #( + let items = [!stream! 0 1]; + [!intersperse! { + items: [!stream! #items #items], // [!stream! [!group! 0 1] [!group! 0 1]] + separator: [!stream! _], + }] as string + ), + "01_01", + ); + // Commands can be used, if they return a valid iterable (e.g. a stream) + preinterpret_assert_eq!( + #([!intersperse! { + items: [!if! false { 0 1 } !else! { 2 3 }], + separator: [!stream! _], + }] as stream as string), "2_3" ); // All inputs can be variables // Inputs can be in any order - preinterpret_assert_eq!({ - [!set! #people = Anna Barbara Charlie] - [!set! #separator = ", "] - [!set! #final_separator = " and "] - [!set! #add_trailing = false] - [!string! [!intersperse! { - separator: #separator, - final_separator: #final_separator, - add_trailing: #add_trailing, - items: #people, - }]] - }, "Anna, Barbara and Charlie"); + preinterpret_assert_eq!( + #( + let people = [!stream! Anna Barbara Charlie]; + let separator = [", "]; + let final_separator = [" and "]; + let add_trailing = false; + [!intersperse! { + separator: separator, + final_separator: final_separator, + add_trailing: add_trailing, + items: people, + }] as string + ), + "Anna, Barbara and Charlie" + ); // Add trailing is executed even if it's irrelevant because there are no items - preinterpret_assert_eq!({ - [!set! #x = "NOT_EXECUTED"] - [!intersperse! { - items: [], - separator: [], - add_trailing: { - [!set! #x = "EXECUTED"] - false - }, - }] - #x - }, "EXECUTED"); + preinterpret_assert_eq!( + #( + let x = "NOT_EXECUTED"; + let _ = [!intersperse! { + items: [], + separator: [], + add_trailing: #( + x = "EXECUTED"; + false + ), + }]; + x + ), + "EXECUTED", + ); } #[test] @@ -316,163 +275,138 @@ fn test_split() { // In this case, drop_empty_start / drop_empty_end are ignored preinterpret_assert_eq!( { - [!debug! [!..split! { - stream: [A::B], - separator: [], + [!debug![!split! { + stream: [!stream! A::B], + separator: [!stream!], }]] }, - "[!stream! [!group! A] [!group! :] [!group! :] [!group! B]]" + "[[!stream! A], [!stream! :], [!stream! :], [!stream! B]]" ); // Double separators are allowed preinterpret_assert_eq!( { - [!debug! [!..split! { - stream: [A::B::C], - separator: [::], + [!debug![!split! { + stream: [!stream! A::B::C], + separator: [!stream! ::], }]] }, - "[!stream! [!group! A] [!group! B] [!group! C]]" + "[[!stream! A], [!stream! B], [!stream! C]]" ); // Trailing separator is ignored by default preinterpret_assert_eq!( { - [!debug! [!..split! { - stream: [Pizza, Mac and Cheese, Hamburger,], - separator: [,], + [!debug![!split! { + stream: [!stream! Pizza, Mac and Cheese, Hamburger,], + separator: [!stream! ,], }]] }, - "[!stream! [!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" + "[[!stream! Pizza], [!stream! Mac and Cheese], [!stream! Hamburger]]" ); // By default, empty groups are included except at the end preinterpret_assert_eq!( { - [!debug! [!..split! { - stream: [::A::B::::C::], - separator: [::], - }]] + [!debug![!split! { + stream: [!stream! ::A::B::::C::], + separator: [!stream! ::], + }] as stream] }, "[!stream! [!group!] [!group! A] [!group! B] [!group!] [!group! C]]" ); // Stream and separator are both interpreted preinterpret_assert_eq!({ [!set! #x = ;] - [!debug! [!..split! { - stream: [;A;;B;C;D #..x E;], + [!debug! [!split! { + stream: [!stream! ;A;;B;C;D #..x E;], separator: #x, drop_empty_start: true, drop_empty_middle: true, drop_empty_end: true, - }]] + }] as stream] }, "[!stream! [!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); // Drop empty false works preinterpret_assert_eq!({ [!set! #x = ;] - [!debug! [!..split! { - stream: [;A;;B;C;D #..x E;], + [!debug! [!split! { + stream: [!stream! ;A;;B;C;D #..x E;], separator: #x, drop_empty_start: false, drop_empty_middle: false, drop_empty_end: false, - }]] + }] as stream] }, "[!stream! [!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]"); // Drop empty middle works preinterpret_assert_eq!( { - [!debug! [!..split! { - stream: [;A;;B;;;;E;], - separator: [;], + [!debug![!split! { + stream: [!stream! ;A;;B;;;;E;], + separator: [!stream! ;], drop_empty_start: false, drop_empty_middle: true, drop_empty_end: false, }]] }, - "[!stream! [!group!] [!group! A] [!group! B] [!group! E] [!group!]]" - ); - // Code blocks are only evaluated once - // (i.e. no "unknown variable B is output in the below") - preinterpret_assert_eq!( - { - [!debug! [!..split! { - stream: { - [A [!raw! #] B [!raw! #] C] - }, - separator: [#], - }]] - }, - "[!stream! [!group! A] [!group! B] [!group! C]]" + "[[!stream!], [!stream! A], [!stream! B], [!stream! E], [!stream!]]" ); } #[test] fn test_comma_split() { preinterpret_assert_eq!( - { [!debug! [!..comma_split! Pizza, Mac and Cheese, Hamburger,]] }, - "[!stream! [!group! Pizza] [!group! Mac and Cheese] [!group! Hamburger]]" + { [!debug! [!comma_split! Pizza, Mac and Cheese, Hamburger,]] }, + "[[!stream! Pizza], [!stream! Mac and Cheese], [!stream! Hamburger]]" ); } #[test] fn test_zip() { preinterpret_assert_eq!( - { [!debug! [!..zip! ([Hello Goodbye] [World Friend])]] }, - "[!stream! (Hello World) (Goodbye Friend)]" + [!debug![ + !zip! [[!stream! Hello "Goodbye"], ["World", "Friend"]] + ]], + r#"[[[!stream! Hello], "World"], ["Goodbye", "Friend"]]"#, ); preinterpret_assert_eq!( { - [!set! #countries = France Germany Italy] + [!set! #countries = "France" "Germany" "Italy"] [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] - [!set! #capitals = Paris Berlin Rome] + [!set! #capitals = "Paris" "Berlin" "Rome"] [!debug! [!zip! { - streams: [#countries #flags #capitals], + streams: [#countries, #flags, #capitals], }]] }, - r#"[!stream! [!group! [France "🇫🇷" Paris] [Germany "🇩🇪" Berlin] [Italy "🇮🇹" Rome]]]"#, + r#"[["France", "🇫🇷", "Paris"], ["Germany", "🇩🇪", "Berlin"], ["Italy", "🇮🇹", "Rome"]]"#, ); preinterpret_assert_eq!( { [!set! #longer = A B C D] [!set! #shorter = 1 2 3] - [!set! #combined = #longer #shorter] - [!debug! [!..zip! { - streams: #combined, + [!debug! [!zip! { + streams: [#longer, #shorter], error_on_length_mismatch: false, }]] }, - r#"[!stream! [!group! A 1] [!group! B 2] [!group! C 3]]"#, + r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); preinterpret_assert_eq!( { [!set! #letters = A B C] - [!set! #numbers = 1 2 3] - [!set! #combined = #letters #numbers] - [!debug! [!..zip! { - streams: { - { #..combined } - }, - }]] - }, - r#"[!stream! { A 1 } { B 2 } { C 3 }]"#, - ); - preinterpret_assert_eq!( - { - [!set! #letters = A B C] - [!set! #numbers = 1 2 3] - [!set! #combined = { #letters #numbers }] - [!debug! [!..zip! { - streams: #..combined, + #(let numbers = [1, 2, 3]) + [!debug! [!zip! { + streams: [#letters, #numbers], }]] }, - r#"[!stream! { A 1 } { B 2 } { C 3 }]"#, + r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); preinterpret_assert_eq!( { [!set! #letters = A B C] - [!set! #numbers = 1 2 3] - [!set! #combined = [#letters #numbers]] - [!debug! [!..zip! #..combined]] + #(let numbers = [1, 2, 3]) + #(let combined = [#letters, #numbers]) + [!debug! [!zip! #combined]] }, - r#"[!stream! [A 1] [B 2] [C 3]]"#, + r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); + preinterpret_assert_eq!([!debug![!zip![]]], r#"[]"#,); } #[test] @@ -480,16 +414,16 @@ fn test_zip_with_for() { preinterpret_assert_eq!( { [!set! #countries = France Germany Italy] - [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] - [!set! #capitals = Paris Berlin Rome] - [!set! #facts = [!for! (#country #flag #capital) in [!zip! (#countries #flags #capitals)] { + #(flags = ["🇫🇷", "🇩🇪", "🇮🇹"]) + [!set! #capitals = "Paris" "Berlin" "Rome"] + [!set! #facts = [!for! [country, flag, capital] in [!zip! [countries, flags, capitals]] { [!string! "=> The capital of " #country " is " #capital " and its flag is " #flag] }]] - [!string! "The facts are:\n" [!intersperse! { + #("The facts are:\n" + [!intersperse! { items: #facts, separator: ["\n"], - }] "\n"] + }] as string + "\n") }, r#"The facts are: => The capital of France is Paris and its flag is 🇫🇷 diff --git a/tests/transforming.rs b/tests/transforming.rs index 6332c73d..af30e62d 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -57,8 +57,8 @@ fn test_variable_parsing() { #..>>..x // Matches stream and appends it flattened: do you agree ? ) = Why Hello Everyone ([!group! This is an exciting adventure] do you agree?)] - [!string! [!intersperse! { items: #x, separator: [_] } ]] - }, "Why_HelloEveryone_This_is_an_exciting_adventure_do_you_agree_?"); + [!debug! #x] + }, "[!stream! Why [!group! Hello Everyone] This is an exciting adventure do you agree ?]"); } #[test] @@ -78,8 +78,8 @@ fn test_ident_transformer() { preinterpret_assert_eq!({ [!set! #x =] [!let! The quick @(#x += @IDENT) fox jumps @(#x += @IDENT) the lazy dog = The quick brown fox jumps over the lazy dog] - [!string! [!intersperse! { items: #x, separator: ["_"] }]] - }, "brown_over"); + [!debug! #x] + }, "[!stream! brown over]"); } #[test] @@ -92,7 +92,7 @@ fn test_literal_transformer() { preinterpret_assert_eq!({ [!set! #x] [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] - [!debug! #..x] + [!debug! #x] }, "[!stream! 'c' 0b1010 r#\"123\"#]"); } @@ -100,18 +100,18 @@ fn test_literal_transformer() { fn test_punct_transformer() { preinterpret_assert_eq!({ [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] - [!debug! #..x] + [!debug! #x] }, "[!stream! !]"); // Test for ' which is treated weirdly by syn / rustc preinterpret_assert_eq!({ [!let! The "quick" fox isn 't brown and doesn @(#x = @PUNCT) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] - [!debug! #..x] + #(x as debug) }, "[!stream! ']"); // Lots of punctuation, most of it ignored preinterpret_assert_eq!({ [!set! #x =] [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] - [!debug! #..x] + [!debug! #x] }, "[!stream! % |]"); } @@ -119,18 +119,18 @@ fn test_punct_transformer() { fn test_group_transformer() { preinterpret_assert_eq!({ [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] - [!debug! #..x] + [!debug! #x] }, "[!stream! fox]"); preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said @[GROUP #..y]! = I said #x!] - [!debug! #..y] + [!debug! #y] }, "[!stream! \"hello\" \"world\"]"); // ... which is equivalent to this: preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said #y! = I said #x!] - [!debug! #..y] + [!debug! #y] }, "[!stream! \"hello\" \"world\"]"); } @@ -138,8 +138,8 @@ fn test_group_transformer() { fn test_none_output_commands_mid_parse() { preinterpret_assert_eq!({ [!let! The "quick" @(#x = @LITERAL) fox [!let! #y = #x] @(#x = @IDENT) = The "quick" "brown" fox jumps] - [!string! "#x = " [!debug! #..x] "; #y = "[!debug! #..y]] - }, "#x = [!stream! jumps]; #y = [!stream! \"brown\"]"); + [!string! "#x = " [!debug! x] "; #y = "[!debug! y]] + }, "#x = [!stream! jumps]; #y = \"brown\""); } #[test] @@ -171,13 +171,13 @@ fn test_exact_transformer() { fn test_parse_command_and_exact_transformer() { // The output stream is additive preinterpret_assert_eq!( - { [!debug! [!parse! [Hello World] as @(@IDENT @IDENT)]] }, + { [!debug! [!parse! [!stream! Hello World] with @(@IDENT @IDENT)]] }, "[!stream! Hello World]" ); // Substreams redirected to a variable are not included in the output preinterpret_assert_eq!( { - [!debug! [!parse! [The quick brown fox] as @( + [!debug! [!parse! [!stream! The quick brown fox] with @( @[EXACT The] quick @IDENT @(#x = @IDENT) )]] }, @@ -189,7 +189,7 @@ fn test_parse_command_and_exact_transformer() { // * That EXACT ignores none-delimited groups, to make it more intuitive preinterpret_assert_eq!({ [!set! #x = [!group! fox]] - [!debug! [!parse! [The quick brown fox is a fox - right?!] as @( + [!debug! [!parse! [!stream! The quick brown fox is a fox - right?!] with @( // The outputs are only from the EXACT transformer The quick @(_ = @IDENT) @[EXACT #x @(_ = @IDENT a) #..x - right?!] )]] From 167fbdde38b0f32d40ab81f176022b22e148bbbb Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 13 Feb 2025 22:17:45 +0000 Subject: [PATCH 091/126] refactor: Got rid of flattened commands and added iterator values --- CHANGELOG.md | 37 +-- src/expressions/array.rs | 43 ++- src/expressions/boolean.rs | 22 +- src/expressions/character.rs | 29 +- src/expressions/expression.rs | 6 +- src/expressions/expression_block.rs | 2 +- src/expressions/float.rs | 30 +- src/expressions/integer.rs | 76 ++--- src/expressions/operations.rs | 2 +- src/expressions/stream.rs | 15 +- src/expressions/string.rs | 22 +- src/expressions/value.rs | 275 +++++++++++++++--- src/interpretation/command.rs | 151 ++-------- .../commands/control_flow_commands.rs | 8 +- src/interpretation/commands/core_commands.rs | 8 +- .../commands/expression_commands.rs | 19 +- src/interpretation/commands/token_commands.rs | 13 +- .../commands/transforming_commands.rs | 2 +- src/interpretation/interpreted_stream.rs | 7 + src/interpretation/interpreter.rs | 5 - src/interpretation/variable.rs | 8 +- src/misc/parse_traits.rs | 14 +- .../core/settings_update_iteration_limit.rs | 2 +- .../settings_update_iteration_limit.stderr | 6 +- .../expressions/large_range.stderr | 6 - ...arge_range.rs => large_range_to_string.rs} | 2 +- .../expressions/large_range_to_string.stderr | 5 + .../destructure_with_flattened_command.rs | 5 - .../destructure_with_flattened_command.stderr | 5 - tests/core.rs | 2 +- tests/expressions.rs | 17 +- tests/tokens.rs | 8 +- 32 files changed, 482 insertions(+), 370 deletions(-) delete mode 100644 tests/compilation_failures/expressions/large_range.stderr rename tests/compilation_failures/expressions/{large_range.rs => large_range_to_string.rs} (62%) create mode 100644 tests/compilation_failures/expressions/large_range_to_string.stderr delete mode 100644 tests/compilation_failures/transforming/destructure_with_flattened_command.rs delete mode 100644 tests/compilation_failures/transforming/destructure_with_flattened_command.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index d33f71a1..d4315923 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,11 +30,11 @@ * Token-stream utility commands: * `[!is_empty! #stream]` * `[!length! #stream]` which gives the number of token trees in the token stream. - * `[!group! ...]` which wraps the tokens in a transparent group. Useful with `!for!`. + * `[!group! ...]` which wraps the tokens in a transparent group. Can be useful if using token streams as iteration sources, e.g. in `!for!`. * `[!intersperse! { ... }]` which inserts separator tokens between each token tree in a stream. * `[!split! ...]` which can be used to split a stream with a given separating stream. * `[!comma_split! ...]` which can be used to split a stream on `,` tokens. - * `[!zip! (#countries #flags #capitals)]` which can be used to combine multiple streams together. + * `[!zip! [#countries #flags #capitals]]` which can be used to combine multiple streams together. * Destructuring commands: * `[!let! = ...]` does destructuring/parsing (see next section). Note `[!let! #..x = ...]` is equivalent to `[!set! #x = ...]` @@ -97,25 +97,7 @@ Inside a transform stream, the following grammar is supported: ### To come -* Consider separating an array `[x, y, z]` from a stream `[!stream! ...]` in the value model - * Add `as ident` casting and support it for array and stream using concat recursive. - * Support an iterator value as: - * (Not a real iterator - takes an `interpreter` for its `next(interpreter)` method) - * Is an input for for loops - * Is an output for zip and intersperse - * Supports casting to/from arrays and streams; with an iteration limit - * Get rid of `OutputKindGroupedStream` and the `[!..command]` thing - * Consider dropping lots of the `group` wrappers? - * Then destructuring and parsing become different: - * Destructuring works over the value model; parsing works over the token stream model. - * Parsers can be embedded inside a destructuring - * Add `..` and `..x` support to the array destructurer - * Support a CastTarget of Array (only supported for array and stream and iterator) -* Support `#(x[..])` syntax for indexing arrays and streams at read time - * Via a post-fix `[...]` operation with high priority - * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) - * `#(x[0..3])` returns a TokenStream - * `#(x[0..=3])` returns a TokenStream +* Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Objects: * Backed by an indexmap @@ -152,8 +134,14 @@ Inside a transform stream, the following grammar is supported: * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` - * Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` +* Support `#(x[..])` syntax for indexing arrays and streams at read time + * Via a post-fix `[..]` operation with high precedence + * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) + * `#(x[0..3])` returns a TokenStream/Array + * `#(x[0..=3])` returns a TokenStream/Array +* Add `..` and `..x` support to the array destructurer * Consider: + * Dropping lots of the `group` wrappers? * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` * Adding `!define_command!` @@ -161,7 +149,10 @@ Inside a transform stream, the following grammar is supported: * `[!is_set! #x]` * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) * Maybe add a `@[REINTERPRET ..]` transformer. -* Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` +* CastTarget expansion: + * Support a CastTarget of `array` (only supported for array and stream and iterator) + * Add `as ident` and `as literal` casting and support it for string, array and stream using concat recursive. + * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Put `[!set! ...]` inside an opt-in feature. * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed diff --git a/src/expressions/array.rs b/src/expressions/array.rs index d7c4a362..f6d6efe8 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -23,18 +23,23 @@ impl ExpressionArray { target_ident, .. } => match target { - CastTarget::Stream => operation.output(self.into_stream_with_grouped_items()?), + CastTarget::Stream => operation.output(self.stream_with_grouped_items()?), CastTarget::Group => operation.output( operation - .output(self.into_stream_with_grouped_items()?) - .into_new_output_stream(Grouping::Grouped, false)?, + .output(self.stream_with_grouped_items()?) + .into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?, ), CastTarget::String => operation.output({ let mut output = String::new(); - self.concat_recursive_into(&mut output, &ConcatBehaviour::standard()); + self.concat_recursive_into(&mut output, &ConcatBehaviour::standard())?; output }), - CastTarget::DebugString => operation.output(self.items).into_debug_string_value(), + CastTarget::DebugString => { + operation.output(self.items).into_debug_string_value()? + } CastTarget::Boolean | CastTarget::Char | CastTarget::Integer(_) @@ -56,12 +61,21 @@ impl ExpressionArray { }) } - pub(crate) fn into_stream_with_grouped_items(self) -> ExecutionResult { - let mut stream = OutputStream::new(); - for item in self.items { - item.output_to(Grouping::Grouped, &mut stream, true)?; + fn stream_with_grouped_items(&self) -> ExecutionResult { + let mut output = OutputStream::new(); + self.output_grouped_items_to(&mut output)?; + Ok(output) + } + + pub(crate) fn output_grouped_items_to(&self, output: &mut OutputStream) -> ExecutionResult<()> { + for item in &self.items { + item.output_to( + Grouping::Grouped, + output, + StreamOutputBehaviour::PermitArrays, + )?; } - Ok(stream) + Ok(()) } pub(super) fn handle_integer_binary_operation( @@ -103,7 +117,11 @@ impl ExpressionArray { }) } - pub(crate) fn concat_recursive_into(self, output: &mut String, behaviour: &ConcatBehaviour) { + pub(crate) fn concat_recursive_into( + self, + output: &mut String, + behaviour: &ConcatBehaviour, + ) -> ExecutionResult<()> { if behaviour.output_array_structure { output.push('['); } @@ -115,12 +133,13 @@ impl ExpressionArray { if !is_first && behaviour.add_space_between_token_trees { output.push(' '); } - item.concat_recursive_into(output, behaviour); + item.concat_recursive_into(output, behaviour)?; is_first = false; } if behaviour.output_array_structure { output.push(']'); } + Ok(()) } } diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index e0743e5a..ded7da28 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -47,16 +47,18 @@ impl ExpressionBoolean { CastTarget::Boolean => operation.output(input), CastTarget::String => operation.output(input.to_string()), CastTarget::DebugString => operation.output(input.to_string()), - CastTarget::Stream => operation.output( - operation - .output(input) - .into_new_output_stream(Grouping::Flattened, false)?, - ), - CastTarget::Group => operation.output( - operation - .output(input) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::Stream => { + operation.output(operation.output(input).into_new_output_stream( + Grouping::Flattened, + StreamOutputBehaviour::Standard, + )?) + } + CastTarget::Group => { + operation.output(operation.output(input).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } }, }) } diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 2bb90606..a43a0cb9 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -46,16 +46,18 @@ impl ExpressionChar { CastTarget::Boolean | CastTarget::Float(_) => return operation.unsupported(self), CastTarget::String => operation.output(char.to_string()), CastTarget::DebugString => operation.output(format!("{:?}", char)), - CastTarget::Stream => operation.output( - operation - .output(char) - .into_new_output_stream(Grouping::Flattened, false)?, - ), - CastTarget::Group => operation.output( - operation - .output(char) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::Stream => { + operation.output(operation.output(char).into_new_output_stream( + Grouping::Flattened, + StreamOutputBehaviour::Standard, + )?) + } + CastTarget::Group => { + operation.output(operation.output(char).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } }, }) } @@ -64,15 +66,16 @@ impl ExpressionChar { self, right: Self, range_limits: OutputSpanned, - ) -> Box + '_> { + ) -> Box { let left = self.value; let right = right.value; + let span_range = range_limits.span_range(); match range_limits.operation { syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(move |x| range_limits.output(x))) + Box::new((left..right).map(move |x| x.to_value(span_range))) } syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(move |x| range_limits.output(x))) + Box::new((left..=right).map(move |x| x.to_value(span_range))) } } } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index b1024745..66c0d797 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -115,7 +115,11 @@ impl Expressionable for Source { _ => {} } } - SourcePeekMatch::Punct(_) => if let Ok(operation) = input.try_parse_or_revert::() { return Ok(NodeExtension::BinaryOperation(operation)) }, + SourcePeekMatch::Punct(_) => { + if let Ok(operation) = input.try_parse_or_revert::() { + return Ok(NodeExtension::BinaryOperation(operation)); + } + } SourcePeekMatch::Ident(ident) if ident == "as" => { let cast_operation = UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index 4548eed8..ef2965a7 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -80,7 +80,7 @@ impl Interpret for &ExpressionBlock { None => Grouping::Grouped, }; self.evaluate(interpreter)? - .output_to(grouping, output, false)?; + .output_to(grouping, output, StreamOutputBehaviour::Standard)?; Ok(()) } } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 4ac0f1af..09281a03 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -173,20 +173,22 @@ impl UntypedFloat { CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input), CastTarget::String => operation.output(input.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value(), + CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } - CastTarget::Stream => operation.output( - operation - .output(self) - .into_new_output_stream(Grouping::Flattened, false)?, - ), - CastTarget::Group => operation.output( - operation - .output(self) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::Stream => { + operation.output(operation.output(self).into_new_output_stream( + Grouping::Flattened, + StreamOutputBehaviour::Standard, + )?) + } + CastTarget::Group => { + operation.output(operation.output(self).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } }, }) } @@ -330,10 +332,10 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value(), + CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, false)?), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, false)?), + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, StreamOutputBehaviour::Standard)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, StreamOutputBehaviour::Standard)?), } }) } diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index ec7eeb10..14e8a078 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -152,7 +152,7 @@ impl ExpressionIntegerValuePair { pub(crate) fn create_range( self, range_limits: OutputSpanned, - ) -> ExecutionResult + '_>> { + ) -> ExecutionResult> { Ok(match self { Self::Untyped(lhs, rhs) => return lhs.create_range(rhs, range_limits), Self::U8(lhs, rhs) => lhs.create_range(rhs, range_limits), @@ -323,17 +323,19 @@ impl UntypedInteger { return operation.execution_err("This cast is not supported") } CastTarget::String => operation.output(input.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value(), - CastTarget::Stream => operation.output( - operation - .output(self) - .into_new_output_stream(Grouping::Flattened, false)?, - ), - CastTarget::Group => operation.output( - operation - .output(self) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::DebugString => operation.output(self).into_debug_string_value()?, + CastTarget::Stream => { + operation.output(operation.output(self).into_new_output_stream( + Grouping::Flattened, + StreamOutputBehaviour::Standard, + )?) + } + CastTarget::Group => { + operation.output(operation.output(self).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } }, }) } @@ -451,15 +453,16 @@ impl UntypedInteger { self, right: Self, range_limits: OutputSpanned, - ) -> ExecutionResult + '_>> { + ) -> ExecutionResult> { let left = self.parse_fallback()?; let right = right.parse_fallback()?; + let span_range = range_limits.span_range(); Ok(match range_limits.operation { syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(move |x| range_limits.output(Self::from_fallback(x)))) + Box::new((left..right).map(move |x| Self::from_fallback(x).to_value(span_range))) } syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(move |x| range_limits.output(Self::from_fallback(x)))) + Box::new((left..=right).map(move |x| Self::from_fallback(x).to_value(span_range))) } }) } @@ -600,14 +603,15 @@ macro_rules! impl_int_operations_except_unary { self, right: Self, range_limits: OutputSpanned, - ) -> Box + '_> { + ) -> Box { let left = self; + let span_range = range_limits.span_range(); match range_limits.operation { syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(move |x| range_limits.output(x))) + Box::new((left..right).map(move |x| x.to_value(span_range))) }, syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(move |x| range_limits.output(x))) + Box::new((left..=right).map(move |x| x.to_value(span_range))) } } } @@ -643,9 +647,9 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value(), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, false)?), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, false)?), + CastTarget::DebugString => operation.output(self).into_debug_string_value()?, + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, StreamOutputBehaviour::Standard)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, StreamOutputBehaviour::Standard)?), } }) } @@ -681,9 +685,9 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value(), - CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, false)?), - CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, false)?), + CastTarget::DebugString => operation.output(self).into_debug_string_value()?, + CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, StreamOutputBehaviour::Standard)?), + CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, StreamOutputBehaviour::Standard)?), } }) } @@ -726,17 +730,19 @@ impl HandleUnaryOperation for u8 { return operation.execution_err("This cast is not supported") } CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value(), - CastTarget::Stream => operation.output( - operation - .output(self) - .into_new_output_stream(Grouping::Flattened, false)?, - ), - CastTarget::Group => operation.output( - operation - .output(self) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::DebugString => operation.output(self).into_debug_string_value()?, + CastTarget::Stream => { + operation.output(operation.output(self).into_new_output_stream( + Grouping::Flattened, + StreamOutputBehaviour::Standard, + )?) + } + CastTarget::Group => { + operation.output(operation.output(self).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } }, }) } diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index f96b564e..80b75c97 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -490,7 +490,7 @@ pub(super) trait HandleCreateRange: Sized { self, right: Self, range_limits: OutputSpanned, - ) -> Box + '_>; + ) -> Box; } impl Operation for syn::RangeLimits { diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs index 2f81ab30..40bbeeb2 100644 --- a/src/expressions/stream.rs +++ b/src/expressions/stream.rs @@ -24,13 +24,16 @@ impl ExpressionStream { self.concat_recursive_into(&mut output, &ConcatBehaviour::standard()); output }), - CastTarget::DebugString => operation.output(self.value).into_debug_string_value(), + CastTarget::DebugString => { + operation.output(self.value).into_debug_string_value()? + } CastTarget::Stream => operation.output(self.value), - CastTarget::Group => operation.output( - operation - .output(self.value) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::Group => { + operation.output(operation.output(self.value).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } CastTarget::Boolean | CastTarget::Char | CastTarget::Integer(_) diff --git a/src/expressions/string.rs b/src/expressions/string.rs index b6a6f644..3ac122c4 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -26,16 +26,18 @@ impl ExpressionString { return operation.unsupported(self) } UnaryOperation::Cast { target, .. } => match target { - CastTarget::Stream => operation.output( - operation - .output(self.value) - .into_new_output_stream(Grouping::Flattened, false)?, - ), - CastTarget::Group => operation.output( - operation - .output(self.value) - .into_new_output_stream(Grouping::Grouped, false)?, - ), + CastTarget::Stream => { + operation.output(operation.output(self.value).into_new_output_stream( + Grouping::Flattened, + StreamOutputBehaviour::Standard, + )?) + } + CastTarget::Group => { + operation.output(operation.output(self.value).into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?) + } CastTarget::String => operation.output(self.value), CastTarget::DebugString => operation.output(format!("{:?}", self.value)), CastTarget::Boolean diff --git a/src/expressions/value.rs b/src/expressions/value.rs index b8370843..ac8bd1b2 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -13,6 +13,7 @@ pub(crate) enum ExpressionValue { UnsupportedLiteral(UnsupportedLiteral), Array(ExpressionArray), Stream(ExpressionStream), + Iterator(ExpressionIterator), } pub(crate) trait ToExpressionValue: Sized { @@ -314,8 +315,9 @@ impl ExpressionValue { match self { ExpressionValue::Array(value) => Ok(ExpressionIterator::new_for_array(value)), ExpressionValue::Stream(value) => Ok(ExpressionIterator::new_for_stream(value)), + ExpressionValue::Iterator(value) => Ok(value), other => other.execution_err(format!( - "{} must be iterable (an array or stream), but it is a {}", + "{} must be iterable (an array or stream or iterator), but it is a {}", place_descriptor, other.value_type(), )), @@ -336,6 +338,7 @@ impl ExpressionValue { ExpressionValue::Stream(value) => value.handle_unary_operation(operation), ExpressionValue::Array(value) => value.handle_unary_operation(operation), ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), + ExpressionValue::Iterator(value) => value.handle_unary_operation(operation), } } @@ -366,6 +369,7 @@ impl ExpressionValue { ExpressionValue::Stream(value) => { value.handle_integer_binary_operation(right, operation) } + ExpressionValue::Iterator(value) => operation.unsupported(value), } } @@ -373,7 +377,7 @@ impl ExpressionValue { self, other: Self, range_limits: &syn::RangeLimits, - ) -> ExecutionResult + '_>> { + ) -> ExecutionResult> { let span_range = SpanRange::new_between(self.span_range().start(), self.span_range().end()); self.expect_value_pair(range_limits, other)? .create_range(range_limits.with_output_span_range(span_range)) @@ -390,6 +394,7 @@ impl ExpressionValue { Self::UnsupportedLiteral(value) => &mut value.span_range, Self::Array(value) => &mut value.span_range, Self::Stream(value) => &mut value.span_range, + Self::Iterator(value) => &mut value.span_range, } } @@ -406,13 +411,13 @@ impl ExpressionValue { pub(crate) fn into_new_output_stream( self, grouping: Grouping, - output_arrays: bool, + behaviour: StreamOutputBehaviour, ) -> ExecutionResult { Ok(match (self, grouping) { (Self::Stream(value), Grouping::Flattened) => value.value, (other, grouping) => { let mut output = OutputStream::new(); - other.output_to(grouping, &mut output, output_arrays)?; + other.output_to(grouping, &mut output, behaviour)?; output } }) @@ -422,7 +427,7 @@ impl ExpressionValue { &self, grouping: Grouping, output: &mut OutputStream, - output_arrays: bool, + behaviour: StreamOutputBehaviour, ) -> ExecutionResult<()> { match grouping { Grouping::Grouped => { @@ -431,13 +436,13 @@ impl ExpressionValue { // * Grouping means -1 is interpreted atomically, rather than as a punct then a number // * Grouping means that a stream is interpreted atomically output.push_grouped( - |inner| self.output_flattened_to(inner, output_arrays), + |inner| self.output_flattened_to(inner, behaviour), Delimiter::None, self.span_range().join_into_span_else_start(), )?; } Grouping::Flattened => { - self.output_flattened_to(output, output_arrays)?; + self.output_flattened_to(output, behaviour)?; } } Ok(()) @@ -446,7 +451,7 @@ impl ExpressionValue { fn output_flattened_to( &self, output: &mut OutputStream, - output_arrays: bool, + behaviour: StreamOutputBehaviour, ) -> ExecutionResult<()> { match self { Self::None { .. } => {} @@ -459,32 +464,43 @@ impl ExpressionValue { output.extend_raw_tokens(literal.lit.to_token_stream()) } Self::Array(array) => { - if output_arrays { - for item in &array.items { - item.output_to(Grouping::Grouped, output, output_arrays)?; - } + if behaviour.should_output_arrays() { + array.output_grouped_items_to(output)? } else { return self.execution_err("Arrays cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `#(XXX as stream)` to cast the array to a stream."); } } Self::Stream(value) => value.value.append_cloned_into(output), + Self::Iterator(iterator) => { + if behaviour.should_output_iterators() { + iterator.clone().output_grouped_items_to(output)? + } else { + return self.execution_err("Iterators cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `#(XXX as stream)` to cast the iterator to a stream."); + } + } }; Ok(()) } - pub(crate) fn into_debug_string_value(self) -> ExpressionValue { + pub(crate) fn into_debug_string_value(self) -> ExecutionResult { let span_range = self.span_range(); - self.concat_recursive(&ConcatBehaviour::debug()) - .to_value(span_range) + let value = self + .concat_recursive(&ConcatBehaviour::debug())? + .to_value(span_range); + Ok(value) } - pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> String { + pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> ExecutionResult { let mut output = String::new(); - self.concat_recursive_into(&mut output, behaviour); - output + self.concat_recursive_into(&mut output, behaviour)?; + Ok(output) } - pub(crate) fn concat_recursive_into(self, output: &mut String, behaviour: &ConcatBehaviour) { + pub(crate) fn concat_recursive_into( + self, + output: &mut String, + behaviour: &ConcatBehaviour, + ) -> ExecutionResult<()> { match self { ExpressionValue::None { .. } => { if behaviour.show_none_values { @@ -495,16 +511,47 @@ impl ExpressionValue { stream.concat_recursive_into(output, behaviour); } ExpressionValue::Array(array) => { - array.concat_recursive_into(output, behaviour); + array.concat_recursive_into(output, behaviour)?; } - _ => { + ExpressionValue::Iterator(iterator) => { + iterator.concat_recursive_into(output, behaviour)?; + } + ExpressionValue::Integer(_) + | ExpressionValue::Float(_) + | ExpressionValue::Char(_) + | ExpressionValue::Boolean(_) + | ExpressionValue::UnsupportedLiteral(_) + | ExpressionValue::String(_) => { // This isn't the most efficient, but it's less code and debug doesn't need to be super efficient. let mut stream = OutputStream::new(); - self.output_flattened_to(&mut stream, false) + self.output_flattened_to(&mut stream, StreamOutputBehaviour::Standard) .expect("Non-composite values should all be able to be outputted to a stream"); stream.concat_recursive_into(output, behaviour); } } + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub(crate) enum StreamOutputBehaviour { + Standard, + PermitArrays, +} + +impl StreamOutputBehaviour { + pub(super) fn should_output_arrays(&self) -> bool { + match self { + Self::Standard => false, + Self::PermitArrays => true, + } + } + + pub(super) fn should_output_iterators(&self) -> bool { + match self { + Self::Standard => false, + Self::PermitArrays => true, + } } } @@ -525,6 +572,7 @@ impl HasValueType for ExpressionValue { Self::UnsupportedLiteral(value) => value.value_type(), Self::Array(value) => value.value_type(), Self::Stream(value) => value.value_type(), + Self::Iterator(value) => value.value_type(), } } } @@ -532,15 +580,16 @@ impl HasValueType for ExpressionValue { impl HasSpanRange for ExpressionValue { fn span_range(&self) -> SpanRange { match self { - Self::None(span_range) => *span_range, - Self::Integer(int) => int.span_range, - Self::Float(float) => float.span_range, - Self::Boolean(bool) => bool.span_range, - Self::String(str) => str.span_range, - Self::Char(char) => char.span_range, - Self::UnsupportedLiteral(lit) => lit.span_range, - Self::Array(array) => array.span_range, - Self::Stream(stream) => stream.span_range, + ExpressionValue::None(span_range) => *span_range, + ExpressionValue::Integer(int) => int.span_range, + ExpressionValue::Float(float) => float.span_range, + ExpressionValue::Boolean(bool) => bool.span_range, + ExpressionValue::String(str) => str.span_range, + ExpressionValue::Char(char) => char.span_range, + ExpressionValue::UnsupportedLiteral(lit) => lit.span_range, + ExpressionValue::Array(array) => array.span_range, + ExpressionValue::Stream(stream) => stream.span_range, + ExpressionValue::Iterator(iterator) => iterator.span_range, } } } @@ -590,7 +639,7 @@ impl EvaluationLiteralPair { pub(super) fn create_range( self, range_limits: OutputSpanned, - ) -> ExecutionResult + '_>> { + ) -> ExecutionResult> { Ok(match self { EvaluationLiteralPair::Integer(pair) => return pair.create_range(range_limits), EvaluationLiteralPair::CharPair(left, right) => left.create_range(right, range_limits), @@ -602,6 +651,7 @@ impl EvaluationLiteralPair { } } +#[derive(Clone)] pub(crate) struct ExpressionIterator { iterator: ExpressionIteratorInner, #[allow(unused)] @@ -623,9 +673,8 @@ impl ExpressionIterator { } } - #[allow(unused)] pub(crate) fn new_custom( - iterator: Box>, + iterator: Box, span_range: SpanRange, ) -> Self { Self { @@ -633,12 +682,168 @@ impl ExpressionIterator { span_range, } } + + pub(super) fn handle_unary_operation( + self, + operation: OutputSpanned, + ) -> ExecutionResult { + Ok(match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { + return operation.unsupported(self) + } + UnaryOperation::Cast { + target, + target_ident, + .. + } => match target { + CastTarget::Stream => operation.output(self.into_stream_with_grouped_items()?), + CastTarget::Group => operation.output( + operation + .output(self.into_stream_with_grouped_items()?) + .into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?, + ), + CastTarget::String => operation.output({ + let mut output = String::new(); + self.concat_recursive_into(&mut output, &ConcatBehaviour::standard())?; + output + }), + CastTarget::DebugString => { + operation.output(self.iterator).into_debug_string_value()? + } + CastTarget::Boolean + | CastTarget::Char + | CastTarget::Integer(_) + | CastTarget::Float(_) => match self.singleton_value() { + Some(value) => value.handle_unary_operation(operation)?, + None => { + return operation.execution_err(format!( + "Only an iterator with one item can be cast to {}", + target_ident, + )) + } + }, + }, + }) + } + + pub(crate) fn singleton_value(mut self) -> Option { + let first = self.next()?; + if self.next().is_none() { + Some(first) + } else { + None + } + } + + fn into_stream_with_grouped_items(self) -> ExecutionResult { + let mut output = OutputStream::new(); + self.output_grouped_items_to(&mut output)?; + Ok(output) + } + + fn output_grouped_items_to(self, output: &mut OutputStream) -> ExecutionResult<()> { + const LIMIT: usize = 10_000; + let span_range = self.span_range; + for (i, item) in self.enumerate() { + if i > LIMIT { + return span_range.execution_err(format!("Only a maximum of {} items can be output to a stream from an iterator, to protect you from infinite loops. This can't currently be reconfigured with the iteration limit.", LIMIT)); + } + item.output_to( + Grouping::Grouped, + output, + StreamOutputBehaviour::PermitArrays, + )?; + } + Ok(()) + } + + pub(crate) fn concat_recursive_into( + self, + output: &mut String, + behaviour: &ConcatBehaviour, + ) -> ExecutionResult<()> { + if behaviour.output_array_structure { + output.push_str("[ "); + } + let max = self.size_hint().1; + let span_range = self.span_range; + for (i, item) in self.enumerate() { + if i >= behaviour.iterator_limit { + if behaviour.error_after_iterator_limit { + return span_range.execution_err(format!("To protect against infinite loops, only a maximum of {} items can be output to a string from an iterator. Try casting `as stream` to avoid this limit. This can't currently be reconfigured with the iteration limit.", behaviour.iterator_limit)); + } else { + if behaviour.output_array_structure { + match max { + Some(max) => output.push_str(&format!( + ", ..<{} further items>", + max.saturating_sub(i) + )), + None => output.push_str(", .."), + } + } + break; + } + } + if i != 0 && behaviour.output_array_structure { + output.push(','); + } + if i != 0 && behaviour.add_space_between_token_trees { + output.push(' '); + } + item.concat_recursive_into(output, behaviour)?; + } + if behaviour.output_array_structure { + output.push(']'); + } + Ok(()) + } +} + +impl ToExpressionValue for ExpressionIteratorInner { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Iterator(ExpressionIterator { + iterator: self, + span_range, + }) + } +} + +impl ToExpressionValue for Box { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Iterator(ExpressionIterator::new_custom(self, span_range)) + } +} + +impl HasValueType for ExpressionIterator { + fn value_type(&self) -> &'static str { + "iterator" + } } +#[derive(Clone)] enum ExpressionIteratorInner { Array( as IntoIterator>::IntoIter), Stream(::IntoIter), - Other(Box>), + Other(Box), +} + +impl + Clone + 'static> CustomExpressionIterator for T { + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +pub(crate) trait CustomExpressionIterator: Iterator { + fn clone_box(&self) -> Box; +} + +impl Clone for Box { + fn clone(&self) -> Self { + (**self).clone_box() + } } impl Iterator for ExpressionIterator { diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 90d7948b..a8a5ea36 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -5,12 +5,12 @@ use crate::internal_prelude::*; #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum CommandOutputKind { None, - FlattenedValue, - GroupedValue, + /// If output to a parent stream, it is flattened + Value, Ident, + /// If output to a parent stream, it is flattened Literal, - FlattenedStream, - GroupedStream, + /// If output to a parent stream, it is flattened Stream, } @@ -20,13 +20,11 @@ pub(crate) trait CommandType { pub(crate) trait OutputKind { type Output; - fn resolve_standard() -> CommandOutputKind; - fn resolve_flattened(error_span_range: SpanRange) -> ParseResult; + fn resolve_enum_kind() -> CommandOutputKind; } struct ExecutionContext<'a> { interpreter: &'a mut Interpreter, - output_kind: CommandOutputKind, delim_span: DelimSpan, } @@ -74,13 +72,9 @@ pub(crate) struct OutputKindNone; impl OutputKind for OutputKindNone { type Output = (); - fn resolve_standard() -> CommandOutputKind { + fn resolve_enum_kind() -> CommandOutputKind { CommandOutputKind::None } - - fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { - error_span_range.parse_err("This command has no output, so cannot be flattened with ..") - } } pub(crate) trait NoOutputCommandDefinition: @@ -111,13 +105,8 @@ pub(crate) struct OutputKindValue; impl OutputKind for OutputKindValue { type Output = TokenTree; - fn resolve_standard() -> CommandOutputKind { - CommandOutputKind::FlattenedValue - } - - fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { - error_span_range - .parse_err("This command outputs a single value, so cannot be flattened with ..") + fn resolve_enum_kind() -> CommandOutputKind { + CommandOutputKind::Value } } @@ -135,12 +124,11 @@ impl CommandInvocationAs for C { context: ExecutionContext, output: &mut OutputStream, ) -> ExecutionResult<()> { - let grouping = match context.output_kind { - CommandOutputKind::FlattenedValue => Grouping::Flattened, - _ => Grouping::Grouped, - }; - self.execute(context.interpreter)? - .output_to(grouping, output, false) + self.execute(context.interpreter)?.output_to( + Grouping::Grouped, + output, + StreamOutputBehaviour::Standard, + ) } fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { @@ -156,14 +144,9 @@ pub(crate) struct OutputKindIdent; impl OutputKind for OutputKindIdent { type Output = Ident; - fn resolve_standard() -> CommandOutputKind { + fn resolve_enum_kind() -> CommandOutputKind { CommandOutputKind::Ident } - - fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { - error_span_range - .parse_err("This command outputs a single ident, so cannot be flattened with ..") - } } pub(crate) trait IdentCommandDefinition: @@ -200,14 +183,9 @@ pub(crate) struct OutputKindLiteral; impl OutputKind for OutputKindLiteral { type Output = Literal; - fn resolve_standard() -> CommandOutputKind { + fn resolve_enum_kind() -> CommandOutputKind { CommandOutputKind::Literal } - - fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { - error_span_range - .parse_err("This command outputs a single literal, so cannot be flattened with ..") - } } pub(crate) trait LiteralCommandDefinition: @@ -238,79 +216,17 @@ impl CommandInvocationAs for C { // OutputKindStream //================= -pub(crate) struct OutputKindGroupedStream; -impl OutputKind for OutputKindGroupedStream { - type Output = OutputStream; - - fn resolve_standard() -> CommandOutputKind { - CommandOutputKind::GroupedStream - } - - fn resolve_flattened(_: SpanRange) -> ParseResult { - Ok(CommandOutputKind::FlattenedStream) - } -} - -pub(crate) trait GroupedStreamCommandDefinition: - Sized + CommandType -{ - const COMMAND_NAME: &'static str; - fn parse(arguments: CommandArguments) -> ParseResult; - fn execute( - self, - interpreter: &mut Interpreter, - output: &mut OutputStream, - ) -> ExecutionResult<()>; -} - -impl CommandInvocationAs for C { - fn execute_into( - self, - context: ExecutionContext, - output: &mut OutputStream, - ) -> ExecutionResult<()> { - match context.output_kind { - CommandOutputKind::FlattenedStream => self.execute(context.interpreter, output), - CommandOutputKind::GroupedStream => output.push_grouped( - |inner| self.execute(context.interpreter, inner), - Delimiter::None, - context.delim_span.join(), - ), - _ => unreachable!(), - } - } - - fn execute_to_value(self, context: ExecutionContext) -> ExecutionResult { - let span_range = context.delim_span.span_range(); - let mut output = OutputStream::new(); - >::execute_into( - self, - context, - &mut output, - )?; - Ok(output.to_value(span_range)) - } -} - -//====================== -// OutputKindControlFlow -//====================== - pub(crate) struct OutputKindStream; impl OutputKind for OutputKindStream { type Output = (); - fn resolve_standard() -> CommandOutputKind { + fn resolve_enum_kind() -> CommandOutputKind { CommandOutputKind::Stream } - - fn resolve_flattened(error_span_range: SpanRange) -> ParseResult { - error_span_range.parse_err("This command always outputs a flattened stream and so cannot be explicitly flattened. If it needs to be grouped, wrap it in a [!group! ..] command") - } } // Control Flow or a command which is unlikely to want grouped output -pub(crate) trait StreamingCommandDefinition: +pub(crate) trait StreamCommandDefinition: Sized + CommandType { const COMMAND_NAME: &'static str; @@ -322,7 +238,7 @@ pub(crate) trait StreamingCommandDefinition: ) -> ExecutionResult<()>; } -impl CommandInvocationAs for C { +impl CommandInvocationAs for C { fn execute_into( self, context: ExecutionContext, @@ -366,18 +282,10 @@ macro_rules! define_command_enums { }) } - pub(crate) fn standard_output_kind(&self) -> CommandOutputKind { - match self { - $( - Self::$command => <$command as CommandType>::OutputKind::resolve_standard(), - )* - } - } - - pub(crate) fn flattened_output_kind(&self, error_span_range: SpanRange) -> ParseResult { + pub(crate) fn resolve_output_kind(&self) -> CommandOutputKind { match self { $( - Self::$command => <$command as CommandType>::OutputKind::resolve_flattened(error_span_range), + Self::$command => <$command as CommandType>::OutputKind::resolve_enum_kind(), )* } } @@ -493,7 +401,6 @@ define_command_enums! { #[derive(Clone)] pub(crate) struct Command { typed: Box, - output_kind: CommandOutputKind, source_group_span: DelimSpan, } @@ -501,20 +408,9 @@ impl Parse for Command { fn parse(input: ParseStream) -> ParseResult { let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; content.parse::()?; - let flattening = if content.peek(Token![.]) { - Some(content.parse::()?) - } else { - None - }; let command_name = content.parse_any_ident()?; - let (command_kind, output_kind) = match CommandKind::for_ident(&command_name) { - Some(command_kind) => { - let output_kind = match flattening { - Some(flattening) => command_kind.flattened_output_kind(flattening.span_range())?, - None => command_kind.standard_output_kind(), - }; - (command_kind, output_kind) - } + let command_kind = match CommandKind::for_ident(&command_name) { + Some(command_kind) => command_kind, None => command_name.span().err( format!( "Expected `[!! ..]`, for one of: {}.\nIf this wasn't intended to be a preinterpret command, you can work around this with [!raw! [!{} ... ]]", @@ -531,7 +427,6 @@ impl Parse for Command { ))?; Ok(Self { typed: Box::new(typed), - output_kind, source_group_span: delim_span, }) } @@ -551,7 +446,6 @@ impl Interpret for Command { ) -> ExecutionResult<()> { let context = ExecutionContext { interpreter, - output_kind: self.output_kind, delim_span: self.source_group_span, }; self.typed.execute_into(context, output) @@ -567,7 +461,6 @@ impl InterpretToValue for Command { ) -> ExecutionResult { let context = ExecutionContext { interpreter, - output_kind: self.output_kind, delim_span: self.source_group_span, }; self.typed.execute_to_value(context) diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 45137350..4ea48163 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -12,7 +12,7 @@ impl CommandType for IfCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for IfCommand { +impl StreamCommandDefinition for IfCommand { const COMMAND_NAME: &'static str = "if"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -88,7 +88,7 @@ impl CommandType for WhileCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for WhileCommand { +impl StreamCommandDefinition for WhileCommand { const COMMAND_NAME: &'static str = "while"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -145,7 +145,7 @@ impl CommandType for LoopCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for LoopCommand { +impl StreamCommandDefinition for LoopCommand { const COMMAND_NAME: &'static str = "loop"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -195,7 +195,7 @@ impl CommandType for ForCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for ForCommand { +impl StreamCommandDefinition for ForCommand { const COMMAND_NAME: &'static str = "for"; fn parse(arguments: CommandArguments) -> ParseResult { diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index a285c2dd..5b819088 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -122,7 +122,7 @@ impl CommandType for RawCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for RawCommand { +impl StreamCommandDefinition for RawCommand { const COMMAND_NAME: &'static str = "raw"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -150,7 +150,7 @@ impl CommandType for StreamCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for StreamCommand { +impl StreamCommandDefinition for StreamCommand { const COMMAND_NAME: &'static str = "stream"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -199,7 +199,7 @@ impl CommandType for ReinterpretCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for ReinterpretCommand { +impl StreamCommandDefinition for ReinterpretCommand { const COMMAND_NAME: &'static str = "reinterpret"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -410,7 +410,7 @@ impl ValueCommandDefinition for DebugCommand { .inner .interpret_to_value(interpreter)? .with_span(self.span) - .into_debug_string_value(); + .into_debug_string_value()?; Ok(value) } } diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs index f290786b..3e2bca2d 100644 --- a/src/interpretation/commands/expression_commands.rs +++ b/src/interpretation/commands/expression_commands.rs @@ -33,24 +33,7 @@ impl ValueCommandDefinition for RangeCommand { let range_limits = self.range_limits; let left = self.left.interpret_to_value(interpreter)?; let right = self.right.interpret_to_value(interpreter)?; - let range_iterator = left.create_range(right, &range_limits)?; - - let (_, length) = range_iterator.size_hint(); - match length { - Some(length) => { - interpreter - .start_iteration_counter(&range_limits) - .add_and_check(length)?; - } - None => { - return range_limits - .execution_err("The range must be between two integers or two characters"); - } - } - - Ok(range_iterator - .collect::>() - .to_value(self.span.span_range())) + Ok(range_iterator.to_value(self.span.span_range())) } } diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 589fddc7..1bc0ca92 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -56,10 +56,10 @@ pub(crate) struct GroupCommand { } impl CommandType for GroupCommand { - type OutputKind = OutputKindGroupedStream; + type OutputKind = OutputKindStream; } -impl GroupedStreamCommandDefinition for GroupCommand { +impl StreamCommandDefinition for GroupCommand { const COMMAND_NAME: &'static str = "group"; fn parse(arguments: CommandArguments) -> ParseResult { @@ -73,9 +73,12 @@ impl GroupedStreamCommandDefinition for GroupCommand { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - // The grouping happens automatically because a non-flattened - // stream command is outputted in a group. - self.arguments.interpret_into(interpreter, output) + let span = self.arguments.span(); + output.push_grouped( + |inner| self.arguments.interpret_into(interpreter, inner), + Delimiter::None, + span, + ) } } diff --git a/src/interpretation/commands/transforming_commands.rs b/src/interpretation/commands/transforming_commands.rs index 3b256788..77a09ed3 100644 --- a/src/interpretation/commands/transforming_commands.rs +++ b/src/interpretation/commands/transforming_commands.rs @@ -12,7 +12,7 @@ impl CommandType for ParseCommand { type OutputKind = OutputKindStream; } -impl StreamingCommandDefinition for ParseCommand { +impl StreamCommandDefinition for ParseCommand { const COMMAND_NAME: &'static str = "parse"; fn parse(arguments: CommandArguments) -> ParseResult { diff --git a/src/interpretation/interpreted_stream.rs b/src/interpretation/interpreted_stream.rs index 2a1c8710..77ac3239 100644 --- a/src/interpretation/interpreted_stream.rs +++ b/src/interpretation/interpreted_stream.rs @@ -23,6 +23,7 @@ enum OutputSegment { OutputGroup(Delimiter, Span, OutputStream), } +#[derive(Clone)] pub(crate) enum OutputTokenTree { TokenTree(TokenTree), OutputGroup(Delimiter, Span, OutputStream), @@ -361,6 +362,8 @@ pub(crate) struct ConcatBehaviour { pub(crate) output_array_structure: bool, pub(crate) unwrap_contents_of_string_like_literals: bool, pub(crate) show_none_values: bool, + pub(crate) iterator_limit: usize, + pub(crate) error_after_iterator_limit: bool, } impl ConcatBehaviour { @@ -371,6 +374,8 @@ impl ConcatBehaviour { output_array_structure: false, unwrap_contents_of_string_like_literals: true, show_none_values: false, + iterator_limit: 1000, + error_after_iterator_limit: true, } } @@ -381,6 +386,8 @@ impl ConcatBehaviour { output_array_structure: true, unwrap_contents_of_string_like_literals: false, show_none_values: true, + iterator_limit: 20, + error_after_iterator_limit: false, } } diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 481bc9e7..95593dff 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -129,11 +129,6 @@ pub(crate) struct IterationCounter<'a, S: HasSpanRange> { } impl IterationCounter<'_, S> { - pub(crate) fn add_and_check(&mut self, count: usize) -> ExecutionResult<()> { - self.count = self.count.wrapping_add(count); - self.check() - } - pub(crate) fn increment_and_check(&mut self) -> ExecutionResult<()> { self.count += 1; self.check() diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 552a3db6..4e2547d1 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -49,9 +49,11 @@ pub(crate) trait IsVariable: HasSpanRange { grouping: Grouping, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.read_existing(interpreter)? - .get(self)? - .output_to(grouping, output, false) + self.read_existing(interpreter)?.get(self)?.output_to( + grouping, + output, + StreamOutputBehaviour::Standard, + ) } fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index adbcb277..1d1c8147 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -37,22 +37,10 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> SourcePeekMatch { if let Some((ident, next)) = next.ident() { if next.punct_matching('!').is_some() { let output_kind = - CommandKind::for_ident(&ident).map(|kind| kind.standard_output_kind()); + CommandKind::for_ident(&ident).map(|kind| kind.resolve_output_kind()); return SourcePeekMatch::Command(output_kind); } } - if let Some((first, next)) = next.punct_matching('.') { - if let Some((_, next)) = next.punct_matching('.') { - if let Some((ident, next)) = next.ident() { - if next.punct_matching('!').is_some() { - let output_kind = CommandKind::for_ident(&ident).and_then(|kind| { - kind.flattened_output_kind(first.span_range()).ok() - }); - return SourcePeekMatch::Command(output_kind); - } - } - } - } } } diff --git a/tests/compilation_failures/core/settings_update_iteration_limit.rs b/tests/compilation_failures/core/settings_update_iteration_limit.rs index eceacad6..406635e9 100644 --- a/tests/compilation_failures/core/settings_update_iteration_limit.rs +++ b/tests/compilation_failures/core/settings_update_iteration_limit.rs @@ -3,6 +3,6 @@ use preinterpret::*; fn main() { preinterpret!{ [!settings! { iteration_limit: 5 }] - [!range! 0..100] + [!loop! {}] }; } \ No newline at end of file diff --git a/tests/compilation_failures/core/settings_update_iteration_limit.stderr b/tests/compilation_failures/core/settings_update_iteration_limit.stderr index 2b365c76..49f34ac4 100644 --- a/tests/compilation_failures/core/settings_update_iteration_limit.stderr +++ b/tests/compilation_failures/core/settings_update_iteration_limit.stderr @@ -1,6 +1,6 @@ error: Iteration limit of 5 exceeded. If needed, the limit can be reconfigured with [!settings! { iteration_limit: X }] - --> tests/compilation_failures/core/settings_update_iteration_limit.rs:6:19 + --> tests/compilation_failures/core/settings_update_iteration_limit.rs:6:17 | -6 | [!range! 0..100] - | ^^ +6 | [!loop! {}] + | ^^ diff --git a/tests/compilation_failures/expressions/large_range.stderr b/tests/compilation_failures/expressions/large_range.stderr deleted file mode 100644 index 8557a88c..00000000 --- a/tests/compilation_failures/expressions/large_range.stderr +++ /dev/null @@ -1,6 +0,0 @@ -error: Iteration limit of 1000 exceeded. - If needed, the limit can be reconfigured with [!settings! { iteration_limit: X }] - --> tests/compilation_failures/expressions/large_range.rs:5:19 - | -5 | [!range! 0..10000000] - | ^^ diff --git a/tests/compilation_failures/expressions/large_range.rs b/tests/compilation_failures/expressions/large_range_to_string.rs similarity index 62% rename from tests/compilation_failures/expressions/large_range.rs rename to tests/compilation_failures/expressions/large_range_to_string.rs index b366f173..029a7365 100644 --- a/tests/compilation_failures/expressions/large_range.rs +++ b/tests/compilation_failures/expressions/large_range_to_string.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - [!range! 0..10000000] + #([!range! 0..100000] as string) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/large_range_to_string.stderr b/tests/compilation_failures/expressions/large_range_to_string.stderr new file mode 100644 index 00000000..db4f0f76 --- /dev/null +++ b/tests/compilation_failures/expressions/large_range_to_string.stderr @@ -0,0 +1,5 @@ +error: To protect against infinite loops, only a maximum of 1000 items can be output to a string from an iterator. Try casting `as stream` to avoid this limit. This can't currently be reconfigured with the iteration limit. + --> tests/compilation_failures/expressions/large_range_to_string.rs:5:11 + | +5 | #([!range! 0..100000] as string) + | ^^^^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/transforming/destructure_with_flattened_command.rs b/tests/compilation_failures/transforming/destructure_with_flattened_command.rs deleted file mode 100644 index 72d60b09..00000000 --- a/tests/compilation_failures/transforming/destructure_with_flattened_command.rs +++ /dev/null @@ -1,5 +0,0 @@ -use preinterpret::*; - -fn main() { - preinterpret!([!let! [!..group! Output] = [!group! Output]]); -} diff --git a/tests/compilation_failures/transforming/destructure_with_flattened_command.stderr b/tests/compilation_failures/transforming/destructure_with_flattened_command.stderr deleted file mode 100644 index f06f18e2..00000000 --- a/tests/compilation_failures/transforming/destructure_with_flattened_command.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: unexpected token - --> tests/compilation_failures/transforming/destructure_with_flattened_command.rs:4:56 - | -4 | preinterpret!([!let! [!..group! Output] = [!group! Output]]); - | ^^^^^^ diff --git a/tests/core.rs b/tests/core.rs index e364848f..de82149a 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -47,7 +47,7 @@ fn test_extend() { preinterpret_assert_eq!( { #(i = 1) - [!set! #output = [!..group!]] + [!set! #output =] [!while! i <= 4 { [!set! #output += #i] [!if! i <= 3 { diff --git a/tests/expressions.rs b/tests/expressions.rs index 4f895dd7..29f3e1f1 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -196,5 +196,20 @@ fn test_range() { "" ); preinterpret_assert_eq!({ [!string! #([!range! 'a'..='f'] as stream)] }, "abcdef"); - preinterpret_assert_eq!({ [!debug! [!range! -1i8..3i8]] }, "[-1i8, 0i8, 1i8, 2i8]"); + preinterpret_assert_eq!( + { [!debug! [!range! -1i8..3i8]] }, + "[ -1i8, 0i8, 1i8, 2i8]" + ); + + // Large ranges are allowed, but are subject to limits at iteration time + preinterpret_assert_eq!({ [!debug! [!range! 0..10000]] }, "[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ..<9980 further items>]"); + preinterpret_assert_eq!( + [!for! i in [!range! 0..10000000] { + [!if! i == 5 { + [!string! #i] + [!break!] + }] + }], + "5" + ); } diff --git a/tests/tokens.rs b/tests/tokens.rs index 46b5c250..4fa6bc4e 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -10,13 +10,13 @@ fn test_tokens_compilation_failures() { } #[test] -fn test_flattened_group_and_is_empty() { +fn test_empty_stream_is_empty() { preinterpret_assert_eq!({ - [!..group!] "hello" [!..group!] [!..group!] + [!stream!] "hello" [!stream!] [!stream!] }, "hello"); preinterpret_assert_eq!([!is_empty!], true); - preinterpret_assert_eq!([!is_empty! [!..group!]], true); - preinterpret_assert_eq!([!is_empty! [!..group!] [!..group!]], true); + preinterpret_assert_eq!([!is_empty! [!stream!]], true); + preinterpret_assert_eq!([!is_empty! [!stream!] [!stream!]], true); preinterpret_assert_eq!([!is_empty! Not Empty], false); preinterpret_assert_eq!({ [!set! #x =] From 80c23246d3d7a681dd740e5586c645c78d6098a7 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 13 Feb 2025 22:35:32 +0000 Subject: [PATCH 092/126] fix: Fix MSRV --- CHANGELOG.md | 4 +++- src/expressions/mod.rs | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4315923..e1f185c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,11 +134,13 @@ Inside a transform stream, the following grammar is supported: * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` -* Support `#(x[..])` syntax for indexing arrays and streams at read time +* Support `#(x[..])` syntax for indexing arrays and streams at read time (see https://doc.rust-lang.org/reference/expressions/range-expr.html) * Via a post-fix `[..]` operation with high precedence * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) + * Consider supporting ranges in the expression tree and range values * `#(x[0..3])` returns a TokenStream/Array * `#(x[0..=3])` returns a TokenStream/Array + * Ditch `[!range! ..]` if it's supported in the expression tree / value * Add `..` and `..x` support to the array destructurer * Consider: * Dropping lots of the `group` wrappers? diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index bad4e0ee..be178f88 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -23,11 +23,8 @@ use float::*; use integer::*; use operations::*; use string::*; -use value::*; pub(crate) use expression::*; pub(crate) use expression_block::*; pub(crate) use stream::*; pub(crate) use value::*; -// For some mysterious reason Rust-analyzer can't resolve this without an explicit export -pub(crate) use value::ExpressionValue; From d5b464e3bc0d865085e23d9861e9a9228a4cec60 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 14 Feb 2025 02:20:44 +0000 Subject: [PATCH 093/126] feature: Ranges are now also in the expression model --- CHANGELOG.md | 24 +- README.md | 6 - src/expressions/character.rs | 18 - src/expressions/evaluation.rs | 104 +++++ src/expressions/expression.rs | 23 +- src/expressions/expression_parsing.rs | 84 +++- src/expressions/integer.rs | 58 --- src/expressions/iterator.rs | 232 ++++++++++ src/expressions/mod.rs | 4 + src/expressions/operations.rs | 8 - src/expressions/range.rs | 397 ++++++++++++++++++ src/expressions/value.rs | 286 ++----------- src/extensions/errors_and_spans.rs | 1 + src/extensions/parsing.rs | 4 + src/interpretation/command.rs | 3 - .../commands/expression_commands.rs | 39 -- src/interpretation/commands/mod.rs | 2 - .../expressions/large_range_to_string.rs | 2 +- .../expressions/large_range_to_string.stderr | 8 +- tests/control_flow.rs | 4 +- tests/expressions.rs | 31 +- tests/tokens.rs | 4 +- 22 files changed, 914 insertions(+), 428 deletions(-) create mode 100644 src/expressions/iterator.rs create mode 100644 src/expressions/range.rs delete mode 100644 src/interpretation/commands/expression_commands.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e1f185c8..ce3b3eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ * `[!settings! { ... }]` can be used to adjust the iteration limit. * Expression commands: * The expression block `#(let x = 123; y /= x; y + 1)` which is discussed in more detail below. - * `[!range! 0..5]` outputs `0 1 2 3 4` * Control flow commands: * `[!if! { ... }]` and `[!if! { ... } !elif! { ... } !else! { ... }]` * `[!while! { ... }]` @@ -54,6 +53,7 @@ The following are recognized values: * String literals * Char literals * Other literals +* Rust [ranges](https://doc.rust-lang.org/reference/expressions/range-expr.html) * Token streams which are defined as `[...]`. The following operators are supported: @@ -97,7 +97,12 @@ Inside a transform stream, the following grammar is supported: ### To come -* Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` +* Support `#(x[..])` syntax for indexing arrays and streams at read time + * Via a post-fix `[..]` operation with high precedence + * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) + * Consider supporting ranges in the expression tree and range values + * `#(x[0..3])` returns a TokenStream/Array + * `#(x[0..=3])` returns a TokenStream/Array * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Objects: * Backed by an indexmap @@ -117,10 +122,13 @@ Inside a transform stream, the following grammar is supported: * Method calls * Also add support for methods (for e.g. exposing functions on syn objects). * `.len()` on stream + * `.push(x)` on array * Consider `.map(|| {})` * TRANSFORMERS => PARSERS cont - * Transformers no longer output - * Scrap `#>>x` etc in favour of `@(#x += ...)` + * Manually search for transform and rename to parse in folder names and file + * Support `@[x = ...]` for individual parsers. Parsers no longer output to a stream past that. + * Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` + * Scrap `#>>x` etc in favour of `@(a = ...) #[x += [a]]` * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. * `@[ANY_GROUP ...]` @@ -134,13 +142,6 @@ Inside a transform stream, the following grammar is supported: * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` -* Support `#(x[..])` syntax for indexing arrays and streams at read time (see https://doc.rust-lang.org/reference/expressions/range-expr.html) - * Via a post-fix `[..]` operation with high precedence - * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) - * Consider supporting ranges in the expression tree and range values - * `#(x[0..3])` returns a TokenStream/Array - * `#(x[0..=3])` returns a TokenStream/Array - * Ditch `[!range! ..]` if it's supported in the expression tree / value * Add `..` and `..x` support to the array destructurer * Consider: * Dropping lots of the `group` wrappers? @@ -152,6 +153,7 @@ Inside a transform stream, the following grammar is supported: * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) * Maybe add a `@[REINTERPRET ..]` transformer. * CastTarget expansion: + * Add `as iterator` and uncomment the test at the end of `test_range()` * Support a CastTarget of `array` (only supported for array and stream and iterator) * Add `as ident` and `as literal` casting and support it for string, array and stream using concat recursive. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` diff --git a/README.md b/README.md index 12c09f23..6f2172e0 100644 --- a/README.md +++ b/README.md @@ -429,12 +429,6 @@ Incremental parsing using a fork of syn ([see issue](https://github.com/dtolnay/ Forking syn may also allow some parts to be made more performant. -### Possible extension: Better performance via lazy execution - -We could tweak some commands to execute lazily. We replace `output: &mut OutputStream` with `output: &mut impl OutputSource` which could either write to the output or be buffered/streamed into other commands. - -This could allow things like `[!zip! ...]` or `[!range! ...]` to execute lazily, assuming the consuming command such as `for` read lazily. - ### Possible extension: User-defined commands * `[!define_command! [!my_command! ] { }]` diff --git a/src/expressions/character.rs b/src/expressions/character.rs index a43a0cb9..6d8a6d0c 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -62,24 +62,6 @@ impl ExpressionChar { }) } - pub(super) fn create_range( - self, - right: Self, - range_limits: OutputSpanned, - ) -> Box { - let left = self.value; - let right = right.value; - let span_range = range_limits.span_range(); - match range_limits.operation { - syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(move |x| x.to_value(span_range))) - } - syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(move |x| x.to_value(span_range))) - } - } - } - pub(super) fn handle_integer_binary_operation( self, _right: ExpressionInteger, diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 001e20ee..05b7f8ac 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -78,6 +78,41 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { }); NextAction::EnterNode(*left_input) } + ExpressionNode::Range { + left, + range_limits, + right, + } => { + match (left, right) { + (None, None) => match range_limits { + syn::RangeLimits::HalfOpen(token) => { + let inner = ExpressionRangeInner::RangeFull { + token: *token, + }; + NextAction::HandleValue(inner.to_value(token.span_range())) + } + syn::RangeLimits::Closed(_) => { + unreachable!("A closed range should have been given a right in continue_range(..)") + } + }, + (None, Some(right)) => { + self.operation_stack.push(EvaluationStackFrame::Range { + range_limits: *range_limits, + state: RangePath::OnRightBranch { left: None }, + }); + NextAction::EnterNode(*right) + } + (Some(left), right) => { + self.operation_stack.push(EvaluationStackFrame::Range { + range_limits: *range_limits, + state: RangePath::OnLeftBranch { + right: *right, + }, + }); + NextAction::EnterNode(*left) + } + } + } }) } @@ -113,6 +148,66 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { NextAction::HandleValue(result) } }, + EvaluationStackFrame::Range { + range_limits, + state, + } => match (state, range_limits) { + (RangePath::OnLeftBranch { right: Some(right) }, range_limits) => { + self.operation_stack.push(EvaluationStackFrame::Range { + range_limits, + state: RangePath::OnRightBranch { left: Some(value) }, + }); + NextAction::EnterNode(right) + } + (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::HalfOpen(token)) => { + let inner = ExpressionRangeInner::RangeFrom { + start_inclusive: value, + token, + }; + NextAction::HandleValue(inner.to_value(token.span_range())) + } + (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::Closed(_)) => { + unreachable!( + "A closed range should have been given a right in continue_range(..)" + ) + } + ( + RangePath::OnRightBranch { left: Some(left) }, + syn::RangeLimits::HalfOpen(token), + ) => { + let inner = ExpressionRangeInner::Range { + start_inclusive: left, + token, + end_exclusive: value, + }; + NextAction::HandleValue(inner.to_value(token.span_range())) + } + ( + RangePath::OnRightBranch { left: Some(left) }, + syn::RangeLimits::Closed(token), + ) => { + let inner = ExpressionRangeInner::RangeInclusive { + start_inclusive: left, + token, + end_inclusive: value, + }; + NextAction::HandleValue(inner.to_value(token.span_range())) + } + (RangePath::OnRightBranch { left: None }, syn::RangeLimits::HalfOpen(token)) => { + let inner = ExpressionRangeInner::RangeTo { + token, + end_exclusive: value, + }; + NextAction::HandleValue(inner.to_value(token.span_range())) + } + (RangePath::OnRightBranch { left: None }, syn::RangeLimits::Closed(token)) => { + let inner = ExpressionRangeInner::RangeToInclusive { + token, + end_inclusive: value, + }; + NextAction::HandleValue(inner.to_value(token.span_range())) + } + }, }) } } @@ -134,6 +229,10 @@ enum EvaluationStackFrame { operation: BinaryOperation, state: BinaryPath, }, + Range { + range_limits: syn::RangeLimits, + state: RangePath, + }, } struct ArrayStackFrame { @@ -165,3 +264,8 @@ enum BinaryPath { OnLeftBranch { right: ExpressionNodeId }, OnRightBranch { left: ExpressionValue }, } + +enum RangePath { + OnLeftBranch { right: Option }, + OnRightBranch { left: Option }, +} diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 66c0d797..5ff92c3f 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -77,7 +77,13 @@ impl Expressionable for Source { is_empty: input.is_current_empty(), } } - SourcePeekMatch::Punct(_) => UnaryAtom::PrefixUnaryOperation(input.parse()?), + SourcePeekMatch::Punct(punct) => { + if punct.as_char() == '.' { + UnaryAtom::Range(input.parse()?) + } else { + UnaryAtom::PrefixUnaryOperation(input.parse()?) + } + } SourcePeekMatch::Ident(_) => match input.try_parse_or_revert() { Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( ExpressionBoolean::for_litbool(bool), @@ -111,14 +117,17 @@ impl Expressionable for Source { ExpressionStackFrame::Group { .. } => { return input.parse_err("Commas are only permitted inside preinterpret arrays []. Preinterpret arrays [a, b] can be used as a drop-in replacement for rust tuples (a, b).") } - // + // Fall through for an unmatched extension _ => {} } } SourcePeekMatch::Punct(_) => { - if let Ok(operation) = input.try_parse_or_revert::() { + if let Ok(operation) = input.try_parse_or_revert() { return Ok(NodeExtension::BinaryOperation(operation)); } + if let Ok(range_limits) = input.try_parse_or_revert() { + return Ok(NodeExtension::Range(range_limits)); + } } SourcePeekMatch::Ident(ident) if ident == "as" => { let cast_operation = @@ -138,7 +147,8 @@ impl Expressionable for Source { // and I want to see if there's an extension (e.g. a cast). // There's nothing matching, so we fall through to an EndOfFrame ExpressionStackFrame::IncompleteUnaryPrefixOperation { .. } - | ExpressionStackFrame::IncompleteBinaryOperation { .. } => { + | ExpressionStackFrame::IncompleteBinaryOperation { .. } + | ExpressionStackFrame::IncompleteRange { .. } => { Ok(NodeExtension::NoValidExtensionForCurrentParent) } } @@ -209,6 +219,11 @@ pub(super) enum ExpressionNode { left_input: ExpressionNodeId, right_input: ExpressionNodeId, }, + Range { + left: Option, + range_limits: syn::RangeLimits, + right: Option, + }, } pub(super) trait Expressionable: Sized { diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index b551922a..4a113652 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -36,6 +36,9 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { WorkItem::TryApplyAlreadyParsedExtension { node, extension } => { self.attempt_extension(node, extension)? } + WorkItem::ContinueRange { lhs, range_limits } => { + self.continue_range(lhs, range_limits)? + } WorkItem::Finished { root } => { return Ok(self.nodes.complete(root)); } @@ -73,6 +76,10 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { operation, }) } + UnaryAtom::Range(range_limits) => WorkItem::ContinueRange { + lhs: None, + range_limits, + }, }) } @@ -108,6 +115,10 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } WorkItem::RequireUnaryAtom } + NodeExtension::Range(range_limits) => WorkItem::ContinueRange { + lhs: Some(node), + range_limits, + }, NodeExtension::EndOfStream | NodeExtension::NoValidExtensionForCurrentParent => { unreachable!("Not possible, as these have minimum precedence") } @@ -163,10 +174,61 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { }); extension.into_post_operation_completion_work_item(node) } + ExpressionStackFrame::IncompleteRange { lhs, range_limits } => { + let node = self.nodes.add_node(ExpressionNode::Range { + left: lhs, + range_limits, + right: Some(node), + }); + extension.into_post_operation_completion_work_item(node) + } }) } } + fn continue_range( + &mut self, + lhs: Option, + range_limits: syn::RangeLimits, + ) -> ParseResult { + // See https://doc.rust-lang.org/reference/expressions/range-expr.html + match &range_limits { + syn::RangeLimits::Closed(_) => { + // A closed range requires a right hand side, so we can just + // go straight to matching a UnaryAtom for it + return Ok( + self.push_stack_frame(ExpressionStackFrame::IncompleteRange { + lhs, + range_limits, + }), + ); + } + syn::RangeLimits::HalfOpen(_) => {} + } + // Otherwise, we have a half-open range, and need to work out whether + // we can parse a UnaryAtom to be the right side of the range or whether + // it will have no right side. + // Some examples of such ranges include: `[3.., 4]`, `[3..]`, `let x = 3..;`, + // `(3..).first()` or even `3...first()` + let can_parse_unary_atom = { + let forked = self.streams.fork_current(); + let mut forked_stack = ParseStreamStack::new(&forked); + K::parse_unary_atom(&mut forked_stack).is_ok() + }; + if can_parse_unary_atom { + // A unary atom can be parsed so let's attempt to complete the range with it + Ok(self.push_stack_frame(ExpressionStackFrame::IncompleteRange { lhs, range_limits })) + } else { + // No unary atom can be parsed, so let's complete the range + let node = self.nodes.add_node(ExpressionNode::Range { + left: lhs, + range_limits, + right: None, + }); + Ok(WorkItem::TryParseAndApplyExtension { node }) + } + } + fn add_leaf(&mut self, leaf: K::Leaf) -> WorkItem { let node = self.nodes.add_node(ExpressionNode::Leaf(leaf)); WorkItem::TryParseAndApplyExtension { node } @@ -433,6 +495,11 @@ pub(super) enum ExpressionStackFrame { lhs: ExpressionNodeId, operation: BinaryOperation, }, + /// A range which will be followed by a rhs + IncompleteRange { + lhs: Option, + range_limits: syn::RangeLimits, + }, } impl ExpressionStackFrame { @@ -441,6 +508,7 @@ impl ExpressionStackFrame { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::IncompleteRange { .. } => OperatorPrecendence::Range, ExpressionStackFrame::IncompleteUnaryPrefixOperation { operation, .. } => { OperatorPrecendence::of_prefix_unary_operation(operation) } @@ -471,6 +539,11 @@ enum WorkItem { node: ExpressionNodeId, extension: NodeExtension, }, + /// Range parsing behaviour is quite unique, so we have a special case for it. + ContinueRange { + lhs: Option, + range_limits: syn::RangeLimits, + }, Finished { root: ExpressionNodeId, }, @@ -484,12 +557,14 @@ pub(super) enum UnaryAtom { is_empty: bool, }, PrefixUnaryOperation(PrefixUnaryOperation), + Range(syn::RangeLimits), } pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), NonTerminalArrayComma, + Range(syn::RangeLimits), EndOfStream, NoValidExtensionForCurrentParent, } @@ -500,6 +575,7 @@ impl NodeExtension { NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), NodeExtension::NonTerminalArrayComma => OperatorPrecendence::NonTerminalComma, + NodeExtension::Range(_) => OperatorPrecendence::Range, NodeExtension::EndOfStream => OperatorPrecendence::MIN, NodeExtension::NoValidExtensionForCurrentParent => OperatorPrecendence::MIN, } @@ -507,9 +583,11 @@ impl NodeExtension { fn into_post_operation_completion_work_item(self, node: ExpressionNodeId) -> WorkItem { match self { - extension @ NodeExtension::PostfixOperation { .. } - | extension @ NodeExtension::BinaryOperation { .. } - | extension @ NodeExtension::EndOfStream => { + // Extensions are independent of depth, so can be re-used without parsing again. + extension @ (NodeExtension::PostfixOperation { .. } + | NodeExtension::BinaryOperation { .. } + | NodeExtension::Range { .. } + | NodeExtension::EndOfStream) => { WorkItem::TryApplyAlreadyParsedExtension { node, extension } } NodeExtension::NonTerminalArrayComma => { diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 14e8a078..a9cf31e8 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -148,27 +148,6 @@ impl ExpressionIntegerValuePair { Self::Isize(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } - - pub(crate) fn create_range( - self, - range_limits: OutputSpanned, - ) -> ExecutionResult> { - Ok(match self { - Self::Untyped(lhs, rhs) => return lhs.create_range(rhs, range_limits), - Self::U8(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::U16(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::U32(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::U64(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::U128(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::Usize(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::I8(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::I16(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::I32(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::I64(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::I128(lhs, rhs) => lhs.create_range(rhs, range_limits), - Self::Isize(lhs, rhs) => lhs.create_range(rhs, range_limits), - }) - } } #[derive(Copy, Clone)] @@ -449,24 +428,6 @@ impl UntypedInteger { }) } - pub(super) fn create_range( - self, - right: Self, - range_limits: OutputSpanned, - ) -> ExecutionResult> { - let left = self.parse_fallback()?; - let right = right.parse_fallback()?; - let span_range = range_limits.span_range(); - Ok(match range_limits.operation { - syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(move |x| Self::from_fallback(x).to_value(span_range))) - } - syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(move |x| Self::from_fallback(x).to_value(span_range))) - } - }) - } - pub(crate) fn from_fallback(value: FallbackInteger) -> Self { Self::new_from_literal(Literal::i128_unsuffixed(value)) } @@ -597,25 +558,6 @@ macro_rules! impl_int_operations_except_unary { }) } } - - impl HandleCreateRange for $integer_type { - fn create_range( - self, - right: Self, - range_limits: OutputSpanned, - ) -> Box { - let left = self; - let span_range = range_limits.span_range(); - match range_limits.operation { - syn::RangeLimits::HalfOpen { .. } => { - Box::new((left..right).map(move |x| x.to_value(span_range))) - }, - syn::RangeLimits::Closed { .. } => { - Box::new((left..=right).map(move |x| x.to_value(span_range))) - } - } - } - } )*}; } diff --git a/src/expressions/iterator.rs b/src/expressions/iterator.rs new file mode 100644 index 00000000..c0d59a2e --- /dev/null +++ b/src/expressions/iterator.rs @@ -0,0 +1,232 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionIterator { + iterator: ExpressionIteratorInner, + #[allow(unused)] + pub(crate) span_range: SpanRange, +} + +impl ExpressionIterator { + pub(crate) fn new_for_array(array: ExpressionArray) -> Self { + Self { + iterator: ExpressionIteratorInner::Array(array.items.into_iter()), + span_range: array.span_range, + } + } + + pub(crate) fn new_for_stream(stream: ExpressionStream) -> Self { + Self { + iterator: ExpressionIteratorInner::Stream(stream.value.into_iter()), + span_range: stream.span_range, + } + } + + pub(crate) fn new_for_range(range: ExpressionRange) -> ExecutionResult { + let iterator = range.inner.into_iterable()?.resolve_iterator()?; + Ok(Self { + iterator: ExpressionIteratorInner::Other(iterator), + span_range: range.span_range, + }) + } + + pub(crate) fn new_custom( + iterator: Box, + span_range: SpanRange, + ) -> Self { + Self { + iterator: ExpressionIteratorInner::Other(iterator), + span_range, + } + } + + pub(super) fn handle_unary_operation( + self, + operation: OutputSpanned, + ) -> ExecutionResult { + Ok(match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { + return operation.unsupported(self) + } + UnaryOperation::Cast { + target, + target_ident, + .. + } => match target { + CastTarget::Stream => operation.output(self.into_stream_with_grouped_items()?), + CastTarget::Group => operation.output( + operation + .output(self.into_stream_with_grouped_items()?) + .into_new_output_stream( + Grouping::Grouped, + StreamOutputBehaviour::Standard, + )?, + ), + CastTarget::String => operation.output({ + let mut output = String::new(); + self.concat_recursive_into(&mut output, &ConcatBehaviour::standard())?; + output + }), + CastTarget::DebugString => { + operation.output(self.iterator).into_debug_string_value()? + } + CastTarget::Boolean + | CastTarget::Char + | CastTarget::Integer(_) + | CastTarget::Float(_) => match self.singleton_value() { + Some(value) => value.handle_unary_operation(operation)?, + None => { + return operation.execution_err(format!( + "Only an iterator with one item can be cast to {}", + target_ident, + )) + } + }, + }, + }) + } + + pub(crate) fn singleton_value(mut self) -> Option { + let first = self.next()?; + if self.next().is_none() { + Some(first) + } else { + None + } + } + + fn into_stream_with_grouped_items(self) -> ExecutionResult { + let mut output = OutputStream::new(); + self.output_grouped_items_to(&mut output)?; + Ok(output) + } + + pub(super) fn output_grouped_items_to(self, output: &mut OutputStream) -> ExecutionResult<()> { + const LIMIT: usize = 10_000; + let span_range = self.span_range; + for (i, item) in self.enumerate() { + if i > LIMIT { + return span_range.execution_err(format!("Only a maximum of {} items can be output to a stream from an iterator, to protect you from infinite loops. This can't currently be reconfigured with the iteration limit.", LIMIT)); + } + item.output_to( + Grouping::Grouped, + output, + StreamOutputBehaviour::PermitArrays, + )?; + } + Ok(()) + } + + pub(crate) fn concat_recursive_into( + self, + output: &mut String, + behaviour: &ConcatBehaviour, + ) -> ExecutionResult<()> { + if behaviour.output_array_structure { + output.push_str("["); + } + let max = self.size_hint().1; + let span_range = self.span_range; + for (i, item) in self.enumerate() { + if i >= behaviour.iterator_limit { + if behaviour.error_after_iterator_limit { + return span_range.execution_err(format!("To protect against infinite loops, only a maximum of {} items can be output to a string from an iterator. Try casting `as stream` to avoid this limit. This can't currently be reconfigured with the iteration limit.", behaviour.iterator_limit)); + } else { + if behaviour.output_array_structure { + match max { + Some(max) => output.push_str(&format!( + ", ..<{} further items>", + max.saturating_sub(i) + )), + None => output.push_str(", .."), + } + } + break; + } + } + if i == 0 { + output.push(' '); + } + if i != 0 && behaviour.output_array_structure { + output.push(','); + } + if i != 0 && behaviour.add_space_between_token_trees { + output.push(' '); + } + item.concat_recursive_into(output, behaviour)?; + } + if behaviour.output_array_structure { + output.push(']'); + } + Ok(()) + } +} + +impl ToExpressionValue for ExpressionIteratorInner { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Iterator(ExpressionIterator { + iterator: self, + span_range, + }) + } +} + +impl ToExpressionValue for Box { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Iterator(ExpressionIterator::new_custom(self, span_range)) + } +} + +impl HasValueType for ExpressionIterator { + fn value_type(&self) -> &'static str { + "iterator" + } +} + +#[derive(Clone)] +enum ExpressionIteratorInner { + Array( as IntoIterator>::IntoIter), + Stream(::IntoIter), + Other(Box), +} + +impl + Clone + 'static> CustomExpressionIterator for T { + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +pub(crate) trait CustomExpressionIterator: Iterator { + fn clone_box(&self) -> Box; +} + +impl Clone for Box { + fn clone(&self) -> Self { + (**self).clone_box() + } +} + +impl Iterator for ExpressionIterator { + type Item = ExpressionValue; + + fn next(&mut self) -> Option { + match &mut self.iterator { + ExpressionIteratorInner::Array(iter) => iter.next(), + ExpressionIteratorInner::Stream(iter) => { + let item = iter.next()?; + let span = item.span(); + let stream: OutputStream = item.into(); + Some(stream.coerce_into_value(span.span_range())) + } + ExpressionIteratorInner::Other(iter) => iter.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + match &self.iterator { + ExpressionIteratorInner::Array(iter) => iter.size_hint(), + ExpressionIteratorInner::Stream(iter) => iter.size_hint(), + ExpressionIteratorInner::Other(iter) => iter.size_hint(), + } + } +} diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index be178f88..13654a59 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -7,7 +7,9 @@ mod expression_block; mod expression_parsing; mod float; mod integer; +mod iterator; mod operations; +mod range; mod stream; mod string; mod value; @@ -22,9 +24,11 @@ use expression_parsing::*; use float::*; use integer::*; use operations::*; +use range::*; use string::*; pub(crate) use expression::*; pub(crate) use expression_block::*; +pub(crate) use iterator::*; pub(crate) use stream::*; pub(crate) use value::*; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 80b75c97..7551151e 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -485,14 +485,6 @@ pub(super) trait HandleBinaryOperation: Sized { ) -> ExecutionResult; } -pub(super) trait HandleCreateRange: Sized { - fn create_range( - self, - right: Self, - range_limits: OutputSpanned, - ) -> Box; -} - impl Operation for syn::RangeLimits { fn symbolic_description(&self) -> &'static str { match self { diff --git a/src/expressions/range.rs b/src/expressions/range.rs new file mode 100644 index 00000000..602f2a55 --- /dev/null +++ b/src/expressions/range.rs @@ -0,0 +1,397 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionRange { + pub(crate) inner: Box, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(crate) span_range: SpanRange, +} + +impl ExpressionRange { + pub(crate) fn concat_recursive_into( + self, + output: &mut String, + behaviour: &ConcatBehaviour, + ) -> ExecutionResult<()> { + match *self.inner { + ExpressionRangeInner::Range { + start_inclusive, + end_exclusive, + .. + } => { + start_inclusive.concat_recursive_into(output, behaviour)?; + output.push_str(".."); + end_exclusive.concat_recursive_into(output, behaviour)?; + } + ExpressionRangeInner::RangeFrom { + start_inclusive, .. + } => { + start_inclusive.concat_recursive_into(output, behaviour)?; + output.push_str(".."); + } + ExpressionRangeInner::RangeTo { end_exclusive, .. } => { + output.push_str(".."); + end_exclusive.concat_recursive_into(output, behaviour)?; + } + ExpressionRangeInner::RangeFull { .. } => { + output.push_str(".."); + } + ExpressionRangeInner::RangeInclusive { + start_inclusive, + end_inclusive, + .. + } => { + start_inclusive.concat_recursive_into(output, behaviour)?; + output.push_str("..="); + end_inclusive.concat_recursive_into(output, behaviour)?; + } + ExpressionRangeInner::RangeToInclusive { end_inclusive, .. } => { + output.push_str("..="); + end_inclusive.concat_recursive_into(output, behaviour)?; + } + } + Ok(()) + } +} + +impl HasSpanRange for ExpressionRange { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl HasValueType for ExpressionRange { + fn value_type(&self) -> &'static str { + self.inner.value_type() + } +} + +/// A representation of the Rust [range expression]. +/// +/// [range expression]: https://doc.rust-lang.org/reference/expressions/range-expr.html +#[derive(Clone)] +pub(crate) enum ExpressionRangeInner { + /// `start .. end` + Range { + start_inclusive: ExpressionValue, + token: Token![..], + end_exclusive: ExpressionValue, + }, + /// `start ..` + RangeFrom { + start_inclusive: ExpressionValue, + token: Token![..], + }, + /// `.. end` + RangeTo { + token: Token![..], + end_exclusive: ExpressionValue, + }, + /// `..` (used inside arrays) + RangeFull { token: Token![..] }, + /// `start ..= end` + RangeInclusive { + start_inclusive: ExpressionValue, + token: Token![..=], + end_inclusive: ExpressionValue, + }, + /// `..= end` + RangeToInclusive { + token: Token![..=], + end_inclusive: ExpressionValue, + }, +} + +impl ExpressionRangeInner { + pub(super) fn into_iterable(self) -> ExecutionResult> { + Ok(match self { + Self::Range { + start_inclusive, + token, + end_exclusive, + } => IterableExpressionRange::RangeFromTo { + start: start_inclusive, + dots: syn::RangeLimits::HalfOpen(token), + end: end_exclusive, + }, + Self::RangeInclusive { + start_inclusive, + token, + end_inclusive, + } => IterableExpressionRange::RangeFromTo { + start: start_inclusive, + dots: syn::RangeLimits::Closed(token), + end: end_inclusive, + }, + Self::RangeFrom { + start_inclusive, + token, + } => IterableExpressionRange::RangeFrom { + start: start_inclusive, + dots: token, + }, + other => { + return other + .operator_span_range() + .execution_err("This range has no start so is not iterable") + } + }) + } + + fn operator_span_range(&self) -> SpanRange { + match self { + Self::Range { token, .. } => token.span_range(), + Self::RangeFrom { token, .. } => token.span_range(), + Self::RangeTo { token, .. } => token.span_range(), + Self::RangeFull { token } => token.span_range(), + Self::RangeInclusive { token, .. } => token.span_range(), + Self::RangeToInclusive { token, .. } => token.span_range(), + } + } +} + +impl HasValueType for ExpressionRangeInner { + fn value_type(&self) -> &'static str { + match self { + Self::Range { .. } => "range start..end", + Self::RangeFrom { .. } => "range start..", + Self::RangeTo { .. } => "range ..end", + Self::RangeFull { .. } => "range ..", + Self::RangeInclusive { .. } => "range start..=end", + Self::RangeToInclusive { .. } => "range ..=end", + } + } +} + +impl ToExpressionValue for ExpressionRangeInner { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Range(ExpressionRange { + inner: Box::new(self), + span_range, + }) + } +} + +pub(super) enum IterableExpressionRange { + // start <= x < end OR start <= x <= end + RangeFromTo { + start: T, + dots: syn::RangeLimits, + end: T, + }, + // start <= x + RangeFrom { + dots: Token![..], + start: T, + }, +} + +impl IterableExpressionRange { + pub(super) fn resolve_iterator(self) -> ExecutionResult> { + match self { + Self::RangeFromTo { start, dots, end } => { + let output_span_range = + SpanRange::new_between(start.span_range().start(), end.span_range().end()); + let pair = start.expect_value_pair(&dots, end)?; + match pair { + EvaluationValuePair::Integer(pair) => match pair { + ExpressionIntegerValuePair::Untyped(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::U8(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::U16(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::U32(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::U64(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::U128(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::Usize(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::I8(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::I16(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::I32(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::I64(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::I128(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + ExpressionIntegerValuePair::Isize(start, end) => { + IterableExpressionRange::RangeFromTo { start, dots, end } + .resolve(output_span_range) + } + }, + EvaluationValuePair::CharPair(start, end) => { + IterableExpressionRange::RangeFromTo { + start: start.value, + dots, + end: end.value, + } + .resolve(output_span_range) + } + _ => { + dots.execution_err( + "The range must be between two integers or two characters", + ) + } + } + } + Self::RangeFrom { start, dots } => { + let output_span_range = + SpanRange::new_between(start.span_range().start(), dots.span_range().end()); + match start { + ExpressionValue::Integer(start) => match start.value { + ExpressionIntegerValue::Untyped(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::U8(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::U16(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::U32(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::U64(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::U128(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::Usize(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::I8(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::I16(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::I32(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::I64(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::I128(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + ExpressionIntegerValue::Isize(start) => { + IterableExpressionRange::RangeFrom { start, dots } + .resolve(output_span_range) + } + }, + ExpressionValue::Char(start) => IterableExpressionRange::RangeFrom { + start: start.value, + dots, + } + .resolve(output_span_range), + _ => { + dots + .execution_err("The range must be from an integer or a character") + } + } + } + } + } +} + +impl IterableExpressionRange { + fn resolve( + self, + output_span_range: SpanRange, + ) -> ExecutionResult> { + match self { + Self::RangeFromTo { start, dots, end } => { + let start = start.parse_fallback()?; + let end = end.parse_fallback()?; + Ok(match dots { + syn::RangeLimits::HalfOpen { .. } => Box::new((start..end).map(move |x| { + UntypedInteger::from_fallback(x).to_value(output_span_range) + })), + syn::RangeLimits::Closed { .. } => Box::new((start..=end).map(move |x| { + UntypedInteger::from_fallback(x).to_value(output_span_range) + })), + }) + } + Self::RangeFrom { start, .. } => { + let start = start.parse_fallback()?; + Ok(Box::new((start..).map(move |x| { + UntypedInteger::from_fallback(x).to_value(output_span_range) + }))) + } + } + } +} + +macro_rules! define_range_resolvers { + ( + $($the_type:ident),* $(,)? + ) => {$( + impl IterableExpressionRange<$the_type> { + fn resolve(self, output_span_range: SpanRange) -> ExecutionResult> { + match self { + Self::RangeFromTo { start, dots, end } => { + Ok(match dots { + syn::RangeLimits::HalfOpen { .. } => { + Box::new((start..end).map(move |x| x.to_value(output_span_range))) + } + syn::RangeLimits::Closed { .. } => { + Box::new((start..=end).map(move |x| x.to_value(output_span_range))) + } + }) + }, + Self::RangeFrom { start, .. } => { + Ok(Box::new((start..).map(move |x| x.to_value(output_span_range)))) + }, + } + } + } + )*}; +} + +define_range_resolvers! { + u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, char, +} diff --git a/src/expressions/value.rs b/src/expressions/value.rs index ac8bd1b2..8cc94bff 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -13,6 +13,7 @@ pub(crate) enum ExpressionValue { UnsupportedLiteral(UnsupportedLiteral), Array(ExpressionArray), Stream(ExpressionStream), + Range(ExpressionRange), Iterator(ExpressionIterator), } @@ -58,7 +59,7 @@ impl ExpressionValue { self, operation: &impl Operation, right: Self, - ) -> ExecutionResult { + ) -> ExecutionResult { Ok(match (self, right) { (ExpressionValue::Integer(left), ExpressionValue::Integer(right)) => { let integer_pair = match (left.value, right.value) { @@ -184,10 +185,10 @@ impl ExpressionValue { return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbolic_description(), left_value.value_type(), right_value.value_type())); } }; - EvaluationLiteralPair::Integer(integer_pair) + EvaluationValuePair::Integer(integer_pair) } (ExpressionValue::Boolean(left), ExpressionValue::Boolean(right)) => { - EvaluationLiteralPair::BooleanPair(left, right) + EvaluationValuePair::BooleanPair(left, right) } (ExpressionValue::Float(left), ExpressionValue::Float(right)) => { let float_pair = match (left.value, right.value) { @@ -223,19 +224,19 @@ impl ExpressionValue { return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbolic_description(), left_value.value_type(), right_value.value_type())); } }; - EvaluationLiteralPair::Float(float_pair) + EvaluationValuePair::Float(float_pair) } (ExpressionValue::String(left), ExpressionValue::String(right)) => { - EvaluationLiteralPair::StringPair(left, right) + EvaluationValuePair::StringPair(left, right) } (ExpressionValue::Char(left), ExpressionValue::Char(right)) => { - EvaluationLiteralPair::CharPair(left, right) + EvaluationValuePair::CharPair(left, right) } (ExpressionValue::Array(left), ExpressionValue::Array(right)) => { - EvaluationLiteralPair::ArrayPair(left, right) + EvaluationValuePair::ArrayPair(left, right) } (ExpressionValue::Stream(left), ExpressionValue::Stream(right)) => { - EvaluationLiteralPair::StreamPair(left, right) + EvaluationValuePair::StreamPair(left, right) } (left, right) => { return operation.execution_err(format!("Cannot infer common type from {} {} {}. Consider using `as` to cast the operands to matching types.", left.value_type(), operation.symbolic_description(), right.value_type())); @@ -316,8 +317,9 @@ impl ExpressionValue { ExpressionValue::Array(value) => Ok(ExpressionIterator::new_for_array(value)), ExpressionValue::Stream(value) => Ok(ExpressionIterator::new_for_stream(value)), ExpressionValue::Iterator(value) => Ok(value), + ExpressionValue::Range(value) => Ok(ExpressionIterator::new_for_range(value)?), other => other.execution_err(format!( - "{} must be iterable (an array or stream or iterator), but it is a {}", + "{} must be iterable (an array, stream, range or iterator), but it is a {}", place_descriptor, other.value_type(), )), @@ -337,6 +339,9 @@ impl ExpressionValue { ExpressionValue::Char(value) => value.handle_unary_operation(operation), ExpressionValue::Stream(value) => value.handle_unary_operation(operation), ExpressionValue::Array(value) => value.handle_unary_operation(operation), + ExpressionValue::Range(range) => { + ExpressionIterator::new_for_range(range)?.handle_unary_operation(operation) + } ExpressionValue::UnsupportedLiteral(value) => operation.unsupported(value), ExpressionValue::Iterator(value) => value.handle_unary_operation(operation), } @@ -370,19 +375,10 @@ impl ExpressionValue { value.handle_integer_binary_operation(right, operation) } ExpressionValue::Iterator(value) => operation.unsupported(value), + ExpressionValue::Range(value) => operation.unsupported(value), } } - pub(crate) fn create_range( - self, - other: Self, - range_limits: &syn::RangeLimits, - ) -> ExecutionResult> { - let span_range = SpanRange::new_between(self.span_range().start(), self.span_range().end()); - self.expect_value_pair(range_limits, other)? - .create_range(range_limits.with_output_span_range(span_range)) - } - fn span_range_mut(&mut self) -> &mut SpanRange { match self { Self::None(span_range) => span_range, @@ -395,6 +391,7 @@ impl ExpressionValue { Self::Array(value) => &mut value.span_range, Self::Stream(value) => &mut value.span_range, Self::Iterator(value) => &mut value.span_range, + Self::Range(value) => &mut value.span_range, } } @@ -478,6 +475,14 @@ impl ExpressionValue { return self.execution_err("Iterators cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `#(XXX as stream)` to cast the iterator to a stream."); } } + Self::Range(range) => { + let iterator = ExpressionIterator::new_for_range(range.clone())?; + if behaviour.should_output_iterators() { + iterator.output_grouped_items_to(output)? + } else { + return self.execution_err("Iterators cannot be output to a stream. You likely wish to use the !for! command or if you wish to output every element, use `#(XXX as stream)` to cast the iterator to a stream."); + } + } }; Ok(()) } @@ -516,6 +521,9 @@ impl ExpressionValue { ExpressionValue::Iterator(iterator) => { iterator.concat_recursive_into(output, behaviour)?; } + ExpressionValue::Range(range) => { + range.concat_recursive_into(output, behaviour)?; + } ExpressionValue::Integer(_) | ExpressionValue::Float(_) | ExpressionValue::Char(_) @@ -573,6 +581,7 @@ impl HasValueType for ExpressionValue { Self::Array(value) => value.value_type(), Self::Stream(value) => value.value_type(), Self::Iterator(value) => value.value_type(), + Self::Range(value) => value.value_type(), } } } @@ -590,6 +599,7 @@ impl HasSpanRange for ExpressionValue { ExpressionValue::Array(array) => array.span_range, ExpressionValue::Stream(stream) => stream.span_range, ExpressionValue::Iterator(iterator) => iterator.span_range, + ExpressionValue::Range(iterator) => iterator.span_range, } } } @@ -610,7 +620,7 @@ impl HasValueType for UnsupportedLiteral { } } -pub(super) enum EvaluationLiteralPair { +pub(super) enum EvaluationValuePair { Integer(ExpressionIntegerValuePair), Float(ExpressionFloatValuePair), BooleanPair(ExpressionBoolean, ExpressionBoolean), @@ -620,7 +630,7 @@ pub(super) enum EvaluationLiteralPair { StreamPair(ExpressionStream, ExpressionStream), } -impl EvaluationLiteralPair { +impl EvaluationValuePair { pub(super) fn handle_paired_binary_operation( self, operation: OutputSpanned, @@ -635,238 +645,4 @@ impl EvaluationLiteralPair { Self::StreamPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } - - pub(super) fn create_range( - self, - range_limits: OutputSpanned, - ) -> ExecutionResult> { - Ok(match self { - EvaluationLiteralPair::Integer(pair) => return pair.create_range(range_limits), - EvaluationLiteralPair::CharPair(left, right) => left.create_range(right, range_limits), - _ => { - return range_limits - .execution_err("The range must be between two integers or two characters") - } - }) - } -} - -#[derive(Clone)] -pub(crate) struct ExpressionIterator { - iterator: ExpressionIteratorInner, - #[allow(unused)] - pub(crate) span_range: SpanRange, -} - -impl ExpressionIterator { - pub(crate) fn new_for_array(array: ExpressionArray) -> Self { - Self { - iterator: ExpressionIteratorInner::Array(array.items.into_iter()), - span_range: array.span_range, - } - } - - pub(crate) fn new_for_stream(stream: ExpressionStream) -> Self { - Self { - iterator: ExpressionIteratorInner::Stream(stream.value.into_iter()), - span_range: stream.span_range, - } - } - - pub(crate) fn new_custom( - iterator: Box, - span_range: SpanRange, - ) -> Self { - Self { - iterator: ExpressionIteratorInner::Other(iterator), - span_range, - } - } - - pub(super) fn handle_unary_operation( - self, - operation: OutputSpanned, - ) -> ExecutionResult { - Ok(match operation.operation { - UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - return operation.unsupported(self) - } - UnaryOperation::Cast { - target, - target_ident, - .. - } => match target { - CastTarget::Stream => operation.output(self.into_stream_with_grouped_items()?), - CastTarget::Group => operation.output( - operation - .output(self.into_stream_with_grouped_items()?) - .into_new_output_stream( - Grouping::Grouped, - StreamOutputBehaviour::Standard, - )?, - ), - CastTarget::String => operation.output({ - let mut output = String::new(); - self.concat_recursive_into(&mut output, &ConcatBehaviour::standard())?; - output - }), - CastTarget::DebugString => { - operation.output(self.iterator).into_debug_string_value()? - } - CastTarget::Boolean - | CastTarget::Char - | CastTarget::Integer(_) - | CastTarget::Float(_) => match self.singleton_value() { - Some(value) => value.handle_unary_operation(operation)?, - None => { - return operation.execution_err(format!( - "Only an iterator with one item can be cast to {}", - target_ident, - )) - } - }, - }, - }) - } - - pub(crate) fn singleton_value(mut self) -> Option { - let first = self.next()?; - if self.next().is_none() { - Some(first) - } else { - None - } - } - - fn into_stream_with_grouped_items(self) -> ExecutionResult { - let mut output = OutputStream::new(); - self.output_grouped_items_to(&mut output)?; - Ok(output) - } - - fn output_grouped_items_to(self, output: &mut OutputStream) -> ExecutionResult<()> { - const LIMIT: usize = 10_000; - let span_range = self.span_range; - for (i, item) in self.enumerate() { - if i > LIMIT { - return span_range.execution_err(format!("Only a maximum of {} items can be output to a stream from an iterator, to protect you from infinite loops. This can't currently be reconfigured with the iteration limit.", LIMIT)); - } - item.output_to( - Grouping::Grouped, - output, - StreamOutputBehaviour::PermitArrays, - )?; - } - Ok(()) - } - - pub(crate) fn concat_recursive_into( - self, - output: &mut String, - behaviour: &ConcatBehaviour, - ) -> ExecutionResult<()> { - if behaviour.output_array_structure { - output.push_str("[ "); - } - let max = self.size_hint().1; - let span_range = self.span_range; - for (i, item) in self.enumerate() { - if i >= behaviour.iterator_limit { - if behaviour.error_after_iterator_limit { - return span_range.execution_err(format!("To protect against infinite loops, only a maximum of {} items can be output to a string from an iterator. Try casting `as stream` to avoid this limit. This can't currently be reconfigured with the iteration limit.", behaviour.iterator_limit)); - } else { - if behaviour.output_array_structure { - match max { - Some(max) => output.push_str(&format!( - ", ..<{} further items>", - max.saturating_sub(i) - )), - None => output.push_str(", .."), - } - } - break; - } - } - if i != 0 && behaviour.output_array_structure { - output.push(','); - } - if i != 0 && behaviour.add_space_between_token_trees { - output.push(' '); - } - item.concat_recursive_into(output, behaviour)?; - } - if behaviour.output_array_structure { - output.push(']'); - } - Ok(()) - } -} - -impl ToExpressionValue for ExpressionIteratorInner { - fn to_value(self, span_range: SpanRange) -> ExpressionValue { - ExpressionValue::Iterator(ExpressionIterator { - iterator: self, - span_range, - }) - } -} - -impl ToExpressionValue for Box { - fn to_value(self, span_range: SpanRange) -> ExpressionValue { - ExpressionValue::Iterator(ExpressionIterator::new_custom(self, span_range)) - } -} - -impl HasValueType for ExpressionIterator { - fn value_type(&self) -> &'static str { - "iterator" - } -} - -#[derive(Clone)] -enum ExpressionIteratorInner { - Array( as IntoIterator>::IntoIter), - Stream(::IntoIter), - Other(Box), -} - -impl + Clone + 'static> CustomExpressionIterator for T { - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } -} - -pub(crate) trait CustomExpressionIterator: Iterator { - fn clone_box(&self) -> Box; -} - -impl Clone for Box { - fn clone(&self) -> Self { - (**self).clone_box() - } -} - -impl Iterator for ExpressionIterator { - type Item = ExpressionValue; - - fn next(&mut self) -> Option { - match &mut self.iterator { - ExpressionIteratorInner::Array(iter) => iter.next(), - ExpressionIteratorInner::Stream(iter) => { - let item = iter.next()?; - let span = item.span(); - let stream: OutputStream = item.into(); - Some(stream.coerce_into_value(span.span_range())) - } - ExpressionIteratorInner::Other(iter) => iter.next(), - } - } - - fn size_hint(&self) -> (usize, Option) { - match &self.iterator { - ExpressionIteratorInner::Array(iter) => iter.size_hint(), - ExpressionIteratorInner::Stream(iter) => iter.size_hint(), - ExpressionIteratorInner::Other(iter) => iter.size_hint(), - } - } } diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index 5e331499..c5a6abcf 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -217,6 +217,7 @@ impl_auto_span_range! { syn::BinOp, syn::UnOp, syn::token::DotDot, + syn::token::DotDotEq, syn::token::Shl, syn::token::Shr, syn::token::AndAnd, diff --git a/src/extensions/parsing.rs b/src/extensions/parsing.rs index 1e56edc1..4943a791 100644 --- a/src/extensions/parsing.rs +++ b/src/extensions/parsing.rs @@ -156,6 +156,10 @@ impl<'a, K> ParseStreamStack<'a, K> { } } + pub(crate) fn fork_current(&self) -> ParseBuffer<'_, K> { + self.current().fork() + } + fn current(&self) -> ParseStream<'_, K> { self.group_stack.last().unwrap_or(self.base) } diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index a8a5ea36..045aaa34 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -373,9 +373,6 @@ define_command_enums! { TitleCommand, InsertSpacesCommand, - // Expression Commands - RangeCommand, - // Control flow commands IfCommand, WhileCommand, diff --git a/src/interpretation/commands/expression_commands.rs b/src/interpretation/commands/expression_commands.rs deleted file mode 100644 index 3e2bca2d..00000000 --- a/src/interpretation/commands/expression_commands.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::internal_prelude::*; - -#[derive(Clone)] -pub(crate) struct RangeCommand { - span: Span, - left: SourceExpression, - range_limits: syn::RangeLimits, - right: SourceExpression, -} - -impl CommandType for RangeCommand { - type OutputKind = OutputKindValue; -} - -impl ValueCommandDefinition for RangeCommand { - const COMMAND_NAME: &'static str = "range"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - span: arguments.command_span(), - left: input.parse()?, - range_limits: input.parse()?, - right: input.parse()?, - }) - }, - "Expected a rust range expression such as [!range! 1..4]", - ) - } - - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let range_limits = self.range_limits; - let left = self.left.interpret_to_value(interpreter)?; - let right = self.right.interpret_to_value(interpreter)?; - let range_iterator = left.create_range(right, &range_limits)?; - Ok(range_iterator.to_value(self.span.span_range())) - } -} diff --git a/src/interpretation/commands/mod.rs b/src/interpretation/commands/mod.rs index 8e132cf7..a88b08ee 100644 --- a/src/interpretation/commands/mod.rs +++ b/src/interpretation/commands/mod.rs @@ -1,13 +1,11 @@ mod concat_commands; mod control_flow_commands; mod core_commands; -mod expression_commands; mod token_commands; mod transforming_commands; pub(crate) use concat_commands::*; pub(crate) use control_flow_commands::*; pub(crate) use core_commands::*; -pub(crate) use expression_commands::*; pub(crate) use token_commands::*; pub(crate) use transforming_commands::*; diff --git a/tests/compilation_failures/expressions/large_range_to_string.rs b/tests/compilation_failures/expressions/large_range_to_string.rs index 029a7365..8b966558 100644 --- a/tests/compilation_failures/expressions/large_range_to_string.rs +++ b/tests/compilation_failures/expressions/large_range_to_string.rs @@ -2,6 +2,6 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - #([!range! 0..100000] as string) + #((0..10000) as iterator as string) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/large_range_to_string.stderr b/tests/compilation_failures/expressions/large_range_to_string.stderr index db4f0f76..54e7f415 100644 --- a/tests/compilation_failures/expressions/large_range_to_string.stderr +++ b/tests/compilation_failures/expressions/large_range_to_string.stderr @@ -1,5 +1,5 @@ -error: To protect against infinite loops, only a maximum of 1000 items can be output to a string from an iterator. Try casting `as stream` to avoid this limit. This can't currently be reconfigured with the iteration limit. - --> tests/compilation_failures/expressions/large_range_to_string.rs:5:11 +error: This type is not supported in cast expressions + --> tests/compilation_failures/expressions/large_range_to_string.rs:5:25 | -5 | #([!range! 0..100000] as string) - | ^^^^^^^^^^^^^^^^^^^ +5 | #((0..10000) as iterator as string) + | ^^^^^^^^ diff --git a/tests/control_flow.rs b/tests/control_flow.rs index d2ae64bb..6c0f7fbd 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -68,7 +68,7 @@ fn test_loop_continue_and_break() { ); preinterpret_assert_eq!( { - [!string! [!for! x in [!range! 65..75] { + [!string! [!for! x in 65..75 { [!if! x % 2 == 0 { [!continue!] }] #(x as u8 as char) }]] @@ -81,7 +81,7 @@ fn test_loop_continue_and_break() { fn test_for() { preinterpret_assert_eq!( { - [!string! [!for! x in [!range! 65..70] { + [!string! [!for! x in 65..70 { #(#x as u8 as char) }]] }, diff --git a/tests/expressions.rs b/tests/expressions.rs index 29f3e1f1..380de9de 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -42,7 +42,7 @@ fn test_basic_evaluate_works() { partial_sum = [!stream! + 2]; [!debug! [!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] ), "[!stream! [!group! 5 + 2] = [!group! 7]]"); - preinterpret_assert_eq!(#(1 + [!range! 1..2] as int), 2); + preinterpret_assert_eq!(#(1 + (1..2) as int), 2); preinterpret_assert_eq!(#("hello" == "world"), false); preinterpret_assert_eq!(#("hello" == "hello"), true); preinterpret_assert_eq!(#('A' as u8 == 65), true); @@ -92,7 +92,7 @@ fn test_very_long_expression_works() { [!settings! { iteration_limit: 100000, }] - #(let expression = [!stream! 0] + [!for! _ in [!range! 0..100000] { + 1 }]) + #(let expression = [!stream! 0] + [!for! _ in 0..100000 { + 1 }]) [!reinterpret! [!raw! #](#expression)] }, 100000 @@ -164,14 +164,14 @@ fn assign_works() { fn test_range() { preinterpret_assert_eq!( #([!intersperse! { - items: [!range! -2..5], + items: -2..5, separator: [" "], }] as string), "-2 -1 0 1 2 3 4" ); preinterpret_assert_eq!( #([!intersperse! { - items: [!range! -2..=5], + items: -2..=5, separator: " ", }] as stream as string), "-2 -1 0 1 2 3 4 5" @@ -180,7 +180,7 @@ fn test_range() { { #(x = 2) #([!intersperse! { - items: [!range! (#x + #x)..=5], + items: (x + x)..=5, separator: " ", }] as stream as string) }, @@ -189,22 +189,28 @@ fn test_range() { preinterpret_assert_eq!( { #([!intersperse! { - items: [!range! 8..=5], + items: 8..=5, separator: " ", }] as stream as string) }, "" ); - preinterpret_assert_eq!({ [!string! #([!range! 'a'..='f'] as stream)] }, "abcdef"); + preinterpret_assert_eq!({ [!string! #(('a'..='f') as stream)] }, "abcdef"); preinterpret_assert_eq!( - { [!debug! [!range! -1i8..3i8]] }, - "[ -1i8, 0i8, 1i8, 2i8]" + { [!debug! -1i8..3i8] }, + "-1i8..3i8" ); // Large ranges are allowed, but are subject to limits at iteration time - preinterpret_assert_eq!({ [!debug! [!range! 0..10000]] }, "[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ..<9980 further items>]"); - preinterpret_assert_eq!( - [!for! i in [!range! 0..10000000] { + preinterpret_assert_eq!({ [!debug! 0..10000] }, "0..10000"); + preinterpret_assert_eq!({ [!debug! ..5 + 5] }, "..10"); + preinterpret_assert_eq!({ [!debug! ..=9] }, "..=9"); + preinterpret_assert_eq!({ [!debug! ..] }, ".."); + preinterpret_assert_eq!({ [!debug![.., ..]] }, "[.., ..]"); + preinterpret_assert_eq!({ [!debug![..[1, 2..], ..]] }, "[..[1, 2..], ..]"); + preinterpret_assert_eq!({ [!debug! 4 + 7..=10] }, "11..=10"); + preinterpret_assert_eq!( + [!for! i in 0..10000000 { [!if! i == 5 { [!string! #i] [!break!] @@ -212,4 +218,5 @@ fn test_range() { }], "5" ); + // preinterpret_assert_eq!({ [!debug! 0..10000 as iterator] }, "[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ..<9980 further items>]"); } diff --git a/tests/tokens.rs b/tests/tokens.rs index 4fa6bc4e..01ab51fc 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -178,7 +178,7 @@ fn complex_cases_for_intersperse_and_input_types() { // Command can be used for items preinterpret_assert_eq!( #([!intersperse! { - items: [!range! 0..4], + items: 0..4, separator: [!stream! _], }] as stream as string), "0_1_2_3" @@ -193,7 +193,7 @@ fn complex_cases_for_intersperse_and_input_types() { }, "0_1_2_3"); // Variable containing iterable can be used for items preinterpret_assert_eq!({ - #(let items = [!range! 0..4]) + #(let items = 0..4) #([!intersperse! { items: #items, separator: [!stream! _], From bd40d1f9a11730e05108d2495ad22a674758642b Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 14 Feb 2025 02:25:05 +0000 Subject: [PATCH 094/126] tweak: Update task list --- CHANGELOG.md | 13 ++++++++----- src/expressions/evaluation.rs | 8 ++------ src/expressions/range.rs | 12 +++--------- style-fix.sh | 4 ++-- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce3b3eae..1d156896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,12 +97,11 @@ Inside a transform stream, the following grammar is supported: ### To come -* Support `#(x[..])` syntax for indexing arrays and streams at read time +* Support `#(x[..])` syntax for indexing arrays at read time (streams to follow in a separate task below after parsers are updated) * Via a post-fix `[..]` operation with high precedence - * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) - * Consider supporting ranges in the expression tree and range values - * `#(x[0..3])` returns a TokenStream/Array - * `#(x[0..=3])` returns a TokenStream/Array + * `#(x[0])` returns the item at that position of the array + * `#(x[0..3])` returns an array + * `#(x[0..=3])` returns an array * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Objects: * Backed by an indexmap @@ -142,6 +141,10 @@ Inside a transform stream, the following grammar is supported: * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` +* Support `#(x[..])` syntax for indexing streams + * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) + * `#(x[0..3])` returns a TokenStream + * `#(x[0..=3])` returns a TokenStream * Add `..` and `..x` support to the array destructurer * Consider: * Dropping lots of the `group` wrappers? diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 05b7f8ac..fdc69847 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -86,9 +86,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { match (left, right) { (None, None) => match range_limits { syn::RangeLimits::HalfOpen(token) => { - let inner = ExpressionRangeInner::RangeFull { - token: *token, - }; + let inner = ExpressionRangeInner::RangeFull { token: *token }; NextAction::HandleValue(inner.to_value(token.span_range())) } syn::RangeLimits::Closed(_) => { @@ -105,9 +103,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { (Some(left), right) => { self.operation_stack.push(EvaluationStackFrame::Range { range_limits: *range_limits, - state: RangePath::OnLeftBranch { - right: *right, - }, + state: RangePath::OnLeftBranch { right: *right }, }); NextAction::EnterNode(*left) } diff --git a/src/expressions/range.rs b/src/expressions/range.rs index 602f2a55..12975c9a 100644 --- a/src/expressions/range.rs +++ b/src/expressions/range.rs @@ -258,11 +258,8 @@ impl IterableExpressionRange { } .resolve(output_span_range) } - _ => { - dots.execution_err( - "The range must be between two integers or two characters", - ) - } + _ => dots + .execution_err("The range must be between two integers or two characters"), } } Self::RangeFrom { start, dots } => { @@ -328,10 +325,7 @@ impl IterableExpressionRange { dots, } .resolve(output_span_range), - _ => { - dots - .execution_err("The range must be from an integer or a character") - } + _ => dots.execution_err("The range must be from an integer or a character"), } } } diff --git a/style-fix.sh b/style-fix.sh index 6bba2585..1d1c4a84 100755 --- a/style-fix.sh +++ b/style-fix.sh @@ -4,5 +4,5 @@ set -e cd "$(dirname "$0")" -cargo fmt; -cargo clippy --fix --tests --allow-dirty --allow-staged; \ No newline at end of file +cargo clippy --fix --tests --allow-dirty --allow-staged; +cargo fmt; \ No newline at end of file From 83d0217c37248d10455b41c8418b4b71a1fedce2 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 14 Feb 2025 22:10:59 +0000 Subject: [PATCH 095/126] refactor: Minor refactors to patterns --- CHANGELOG.md | 41 +++++++++++++++++-- src/expressions/expression.rs | 2 +- src/expressions/expression_block.rs | 27 ++++++++---- src/expressions/expression_parsing.rs | 4 +- src/expressions/mod.rs | 12 +++--- .../commands/control_flow_commands.rs | 2 +- src/interpretation/variable.rs | 41 +++++++------------ src/transformation/destructuring.rs | 34 +++++++-------- tests/control_flow.rs | 8 ++-- tests/core.rs | 2 +- tests/expressions.rs | 8 ++-- tests/tokens.rs | 2 +- 12 files changed, 110 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d156896..68d3ab71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,26 @@ Inside a transform stream, the following grammar is supported: * `#(x[0])` returns the item at that position of the array * `#(x[0..3])` returns an array * `#(x[0..=3])` returns an array + * ... and `#(x[0] = y)` can be used to set the item + * Considering allowing assignments inside an expression + * `let XX =` and `YY += y` are actually totally different... + * With `let XX =`, `XX` is a _pattern_ and creates new variables. + * With `YY += y` we're inside an expression already, and it only works with existing variables. The expression `YY` is converted into a destructuring at execution time. + Note that the value side always executes first, before the place is executed. + * Test cases: +```rust + // These examples should compile: + let mut a = [0; 5]; + let mut b: u32 = 3; + let c; + let out = c = (a[2], _) = (4, 5); + let out = a[1] += 2; + let out = b = 2; + // In the below, a = [0, 5], showing the right side executes first + let mut a = [0; 2]; + let mut b = 0; + a[b] += { b += 1; 5 }; +``` * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Objects: * Backed by an indexmap @@ -119,13 +139,27 @@ Inside a transform stream, the following grammar is supported: * When we parse a `#x` (`@(x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. * Output to final output as unwrapped content * Method calls + * Mutable methods notes: + * They require either: + * Reference semantics (e.g. using `Rc>` inside Object, Array) with explicit cloning + * AND/OR Place semantics (e.g. each type supports being either a value or a reference to a path, so that an operator e.g. `+` can mutate) + * I think we want reference semantics for object and array anyway. Unclear for stream. + * Ideally we'd arrange it so that `x += ["Hello"] + ["World]` would append Hello and World; but `x += (["Hello"] + ["World])` would behave differently. + * I think that means that `+=` becomes an operator inside an expression, and its LHS is a `PlaceValue` (`PlaceValue::Stream` or `PlaceValue::Array`) * Also add support for methods (for e.g. exposing functions on syn objects). * `.len()` on stream * `.push(x)` on array * Consider `.map(|| {})` * TRANSFORMERS => PARSERS cont - * Manually search for transform and rename to parse in folder names and file - * Support `@[x = ...]` for individual parsers. Parsers no longer output to a stream past that. + * Manually search for transform and rename to parse in folder names and file. + * Parsers no longer output to a stream past that. + Instead, they act like a `StreamPattern` which needs to: + * Define the variables it binds up front `{ x, y }` + * Can't mutate any variables in ancestor frames (but can potentially read them) +```rust,ignore +@{ x, y }(... destructuring ...) +``` + * Support `@[x = ...]` and `@[let x = ...]` for individual parsers. * Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` * Scrap `#>>x` etc in favour of `@(a = ...) #[x += [a]]` * `@TOKEN_TREE` @@ -148,6 +182,7 @@ Inside a transform stream, the following grammar is supported: * Add `..` and `..x` support to the array destructurer * Consider: * Dropping lots of the `group` wrappers? + * If any types should have reference semantics instead of clone/value semantics? * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` * Adding `!define_command!` @@ -160,7 +195,7 @@ Inside a transform stream, the following grammar is supported: * Support a CastTarget of `array` (only supported for array and stream and iterator) * Add `as ident` and `as literal` casting and support it for string, array and stream using concat recursive. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` -* Put `[!set! ...]` inside an opt-in feature. +* Put `[!set! ...]` inside an opt-in feature because it's quite confusing. * TODO check * Check all `#[allow(unused)]` and remove any which aren't needed * Add benches inspired by this: https://github.com/dtolnay/quote/tree/master/benches diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 5ff92c3f..38f7918e 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -29,7 +29,7 @@ impl InterpretToValue for &SourceExpression { pub(super) enum SourceExpressionLeaf { Command(Command), - VariablePath(VariablePath), + VariablePath(VariableOrField), Variable(GroupedVariable), ExpressionBlock(ExpressionBlock), Value(ExpressionValue), diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index ef2965a7..d1a08a70 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -155,6 +155,18 @@ impl InterpretToValue for &Statement { } } +/// In a rust expression, assignments are allowed in the middle of an expression. +/// +/// But the following is rather hard to parse in a streaming manner, +/// due to ambiguity and right-associativity of = +/// ```rust,ignore +/// let a; +/// let b; +/// // When the = (4,) is revealed, the `(b,)` changes from a value to a destructuring +/// let out = a = (b,) = (4,); +/// // When the += 2 is revealed, b changes from a value to a place +/// let out = b += 2; +/// ``` #[derive(Clone)] pub(crate) struct AssignmentStatement { destination: Destination, @@ -181,13 +193,10 @@ impl InterpretToValue for &AssignmentStatement { interpreter: &mut Interpreter, ) -> ExecutionResult { match &self.destination { - Destination::LetBinding { - let_token, - destructuring, - } => { + Destination::LetBinding { let_token, pattern } => { let value = self.expression.interpret_to_value(interpreter)?; let mut span_range = value.span_range(); - destructuring.handle_destructure(interpreter, value)?; + pattern.handle_destructure(interpreter, value)?; span_range.set_start(let_token.span); Ok(ExpressionValue::None(span_range)) } @@ -197,6 +206,8 @@ impl InterpretToValue for &AssignmentStatement { let right = self.expression.interpret_to_value(interpreter)?; operation.evaluate(left, right)? } else { + // First, check path already exists + let _ = path.get_value(interpreter)?; self.expression.interpret_to_value(interpreter)? }; let mut span_range = value.span_range(); @@ -212,10 +223,10 @@ impl InterpretToValue for &AssignmentStatement { enum Destination { LetBinding { let_token: Token![let], - destructuring: Destructuring, + pattern: Pattern, }, ExistingVariable { - path: VariablePath, + path: VariableOrField, operation: Option, }, } @@ -226,7 +237,7 @@ impl Parse for Destination { let let_token = input.parse()?; Ok(Self::LetBinding { let_token, - destructuring: input.parse()?, + pattern: input.parse()?, }) } else { Ok(Self::ExistingVariable { diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 4a113652..82e637d6 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -297,7 +297,9 @@ impl ExpressionNodes { enum OperatorPrecendence { // return, break, closures Jump, - // In arrays (this is a preinterpret addition) + // [PREINTERPRET ADDITION] + // Assignment should bind more tightly than arrays. + // e.g. [x = 3, 2] should parse as [(x = 3), 2] rather than [x = (3, 2)] NonTerminalComma, /// = += -= *= /= %= &= |= ^= <<= >>= Assign, diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 13654a59..6a16242f 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -14,6 +14,12 @@ mod stream; mod string; mod value; +pub(crate) use expression::*; +pub(crate) use expression_block::*; +pub(crate) use iterator::*; +pub(crate) use stream::*; +pub(crate) use value::*; + // Marked as use for expression sub-modules to use with a `use super::*` statement use crate::internal_prelude::*; use array::*; @@ -26,9 +32,3 @@ use integer::*; use operations::*; use range::*; use string::*; - -pub(crate) use expression::*; -pub(crate) use expression_block::*; -pub(crate) use iterator::*; -pub(crate) use stream::*; -pub(crate) use value::*; diff --git a/src/interpretation/commands/control_flow_commands.rs b/src/interpretation/commands/control_flow_commands.rs index 4ea48163..44993341 100644 --- a/src/interpretation/commands/control_flow_commands.rs +++ b/src/interpretation/commands/control_flow_commands.rs @@ -184,7 +184,7 @@ impl StreamCommandDefinition for LoopCommand { #[derive(Clone)] pub(crate) struct ForCommand { - destructuring: Destructuring, + destructuring: Pattern, #[allow(unused)] in_token: Token![in], input: SourceExpression, diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 4e2547d1..45891ab1 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -240,42 +240,31 @@ impl core::fmt::Display for FlattenedVariable { // An identifier for a variable path in an expression #[derive(Clone)] -pub(crate) struct VariablePath { - root: Ident, - fields: Vec<(Token![.], Ident)>, +pub(crate) struct VariableOrField { + ident: Ident, } -impl Parse for VariablePath { +impl Parse for VariableOrField { fn parse(input: ParseStream) -> ParseResult { Ok(Self { - root: input.parse()?, - fields: { - let mut fields = vec![]; - while input.peek(Token![.]) { - fields.push((input.parse()?, input.parse()?)); - } - fields - }, + ident: input.parse()?, }) } } -impl IsVariable for VariablePath { +impl IsVariable for VariableOrField { fn get_name(&self) -> String { - self.root.to_string() + self.ident.to_string() } } -impl HasSpanRange for VariablePath { - fn span_range(&self) -> SpanRange { - match self.fields.last() { - Some((_, ident)) => SpanRange::new_between(self.root.span(), ident.span()), - None => self.root.span_range(), - } +impl HasSpan for VariableOrField { + fn span(&self) -> Span { + self.ident.span() } } -impl InterpretToValue for &VariablePath { +impl InterpretToValue for &VariableOrField { type OutputValue = ExpressionValue; fn interpret_to_value( @@ -287,11 +276,11 @@ impl InterpretToValue for &VariablePath { } #[derive(Clone)] -pub(crate) struct VariableDestructuring { +pub(crate) struct VariablePattern { name: Ident, } -impl Parse for VariableDestructuring { +impl Parse for VariablePattern { fn parse(input: ParseStream) -> ParseResult { Ok(Self { name: input.parse()?, @@ -299,19 +288,19 @@ impl Parse for VariableDestructuring { } } -impl IsVariable for VariableDestructuring { +impl IsVariable for VariablePattern { fn get_name(&self) -> String { self.name.to_string() } } -impl HasSpan for VariableDestructuring { +impl HasSpan for VariablePattern { fn span(&self) -> Span { self.name.span() } } -impl HandleDestructure for VariableDestructuring { +impl HandleDestructure for VariablePattern { fn handle_destructure( &self, interpreter: &mut Interpreter, diff --git a/src/transformation/destructuring.rs b/src/transformation/destructuring.rs index 0a156b2f..faa72709 100644 --- a/src/transformation/destructuring.rs +++ b/src/transformation/destructuring.rs @@ -9,25 +9,25 @@ pub(crate) trait HandleDestructure { } #[derive(Clone)] -pub(crate) enum Destructuring { - Variable(VariableDestructuring), - Array(ArrayDestructuring), +pub(crate) enum Pattern { + Variable(VariablePattern), + Array(ArrayPattern), Stream(ExplicitTransformStream), #[allow(unused)] Discarded(Token![_]), } -impl Parse for Destructuring { +impl Parse for Pattern { fn parse(input: ParseStream) -> ParseResult { let lookahead = input.lookahead1(); if lookahead.peek(syn::Ident) { - Ok(Destructuring::Variable(input.parse()?)) + Ok(Pattern::Variable(input.parse()?)) } else if lookahead.peek(syn::token::Bracket) { - Ok(Destructuring::Array(input.parse()?)) + Ok(Pattern::Array(input.parse()?)) } else if lookahead.peek(Token![@]) { - Ok(Destructuring::Stream(input.parse()?)) + Ok(Pattern::Stream(input.parse()?)) } else if lookahead.peek(Token![_]) { - Ok(Destructuring::Discarded(input.parse()?)) + Ok(Pattern::Discarded(input.parse()?)) } else if input.peek(Token![#]) { return input.parse_err("Use `var` instead of `#var` in a destructuring"); } else { @@ -36,29 +36,29 @@ impl Parse for Destructuring { } } -impl HandleDestructure for Destructuring { +impl HandleDestructure for Pattern { fn handle_destructure( &self, interpreter: &mut Interpreter, value: ExpressionValue, ) -> ExecutionResult<()> { match self { - Destructuring::Variable(variable) => variable.handle_destructure(interpreter, value), - Destructuring::Array(array) => array.handle_destructure(interpreter, value), - Destructuring::Stream(stream) => stream.handle_destructure(interpreter, value), - Destructuring::Discarded(_) => Ok(()), + Pattern::Variable(variable) => variable.handle_destructure(interpreter, value), + Pattern::Array(array) => array.handle_destructure(interpreter, value), + Pattern::Stream(stream) => stream.handle_destructure(interpreter, value), + Pattern::Discarded(_) => Ok(()), } } } #[derive(Clone)] -pub struct ArrayDestructuring { +pub struct ArrayPattern { #[allow(unused)] delim_span: DelimSpan, - items: Punctuated, + items: Punctuated, } -impl Parse for ArrayDestructuring { +impl Parse for ArrayPattern { fn parse(input: ParseStream) -> ParseResult { let (delim_span, inner) = input.parse_specific_group(Delimiter::Bracket)?; Ok(Self { @@ -68,7 +68,7 @@ impl Parse for ArrayDestructuring { } } -impl HandleDestructure for ArrayDestructuring { +impl HandleDestructure for ArrayPattern { fn handle_destructure( &self, interpreter: &mut Interpreter, diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 6c0f7fbd..7580e66f 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -16,11 +16,11 @@ fn test_control_flow_compilation_failures() { fn test_if() { preinterpret_assert_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); preinterpret_assert_eq!({ - #(x = 1 == 2) + #(let x = 1 == 2) [!if! #x { "YES" } !else! { "NO" }] }, "NO"); preinterpret_assert_eq!({ - #(x = 1; y = 2) + #(let x = 1; let y = 2) [!if! #x == #y { "YES" } !else! { "NO" }] }, "NO"); preinterpret_assert_eq!({ @@ -47,7 +47,7 @@ fn test_if() { #[test] fn test_while() { preinterpret_assert_eq!({ - #(x = 0) + #(let x = 0) [!while! #x < 5 { #(x += 1) }] #x }, 5); @@ -57,7 +57,7 @@ fn test_while() { fn test_loop_continue_and_break() { preinterpret_assert_eq!( { - #(x = 0) + #(let x = 0) [!loop! { #(x += 1) [!if! x >= 10 { [!break!] }] diff --git a/tests/core.rs b/tests/core.rs index de82149a..80c4b009 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -46,7 +46,7 @@ fn test_extend() { ); preinterpret_assert_eq!( { - #(i = 1) + #(let i = 1) [!set! #output =] [!while! i <= 4 { [!set! #output += #i] diff --git a/tests/expressions.rs b/tests/expressions.rs index 380de9de..63607ec0 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -37,9 +37,9 @@ fn test_basic_evaluate_works() { preinterpret_assert_eq!(#(123 != 456), true); preinterpret_assert_eq!(#(123 >= 456), false); preinterpret_assert_eq!(#(123 > 456), false); - preinterpret_assert_eq!(#(six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); + preinterpret_assert_eq!(#(let six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); preinterpret_assert_eq!(#( - partial_sum = [!stream! + 2]; + let partial_sum = [!stream! + 2]; [!debug! [!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] ), "[!stream! [!group! 5 + 2] = [!group! 7]]"); preinterpret_assert_eq!(#(1 + (1..2) as int), 2); @@ -122,7 +122,7 @@ fn boolean_operators_short_circuit() { // For comparison, the & operator does _not_ short-circuit preinterpret_assert_eq!( #( - is_lazy = true; + let is_lazy = true; let _ = false & #(is_lazy = false; true); #is_lazy ), @@ -178,7 +178,7 @@ fn test_range() { ); preinterpret_assert_eq!( { - #(x = 2) + #(let x = 2) #([!intersperse! { items: (x + x)..=5, separator: " ", diff --git a/tests/tokens.rs b/tests/tokens.rs index 01ab51fc..f03d88a4 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -414,7 +414,7 @@ fn test_zip_with_for() { preinterpret_assert_eq!( { [!set! #countries = France Germany Italy] - #(flags = ["🇫🇷", "🇩🇪", "🇮🇹"]) + #(let flags = ["🇫🇷", "🇩🇪", "🇮🇹"]) [!set! #capitals = "Paris" "Berlin" "Rome"] [!set! #facts = [!for! [country, flag, capital] in [!zip! [countries, flags, capitals]] { [!string! "=> The capital of " #country " is " #capital " and its flag is " #flag] From 2defae9bc5daeb2194da5c3b5551bf39cb9a571f Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 14 Feb 2025 22:21:03 +0000 Subject: [PATCH 096/126] fix: Fix compilation error test --- .../expressions/code_blocks_are_not_reevaluated.rs | 11 +++++++---- .../code_blocks_are_not_reevaluated.stderr | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs index 73c69dbc..b5541945 100644 --- a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs @@ -2,9 +2,12 @@ use preinterpret::*; fn main() { let _ = preinterpret!{ - // We don't get a re-evaluation. Instead, we get a parse error, because we end up - // with let _ = [!error! "This was a re-evaluation"]; which is a parse error in - // normal rust land. - #(indirect = [!raw! [!error! "This was a re-evaluation"]]; #(indirect)) + #( + let indirect = [!raw! [!error! "This was a re-evaluation"]]; + // We don't get a re-evaluation. Instead, we get a parse error, because we end up + // with let _ = [!error! "This was a re-evaluation"]; which is a parse error in + // normal rust land. + #(indirect) + ) }; } \ No newline at end of file diff --git a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr index e9924386..d561ddf6 100644 --- a/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr +++ b/tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.stderr @@ -1,5 +1,5 @@ error: expected one of `(`, `[`, or `{`, found `"This was a re-evaluation"` - --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:8:38 + --> tests/compilation_failures/expressions/code_blocks_are_not_reevaluated.rs:6:44 | -8 | #(indirect = [!raw! [!error! "This was a re-evaluation"]]; #(indirect)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected one of `(`, `[`, or `{` +6 | let indirect = [!raw! [!error! "This was a re-evaluation"]]; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected one of `(`, `[`, or `{` From 517f5243b0c9171d8326476860b085e98d4a17bf Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 15 Feb 2025 22:55:10 +0000 Subject: [PATCH 097/126] refactor: Assignments are inside expressions --- CHANGELOG.md | 34 +++++ src/expressions/evaluation.rs | 175 +++++++++++++++++++++++--- src/expressions/expression.rs | 70 ++++++++--- src/expressions/expression_block.rs | 128 ++++--------------- src/expressions/expression_parsing.rs | 60 ++++++++- src/expressions/operations.rs | 156 +++++++++++++++++++++++ src/extensions/errors_and_spans.rs | 10 ++ src/interpretation/variable.rs | 10 +- tests/control_flow.rs | 3 +- 9 files changed, 500 insertions(+), 146 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68d3ab71..05c687a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,6 +108,19 @@ Inside a transform stream, the following grammar is supported: * With `let XX =`, `XX` is a _pattern_ and creates new variables. * With `YY += y` we're inside an expression already, and it only works with existing variables. The expression `YY` is converted into a destructuring at execution time. Note that the value side always executes first, before the place is executed. + * Implementation realisations: + * Rust reference very good: https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions + * For the += operators etc: + * The left hand side must resolve to a _single_ VariableData, e.g. `z` or + `x.y["z"]` + * In the rust reference, this is a "Place Expression" + * We will have a `handle_assign_paired_binary_operation(&mut self, operation, other: Self)` (which for value types can resolve to the non-assign version) + * For the = operator: + * The left hand side will resolve non-Variable expressions, and be effectively + left with something which can form a destructuring of the right hand side. + e.g. `[x.y, z[x.a][12]] = [1, 2]` + * In the rust reference, this is an "Assignee Expression" and is a generalization + of place expressions * Test cases: ```rust // These examples should compile: @@ -121,6 +134,27 @@ Inside a transform stream, the following grammar is supported: let mut a = [0; 2]; let mut b = 0; a[b] += { b += 1; 5 }; + // This works: + let (x, y); + [x, .., y] = [1, 2, 3, 4]; // x = 1, y = 4 + // This works... + // In other words, the assignee operation is executed incrementally, + // The first assignment arr[0] = 1 occurs before being overwritten by + // the arr[0] = 5 in the second section. + let mut arr = [0; 2]; + (arr[0], arr[{arr[0] = 5; 1}]) = (1, 1); + assert_eq!(arr, [5, 1]); + // This doesn't work - two errors: + // error[E0308]: mismatched types: expected `[{integer}]`, found `[{integer}; 4]` + // error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + // (it looks like you can't just overwrite array subslices) + let arr = [0; 5]; + arr[1..=4] = [1, 2, 3, 4]; + // This doesn't work. + // error[E0368]: binary assignment operation `+=` cannot be applied to type `({integer}, {integer})` + // https://doc.rust-lang.org/error_codes/E0368.html + let (a, b) = (1, 1); + (a, b) += (1, 2); ``` * Variable typing (stream / value / object to start with), including an `object` type, like a JS object: * Objects: diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index fdc69847..e278bac9 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -5,8 +5,8 @@ pub(super) struct ExpressionEvaluator<'a, K: Expressionable> { operation_stack: Vec, } -impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { - pub(super) fn new(nodes: &'a [ExpressionNode]) -> Self { +impl<'a> ExpressionEvaluator<'a, Source> { + pub(super) fn new(nodes: &'a [ExpressionNode]) -> Self { Self { nodes, operation_stack: Vec::new(), @@ -16,9 +16,9 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { pub(super) fn evaluate( mut self, root: ExpressionNodeId, - evaluation_context: &mut K::EvaluationContext, + interpreter: &mut Interpreter, ) -> ExecutionResult { - let mut next = self.begin_node_evaluation(root, evaluation_context)?; + let mut next = self.begin_node_evaluation(root, interpreter)?; loop { match next { @@ -27,10 +27,10 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { Some(top) => top, None => return Ok(value), }; - next = self.continue_node_evaluation(top_of_stack, value)?; + next = self.handle_value(top_of_stack, value, interpreter)?; } - NextAction::EnterNode(next_node) => { - next = self.begin_node_evaluation(next_node, evaluation_context)?; + NextAction::EnterValueNode(next_node) => { + next = self.begin_node_evaluation(next_node, interpreter)?; } } } @@ -39,17 +39,17 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { fn begin_node_evaluation( &mut self, node_id: ExpressionNodeId, - evaluation_context: &mut K::EvaluationContext, + interpreter: &mut Interpreter, ) -> ExecutionResult { Ok(match &self.nodes[node_id.0] { ExpressionNode::Leaf(leaf) => { - NextAction::HandleValue(K::evaluate_leaf(leaf, evaluation_context)?) + NextAction::HandleValue(Source::evaluate_leaf(leaf, interpreter)?) } ExpressionNode::Grouped { delim_span, inner } => { self.operation_stack.push(EvaluationStackFrame::Group { span: delim_span.join(), }); - NextAction::EnterNode(*inner) + NextAction::EnterValueNode(*inner) } ExpressionNode::Array { delim_span, items } => ArrayStackFrame { span: delim_span.join(), @@ -62,7 +62,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { .push(EvaluationStackFrame::UnaryOperation { operation: operation.clone(), }); - NextAction::EnterNode(*input) + NextAction::EnterValueNode(*input) } ExpressionNode::BinaryOperation { operation, @@ -76,7 +76,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { right: *right_input, }, }); - NextAction::EnterNode(*left_input) + NextAction::EnterValueNode(*left_input) } ExpressionNode::Range { left, @@ -98,24 +98,49 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { range_limits: *range_limits, state: RangePath::OnRightBranch { left: None }, }); - NextAction::EnterNode(*right) + NextAction::EnterValueNode(*right) } (Some(left), right) => { self.operation_stack.push(EvaluationStackFrame::Range { range_limits: *range_limits, state: RangePath::OnLeftBranch { right: *right }, }); - NextAction::EnterNode(*left) + NextAction::EnterValueNode(*left) } } } + ExpressionNode::Assignment { + assignee, + equals_token, + value, + } => { + self.operation_stack + .push(EvaluationStackFrame::AssignmentValue { + assignee: *assignee, + equals_token: *equals_token, + }); + NextAction::EnterValueNode(*value) + } + ExpressionNode::CompoundAssignment { + place, + operation, + value, + } => { + self.operation_stack + .push(EvaluationStackFrame::CompoundAssignmentValue { + place: *place, + operation: *operation, + }); + NextAction::EnterValueNode(*value) + } }) } - fn continue_node_evaluation( + fn handle_value( &mut self, top_of_stack: EvaluationStackFrame, value: ExpressionValue, + interpreter: &mut Interpreter, ) -> ExecutionResult { Ok(match top_of_stack { EvaluationStackFrame::Group { span } => NextAction::HandleValue(value.with_span(span)), @@ -136,7 +161,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { operation, state: BinaryPath::OnRightBranch { left: value }, }); - NextAction::EnterNode(right) + NextAction::EnterValueNode(right) } } BinaryPath::OnRightBranch { left } => { @@ -153,7 +178,7 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { range_limits, state: RangePath::OnRightBranch { left: Some(value) }, }); - NextAction::EnterNode(right) + NextAction::EnterValueNode(right) } (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::HalfOpen(token)) => { let inner = ExpressionRangeInner::RangeFrom { @@ -204,13 +229,117 @@ impl<'a, K: Expressionable> ExpressionEvaluator<'a, K> { NextAction::HandleValue(inner.to_value(token.span_range())) } }, + EvaluationStackFrame::AssignmentValue { + assignee, + equals_token, + } => { + let span_range = + self.handle_assignment(assignee, equals_token, value, interpreter)?; + NextAction::HandleValue(ExpressionValue::None(span_range)) + } + EvaluationStackFrame::CompoundAssignmentValue { place, operation } => { + let span_range = + self.handle_compound_assignment(place, operation, value, interpreter)?; + NextAction::HandleValue(ExpressionValue::None(span_range)) + } + }) + } + + fn handle_assignment( + &mut self, + assignee: ExpressionNodeId, + equals_token: Token![=], + value: ExpressionValue, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + // TODO: When we add place resolution, we likely wish to make use of the execution stack. + let assignee = self.resolve_assignee_or_place(assignee)?; + Ok(match assignee { + Assignee::Place(place) => { + let span_range = SpanRange::new_between( + place.variable.span_range().start(), + value.span_range().end(), + ); + place.variable.set_value(interpreter, value)?; + span_range + } + Assignee::CompoundWip => { + return equals_token.execution_err("Compound assignment is not yet supported") + } }) } + + fn handle_compound_assignment( + &mut self, + place: ExpressionNodeId, + operation: CompoundAssignmentOperation, + value: ExpressionValue, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + // TODO: When we add place resolution, we likely wish to make use of the execution stack. + let assignee = self.resolve_assignee_or_place(place)?; + Ok(match assignee { + Assignee::Place(place) => { + let span_range = SpanRange::new_between( + place.variable.span_range().start(), + value.span_range().end(), + ); + let left = place.variable.interpret_to_value(interpreter)?; + place + .variable + .set_value(interpreter, operation.to_binary().evaluate(left, value)?)?; + span_range + } + Assignee::CompoundWip => { + return operation + .execution_err("Compound values are not supported for operation assignment") + } + }) + } + + /// See the [rust reference] for a good description of assignee vs place. + /// + /// [rust reference]: https://doc.rust-lang.org/reference/expressions/assignment-expressions.html#assignee-vs-place + fn resolve_assignee_or_place( + &self, + mut assignee: ExpressionNodeId, + ) -> ExecutionResult> { + let resolved = loop { + match &self.nodes[assignee.0] { + ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => { + break Assignee::Place(Place { variable }); + } + ExpressionNode::Array { .. } => { + break Assignee::CompoundWip; + } + ExpressionNode::Grouped { inner, .. } => { + assignee = *inner; + continue; + } + other => { + return other + .operator_span_range() + .execution_err("This type of expression is not supported as an assignee"); + } + }; + }; + Ok(resolved) + } +} + +enum Assignee<'a> { + Place(Place<'a>), + CompoundWip, // To come +} + +struct Place<'a> { + variable: &'a VariableIdentifier, + // To come: Array index, field access, etc. } enum NextAction { HandleValue(ExpressionValue), - EnterNode(ExpressionNodeId), + EnterValueNode(ExpressionNodeId), } enum EvaluationStackFrame { @@ -229,6 +358,14 @@ enum EvaluationStackFrame { range_limits: syn::RangeLimits, state: RangePath, }, + AssignmentValue { + assignee: ExpressionNodeId, + equals_token: Token![=], + }, + CompoundAssignmentValue { + place: ExpressionNodeId, + operation: CompoundAssignmentOperation, + }, } struct ArrayStackFrame { @@ -246,7 +383,7 @@ impl ArrayStackFrame { { Some(next) => { operation_stack.push(EvaluationStackFrame::Array(self)); - NextAction::EnterNode(next) + NextAction::EnterValueNode(next) } None => NextAction::HandleValue(ExpressionValue::Array(ExpressionArray { items: self.evaluated_items, diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 38f7918e..9794fa31 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -23,18 +23,30 @@ impl InterpretToValue for &SourceExpression { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - Source::evaluate(&self.inner, interpreter) + ExpressionEvaluator::new(&self.inner.nodes).evaluate(self.inner.root, interpreter) } } pub(super) enum SourceExpressionLeaf { Command(Command), - VariablePath(VariableOrField), - Variable(GroupedVariable), + Variable(VariableIdentifier), + MarkedVariable(GroupedVariable), ExpressionBlock(ExpressionBlock), Value(ExpressionValue), } +impl HasSpanRange for SourceExpressionLeaf { + fn span_range(&self) -> SpanRange { + match self { + SourceExpressionLeaf::Command(command) => command.span_range(), + SourceExpressionLeaf::Variable(variable) => variable.span_range(), + SourceExpressionLeaf::MarkedVariable(variable) => variable.span_range(), + SourceExpressionLeaf::ExpressionBlock(block) => block.span_range(), + SourceExpressionLeaf::Value(value) => value.span_range(), + } + } +} + impl Expressionable for Source { type Leaf = SourceExpressionLeaf; type EvaluationContext = Interpreter; @@ -43,7 +55,7 @@ impl Expressionable for Source { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), SourcePeekMatch::Variable(Grouping::Grouped) => { - UnaryAtom::Leaf(Self::Leaf::Variable(input.parse()?)) + UnaryAtom::Leaf(Self::Leaf::MarkedVariable(input.parse()?)) } SourcePeekMatch::Variable(Grouping::Flattened) => { return input.parse_err( @@ -88,7 +100,7 @@ impl Expressionable for Source { Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( ExpressionBoolean::for_litbool(bool), ))), - Err(_) => UnaryAtom::Leaf(Self::Leaf::VariablePath(input.parse()?)), + Err(_) => UnaryAtom::Leaf(Self::Leaf::Variable(input.parse()?)), }, SourcePeekMatch::Literal(_) => { let value = ExpressionValue::for_syn_lit(input.parse()?); @@ -122,12 +134,18 @@ impl Expressionable for Source { } } SourcePeekMatch::Punct(_) => { + if let Ok(operation) = input.try_parse_or_revert() { + return Ok(NodeExtension::CompoundAssignmentOperation(operation)); + } if let Ok(operation) = input.try_parse_or_revert() { return Ok(NodeExtension::BinaryOperation(operation)); } if let Ok(range_limits) = input.try_parse_or_revert() { return Ok(NodeExtension::Range(range_limits)); } + if let Ok(eq) = input.try_parse_or_revert() { + return Ok(NodeExtension::AssignmentOperation(eq)); + } } SourcePeekMatch::Ident(ident) if ident == "as" => { let cast_operation = @@ -148,7 +166,9 @@ impl Expressionable for Source { // There's nothing matching, so we fall through to an EndOfFrame ExpressionStackFrame::IncompleteUnaryPrefixOperation { .. } | ExpressionStackFrame::IncompleteBinaryOperation { .. } - | ExpressionStackFrame::IncompleteRange { .. } => { + | ExpressionStackFrame::IncompleteRange { .. } + | ExpressionStackFrame::IncompleteAssignment { .. } + | ExpressionStackFrame::IncompleteCompoundAssignment { .. } => { Ok(NodeExtension::NoValidExtensionForCurrentParent) } } @@ -162,8 +182,10 @@ impl Expressionable for Source { SourceExpressionLeaf::Command(command) => { command.clone().interpret_to_value(interpreter)? } - SourceExpressionLeaf::Variable(variable) => variable.interpret_to_value(interpreter)?, - SourceExpressionLeaf::VariablePath(variable_path) => { + SourceExpressionLeaf::MarkedVariable(variable) => { + variable.interpret_to_value(interpreter)? + } + SourceExpressionLeaf::Variable(variable_path) => { variable_path.interpret_to_value(interpreter)? } SourceExpressionLeaf::ExpressionBlock(block) => { @@ -224,6 +246,31 @@ pub(super) enum ExpressionNode { range_limits: syn::RangeLimits, right: Option, }, + Assignment { + assignee: ExpressionNodeId, + equals_token: Token![=], + value: ExpressionNodeId, + }, + CompoundAssignment { + place: ExpressionNodeId, + operation: CompoundAssignmentOperation, + value: ExpressionNodeId, + }, +} + +impl ExpressionNode { + pub(super) fn operator_span_range(&self) -> SpanRange { + match self { + ExpressionNode::Leaf(leaf) => leaf.span_range(), + ExpressionNode::Grouped { delim_span, .. } => delim_span.span_range(), + ExpressionNode::Array { delim_span, .. } => delim_span.span_range(), + ExpressionNode::UnaryOperation { operation, .. } => operation.span_range(), + ExpressionNode::BinaryOperation { operation, .. } => operation.span_range(), + ExpressionNode::Range { range_limits, .. } => range_limits.span_range(), + ExpressionNode::Assignment { equals_token, .. } => equals_token.span_range(), + ExpressionNode::CompoundAssignment { operation, .. } => operation.span_range(), + } + } } pub(super) trait Expressionable: Sized { @@ -240,11 +287,4 @@ pub(super) trait Expressionable: Sized { leaf: &Self::Leaf, context: &mut Self::EvaluationContext, ) -> ExecutionResult; - - fn evaluate( - expression: &Expression, - context: &mut Self::EvaluationContext, - ) -> ExecutionResult { - ExpressionEvaluator::new(&expression.nodes).evaluate(expression.root, context) - } } diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index d1a08a70..be454a83 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -102,41 +102,16 @@ impl InterpretToValue for &ExpressionBlock { #[derive(Clone)] pub(crate) enum Statement { - Assignment(AssignmentStatement), + LetStatement(LetStatement), Expression(SourceExpression), } impl Parse for Statement { fn parse(input: ParseStream) -> ParseResult { - Ok(match input.cursor().ident() { - // let or some ident for a variable - // It may be the start of an assignment. - Some((ident, _)) - if { - let str = ident.to_string(); - str != "true" && str != "false" - } => - { - let forked = input.fork(); - match forked.call(|input| { - let destination = input.parse()?; - let equals = input.parse()?; - Ok((destination, equals)) - }) { - Ok((destination, equals)) => { - input.advance_to(&forked); - // We commit to the fork after successfully parsing the destination and equals. - // This gives better error messages, if there is an error in the expression itself. - Statement::Assignment(AssignmentStatement { - destination, - equals, - expression: input.parse()?, - }) - } - Err(_) => Statement::Expression(input.parse()?), - } - } - _ => Statement::Expression(input.parse()?), + Ok(if input.peek(Token![let]) { + Statement::LetStatement(input.parse()?) + } else { + Statement::Expression(input.parse()?) }) } } @@ -149,7 +124,7 @@ impl InterpretToValue for &Statement { interpreter: &mut Interpreter, ) -> ExecutionResult { match self { - Statement::Assignment(assignment) => assignment.interpret_to_value(interpreter), + Statement::LetStatement(assignment) => assignment.interpret_to_value(interpreter), Statement::Expression(expression) => expression.interpret_to_value(interpreter), } } @@ -168,97 +143,42 @@ impl InterpretToValue for &Statement { /// let out = b += 2; /// ``` #[derive(Clone)] -pub(crate) struct AssignmentStatement { - destination: Destination, +pub(crate) struct LetStatement { + let_token: Token![let], + pattern: Pattern, #[allow(unused)] equals: Token![=], expression: SourceExpression, } -impl Parse for AssignmentStatement { +impl Parse for LetStatement { fn parse(input: ParseStream) -> ParseResult { Ok(Self { - destination: input.parse()?, + let_token: input.parse()?, + pattern: input.parse()?, equals: input.parse()?, expression: input.parse()?, }) } } -impl InterpretToValue for &AssignmentStatement { +impl InterpretToValue for &LetStatement { type OutputValue = ExpressionValue; fn interpret_to_value( self, interpreter: &mut Interpreter, ) -> ExecutionResult { - match &self.destination { - Destination::LetBinding { let_token, pattern } => { - let value = self.expression.interpret_to_value(interpreter)?; - let mut span_range = value.span_range(); - pattern.handle_destructure(interpreter, value)?; - span_range.set_start(let_token.span); - Ok(ExpressionValue::None(span_range)) - } - Destination::ExistingVariable { path, operation } => { - let value = if let Some(operation) = operation { - let left = path.interpret_to_value(interpreter)?; - let right = self.expression.interpret_to_value(interpreter)?; - operation.evaluate(left, right)? - } else { - // First, check path already exists - let _ = path.get_value(interpreter)?; - self.expression.interpret_to_value(interpreter)? - }; - let mut span_range = value.span_range(); - path.set_value(interpreter, value)?; - span_range.set_start(path.span_range().start()); - Ok(ExpressionValue::None(span_range)) - } - } - } -} - -#[derive(Clone)] -enum Destination { - LetBinding { - let_token: Token![let], - pattern: Pattern, - }, - ExistingVariable { - path: VariableOrField, - operation: Option, - }, -} - -impl Parse for Destination { - fn parse(input: ParseStream) -> ParseResult { - if input.peek(Token![let]) { - let let_token = input.parse()?; - Ok(Self::LetBinding { - let_token, - pattern: input.parse()?, - }) - } else { - Ok(Self::ExistingVariable { - path: input.parse()?, - operation: if input.peek(Token![=]) { - None - } else { - let operator_char = match input.cursor().punct() { - Some((operator, _)) => operator.as_char(), - None => 'X', - }; - match operator_char { - '+' | '-' | '*' | '/' | '%' | '&' | '|' | '^' => {} - _ => { - return input - .parse_err("Expected = or one of += -= *= /= %= &= |= or ^=") - } - } - Some(input.parse()?) - }, - }) - } + let LetStatement { + let_token, + pattern, + expression, + .. + } = self; + let value = expression.interpret_to_value(interpreter)?; + let mut span_range = value.span_range(); + pattern.handle_destructure(interpreter, value)?; + span_range.set_start(let_token.span); + Ok(ExpressionValue::None(span_range)) } } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 82e637d6..cb42d94e 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -119,6 +119,18 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { lhs: Some(node), range_limits, }, + NodeExtension::AssignmentOperation(equals_token) => { + self.push_stack_frame(ExpressionStackFrame::IncompleteAssignment { + assignee: node, + equals_token, + }) + } + NodeExtension::CompoundAssignmentOperation(operation) => { + self.push_stack_frame(ExpressionStackFrame::IncompleteCompoundAssignment { + place: node, + operation, + }) + } NodeExtension::EndOfStream | NodeExtension::NoValidExtensionForCurrentParent => { unreachable!("Not possible, as these have minimum precedence") } @@ -182,6 +194,25 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { }); extension.into_post_operation_completion_work_item(node) } + ExpressionStackFrame::IncompleteAssignment { + assignee, + equals_token, + } => { + let node = self.nodes.add_node(ExpressionNode::Assignment { + assignee, + equals_token, + value: node, + }); + extension.into_post_operation_completion_work_item(node) + } + ExpressionStackFrame::IncompleteCompoundAssignment { place, operation } => { + let node = self.nodes.add_node(ExpressionNode::CompoundAssignment { + place, + operation, + value: node, + }); + extension.into_post_operation_completion_work_item(node) + } }) } } @@ -497,6 +528,22 @@ pub(super) enum ExpressionStackFrame { lhs: ExpressionNodeId, operation: BinaryOperation, }, + /// An incomplete assignment operation + /// It's left side is an assignee expression, according to the [rust reference]. + /// + /// [rust reference]: https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions + IncompleteAssignment { + assignee: ExpressionNodeId, + equals_token: Token![=], + }, + /// An incomplete assignment operation + /// It's left side is a place expression, according to the [rust reference]. + /// + /// [rust reference]: https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions + IncompleteCompoundAssignment { + place: ExpressionNodeId, + operation: CompoundAssignmentOperation, + }, /// A range which will be followed by a rhs IncompleteRange { lhs: Option, @@ -511,6 +558,10 @@ impl ExpressionStackFrame { ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteRange { .. } => OperatorPrecendence::Range, + ExpressionStackFrame::IncompleteAssignment { .. } => OperatorPrecendence::Assign, + ExpressionStackFrame::IncompleteCompoundAssignment { .. } => { + OperatorPrecendence::Assign + } ExpressionStackFrame::IncompleteUnaryPrefixOperation { operation, .. } => { OperatorPrecendence::of_prefix_unary_operation(operation) } @@ -567,6 +618,8 @@ pub(super) enum NodeExtension { BinaryOperation(BinaryOperation), NonTerminalArrayComma, Range(syn::RangeLimits), + AssignmentOperation(Token![=]), + CompoundAssignmentOperation(CompoundAssignmentOperation), EndOfStream, NoValidExtensionForCurrentParent, } @@ -579,16 +632,21 @@ impl NodeExtension { NodeExtension::NonTerminalArrayComma => OperatorPrecendence::NonTerminalComma, NodeExtension::Range(_) => OperatorPrecendence::Range, NodeExtension::EndOfStream => OperatorPrecendence::MIN, + NodeExtension::AssignmentOperation(_) => OperatorPrecendence::Assign, + NodeExtension::CompoundAssignmentOperation(_) => OperatorPrecendence::Assign, NodeExtension::NoValidExtensionForCurrentParent => OperatorPrecendence::MIN, } } fn into_post_operation_completion_work_item(self, node: ExpressionNodeId) -> WorkItem { match self { - // Extensions are independent of depth, so can be re-used without parsing again. + // These extensions are valid/correct for any parent, + // so can be re-used without parsing again. extension @ (NodeExtension::PostfixOperation { .. } | NodeExtension::BinaryOperation { .. } | NodeExtension::Range { .. } + | NodeExtension::AssignmentOperation { .. } + | NodeExtension::CompoundAssignmentOperation { .. } | NodeExtension::EndOfStream) => { WorkItem::TryApplyAlreadyParsedExtension { node, extension } } diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 7551151e..f53e1778 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -236,6 +236,18 @@ pub(crate) enum BinaryOperation { Integer(IntegerBinaryOperation), } +impl From for BinaryOperation { + fn from(operation: PairedBinaryOperation) -> Self { + Self::Paired(operation) + } +} + +impl From for BinaryOperation { + fn from(operation: IntegerBinaryOperation) -> Self { + Self::Integer(operation) + } +} + impl SynParse for BinaryOperation { fn parse(input: SynParseStream) -> SynResult { // In line with Syn's BinOp, we use peek instead of lookahead @@ -499,3 +511,147 @@ impl HasSpanRange for syn::RangeLimits { self.span_range_from_iterating_over_all_tokens() } } + +#[derive(Clone, Copy)] +pub(crate) enum CompoundAssignmentOperation { + Add(Token![+=]), + Sub(Token![-=]), + Mul(Token![*=]), + Div(Token![/=]), + Rem(Token![%=]), + BitAnd(Token![&=]), + BitOr(Token![|=]), + BitXor(Token![^=]), + Shl(Token![<<=]), + Shr(Token![>>=]), +} + +impl SynParse for CompoundAssignmentOperation { + fn parse(input: SynParseStream) -> SynResult { + // In line with Syn's BinOp, we use peek instead of lookahead + // ...I assume for slightly increased performance + // ...Or because 30 alternative options in the error message is too many + if input.peek(Token![+=]) { + Ok(Self::Add(input.parse()?)) + } else if input.peek(Token![-=]) { + Ok(Self::Sub(input.parse()?)) + } else if input.peek(Token![*=]) { + Ok(Self::Mul(input.parse()?)) + } else if input.peek(Token![/=]) { + Ok(Self::Div(input.parse()?)) + } else if input.peek(Token![%=]) { + Ok(Self::Rem(input.parse()?)) + } else if input.peek(Token![&=]) { + Ok(Self::BitAnd(input.parse()?)) + } else if input.peek(Token![|=]) { + Ok(Self::BitOr(input.parse()?)) + } else if input.peek(Token![^=]) { + Ok(Self::BitXor(input.parse()?)) + } else if input.peek(Token![<<=]) { + Ok(Self::Shl(input.parse()?)) + } else if input.peek(Token![>>=]) { + Ok(Self::Shr(input.parse()?)) + } else { + Err(input.error("Expected one of += -= *= /= %= &= |= ^= <<= or >>=")) + } + } +} + +impl CompoundAssignmentOperation { + pub(crate) fn to_binary(self) -> BinaryOperation { + match self { + CompoundAssignmentOperation::Add(token) => { + let token = create_single_token('+', token.spans[0]); + PairedBinaryOperation::Addition(token).into() + } + CompoundAssignmentOperation::Sub(token) => { + let token = create_single_token('-', token.spans[0]); + PairedBinaryOperation::Subtraction(token).into() + } + CompoundAssignmentOperation::Mul(token) => { + let token = create_single_token('*', token.spans[0]); + PairedBinaryOperation::Multiplication(token).into() + } + CompoundAssignmentOperation::Div(token) => { + let token = create_single_token('/', token.spans[0]); + PairedBinaryOperation::Division(token).into() + } + CompoundAssignmentOperation::Rem(token) => { + let token = create_single_token('%', token.spans[0]); + PairedBinaryOperation::Remainder(token).into() + } + CompoundAssignmentOperation::BitAnd(token) => { + let token = create_single_token('&', token.spans[0]); + PairedBinaryOperation::BitAnd(token).into() + } + CompoundAssignmentOperation::BitOr(token) => { + let token = create_single_token('^', token.spans[0]); + PairedBinaryOperation::BitOr(token).into() + } + CompoundAssignmentOperation::BitXor(token) => { + let token = create_single_token('|', token.spans[0]); + PairedBinaryOperation::BitXor(token).into() + } + CompoundAssignmentOperation::Shl(token) => { + let token = create_double_token('<', token.spans[0], '<', token.spans[1]); + IntegerBinaryOperation::ShiftLeft(token).into() + } + CompoundAssignmentOperation::Shr(token) => { + let token = create_double_token('>', token.spans[0], '>', token.spans[1]); + IntegerBinaryOperation::ShiftRight(token).into() + } + } + } +} + +fn create_single_token(char: char, span: Span) -> T { + let stream = Punct::new(char, Spacing::Alone) + .with_span(span) + .to_token_stream(); + T::parse.parse2(stream).unwrap() +} + +fn create_double_token(char1: char, span1: Span, char2: char, span2: Span) -> T { + let mut stream = TokenStream::new(); + Punct::new(char1, Spacing::Alone) + .with_span(span1) + .to_tokens(&mut stream); + Punct::new(char2, Spacing::Alone) + .with_span(span2) + .to_tokens(&mut stream); + T::parse.parse2(stream).unwrap() +} + +impl Operation for CompoundAssignmentOperation { + fn symbolic_description(&self) -> &'static str { + match self { + CompoundAssignmentOperation::Add(_) => "+=", + CompoundAssignmentOperation::Sub(_) => "-=", + CompoundAssignmentOperation::Mul(_) => "*=", + CompoundAssignmentOperation::Div(_) => "/=", + CompoundAssignmentOperation::Rem(_) => "%=", + CompoundAssignmentOperation::BitAnd(_) => "&=", + CompoundAssignmentOperation::BitOr(_) => "|=", + CompoundAssignmentOperation::BitXor(_) => "^=", + CompoundAssignmentOperation::Shl(_) => "<<=", + CompoundAssignmentOperation::Shr(_) => ">>=", + } + } +} + +impl HasSpanRange for CompoundAssignmentOperation { + fn span_range(&self) -> SpanRange { + match self { + CompoundAssignmentOperation::Add(op) => op.span_range(), + CompoundAssignmentOperation::Sub(op) => op.span_range(), + CompoundAssignmentOperation::Mul(op) => op.span_range(), + CompoundAssignmentOperation::Div(op) => op.span_range(), + CompoundAssignmentOperation::Rem(op) => op.span_range(), + CompoundAssignmentOperation::BitAnd(op) => op.span_range(), + CompoundAssignmentOperation::BitOr(op) => op.span_range(), + CompoundAssignmentOperation::BitXor(op) => op.span_range(), + CompoundAssignmentOperation::Shl(op) => op.span_range(), + CompoundAssignmentOperation::Shr(op) => op.span_range(), + } + } +} diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index c5a6abcf..a094862d 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -229,6 +229,16 @@ impl_auto_span_range! { syn::token::Ge, syn::token::Gt, syn::token::Comma, + syn::token::PlusEq, + syn::token::MinusEq, + syn::token::StarEq, + syn::token::SlashEq, + syn::token::PercentEq, + syn::token::AndEq, + syn::token::OrEq, + syn::token::CaretEq, + syn::token::ShlEq, + syn::token::ShrEq, } macro_rules! single_span_token { diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 45891ab1..8988feff 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -240,11 +240,11 @@ impl core::fmt::Display for FlattenedVariable { // An identifier for a variable path in an expression #[derive(Clone)] -pub(crate) struct VariableOrField { +pub(crate) struct VariableIdentifier { ident: Ident, } -impl Parse for VariableOrField { +impl Parse for VariableIdentifier { fn parse(input: ParseStream) -> ParseResult { Ok(Self { ident: input.parse()?, @@ -252,19 +252,19 @@ impl Parse for VariableOrField { } } -impl IsVariable for VariableOrField { +impl IsVariable for VariableIdentifier { fn get_name(&self) -> String { self.ident.to_string() } } -impl HasSpan for VariableOrField { +impl HasSpan for VariableIdentifier { fn span(&self) -> Span { self.ident.span() } } -impl InterpretToValue for &VariableOrField { +impl InterpretToValue for &VariableIdentifier { type OutputValue = ExpressionValue; fn interpret_to_value( diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 7580e66f..eb9e11b4 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -1,5 +1,4 @@ #![allow(clippy::identity_op)] // https://github.com/rust-lang/rust-clippy/issues/13924 -#![allow(clippy::zero_prefixed_literal)] // https://github.com/rust-lang/rust-clippy/issues/14199 #[path = "helpers/prelude.rs"] mod prelude; @@ -48,7 +47,7 @@ fn test_if() { fn test_while() { preinterpret_assert_eq!({ #(let x = 0) - [!while! #x < 5 { #(x += 1) }] + [!while! x < 5 { #(x += 1) }] #x }, 5); } From ece684fc84f6a7a140677b7db20fdb6ea39b119e Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 16 Feb 2025 01:09:08 +0000 Subject: [PATCH 098/126] feature: Indexing works for reading --- CHANGELOG.md | 4 +- src/expressions/array.rs | 23 ++++ src/expressions/evaluation.rs | 108 +++++++++++++------ src/expressions/expression.rs | 29 ++++- src/expressions/expression_block.rs | 8 +- src/expressions/expression_parsing.rs | 37 ++++++- src/expressions/operations.rs | 23 ++++ src/expressions/range.rs | 4 +- src/expressions/value.rs | 35 ++++-- src/extensions/errors_and_spans.rs | 80 ++++++++++++++ src/interpretation/command.rs | 14 +-- src/interpretation/commands/core_commands.rs | 5 +- src/interpretation/interpreter.rs | 66 ++++++------ src/interpretation/source_code_block.rs | 10 +- src/interpretation/variable.rs | 10 +- src/misc/field_inputs.rs | 4 +- src/misc/parse_traits.rs | 22 ++++ src/transformation/destructuring.rs | 6 +- src/transformation/fields.rs | 2 +- src/transformation/transform_stream.rs | 19 ++-- src/transformation/transformer.rs | 14 +-- src/transformation/transformers.rs | 2 +- src/transformation/variable_binding.rs | 8 +- tests/expressions.rs | 10 ++ 24 files changed, 411 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c687a3..4b6d0d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,8 +98,6 @@ Inside a transform stream, the following grammar is supported: ### To come * Support `#(x[..])` syntax for indexing arrays at read time (streams to follow in a separate task below after parsers are updated) - * Via a post-fix `[..]` operation with high precedence - * `#(x[0])` returns the item at that position of the array * `#(x[0..3])` returns an array * `#(x[0..=3])` returns an array * ... and `#(x[0] = y)` can be used to set the item @@ -213,7 +211,7 @@ Inside a transform stream, the following grammar is supported: * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) * `#(x[0..3])` returns a TokenStream * `#(x[0..=3])` returns a TokenStream -* Add `..` and `..x` support to the array destructurer +* Add `..` and `.., x` support to the array pattern * Consider: * Dropping lots of the `group` wrappers? * If any types should have reference semantics instead of clone/value semantics? diff --git a/src/expressions/array.rs b/src/expressions/array.rs index f6d6efe8..2a7e69aa 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -117,6 +117,29 @@ impl ExpressionArray { }) } + pub(super) fn handle_index_access( + &self, + access: IndexAccess, + index: ExpressionValue, + ) -> ExecutionResult { + Ok(match index { + ExpressionValue::Integer(int) => { + let span_range = int.span_range; + let index = int.expect_usize()?; + if index < self.items.len() { + self.items[index].clone().with_span(access.span()) + } else { + return span_range.execution_err(format!( + "Index of {} is out of range of the array length of {}", + index, + self.items.len() + )); + } + } + _ => return index.execution_err("The index must be an integer"), + }) + } + pub(crate) fn concat_recursive_into( self, output: &mut String, diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index e278bac9..bb5edd9f 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -78,6 +78,23 @@ impl<'a> ExpressionEvaluator<'a, Source> { }); NextAction::EnterValueNode(*left_input) } + ExpressionNode::Property { node, access } => { + self.operation_stack.push(EvaluationStackFrame::Property { + access: access.clone(), + }); + NextAction::EnterValueNode(*node) + } + ExpressionNode::Index { + node, + access, + index, + } => { + self.operation_stack.push(EvaluationStackFrame::Index { + access: access.clone(), + state: IndexPath::OnObjectBranch { index: *index }, + }); + NextAction::EnterValueNode(*node) + } ExpressionNode::Range { left, range_limits, @@ -169,6 +186,23 @@ impl<'a> ExpressionEvaluator<'a, Source> { NextAction::HandleValue(result) } }, + EvaluationStackFrame::Property { access } => { + let result = value.handle_property_access(access)?; + NextAction::HandleValue(result) + } + EvaluationStackFrame::Index { access, state } => match state { + IndexPath::OnObjectBranch { index } => { + self.operation_stack.push(EvaluationStackFrame::Index { + access, + state: IndexPath::OnIndexBranch { object: value }, + }); + NextAction::EnterValueNode(index) + } + IndexPath::OnIndexBranch { object } => { + let result = object.handle_index_access(access, value)?; + NextAction::HandleValue(result) + } + }, EvaluationStackFrame::Range { range_limits, state, @@ -233,14 +267,12 @@ impl<'a> ExpressionEvaluator<'a, Source> { assignee, equals_token, } => { - let span_range = - self.handle_assignment(assignee, equals_token, value, interpreter)?; - NextAction::HandleValue(ExpressionValue::None(span_range)) + // TODO: This should be replaced with setting a stack frame for AssignmentResolution + self.handle_assignment(assignee, equals_token, value, interpreter)? } EvaluationStackFrame::CompoundAssignmentValue { place, operation } => { - let span_range = - self.handle_compound_assignment(place, operation, value, interpreter)?; - NextAction::HandleValue(ExpressionValue::None(span_range)) + // TODO: This should be replaced with setting a stack frame for CompoundAssignmentResolution + self.handle_compound_assignment(place, operation, value, interpreter)? } }) } @@ -251,17 +283,15 @@ impl<'a> ExpressionEvaluator<'a, Source> { equals_token: Token![=], value: ExpressionValue, interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { // TODO: When we add place resolution, we likely wish to make use of the execution stack. - let assignee = self.resolve_assignee_or_place(assignee)?; + let assignee = self.resolve_assignee_or_place(assignee, interpreter)?; Ok(match assignee { Assignee::Place(place) => { - let span_range = SpanRange::new_between( - place.variable.span_range().start(), - value.span_range().end(), - ); - place.variable.set_value(interpreter, value)?; - span_range + let span_range = + SpanRange::new_between(place.span_range().start(), value.span_range().end()); + place.set(value)?; + NextAction::HandleValue(ExpressionValue::None(span_range)) } Assignee::CompoundWip => { return equals_token.execution_err("Compound assignment is not yet supported") @@ -275,20 +305,16 @@ impl<'a> ExpressionEvaluator<'a, Source> { operation: CompoundAssignmentOperation, value: ExpressionValue, interpreter: &mut Interpreter, - ) -> ExecutionResult { + ) -> ExecutionResult { // TODO: When we add place resolution, we likely wish to make use of the execution stack. - let assignee = self.resolve_assignee_or_place(place)?; + let assignee = self.resolve_assignee_or_place(place, interpreter)?; Ok(match assignee { Assignee::Place(place) => { - let span_range = SpanRange::new_between( - place.variable.span_range().start(), - value.span_range().end(), - ); - let left = place.variable.interpret_to_value(interpreter)?; - place - .variable - .set_value(interpreter, operation.to_binary().evaluate(left, value)?)?; - span_range + let span_range = + SpanRange::new_between(place.span_range().start(), value.span_range().end()); + let left = place.get_cloned()?.clone(); + place.set(operation.to_binary().evaluate(left, value)?)?; + NextAction::HandleValue(ExpressionValue::None(span_range)) } Assignee::CompoundWip => { return operation @@ -303,11 +329,18 @@ impl<'a> ExpressionEvaluator<'a, Source> { fn resolve_assignee_or_place( &self, mut assignee: ExpressionNodeId, - ) -> ExecutionResult> { + interpreter: &mut Interpreter, + ) -> ExecutionResult { let resolved = loop { match &self.nodes[assignee.0] { ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => { - break Assignee::Place(Place { variable }); + break Assignee::Place(variable.read_existing(interpreter)?); + } + ExpressionNode::Index { .. } => { + todo!() + } + ExpressionNode::Property { .. } => { + todo!() } ExpressionNode::Array { .. } => { break Assignee::CompoundWip; @@ -327,16 +360,11 @@ impl<'a> ExpressionEvaluator<'a, Source> { } } -enum Assignee<'a> { - Place(Place<'a>), +enum Assignee { + Place(VariableData), CompoundWip, // To come } -struct Place<'a> { - variable: &'a VariableIdentifier, - // To come: Array index, field access, etc. -} - enum NextAction { HandleValue(ExpressionValue), EnterValueNode(ExpressionNodeId), @@ -354,6 +382,13 @@ enum EvaluationStackFrame { operation: BinaryOperation, state: BinaryPath, }, + Property { + access: PropertyAccess, + }, + Index { + access: IndexAccess, + state: IndexPath, + }, Range { range_limits: syn::RangeLimits, state: RangePath, @@ -398,6 +433,11 @@ enum BinaryPath { OnRightBranch { left: ExpressionValue }, } +enum IndexPath { + OnObjectBranch { index: ExpressionNodeId }, + OnIndexBranch { object: ExpressionValue }, +} + enum RangePath { OnLeftBranch { right: Option }, OnRightBranch { left: Option }, diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 9794fa31..a97da681 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -116,6 +116,12 @@ impl Expressionable for Source { ) -> ParseResult { // We fall through if we have no match match input.peek_grammar() { + SourcePeekMatch::Group(Delimiter::Bracket) => { + let (_, delim_span) = input.parse_and_enter_group()?; + return Ok(NodeExtension::Index(IndexAccess { + brackets: Brackets { delim_span }, + })); + } SourcePeekMatch::Punct(punct) if punct.as_char() == ',' => { match parent_stack_frame { ExpressionStackFrame::Array { .. } => { @@ -133,7 +139,13 @@ impl Expressionable for Source { _ => {} } } - SourcePeekMatch::Punct(_) => { + SourcePeekMatch::Punct(punct) => { + if punct.as_char() == '.' && input.peek2(syn::Ident) { + return Ok(NodeExtension::Property(PropertyAccess { + dot: input.parse()?, + property: input.parse()?, + })); + } if let Ok(operation) = input.try_parse_or_revert() { return Ok(NodeExtension::CompoundAssignmentOperation(operation)); } @@ -160,7 +172,9 @@ impl Expressionable for Source { match parent_stack_frame { ExpressionStackFrame::Root => Ok(NodeExtension::NoValidExtensionForCurrentParent), ExpressionStackFrame::Group { .. } => input.parse_err("Expected ) or operator"), - ExpressionStackFrame::Array { .. } => input.parse_err("Expected comma, ], or operator"), + ExpressionStackFrame::Array { .. } | ExpressionStackFrame::IncompleteIndex { .. } => { + input.parse_err("Expected comma, ], or operator") + } // e.g. I've just matched the true in !true or false || true, // and I want to see if there's an extension (e.g. a cast). // There's nothing matching, so we fall through to an EndOfFrame @@ -241,6 +255,15 @@ pub(super) enum ExpressionNode { left_input: ExpressionNodeId, right_input: ExpressionNodeId, }, + Property { + node: ExpressionNodeId, + access: PropertyAccess, + }, + Index { + node: ExpressionNodeId, + access: IndexAccess, + index: ExpressionNodeId, + }, Range { left: Option, range_limits: syn::RangeLimits, @@ -264,6 +287,8 @@ impl ExpressionNode { ExpressionNode::Leaf(leaf) => leaf.span_range(), ExpressionNode::Grouped { delim_span, .. } => delim_span.span_range(), ExpressionNode::Array { delim_span, .. } => delim_span.span_range(), + ExpressionNode::Property { access, .. } => access.span_range(), + ExpressionNode::Index { access, .. } => access.span_range(), ExpressionNode::UnaryOperation { operation, .. } => operation.span_range(), ExpressionNode::BinaryOperation { operation, .. } => operation.span_range(), ExpressionNode::Range { range_limits, .. } => range_limits.span_range(), diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index be454a83..1c9bbc93 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -4,7 +4,7 @@ use super::*; pub(crate) struct ExpressionBlock { marker: Token![#], flattening: Option, - delim_span: DelimSpan, + parentheses: Parentheses, standard_statements: Vec<(Statement, Token![;])>, return_statement: Option, } @@ -17,7 +17,7 @@ impl Parse for ExpressionBlock { } else { None }; - let (delim_span, inner) = input.parse_specific_group(Delimiter::Parenthesis)?; + let (parentheses, inner) = input.parse_parentheses()?; let mut standard_statements = Vec::new(); let return_statement = loop { if inner.is_empty() { @@ -35,7 +35,7 @@ impl Parse for ExpressionBlock { Ok(Self { marker, flattening, - delim_span, + parentheses, standard_statements, return_statement, }) @@ -44,7 +44,7 @@ impl Parse for ExpressionBlock { impl HasSpanRange for ExpressionBlock { fn span_range(&self) -> SpanRange { - SpanRange::new_between(self.marker.span, self.delim_span.close()) + SpanRange::new_between(self.marker.span, self.parentheses.close()) } } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index cb42d94e..6a572e3a 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -115,6 +115,14 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } WorkItem::RequireUnaryAtom } + NodeExtension::Property(access) => WorkItem::TryParseAndApplyExtension { + node: self + .nodes + .add_node(ExpressionNode::Property { node, access }), + }, + NodeExtension::Index(access) => { + self.push_stack_frame(ExpressionStackFrame::IncompleteIndex { node, access }) + } NodeExtension::Range(range_limits) => WorkItem::ContinueRange { lhs: Some(node), range_limits, @@ -186,6 +194,19 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { }); extension.into_post_operation_completion_work_item(node) } + ExpressionStackFrame::IncompleteIndex { + node: source, + access, + } => { + assert!(matches!(extension, NodeExtension::EndOfStream)); + self.streams.exit_group(); + let node = self.nodes.add_node(ExpressionNode::Index { + node: source, + access, + index: node, + }); + WorkItem::TryParseAndApplyExtension { node } + } ExpressionStackFrame::IncompleteRange { lhs, range_limits } => { let node = self.nodes.add_node(ExpressionNode::Range { left: lhs, @@ -281,7 +302,7 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { fn parent_precedence(&self) -> OperatorPrecendence { self.expression_stack .last() - .map(|s| s.precedence()) + .map(|s| s.precedence_to_bind_to_child()) .unwrap() } } @@ -528,6 +549,11 @@ pub(super) enum ExpressionStackFrame { lhs: ExpressionNodeId, operation: BinaryOperation, }, + /// An incomplete indexing access + IncompleteIndex { + node: ExpressionNodeId, + access: IndexAccess, + }, /// An incomplete assignment operation /// It's left side is an assignee expression, according to the [rust reference]. /// @@ -552,11 +578,12 @@ pub(super) enum ExpressionStackFrame { } impl ExpressionStackFrame { - fn precedence(&self) -> OperatorPrecendence { + fn precedence_to_bind_to_child(&self) -> OperatorPrecendence { match self { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::IncompleteIndex { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteRange { .. } => OperatorPrecendence::Range, ExpressionStackFrame::IncompleteAssignment { .. } => OperatorPrecendence::Assign, ExpressionStackFrame::IncompleteCompoundAssignment { .. } => { @@ -617,6 +644,8 @@ pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), NonTerminalArrayComma, + Property(PropertyAccess), + Index(IndexAccess), Range(syn::RangeLimits), AssignmentOperation(Token![=]), CompoundAssignmentOperation(CompoundAssignmentOperation), @@ -630,6 +659,8 @@ impl NodeExtension { NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), NodeExtension::NonTerminalArrayComma => OperatorPrecendence::NonTerminalComma, + NodeExtension::Property { .. } => OperatorPrecendence::Unambiguous, + NodeExtension::Index { .. } => OperatorPrecendence::Unambiguous, NodeExtension::Range(_) => OperatorPrecendence::Range, NodeExtension::EndOfStream => OperatorPrecendence::MIN, NodeExtension::AssignmentOperation(_) => OperatorPrecendence::Assign, @@ -644,6 +675,8 @@ impl NodeExtension { // so can be re-used without parsing again. extension @ (NodeExtension::PostfixOperation { .. } | NodeExtension::BinaryOperation { .. } + | NodeExtension::Property { .. } + | NodeExtension::Index { .. } | NodeExtension::Range { .. } | NodeExtension::AssignmentOperation { .. } | NodeExtension::CompoundAssignmentOperation { .. } diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index f53e1778..509b3137 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -655,3 +655,26 @@ impl HasSpanRange for CompoundAssignmentOperation { } } } + +#[derive(Clone)] +pub(super) struct PropertyAccess { + pub(super) dot: Token![.], + pub(super) property: Ident, +} + +impl HasSpanRange for PropertyAccess { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.dot.span, self.property.span()) + } +} + +#[derive(Clone)] +pub(super) struct IndexAccess { + pub(super) brackets: Brackets, +} + +impl HasSpan for IndexAccess { + fn span(&self) -> Span { + self.brackets.join() + } +} diff --git a/src/expressions/range.rs b/src/expressions/range.rs index 12975c9a..32322bf9 100644 --- a/src/expressions/range.rs +++ b/src/expressions/range.rs @@ -196,7 +196,7 @@ impl IterableExpressionRange { SpanRange::new_between(start.span_range().start(), end.span_range().end()); let pair = start.expect_value_pair(&dots, end)?; match pair { - EvaluationValuePair::Integer(pair) => match pair { + ExpressionValuePair::Integer(pair) => match pair { ExpressionIntegerValuePair::Untyped(start, end) => { IterableExpressionRange::RangeFromTo { start, dots, end } .resolve(output_span_range) @@ -250,7 +250,7 @@ impl IterableExpressionRange { .resolve(output_span_range) } }, - EvaluationValuePair::CharPair(start, end) => { + ExpressionValuePair::CharPair(start, end) => { IterableExpressionRange::RangeFromTo { start: start.value, dots, diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 8cc94bff..923923cc 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -59,7 +59,7 @@ impl ExpressionValue { self, operation: &impl Operation, right: Self, - ) -> ExecutionResult { + ) -> ExecutionResult { Ok(match (self, right) { (ExpressionValue::Integer(left), ExpressionValue::Integer(right)) => { let integer_pair = match (left.value, right.value) { @@ -185,10 +185,10 @@ impl ExpressionValue { return operation.execution_err(format!("The {} operator cannot infer a common integer operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbolic_description(), left_value.value_type(), right_value.value_type())); } }; - EvaluationValuePair::Integer(integer_pair) + ExpressionValuePair::Integer(integer_pair) } (ExpressionValue::Boolean(left), ExpressionValue::Boolean(right)) => { - EvaluationValuePair::BooleanPair(left, right) + ExpressionValuePair::BooleanPair(left, right) } (ExpressionValue::Float(left), ExpressionValue::Float(right)) => { let float_pair = match (left.value, right.value) { @@ -224,19 +224,19 @@ impl ExpressionValue { return operation.execution_err(format!("The {} operator cannot infer a common float operand type from {} and {}. Consider using `as` to cast to matching types.", operation.symbolic_description(), left_value.value_type(), right_value.value_type())); } }; - EvaluationValuePair::Float(float_pair) + ExpressionValuePair::Float(float_pair) } (ExpressionValue::String(left), ExpressionValue::String(right)) => { - EvaluationValuePair::StringPair(left, right) + ExpressionValuePair::StringPair(left, right) } (ExpressionValue::Char(left), ExpressionValue::Char(right)) => { - EvaluationValuePair::CharPair(left, right) + ExpressionValuePair::CharPair(left, right) } (ExpressionValue::Array(left), ExpressionValue::Array(right)) => { - EvaluationValuePair::ArrayPair(left, right) + ExpressionValuePair::ArrayPair(left, right) } (ExpressionValue::Stream(left), ExpressionValue::Stream(right)) => { - EvaluationValuePair::StreamPair(left, right) + ExpressionValuePair::StreamPair(left, right) } (left, right) => { return operation.execution_err(format!("Cannot infer common type from {} {} {}. Consider using `as` to cast the operands to matching types.", left.value_type(), operation.symbolic_description(), right.value_type())); @@ -379,6 +379,21 @@ impl ExpressionValue { } } + pub(super) fn handle_index_access( + self, + access: IndexAccess, + index: Self, + ) -> ExecutionResult { + match self { + ExpressionValue::Array(array) => array.handle_index_access(access, index), + other => access.execution_err(format!("Cannot index into a {}", other.value_type())), + } + } + + pub(super) fn handle_property_access(self, access: PropertyAccess) -> ExecutionResult { + access.execution_err("Fields are not supported") + } + fn span_range_mut(&mut self) -> &mut SpanRange { match self { Self::None(span_range) => span_range, @@ -620,7 +635,7 @@ impl HasValueType for UnsupportedLiteral { } } -pub(super) enum EvaluationValuePair { +pub(super) enum ExpressionValuePair { Integer(ExpressionIntegerValuePair), Float(ExpressionFloatValuePair), BooleanPair(ExpressionBoolean, ExpressionBoolean), @@ -630,7 +645,7 @@ pub(super) enum EvaluationValuePair { StreamPair(ExpressionStream, ExpressionStream), } -impl EvaluationValuePair { +impl ExpressionValuePair { pub(super) fn handle_paired_binary_operation( self, operation: OutputSpanned, diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index a094862d..cad5d943 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -179,6 +179,85 @@ impl HasSpan for Literal { } } +/// [ ... ] +#[derive(Copy, Clone)] +pub(crate) struct Brackets { + pub(crate) delim_span: DelimSpan, +} + +impl core::ops::Deref for Brackets { + type Target = DelimSpan; + + fn deref(&self) -> &Self::Target { + &self.delim_span + } +} + +impl HasSpan for Brackets { + fn span(&self) -> Span { + self.delim_span.span() + } +} + +/// { ... } +#[derive(Copy, Clone)] +pub(crate) struct Braces { + pub(crate) delim_span: DelimSpan, +} + +impl core::ops::Deref for Braces { + type Target = DelimSpan; + + fn deref(&self) -> &Self::Target { + &self.delim_span + } +} + +impl HasSpan for Braces { + fn span(&self) -> Span { + self.delim_span.span() + } +} + +/// ( ... ) +#[derive(Copy, Clone)] +pub(crate) struct Parentheses { + pub(crate) delim_span: DelimSpan, +} + +impl core::ops::Deref for Parentheses { + type Target = DelimSpan; + + fn deref(&self) -> &Self::Target { + &self.delim_span + } +} + +impl HasSpan for Parentheses { + fn span(&self) -> Span { + self.delim_span.span() + } +} + +#[derive(Copy, Clone)] +pub(crate) struct TransparentDelimiters { + pub(crate) delim_span: DelimSpan, +} + +impl core::ops::Deref for TransparentDelimiters { + type Target = DelimSpan; + + fn deref(&self) -> &Self::Target { + &self.delim_span + } +} + +impl HasSpan for TransparentDelimiters { + fn span(&self) -> Span { + self.delim_span.span() + } +} + pub(crate) trait SlowSpanRange { /// This name is purposefully very long to discourage use, as it can cause nasty performance issues fn span_range_from_iterating_over_all_tokens(&self) -> SpanRange; @@ -216,6 +295,7 @@ macro_rules! impl_auto_span_range { impl_auto_span_range! { syn::BinOp, syn::UnOp, + syn::token::Dot, syn::token::DotDot, syn::token::DotDotEq, syn::token::Shl, diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 045aaa34..fa2593f7 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -398,12 +398,12 @@ define_command_enums! { #[derive(Clone)] pub(crate) struct Command { typed: Box, - source_group_span: DelimSpan, + brackets: Brackets, } impl Parse for Command { fn parse(input: ParseStream) -> ParseResult { - let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; + let (brackets, content) = input.parse_brackets()?; content.parse::()?; let command_name = content.parse_any_ident()?; let command_kind = match CommandKind::for_ident(&command_name) { @@ -420,18 +420,18 @@ impl Parse for Command { let typed = command_kind.parse_command(CommandArguments::new( &content, command_name, - delim_span.join(), + brackets.join(), ))?; Ok(Self { typed: Box::new(typed), - source_group_span: delim_span, + brackets, }) } } impl HasSpan for Command { fn span(&self) -> Span { - self.source_group_span.join() + self.brackets.join() } } @@ -443,7 +443,7 @@ impl Interpret for Command { ) -> ExecutionResult<()> { let context = ExecutionContext { interpreter, - delim_span: self.source_group_span, + delim_span: self.brackets.delim_span, }; self.typed.execute_into(context, output) } @@ -458,7 +458,7 @@ impl InterpretToValue for Command { ) -> ExecutionResult { let context = ExecutionContext { interpreter, - delim_span: self.source_group_span, + delim_span: self.brackets.delim_span, }; self.typed.execute_to_value(context) } diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 5b819088..036d4156 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -95,10 +95,7 @@ impl NoOutputCommandDefinition for SetCommand { variable, content, .. } => { let variable_data = variable.get_existing_for_mutation(interpreter)?; - content.interpret_into( - interpreter, - variable_data.get_mut_stream(&variable)?.deref_mut(), - )?; + content.interpret_into(interpreter, variable_data.get_mut_stream()?.deref_mut())?; } SetArguments::SetVariablesEmpty { variables } => { for variable in variables { diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 95593dff..fb5e81dc 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -13,65 +13,67 @@ pub(crate) struct Interpreter { #[derive(Clone)] pub(crate) struct VariableData { value: Rc>, + /// In the store, this is the span range of the original let variable declaration. + /// When this is a variable reference, this is the span range of the variable + /// which created the reference. + span_range: SpanRange, } impl VariableData { - fn new(tokens: ExpressionValue) -> Self { + fn new(tokens: ExpressionValue, span_range: SpanRange) -> Self { Self { value: Rc::new(RefCell::new(tokens)), + span_range, } } - pub(crate) fn get<'d>( - &'d self, - variable: &(impl IsVariable + ?Sized), - ) -> ExecutionResult> { + pub(crate) fn get_ref(&self) -> ExecutionResult> { self.value.try_borrow().map_err(|_| { - variable - .error("The variable cannot be read if it is currently being modified") - .into() + self.execution_error("The variable cannot be read if it is currently being modified") }) } - pub(crate) fn get_mut<'d>( - &'d self, - variable: &(impl IsVariable + ?Sized), - ) -> ExecutionResult> { + // Gets the cloned expression value, setting the span range appropriately + pub(crate) fn get_cloned(&self) -> ExecutionResult { + Ok(self.get_ref()?.clone().with_span_range(self.span_range)) + } + + pub(crate) fn get_mut(&self) -> ExecutionResult> { self.value.try_borrow_mut().map_err(|_| { - variable.execution_error( + self.execution_error( "The variable cannot be modified if it is already currently being modified", ) }) } - pub(crate) fn get_mut_stream<'d>( - &'d self, - variable: &(impl IsVariable + ?Sized), - ) -> ExecutionResult> { - let mut_guard = self.get_mut(variable)?; + pub(crate) fn get_mut_stream(&self) -> ExecutionResult> { + let mut_guard = self.get_mut()?; RefMut::filter_map(mut_guard, |mut_guard| match mut_guard { ExpressionValue::Stream(stream) => Some(&mut stream.value), _ => None, }) - .map_err(|_| variable.execution_error("The variable is not a stream")) + .map_err(|_| self.execution_error("The variable is not a stream")) } - pub(crate) fn set( - &self, - variable: &(impl IsVariable + ?Sized), - content: ExpressionValue, - ) -> ExecutionResult<()> { - *self.get_mut(variable)? = content; + pub(crate) fn set(&self, content: ExpressionValue) -> ExecutionResult<()> { + *self.get_mut()? = content; Ok(()) } - pub(crate) fn cheap_clone(&self) -> Self { + pub(crate) fn cheap_clone(&self, span_range: SpanRange) -> Self { Self { value: self.value.clone(), + span_range, } } } +impl HasSpanRange for VariableData { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + impl Interpreter { pub(crate) fn new() -> Self { Self { @@ -87,10 +89,10 @@ impl Interpreter { ) -> ExecutionResult<()> { match self.variable_data.entry(variable.get_name()) { Entry::Occupied(mut entry) => { - entry.get_mut().set(variable, value)?; + entry.get_mut().set(value)?; } Entry::Vacant(entry) => { - entry.insert(VariableData::new(value)); + entry.insert(VariableData::new(value, variable.span_range())); } } Ok(()) @@ -100,10 +102,12 @@ impl Interpreter { &self, variable: &(impl IsVariable + ?Sized), make_error: impl FnOnce() -> SynError, - ) -> ExecutionResult<&VariableData> { - self.variable_data + ) -> ExecutionResult { + let data = self + .variable_data .get(&variable.get_name()) - .ok_or_else(|| make_error().into()) + .ok_or_else(make_error)?; + Ok(data.cheap_clone(variable.span_range())) } pub(crate) fn start_iteration_counter<'s, S: HasSpanRange>( diff --git a/src/interpretation/source_code_block.rs b/src/interpretation/source_code_block.rs index 12a33327..d3c2bdbb 100644 --- a/src/interpretation/source_code_block.rs +++ b/src/interpretation/source_code_block.rs @@ -3,21 +3,21 @@ use crate::internal_prelude::*; /// A group `{ ... }` representing code which can be interpreted #[derive(Clone)] pub(crate) struct SourceCodeBlock { - delim_span: DelimSpan, + braces: Braces, inner: SourceStream, } impl Parse for SourceCodeBlock { fn parse(input: ParseStream) -> ParseResult { - let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; - let inner = content.parse_with_context(delim_span.join())?; - Ok(Self { delim_span, inner }) + let (braces, content) = input.parse_braces()?; + let inner = content.parse_with_context(braces.join())?; + Ok(Self { braces, inner }) } } impl HasSpan for SourceCodeBlock { fn span(&self) -> Span { - self.delim_span.join() + self.braces.span() } } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 8988feff..782823e5 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -31,13 +31,15 @@ pub(crate) trait IsVariable: HasSpanRange { &self, interpreter: &Interpreter, ) -> ExecutionResult { - Ok(self.read_existing(interpreter)?.cheap_clone()) + Ok(self + .read_existing(interpreter)? + .cheap_clone(self.span_range())) } fn get_value(&self, interpreter: &Interpreter) -> ExecutionResult { let value = self .read_existing(interpreter)? - .get(self)? + .get_cloned()? .clone() .with_span_range(self.span_range()); Ok(value) @@ -49,14 +51,14 @@ pub(crate) trait IsVariable: HasSpanRange { grouping: Grouping, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.read_existing(interpreter)?.get(self)?.output_to( + self.read_existing(interpreter)?.get_ref()?.output_to( grouping, output, StreamOutputBehaviour::Standard, ) } - fn read_existing<'i>(&self, interpreter: &'i Interpreter) -> ExecutionResult<&'i VariableData> { + fn read_existing(&self, interpreter: &Interpreter) -> ExecutionResult { interpreter.get_existing_variable_data(self, || { self.error("The variable does not already exist in the current scope") }) diff --git a/src/misc/field_inputs.rs b/src/misc/field_inputs.rs index 02d36ff9..c76197b5 100644 --- a/src/misc/field_inputs.rs +++ b/src/misc/field_inputs.rs @@ -33,7 +33,7 @@ macro_rules! define_field_inputs { let mut $optional_field: Option<$optional_type> = None; )* - let (delim_span, content) = input.parse_specific_group(Delimiter::Brace)?; + let (braces, content) = input.parse_braces()?; while !content.is_empty() { let ident = content.parse_any_ident()?; @@ -72,7 +72,7 @@ macro_rules! define_field_inputs { )* if !missing_fields.is_empty() { - return delim_span.join().parse_err(format!( + return braces.parse_err(format!( "required fields are missing: {}", missing_fields.join(", ") )); diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 1d1c8147..970884c3 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -325,6 +325,28 @@ impl<'a, K> ParseBuffer<'a, K> { ) } + pub(crate) fn parse_braces(&self) -> ParseResult<(Braces, ParseBuffer)> { + let (delim_span, inner) = self.parse_specific_group(Delimiter::Brace)?; + Ok((Braces { delim_span }, inner)) + } + + pub(crate) fn parse_brackets(&self) -> ParseResult<(Brackets, ParseBuffer)> { + let (delim_span, inner) = self.parse_specific_group(Delimiter::Bracket)?; + Ok((Brackets { delim_span }, inner)) + } + + pub(crate) fn parse_parentheses(&self) -> ParseResult<(Parentheses, ParseBuffer)> { + let (delim_span, inner) = self.parse_specific_group(Delimiter::Parenthesis)?; + Ok((Parentheses { delim_span }, inner)) + } + + pub(crate) fn parse_transparent_group( + &self, + ) -> ParseResult<(TransparentDelimiters, ParseBuffer)> { + let (delim_span, inner) = self.parse_specific_group(Delimiter::None)?; + Ok((TransparentDelimiters { delim_span }, inner)) + } + pub(crate) fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { Err(self.parse_error(message)) } diff --git a/src/transformation/destructuring.rs b/src/transformation/destructuring.rs index faa72709..67a2f130 100644 --- a/src/transformation/destructuring.rs +++ b/src/transformation/destructuring.rs @@ -54,15 +54,15 @@ impl HandleDestructure for Pattern { #[derive(Clone)] pub struct ArrayPattern { #[allow(unused)] - delim_span: DelimSpan, + brackets: Brackets, items: Punctuated, } impl Parse for ArrayPattern { fn parse(input: ParseStream) -> ParseResult { - let (delim_span, inner) = input.parse_specific_group(Delimiter::Bracket)?; + let (brackets, inner) = input.parse_brackets()?; Ok(Self { - delim_span, + brackets, items: inner.parse_terminated()?, }) } diff --git a/src/transformation/fields.rs b/src/transformation/fields.rs index bc05e28f..1323334b 100644 --- a/src/transformation/fields.rs +++ b/src/transformation/fields.rs @@ -79,7 +79,7 @@ impl FieldsParseDefinition { error_span_range: SpanRange, ) -> ParseResult { let mut builder = new_builder; - let (_, content) = input.parse_specific_group(Delimiter::Brace)?; + let (_, content) = input.parse_braces()?; let mut required_field_names: BTreeSet<_> = field_definitions .0 diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index 221c596b..e3551b76 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -137,8 +137,15 @@ impl HandleTransformation for TransformGroup { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - let (_, inner) = input.parse_specific_group(self.delimiter)?; - self.inner.handle_transform(&inner, interpreter, output) + // Because `None` is ignored by Syn at parsing time, we can effectively be most permissive by ignoring them. + // This removes a bit of a footgun for users. + // If they really want to check for a None group, they can embed `@[GROUP ...]` transformer. + if self.delimiter == Delimiter::None { + self.inner.handle_transform(input, interpreter, output) + } else { + let (_, inner) = input.parse_specific_group(self.delimiter)?; + self.inner.handle_transform(&inner, interpreter, output) + } } } @@ -147,7 +154,7 @@ pub(crate) struct ExplicitTransformStream { #[allow(unused)] transformer_token: Token![@], #[allow(unused)] - delim_span: DelimSpan, + parentheses: Parentheses, arguments: ExplicitTransformStreamArguments, } @@ -214,11 +221,11 @@ impl Parse for ExplicitTransformStreamArguments { impl Parse for ExplicitTransformStream { fn parse(input: ParseStream) -> ParseResult { let transformer_token = input.parse()?; - let (delim_span, content) = input.parse_specific_group(Delimiter::Parenthesis)?; + let (parentheses, content) = input.parse_parentheses()?; Ok(Self { transformer_token, - delim_span, + parentheses, arguments: content.parse()?, }) } @@ -249,7 +256,7 @@ impl HandleTransformation for ExplicitTransformStream { content.handle_transform( input, interpreter, - variable_data.get_mut_stream(variable)?.deref_mut(), + variable_data.get_mut_stream()?.deref_mut(), )?; } ExplicitTransformStreamArguments::Discard { content, .. } => { diff --git a/src/transformation/transformer.rs b/src/transformation/transformer.rs index 40b69939..34b777ff 100644 --- a/src/transformation/transformer.rs +++ b/src/transformation/transformer.rs @@ -85,7 +85,7 @@ pub(crate) struct Transformer { transformer_token: Token![@], instance: NamedTransformer, #[allow(unused)] - source_group_span: Option, + source_brackets: Option, } impl Parse for Transformer { @@ -96,9 +96,9 @@ impl Parse for Transformer { let ident = input.parse_any_ident()?; (ident, None) } else if input.peek_specific_group(Delimiter::Bracket) { - let (delim_span, content) = input.parse_specific_group(Delimiter::Bracket)?; + let (brackets, content) = input.parse_brackets()?; let ident = content.parse_any_ident()?; - (ident, Some((content, delim_span))) + (ident, Some((content, brackets))) } else { return input.parse_err("Expected @TRANSFORMER or @[TRANSFORMER ...arguments...]"); }; @@ -114,12 +114,12 @@ impl Parse for Transformer { }; match arguments { - Some((buffer, delim_span)) => { - let arguments = TransformerArguments::new(&buffer, name, delim_span.join()); + Some((buffer, brackets)) => { + let arguments = TransformerArguments::new(&buffer, name, brackets.join()); Ok(Self { transformer_token, instance: transformer_kind.parse_instance(arguments)?, - source_group_span: Some(delim_span), + source_brackets: Some(brackets), }) } None => { @@ -142,7 +142,7 @@ impl Parse for Transformer { Ok(Self { transformer_token, instance, - source_group_span: None, + source_brackets: None, }) } } diff --git a/src/transformation/transformers.rs b/src/transformation/transformers.rs index c29c0a4b..2fe3a07e 100644 --- a/src/transformation/transformers.rs +++ b/src/transformation/transformers.rs @@ -95,7 +95,7 @@ impl TransformerDefinition for GroupTransformer { interpreter: &mut Interpreter, output: &mut OutputStream, ) -> ExecutionResult<()> { - let (_, inner) = input.parse_specific_group(Delimiter::None)?; + let (_, inner) = input.parse_transparent_group()?; self.inner.handle_transform(&inner, interpreter, output) } } diff --git a/src/transformation/variable_binding.rs b/src/transformation/variable_binding.rs index 7db29363..0a1fcc0b 100644 --- a/src/transformation/variable_binding.rs +++ b/src/transformation/variable_binding.rs @@ -189,17 +189,17 @@ impl HandleTransformation for VariableBinding { let variable_data = self.get_existing_for_mutation(interpreter)?; input .parse::()? - .push_as_token_tree(variable_data.get_mut_stream(self)?.deref_mut()); + .push_as_token_tree(variable_data.get_mut_stream()?.deref_mut()); } VariableBinding::GroupedAppendFlattened { .. } => { let variable_data = self.get_existing_for_mutation(interpreter)?; input .parse::()? - .flatten_into(variable_data.get_mut_stream(self)?.deref_mut()); + .flatten_into(variable_data.get_mut_stream()?.deref_mut()); } VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { let variable_data = self.get_existing_for_mutation(interpreter)?; - variable_data.get_mut_stream(self)?.push_grouped( + variable_data.get_mut_stream()?.push_grouped( |inner| until.handle_parse_into(input, inner), Delimiter::None, marker.span, @@ -207,7 +207,7 @@ impl HandleTransformation for VariableBinding { } VariableBinding::FlattenedAppendFlattened { until, .. } => { let variable_data = self.get_existing_for_mutation(interpreter)?; - until.handle_parse_into(input, variable_data.get_mut_stream(self)?.deref_mut())?; + until.handle_parse_into(input, variable_data.get_mut_stream()?.deref_mut())?; } } Ok(()) diff --git a/tests/expressions.rs b/tests/expressions.rs index 63607ec0..6777b435 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -220,3 +220,13 @@ fn test_range() { ); // preinterpret_assert_eq!({ [!debug! 0..10000 as iterator] }, "[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ..<9980 further items>]"); } + +#[test] +fn test_indexing() { + preinterpret_assert_eq!( + #(let x = [1, 2, 3]; x[1]), 2 + ); + preinterpret_assert_eq!( + #(let x = [1, 2, 3]; x[x[0] + x[x[1] - 1] - 1]), 3 + ); +} From cc49e09930b664938f3459137ddc0f18ab367101 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sun, 16 Feb 2025 09:48:59 +0000 Subject: [PATCH 099/126] refactor: Create variable reference abstraction --- CHANGELOG.md | 10 ++ src/expressions/evaluation.rs | 8 +- src/expressions/value.rs | 6 + src/interpretation/commands/core_commands.rs | 11 +- src/interpretation/interpreter.rs | 135 +++++++++++------- src/interpretation/variable.rs | 55 ++----- src/transformation/transform_stream.rs | 6 +- src/transformation/variable_binding.rs | 20 +-- .../core/extend_variable_and_then_set_it.rs | 2 +- .../extend_variable_and_then_set_it.stderr | 6 +- 10 files changed, 139 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b6d0d62..1ce001fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,11 +97,21 @@ Inside a transform stream, the following grammar is supported: ### To come +* Merge `GroupedVariable` and `ExpressionBlock` into an `ExplicitExpression`: + * Either `#ident` or `#(...)` or `#{ ... }`... + the latter defines a new variable stack frame, just like Rust + * To avoid confusion (such as below) and teach the user to only include #var where + necessary, only expression _blocks_ are allowed in an expression. + * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal + rust because the inside is a `{ .. }` which defines a new scope. * Support `#(x[..])` syntax for indexing arrays at read time (streams to follow in a separate task below after parsers are updated) * `#(x[0..3])` returns an array * `#(x[0..=3])` returns an array * ... and `#(x[0] = y)` can be used to set the item * Considering allowing assignments inside an expression + * Implementation notes: + * Separate `VariableData` and `VariableReference`, separate `let` and `set` + * Three separate stacks for various calculations in evaluation * `let XX =` and `YY += y` are actually totally different... * With `let XX =`, `XX` is a _pattern_ and creates new variables. * With `YY += y` we're inside an expression already, and it only works with existing variables. The expression `YY` is converted into a destructuring at execution time. diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index bb5edd9f..6ed2f5b8 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -312,7 +312,9 @@ impl<'a> ExpressionEvaluator<'a, Source> { Assignee::Place(place) => { let span_range = SpanRange::new_between(place.span_range().start(), value.span_range().end()); - let left = place.get_cloned()?.clone(); + // TODO - replace with handling a compound operation for better performance + // of e.g. arrays or streams + let left = place.get_value_cloned()?.clone(); place.set(operation.to_binary().evaluate(left, value)?)?; NextAction::HandleValue(ExpressionValue::None(span_range)) } @@ -334,7 +336,7 @@ impl<'a> ExpressionEvaluator<'a, Source> { let resolved = loop { match &self.nodes[assignee.0] { ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => { - break Assignee::Place(variable.read_existing(interpreter)?); + break Assignee::Place(variable.reference(interpreter)?); } ExpressionNode::Index { .. } => { todo!() @@ -361,7 +363,7 @@ impl<'a> ExpressionEvaluator<'a, Source> { } enum Assignee { - Place(VariableData), + Place(VariableReference), CompoundWip, // To come } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 923923cc..a1b53a53 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -556,6 +556,12 @@ impl ExpressionValue { } } +impl ToExpressionValue for ExpressionValue { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + self.with_span_range(span_range) + } +} + #[derive(Clone, Copy)] pub(crate) enum StreamOutputBehaviour { Standard, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 036d4156..67d92b0b 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -89,17 +89,20 @@ impl NoOutputCommandDefinition for SetCommand { variable, content, .. } => { let content = content.interpret_to_new_stream(interpreter)?; - variable.set_stream(interpreter, content)?; + variable.define(interpreter, content); } SetArguments::ExtendVariable { variable, content, .. } => { - let variable_data = variable.get_existing_for_mutation(interpreter)?; - content.interpret_into(interpreter, variable_data.get_mut_stream()?.deref_mut())?; + let variable_data = variable.reference(interpreter)?; + content.interpret_into( + interpreter, + variable_data.get_value_stream_mut()?.deref_mut(), + )?; } SetArguments::SetVariablesEmpty { variables } => { for variable in variables { - variable.set_stream(interpreter, OutputStream::new())?; + variable.define(interpreter, OutputStream::new()); } } SetArguments::Discard { content, .. } => { diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index fb5e81dc..0d6951e8 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -2,52 +2,44 @@ use crate::internal_prelude::*; use super::IsVariable; use std::cell::*; -use std::collections::hash_map::Entry; use std::rc::Rc; pub(crate) struct Interpreter { config: InterpreterConfig, - variable_data: HashMap, + variable_data: VariableData, } #[derive(Clone)] -pub(crate) struct VariableData { - value: Rc>, - /// In the store, this is the span range of the original let variable declaration. - /// When this is a variable reference, this is the span range of the variable - /// which created the reference. - span_range: SpanRange, +pub(crate) struct VariableReference { + data: Rc>, + variable_span_range: SpanRange, } -impl VariableData { - fn new(tokens: ExpressionValue, span_range: SpanRange) -> Self { - Self { - value: Rc::new(RefCell::new(tokens)), - span_range, - } - } - - pub(crate) fn get_ref(&self) -> ExecutionResult> { - self.value.try_borrow().map_err(|_| { +impl VariableReference { + pub(crate) fn get_value_ref(&self) -> ExecutionResult> { + self.data.try_borrow().map_err(|_| { self.execution_error("The variable cannot be read if it is currently being modified") }) } // Gets the cloned expression value, setting the span range appropriately - pub(crate) fn get_cloned(&self) -> ExecutionResult { - Ok(self.get_ref()?.clone().with_span_range(self.span_range)) + pub(crate) fn get_value_cloned(&self) -> ExecutionResult { + Ok(self + .get_value_ref()? + .clone() + .with_span_range(self.variable_span_range)) } - pub(crate) fn get_mut(&self) -> ExecutionResult> { - self.value.try_borrow_mut().map_err(|_| { + pub(crate) fn get_value_mut(&self) -> ExecutionResult> { + self.data.try_borrow_mut().map_err(|_| { self.execution_error( "The variable cannot be modified if it is already currently being modified", ) }) } - pub(crate) fn get_mut_stream(&self) -> ExecutionResult> { - let mut_guard = self.get_mut()?; + pub(crate) fn get_value_stream_mut(&self) -> ExecutionResult> { + let mut_guard = self.get_value_mut()?; RefMut::filter_map(mut_guard, |mut_guard| match mut_guard { ExpressionValue::Stream(stream) => Some(&mut stream.value), _ => None, @@ -55,22 +47,15 @@ impl VariableData { .map_err(|_| self.execution_error("The variable is not a stream")) } - pub(crate) fn set(&self, content: ExpressionValue) -> ExecutionResult<()> { - *self.get_mut()? = content; + pub(crate) fn set(&self, content: impl ToExpressionValue) -> ExecutionResult<()> { + *self.get_value_mut()? = content.to_value(self.variable_span_range); Ok(()) } - - pub(crate) fn cheap_clone(&self, span_range: SpanRange) -> Self { - Self { - value: self.value.clone(), - span_range, - } - } } -impl HasSpanRange for VariableData { +impl HasSpanRange for VariableReference { fn span_range(&self) -> SpanRange { - self.span_range + self.variable_span_range } } @@ -78,36 +63,24 @@ impl Interpreter { pub(crate) fn new() -> Self { Self { config: Default::default(), - variable_data: Default::default(), + variable_data: VariableData::new(), } } - pub(crate) fn set_variable( + pub(crate) fn define_variable( &mut self, variable: &(impl IsVariable + ?Sized), value: ExpressionValue, - ) -> ExecutionResult<()> { - match self.variable_data.entry(variable.get_name()) { - Entry::Occupied(mut entry) => { - entry.get_mut().set(value)?; - } - Entry::Vacant(entry) => { - entry.insert(VariableData::new(value, variable.span_range())); - } - } - Ok(()) + ) { + self.variable_data.define_variable(variable, value) } - pub(crate) fn get_existing_variable_data( + pub(crate) fn get_variable_reference( &self, variable: &(impl IsVariable + ?Sized), make_error: impl FnOnce() -> SynError, - ) -> ExecutionResult { - let data = self - .variable_data - .get(&variable.get_name()) - .ok_or_else(make_error)?; - Ok(data.cheap_clone(variable.span_range())) + ) -> ExecutionResult { + self.variable_data.get_reference(variable, make_error) } pub(crate) fn start_iteration_counter<'s, S: HasSpanRange>( @@ -126,6 +99,60 @@ impl Interpreter { } } +struct VariableData { + variable_data: HashMap, +} + +impl VariableData { + fn new() -> Self { + Self { + variable_data: HashMap::new(), + } + } + + fn define_variable(&mut self, variable: &(impl IsVariable + ?Sized), value: ExpressionValue) { + self.variable_data.insert( + variable.get_name(), + VariableContent::new(value, variable.span_range()), + ); + } + + fn get_reference( + &self, + variable: &(impl IsVariable + ?Sized), + make_error: impl FnOnce() -> SynError, + ) -> ExecutionResult { + let reference = self + .variable_data + .get(&variable.get_name()) + .ok_or_else(make_error)? + .create_reference(variable); + Ok(reference) + } +} + +struct VariableContent { + value: Rc>, + #[allow(unused)] + definition_span_range: SpanRange, +} + +impl VariableContent { + fn new(tokens: ExpressionValue, definition_span_range: SpanRange) -> Self { + Self { + value: Rc::new(RefCell::new(tokens)), + definition_span_range, + } + } + + fn create_reference(&self, variable: &(impl IsVariable + ?Sized)) -> VariableReference { + VariableReference { + data: self.value.clone(), + variable_span_range: variable.span_range(), + } + } +} + pub(crate) struct IterationCounter<'a, S: HasSpanRange> { span_source: &'a S, count: usize, diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 782823e5..cc49b762 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -3,46 +3,16 @@ use crate::internal_prelude::*; pub(crate) trait IsVariable: HasSpanRange { fn get_name(&self) -> String; - fn set_stream( - &self, - interpreter: &mut Interpreter, - stream: OutputStream, - ) -> ExecutionResult<()> { - interpreter.set_variable(self, stream.to_value(self.span_range())) - } - - fn set_coerced_stream( - &self, - interpreter: &mut Interpreter, - stream: OutputStream, - ) -> ExecutionResult<()> { - interpreter.set_variable(self, stream.coerce_into_value(self.span_range())) + fn define(&self, interpreter: &mut Interpreter, value_source: impl ToExpressionValue) { + interpreter.define_variable(self, value_source.to_value(self.span_range())) } - fn set_value( - &self, - interpreter: &mut Interpreter, - value: ExpressionValue, - ) -> ExecutionResult<()> { - interpreter.set_variable(self, value) - } - - fn get_existing_for_mutation( - &self, - interpreter: &Interpreter, - ) -> ExecutionResult { - Ok(self - .read_existing(interpreter)? - .cheap_clone(self.span_range())) + fn define_coerced(&self, interpreter: &mut Interpreter, content: OutputStream) { + interpreter.define_variable(self, content.coerce_into_value(self.span_range())) } - fn get_value(&self, interpreter: &Interpreter) -> ExecutionResult { - let value = self - .read_existing(interpreter)? - .get_cloned()? - .clone() - .with_span_range(self.span_range()); - Ok(value) + fn get_cloned_value(&self, interpreter: &Interpreter) -> ExecutionResult { + self.reference(interpreter)?.get_value_cloned() } fn substitute_into( @@ -51,15 +21,15 @@ pub(crate) trait IsVariable: HasSpanRange { grouping: Grouping, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.read_existing(interpreter)?.get_ref()?.output_to( + self.reference(interpreter)?.get_value_ref()?.output_to( grouping, output, StreamOutputBehaviour::Standard, ) } - fn read_existing(&self, interpreter: &Interpreter) -> ExecutionResult { - interpreter.get_existing_variable_data(self, || { + fn reference(&self, interpreter: &Interpreter) -> ExecutionResult { + interpreter.get_variable_reference(self, || { self.error("The variable does not already exist in the current scope") }) } @@ -161,7 +131,7 @@ impl InterpretToValue for &GroupedVariable { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - self.get_value(interpreter) + self.get_cloned_value(interpreter) } } @@ -273,7 +243,7 @@ impl InterpretToValue for &VariableIdentifier { self, interpreter: &mut Interpreter, ) -> ExecutionResult { - self.get_value(interpreter) + self.get_cloned_value(interpreter) } } @@ -308,6 +278,7 @@ impl HandleDestructure for VariablePattern { interpreter: &mut Interpreter, value: ExpressionValue, ) -> ExecutionResult<()> { - self.set_value(interpreter, value) + self.define(interpreter, value); + Ok(()) } } diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index e3551b76..e30f2fdf 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -247,16 +247,16 @@ impl HandleTransformation for ExplicitTransformStream { } => { let mut new_output = OutputStream::new(); content.handle_transform(input, interpreter, &mut new_output)?; - variable.set_stream(interpreter, new_output)?; + variable.define(interpreter, new_output); } ExplicitTransformStreamArguments::ExtendToVariable { variable, content, .. } => { - let variable_data = variable.get_existing_for_mutation(interpreter)?; + let reference = variable.reference(interpreter)?; content.handle_transform( input, interpreter, - variable_data.get_mut_stream()?.deref_mut(), + reference.get_value_stream_mut()?.deref_mut(), )?; } ExplicitTransformStreamArguments::Discard { content, .. } => { diff --git a/src/transformation/variable_binding.rs b/src/transformation/variable_binding.rs index 0a1fcc0b..c70d1981 100644 --- a/src/transformation/variable_binding.rs +++ b/src/transformation/variable_binding.rs @@ -178,36 +178,36 @@ impl HandleTransformation for VariableBinding { match self { VariableBinding::Grouped { .. } => { let content = input.parse::()?.into_interpreted(); - self.set_coerced_stream(interpreter, content)?; + self.define_coerced(interpreter, content); } VariableBinding::Flattened { until, .. } => { let mut content = OutputStream::new(); until.handle_parse_into(input, &mut content)?; - self.set_coerced_stream(interpreter, content)?; + self.define_coerced(interpreter, content); } VariableBinding::GroupedAppendGrouped { .. } => { - let variable_data = self.get_existing_for_mutation(interpreter)?; + let reference = self.reference(interpreter)?; input .parse::()? - .push_as_token_tree(variable_data.get_mut_stream()?.deref_mut()); + .push_as_token_tree(reference.get_value_stream_mut()?.deref_mut()); } VariableBinding::GroupedAppendFlattened { .. } => { - let variable_data = self.get_existing_for_mutation(interpreter)?; + let reference = self.reference(interpreter)?; input .parse::()? - .flatten_into(variable_data.get_mut_stream()?.deref_mut()); + .flatten_into(reference.get_value_stream_mut()?.deref_mut()); } VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { - let variable_data = self.get_existing_for_mutation(interpreter)?; - variable_data.get_mut_stream()?.push_grouped( + let reference = self.reference(interpreter)?; + reference.get_value_stream_mut()?.push_grouped( |inner| until.handle_parse_into(input, inner), Delimiter::None, marker.span, )?; } VariableBinding::FlattenedAppendFlattened { until, .. } => { - let variable_data = self.get_existing_for_mutation(interpreter)?; - until.handle_parse_into(input, variable_data.get_mut_stream()?.deref_mut())?; + let reference = self.reference(interpreter)?; + until.handle_parse_into(input, reference.get_value_stream_mut()?.deref_mut())?; } } Ok(()) diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.rs b/tests/compilation_failures/core/extend_variable_and_then_set_it.rs index 8ca1f9b2..ca180ac7 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_set_it.rs +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.rs @@ -3,6 +3,6 @@ use preinterpret::*; fn main() { preinterpret! { [!set! #variable = Hello] - [!set! #variable += World [!set! #variable = Hello2]] + [!set! #variable += World #(variable = [!stream! Hello2])] } } \ No newline at end of file diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr index eba38ba1..5e395d4a 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr @@ -1,5 +1,5 @@ error: The variable cannot be modified if it is already currently being modified - --> tests/compilation_failures/core/extend_variable_and_then_set_it.rs:6:42 + --> tests/compilation_failures/core/extend_variable_and_then_set_it.rs:6:37 | -6 | [!set! #variable += World [!set! #variable = Hello2]] - | ^^^^^^^^^ +6 | [!set! #variable += World #(variable = [!stream! Hello2])] + | ^^^^^^^^ From aed0f8557a690fc805c607ce0c0bfe987834bd78 Mon Sep 17 00:00:00 2001 From: David Edey Date: Mon, 17 Feb 2025 14:19:38 +0000 Subject: [PATCH 100/126] tweak: #variable syntax is no longer allowed in expressions --- CHANGELOG.md | 225 +++++++++--------- src/expressions/expression.rs | 11 +- src/expressions/expression_parsing.rs | 21 +- .../control_flow/error_after_continue.rs | 4 +- .../core/error_span_repeat.rs | 2 +- .../core/error_span_repeat.stderr | 6 +- ...rsperse_stream_input_variable_issue.stderr | 2 +- tests/control_flow.rs | 6 +- tests/expressions.rs | 18 +- tests/tokens.rs | 28 +-- tests/transforming.rs | 16 +- 11 files changed, 169 insertions(+), 170 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ce001fa..dab3ee54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -193,48 +193,73 @@ Inside a transform stream, the following grammar is supported: * `.push(x)` on array * Consider `.map(|| {})` * TRANSFORMERS => PARSERS cont + * Re-read the `Parsers Revisited` section below * Manually search for transform and rename to parse in folder names and file. - * Parsers no longer output to a stream past that. - Instead, they act like a `StreamPattern` which needs to: - * Define the variables it binds up front `{ x, y }` - * Can't mutate any variables in ancestor frames (but can potentially read them) -```rust,ignore -@{ x, y }(... destructuring ...) -``` - * Support `@[x = ...]` and `@[let x = ...]` for individual parsers. - * Scrap `[!let!]` and `[!parse! ..]` in favour of `#(let = #x)` - * Scrap `#>>x` etc in favour of `@(a = ...) #[x += [a]]` + * Parsers no longer output to a stream. + * We let `#(let x)` INSIDE a `@(...)` bind to the same scope as its surroundings... + Now some repetition like `@{..}*` needs to introduce its own scope. + ==> Annoyingly, a `@(..)*` has to really push/output to an array internally to have good developer experience; which means we need to solve the "relatively performant staged/reverted interpreter state" problem regardless; and putting an artificial limitation on conditional parse streams to not do that doesn't really work. TBC - needs more consideration. + ==> Maybe there is an alternative such as: + * `@(..)*` returns an array of some output of it's inner thing + * But things don't output so what does that mean?? + * I think in practice it will be quite an annoying restriction anyway + * OLD: Support `@[x = ...]` and `@[let x = ...]` for individual parsers. + * CHANGE OF THOUGHT: instead, support `#(x = @IDENT)` where `#` blocks inside parsers can embed @parsers and consume from a parse stream. + * This makes it kinda like a `Parse` implementation code block. + * This lets us just support variable definitions in expression statements. + * Don't support `@(x = ...)` - instead we can have `#(x = @[STREAM ...])` + * This can capture the original tokens by using `let forked = input.fork()` + and then `let end_cursor = input.end();` and then consuming `TokenTree`s + from `forked` until `forked.cursor >= end_cursor` (making use of the + PartialEq implementation) + * Revisit `parse` + * Scrap `#>>x` etc in favour of `#(a.push(@[XXX]))` + * Scrap `[!let!]` in favour of `#(let = #x)` * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. + * `@INFER_TOKEN_TREE` - Infers values, falls back to Stream * `@[ANY_GROUP ...]` * `@REST` - * `@[UNTIL xxxx]` - For now - takes a raw stream which is turned into an ExactStream. * `@[FIELDS { ... }]` and `@[SUBFIELDS { ... }]` - * Add ability to add scope to interpreter state (copy on write?) (and commit/revert) and can then add: + * Add ability to add scope to interpreter state (see `Parsers Revisited`) and can then add: * `@[OPTIONAL ...]` and `@(...)?` - * `@(REPEATED { ... })` (see below) - * Potentially change `@UNTIL` to take a transform stream instead of a raw stream. + * `@[REPEATED { ... }]` (see below) + * `@[UNTIL @{...}]` - takes an explicit parse block or parser. Reverts any state change * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` +```rust +@[REPEATED { + item: @(...), // Captured into #item variable + separator?: @(), // Captured into #separator variable + min?: 0, + max?: 1000000, + handle_item?: { #item }, // Default is to output the grouped item. #()+ instead uses `{ (#..item) }` + handle_separator?: { }, // Default is to not output the separator +}] +``` * Support `#(x[..])` syntax for indexing streams * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) * `#(x[0..3])` returns a TokenStream * `#(x[0..=3])` returns a TokenStream * Add `..` and `.., x` support to the array pattern * Consider: + * Moving control flow (`for` and `while`) to the expression side? * Dropping lots of the `group` wrappers? * If any types should have reference semantics instead of clone/value semantics? * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` * Adding `!define_command!` * Adding `!define_transformer!` + * Whether `preinterpret` should start in expression mode? + => Or whether to have `preinterpet::stream` / `preinterpret::run` / `preinterpret::define_macro` options? + => Maybe `preinterpret::preinterpret` is marked as deprecated; starts in `stream` mode, and enables `[!set!]`? * `[!is_set! #x]` * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) * Maybe add a `@[REINTERPRET ..]` transformer. * CastTarget expansion: * Add `as iterator` and uncomment the test at the end of `test_range()` - * Support a CastTarget of `array` (only supported for array and stream and iterator) + * Support a CastTarget of `array` using `into_iterator()`. * Add `as ident` and `as literal` casting and support it for string, array and stream using concat recursive. * Add casts of other integers to char, via `char::from_u32(u32::try_from(x))` * Put `[!set! ...]` inside an opt-in feature because it's quite confusing. @@ -251,11 +276,8 @@ Inside a transform stream, the following grammar is supported: * Those taking some { fields } * Those taking some custom syntax, e.g. `!set!`, `!if!`, `!while!` etc -### Transformers Revisited +### Parsers Revisited ```rust -// * The current situation feels unclear/arbitrary/inflexible. -// * Better would be slightly more explicit. -// // PROPOSAL (subject to the object proposal above): // * Four modes: // * Output stream mode @@ -267,7 +289,7 @@ Inside a transform stream, the following grammar is supported: // * Idents mean variables. // * `let #x; let _ = ; ; if {} else {}` // * Error not to discard a non-None value with `let _ = ` -// * Not allowed to parse. +// * If embedded into a parse stream, it's allowed to parse by embedding parsers, e.g. @[STREAM ...] // * Only last statement can (optionally) output, with a value model of: // * Leaf(Bool | Int | Float | String) // * Object @@ -275,126 +297,95 @@ Inside a transform stream, the following grammar is supported: // * None // * The actual type is only known at evaluation time. // * #var is equivalent to #(var) -// * @[XXX] or @[hello.world = XXX] +// * Named parser @XXX or @[XXX] or @[XXX ] // * Can parse; has no output. // * XXX is: // * A named destructurer (possibly taking further input) // * An { .. } object destructurer // * A @(...) transformer stream (output = input) -// * @() = Transformer stream -// * Can parse +// * @() or @{} = Parse block +// * Expression/command outputs are parsed exactly. +// * Can return a value with `#(return X)` +// * (`@[STREAM ...]` is broadly equivalent, but does output the input stream) // * Its only output is an input stream reference // * ...and only if it's redirected inside a @[x = $(...)] // * [!command! ...] // * Can output but does not parse. -// * Every transformer has a typed output, which is either: -// * EITHER its token stream (for simple matchers) +// * Every named parser has a typed output, which is: +// * EITHER its input token stream (for simple matchers, e.g. @IDENT) // * OR an #output OBJECT with at least two properties: // * input => all matched characters (a slice reference which can be dropped...) // (it might only be possible to performantly capture this after the syn fork) -// * stream => A lazy function, used to handle the output when #x is in the final output... +// * output_to => A lazy function, used to handle the output when #x is in the final output... // likely `input` or an error depending on the case. +// * into_iterator // * ... other properties, depending on the TRANSFORMER: // * e.g. a Rust ITEM might have quite a few (mostly lazy) -// * Drop @XXX syntax. Require: @[ ... ] instead, one of: -// * @[XXX] or equivalently @[let _ = XXX ...] -// * @[let x = XXX] or @[let x = XXX { ... }] -// * @[let x.field = XXX ...] -// * @[x = XXX] -// * @[x.field += IDENT] -// * Explicit stream: @(...) -/// * Has an explicit output property via command output, e.g. [!output! ...] which appends both: -// * Command output -// * Output of inner transformer streams. -// * It can be treated as a transformer and its output can be redirected with @[#x = @(...)] syntax. -// But, on trying a few examples, we don't want to require such redirection. -// * QUESTION: Do we actually use square brackets, i.e. @[#x = ...] (i.e. it's just without the IDENT) -// * OR maybe as an explicit [ ... ] stream: @[#x = [...]] -// * For inline destructurings (e.g. in for loops), we allow an implicit inner ... stream -// * EXACT then is just used for some sections where we're reading from variables. +// * Repetitions (including optional), e.g. @IDENT,* or @[IDENT ...],* or @{...},* +// * Output their contents in a `Repeated` type, with an `items` property, and an into_iterator implementation? +// * Transform streams can't be repeated, only transform blocks (because they create a new interpreter frame) +// * There is still an issue with "what happens to mutated state when a repetition is not possible?" +// ... this is also true in a case statement... We need some way to rollback in these cases: +// => Easier - Use of efficient-ish immutable data structures, e.g. ImmutableList, Copy-on-write leaf types etc +// ... so that we can clone them cheaply (e.g. https://crates.io/crates/im-rc) +// => Middle (best?) - Any changes to variables outside the scope of a given refutable parser => it's a fatal error +// to parse further until that scope is closed. (this needs to handle nested repetitions) +// => Hardest - Some kind of storing of diffs / layered stores without any O(N^2) behaviours // -// EXAMPLE -#(parsed = []) -[!parse! [...] as - @( - #(let item = {}) - impl @[item.trait = IDENT] for @[item.type = IDENT] - #(parsed.push(item)) - ),* -] -[!stream_for! { trait, type } in parsed.output { - impl #trait for #type {} +// EXAMPLE (Updated, 17th February) +[!parse! { + input: [...], + parser: #( + let parsed = @{ + impl #(let the_trait = @IDENT) for #(let the_type = @IDENT) + #(return { the_trait, the_type }) + },* + ), }] - -``` -### Transformer Notes WIP -```rust -// FINAL DESIGN --- -// ALL THESE CAN BE SUPPORTED: -[!for! (#x #y) in [!parse! #input as @(impl @IDENT for @IDENT),*] { - +// EXAMPLE IN PATTERN POSITION +#( + let [!parser! // Takes an expression, which could include a #(...) + let parsed = @( + #(let item = {}) + impl #(item.the_trait = @IDENT) for #(item.the_type = @IDENT) + #(return item) + )* + ] = [...] +) + +[!for! { the_trait, the_type } in parsed { + impl #the_trait for #the_type {} }] -// NICE -[!parse! #input as @[REPEATED { - item: @(impl @(#trait = @IDENT) for @(#type = @TYPE)), - item_output: { - impl BLAH BLAH { - ... - } - } -}]] -// ALSO NICE -[!define_transformer! @INPUT = @(impl @IDENT for @IDENT),*] -[!for! (#x #y) in [!parse! #input as @INPUT] { - +// Example inlined: +[!for! { the_trait, the_type } in [!parse! { + input: [...], + parser: @( // This is implicitly an expression, which returns a `Repeated` type with an `into_iterator` over its items. + impl #(let the_trait = @IDENT) for #(let the_type = @IDENT) + #(return { the_trait, the_type }) + ),* +}] { + impl #the_trait for #the_type {} }] -// MAYBE - probably not though... -[!parse_for! #input as @(impl @(#x = @IDENT) for @(#y = @IDENT)),* { +// Example pre-defined: +[!parser! @IMPL_ITEM { + @(impl); + let the_trait = @IDENT; + @(for); + let the_type = @IDENT; + return { the_trait, the_type }; }] -// WHAT WIZARDRY IS THIS -[!define_transformer! @[REPEAT_EXACT @[FIELDS { - matcher: @(#matcher = $(@REST)), - repetitions: @(#repetitions = @LITERAL), -}]] = @[REINTERPRET [!for! #_ in [!range! 0..#repetitions] { $(#..matcher) }]]] - -// SYNTAX PREFERENCE - output by default; use @ for destructurers -// * @( ... ) destructure stream, has an output -// * @( ... )? optional destructure stream, has an output -// * Can similarly have @(...),+ which handles a trailing , -// * @X shorthand for @[X] for destructurers which can take no input, e.g. IDENT, TOKEN_TREE, TYPE etc -// => NOTE: Each destructurer should return just its tokens by default if it has no arguments. -// => It can also have its output over-written or other things outputted using e.g. @[TYPE { is_prefixed: X, parts: #(...), output: { #output } }] -// * #x is shorthand for @(#x = @TOKEN_OR_GROUP_CONTENT) -// * #..x) is shorthand for @(#x = @REST) and #..x, is shorthand for @(#x = @[UNTIL_TOKEN ,]) -// * @(_ = ...) -// * @(#x = impl @IDENT for @IDENT) -// * @(#x += impl @IDENT for @IDENT) -// * @[REPEATED { ... }] -// * Can embed commands to output stuff too -// * Can output a group with: @(#x = @IDENT for @IDENT) [!output! #x] - -// In this model, REPEATED is really clean and looks like this: -@[REPEATED { - item: @(...), // Captured into #item variable - separator?: @(), // Captured into #separator variable - min?: 0, - max?: 1000000, - handle_item?: { #item }, // Default is to output the grouped item. #()+ instead uses `{ (#..item) }` - handle_separator?: { }, // Default is to not output the separator +[!for! { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] { + impl #the_trait for #the_type {} }] -// How does optional work? -// @(#x = @(@IDENT)?) -// Along with: -// [!object! { #x, my_var: #y, #z }] -// And if some field #z isn't set, it's outputted as null. -// Field access can be with #(variable.field) - -// Do we want something like !parse_for!? It needs to execute lazily - how? -// > Probably by passing some `OnOutput` hook to an output stream method -[!parse_for! #input as @(impl @(#x = @IDENT) for @(#y = @IDENT)),+ { - +// Example pre-defined 2: +[!parser! @[IMPL_ITEM /* argument parse stream */] { + @(impl #(let the_trait = @IDENT) for #(let the_type = @IDENT)); + { the_trait, the_type } +}] +[!for! { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] { + impl #the_trait for #the_type {} }] ``` diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index a97da681..203e651b 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -30,7 +30,6 @@ impl InterpretToValue for &SourceExpression { pub(super) enum SourceExpressionLeaf { Command(Command), Variable(VariableIdentifier), - MarkedVariable(GroupedVariable), ExpressionBlock(ExpressionBlock), Value(ExpressionValue), } @@ -40,7 +39,6 @@ impl HasSpanRange for SourceExpressionLeaf { match self { SourceExpressionLeaf::Command(command) => command.span_range(), SourceExpressionLeaf::Variable(variable) => variable.span_range(), - SourceExpressionLeaf::MarkedVariable(variable) => variable.span_range(), SourceExpressionLeaf::ExpressionBlock(block) => block.span_range(), SourceExpressionLeaf::Value(value) => value.span_range(), } @@ -55,11 +53,13 @@ impl Expressionable for Source { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => UnaryAtom::Leaf(Self::Leaf::Command(input.parse()?)), SourcePeekMatch::Variable(Grouping::Grouped) => { - UnaryAtom::Leaf(Self::Leaf::MarkedVariable(input.parse()?)) + return input.parse_err( + "In an expression, the # variable prefix is not allowed. The # prefix should only be used when embedding a variable into an output stream.", + ) } SourcePeekMatch::Variable(Grouping::Flattened) => { return input.parse_err( - "Remove the .. prefix. Flattened variables are not supported in an expression.", + "In an expression, the #.. variable prefix is not allowed. The # prefix should only be used when embedding a variable into an output sream.", ) } SourcePeekMatch::ExpressionBlock(_) => { @@ -196,9 +196,6 @@ impl Expressionable for Source { SourceExpressionLeaf::Command(command) => { command.clone().interpret_to_value(interpreter)? } - SourceExpressionLeaf::MarkedVariable(variable) => { - variable.interpret_to_value(interpreter)? - } SourceExpressionLeaf::Variable(variable_path) => { variable_path.interpret_to_value(interpreter)? } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 6a572e3a..b3488b9f 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -1,5 +1,24 @@ use super::*; - + +/// ## Overview +/// +/// Expression parsing is quite complicated, but this is possibly slightly more complicated in some ways than it needs to be. +/// Its design is intended to make expression parsing more intuitive (although I'm not sure that's been achieved really), +/// and to avoid recursion by making use of an explicit stack frame approach with [`ExpressionStackFrame`]s. +/// +/// This allows parsing of very long expressions (e.g. a sum of 1 million terms) without hitting recursion limits. +/// +/// ## Intuition +/// +/// You can think of these frames in two ways - as: +/// (a) the local variables of a function in an un-flattened parser call stack +/// (b) the specifics of a parent operator, which can allow judging if/when an introduced expression with a possible +/// extension should either bind to its parent (ignoring the extension for now, and retrying the extension with +/// its parent) or bind to the extension itself. +/// +/// ## Examples +/// +/// See the rust doc on the [`ExpressionStackFrame`] for further details. pub(super) struct ExpressionParser<'a, K: Expressionable> { streams: ParseStreamStack<'a, K>, nodes: ExpressionNodes, diff --git a/tests/compilation_failures/control_flow/error_after_continue.rs b/tests/compilation_failures/control_flow/error_after_continue.rs index 4c1ae919..04d06ac5 100644 --- a/tests/compilation_failures/control_flow/error_after_continue.rs +++ b/tests/compilation_failures/control_flow/error_after_continue.rs @@ -5,9 +5,9 @@ fn main() { #(let x = 0) [!while! true { #(x += 1) - [!if! #x == 3 { + [!if! x == 3 { [!continue!] - } !elif! #x >= 3 { + } !elif! x >= 3 { // This checks that the "continue" flag is consumed, // and future errors propagate correctly. [!error! { diff --git a/tests/compilation_failures/core/error_span_repeat.rs b/tests/compilation_failures/core/error_span_repeat.rs index e8c991e4..36832d75 100644 --- a/tests/compilation_failures/core/error_span_repeat.rs +++ b/tests/compilation_failures/core/error_span_repeat.rs @@ -3,7 +3,7 @@ use preinterpret::*; macro_rules! assert_input_length_of_3 { ($($input:literal)+) => {preinterpret!{ [!set! #input_length = [!length! $($input)+]]; - [!if! (#input_length != 3) { + [!if! input_length != 3 { [!error! { message: [!string! "Expected 3 inputs, got " #input_length], spans: [!stream! $($input)+], diff --git a/tests/compilation_failures/core/error_span_repeat.stderr b/tests/compilation_failures/core/error_span_repeat.stderr index 44b72dc4..93939f2f 100644 --- a/tests/compilation_failures/core/error_span_repeat.stderr +++ b/tests/compilation_failures/core/error_span_repeat.stderr @@ -1,8 +1,8 @@ error: Cannot infer common type from stream != untyped integer. Consider using `as` to cast the operands to matching types. - --> tests/compilation_failures/core/error_span_repeat.rs:6:30 + --> tests/compilation_failures/core/error_span_repeat.rs:6:28 | -6 | [!if! (#input_length != 3) { - | ^^ +6 | [!if! input_length != 3 { + | ^^ ... 16 | assert_input_length_of_3!(42 101 666 1024); | ------------------------------------------ in this macro invocation diff --git a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr index bf3d688a..3c756a08 100644 --- a/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr +++ b/tests/compilation_failures/tokens/intersperse_stream_input_variable_issue.stderr @@ -1,4 +1,4 @@ -error: Remove the .. prefix. Flattened variables are not supported in an expression. +error: In an expression, the #.. variable prefix is not allowed. The # prefix should only be used when embedding a variable into an output sream. Occurred whilst parsing [!intersperse! ...] - Expected: { // An array or stream (by coerced token-tree) to intersperse items: ["Hello", "World"], diff --git a/tests/control_flow.rs b/tests/control_flow.rs index eb9e11b4..820dc08d 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -16,11 +16,11 @@ fn test_if() { preinterpret_assert_eq!([!if! (1 == 2) { "YES" } !else! { "NO" }], "NO"); preinterpret_assert_eq!({ #(let x = 1 == 2) - [!if! #x { "YES" } !else! { "NO" }] + [!if! x { "YES" } !else! { "NO" }] }, "NO"); preinterpret_assert_eq!({ #(let x = 1; let y = 2) - [!if! #x == #y { "YES" } !else! { "NO" }] + [!if! x == y { "YES" } !else! { "NO" }] }, "NO"); preinterpret_assert_eq!({ 0 @@ -81,7 +81,7 @@ fn test_for() { preinterpret_assert_eq!( { [!string! [!for! x in 65..70 { - #(#x as u8 as char) + #(x as u8 as char) }]] }, "ABCDE" diff --git a/tests/expressions.rs b/tests/expressions.rs index 6777b435..a7b83f5a 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -37,7 +37,7 @@ fn test_basic_evaluate_works() { preinterpret_assert_eq!(#(123 != 456), true); preinterpret_assert_eq!(#(123 >= 456), false); preinterpret_assert_eq!(#(123 > 456), false); - preinterpret_assert_eq!(#(let six_as_sum = 3 + 3; #six_as_sum * #six_as_sum), 36); + preinterpret_assert_eq!(#(let six_as_sum = 3 + 3; six_as_sum * six_as_sum), 36); preinterpret_assert_eq!(#( let partial_sum = [!stream! + 2]; [!debug! [!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] @@ -106,7 +106,7 @@ fn boolean_operators_short_circuit() { #( let is_lazy = true; let _ = false && #(is_lazy = false; true); - #is_lazy + is_lazy ), true ); @@ -115,7 +115,7 @@ fn boolean_operators_short_circuit() { #( let is_lazy = true; let _ = true || #(is_lazy = false; true); - #is_lazy + is_lazy ), true ); @@ -124,7 +124,7 @@ fn boolean_operators_short_circuit() { #( let is_lazy = true; let _ = false & #(is_lazy = false; true); - #is_lazy + is_lazy ), false ); @@ -135,7 +135,7 @@ fn assign_works() { preinterpret_assert_eq!( #( let x = 5 + 5; - [!debug! #x] + [!debug! x] ), "10" ); @@ -143,8 +143,8 @@ fn assign_works() { #( let x = 10; x /= 1 + 1; // 10 / (1 + 1) - x += 2 + #x; // 5 + (2 + 5) - #x + x += 2 + x; // 5 + (2 + 5) + x ), 12 ); @@ -153,8 +153,8 @@ fn assign_works() { preinterpret_assert_eq!( #( let x = 2; - x += #x; - #x + x += x; + x ), 4 ); diff --git a/tests/tokens.rs b/tests/tokens.rs index f03d88a4..27d98800 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -175,19 +175,11 @@ fn complex_cases_for_intersperse_and_input_types() { ), "A(0)B(1)C(2)D(3)E(4)F(5)G(6)", ); - // Command can be used for items - preinterpret_assert_eq!( - #([!intersperse! { - items: 0..4, - separator: [!stream! _], - }] as stream as string), - "0_1_2_3" - ); // Variable containing stream be used for items preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] #([!intersperse! { - items: #items, + items: items, separator: [!stream! _], }] as stream as string) }, "0_1_2_3"); @@ -195,7 +187,7 @@ fn complex_cases_for_intersperse_and_input_types() { preinterpret_assert_eq!({ #(let items = 0..4) #([!intersperse! { - items: #items, + items: items, separator: [!stream! _], }] as stream as string) }, "0_1_2_3"); @@ -317,7 +309,7 @@ fn test_split() { [!set! #x = ;] [!debug! [!split! { stream: [!stream! ;A;;B;C;D #..x E;], - separator: #x, + separator: x, drop_empty_start: true, drop_empty_middle: true, drop_empty_end: true, @@ -328,7 +320,7 @@ fn test_split() { [!set! #x = ;] [!debug! [!split! { stream: [!stream! ;A;;B;C;D #..x E;], - separator: #x, + separator: x, drop_empty_start: false, drop_empty_middle: false, drop_empty_end: false, @@ -371,7 +363,7 @@ fn test_zip() { [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] [!set! #capitals = "Paris" "Berlin" "Rome"] [!debug! [!zip! { - streams: [#countries, #flags, #capitals], + streams: [countries, flags, capitals], }]] }, r#"[["France", "🇫🇷", "Paris"], ["Germany", "🇩🇪", "Berlin"], ["Italy", "🇮🇹", "Rome"]]"#, @@ -381,7 +373,7 @@ fn test_zip() { [!set! #longer = A B C D] [!set! #shorter = 1 2 3] [!debug! [!zip! { - streams: [#longer, #shorter], + streams: [longer, shorter], error_on_length_mismatch: false, }]] }, @@ -392,7 +384,7 @@ fn test_zip() { [!set! #letters = A B C] #(let numbers = [1, 2, 3]) [!debug! [!zip! { - streams: [#letters, #numbers], + streams: [letters, numbers], }]] }, r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, @@ -401,8 +393,8 @@ fn test_zip() { { [!set! #letters = A B C] #(let numbers = [1, 2, 3]) - #(let combined = [#letters, #numbers]) - [!debug! [!zip! #combined]] + #(let combined = [letters, numbers]) + [!debug! [!zip! combined]] }, r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); @@ -421,7 +413,7 @@ fn test_zip_with_for() { }]] #("The facts are:\n" + [!intersperse! { - items: #facts, + items: facts, separator: ["\n"], }] as string + "\n") }, diff --git a/tests/transforming.rs b/tests/transforming.rs index af30e62d..29995d47 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -57,7 +57,7 @@ fn test_variable_parsing() { #..>>..x // Matches stream and appends it flattened: do you agree ? ) = Why Hello Everyone ([!group! This is an exciting adventure] do you agree?)] - [!debug! #x] + [!debug! x] }, "[!stream! Why [!group! Hello Everyone] This is an exciting adventure do you agree ?]"); } @@ -78,7 +78,7 @@ fn test_ident_transformer() { preinterpret_assert_eq!({ [!set! #x =] [!let! The quick @(#x += @IDENT) fox jumps @(#x += @IDENT) the lazy dog = The quick brown fox jumps over the lazy dog] - [!debug! #x] + [!debug! x] }, "[!stream! brown over]"); } @@ -92,7 +92,7 @@ fn test_literal_transformer() { preinterpret_assert_eq!({ [!set! #x] [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] - [!debug! #x] + [!debug! x] }, "[!stream! 'c' 0b1010 r#\"123\"#]"); } @@ -100,7 +100,7 @@ fn test_literal_transformer() { fn test_punct_transformer() { preinterpret_assert_eq!({ [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] - [!debug! #x] + [!debug! x] }, "[!stream! !]"); // Test for ' which is treated weirdly by syn / rustc preinterpret_assert_eq!({ @@ -111,7 +111,7 @@ fn test_punct_transformer() { preinterpret_assert_eq!({ [!set! #x =] [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] - [!debug! #x] + [!debug! x] }, "[!stream! % |]"); } @@ -119,18 +119,18 @@ fn test_punct_transformer() { fn test_group_transformer() { preinterpret_assert_eq!({ [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] - [!debug! #x] + [!debug! x] }, "[!stream! fox]"); preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said @[GROUP #..y]! = I said #x!] - [!debug! #y] + [!debug! y] }, "[!stream! \"hello\" \"world\"]"); // ... which is equivalent to this: preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said #y! = I said #x!] - [!debug! #y] + [!debug! y] }, "[!stream! \"hello\" \"world\"]"); } From 276d1c9cc612dd5546eff6d9851d35353f418f9c Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 18 Feb 2025 00:34:50 +0000 Subject: [PATCH 101/126] feature: Added index and array place destructuring --- CHANGELOG.md | 103 +- src/expressions/array.rs | 31 +- src/expressions/evaluation.rs | 943 +++++++++++++----- src/expressions/expression.rs | 20 +- src/expressions/expression_parsing.rs | 15 +- src/expressions/float.rs | 13 +- src/expressions/integer.rs | 13 +- src/expressions/mod.rs | 2 +- src/expressions/operations.rs | 4 +- src/expressions/value.rs | 15 +- src/extensions/errors_and_spans.rs | 8 +- src/internal_prelude.rs | 2 +- src/interpretation/commands/core_commands.rs | 2 +- src/interpretation/interpreter.rs | 141 ++- src/transformation/transform_stream.rs | 2 +- src/transformation/variable_binding.rs | 20 +- ...d_variable_and_then_extend_it_again.stderr | 2 +- .../extend_variable_and_then_set_it.stderr | 2 +- ...ay_place_destructure_element_mismatch_1.rs | 7 + ...lace_destructure_element_mismatch_1.stderr | 5 + ...ay_place_destructure_element_mismatch_2.rs | 7 + ...lace_destructure_element_mismatch_2.stderr | 5 + ...ay_place_destructure_element_mismatch_3.rs | 7 + ...lace_destructure_element_mismatch_3.stderr | 5 + ...ray_place_destructure_multiple_dot_dots.rs | 7 + ...place_destructure_multiple_dot_dots.stderr | 5 + .../array_place_destructure_multiple_muts.rs | 7 + ...ray_place_destructure_multiple_muts.stderr | 5 + .../expressions/discard_in_value_position.rs | 7 + .../discard_in_value_position.stderr | 5 + .../expressions/index_into_discarded_place.rs | 7 + .../index_into_discarded_place.stderr | 5 + tests/expressions.rs | 94 ++ 33 files changed, 1096 insertions(+), 420 deletions(-) create mode 100644 tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.rs create mode 100644 tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr create mode 100644 tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.rs create mode 100644 tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr create mode 100644 tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.rs create mode 100644 tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr create mode 100644 tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.rs create mode 100644 tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.stderr create mode 100644 tests/compilation_failures/expressions/array_place_destructure_multiple_muts.rs create mode 100644 tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr create mode 100644 tests/compilation_failures/expressions/discard_in_value_position.rs create mode 100644 tests/compilation_failures/expressions/discard_in_value_position.stderr create mode 100644 tests/compilation_failures/expressions/index_into_discarded_place.rs create mode 100644 tests/compilation_failures/expressions/index_into_discarded_place.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index dab3ee54..efb94a79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,90 +96,22 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come - -* Merge `GroupedVariable` and `ExpressionBlock` into an `ExplicitExpression`: - * Either `#ident` or `#(...)` or `#{ ... }`... - the latter defines a new variable stack frame, just like Rust - * To avoid confusion (such as below) and teach the user to only include #var where - necessary, only expression _blocks_ are allowed in an expression. - * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal - rust because the inside is a `{ .. }` which defines a new scope. * Support `#(x[..])` syntax for indexing arrays at read time (streams to follow in a separate task below after parsers are updated) * `#(x[0..3])` returns an array * `#(x[0..=3])` returns an array - * ... and `#(x[0] = y)` can be used to set the item - * Considering allowing assignments inside an expression - * Implementation notes: - * Separate `VariableData` and `VariableReference`, separate `let` and `set` - * Three separate stacks for various calculations in evaluation - * `let XX =` and `YY += y` are actually totally different... - * With `let XX =`, `XX` is a _pattern_ and creates new variables. - * With `YY += y` we're inside an expression already, and it only works with existing variables. The expression `YY` is converted into a destructuring at execution time. - Note that the value side always executes first, before the place is executed. - * Implementation realisations: - * Rust reference very good: https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions - * For the += operators etc: - * The left hand side must resolve to a _single_ VariableData, e.g. `z` or - `x.y["z"]` - * In the rust reference, this is a "Place Expression" - * We will have a `handle_assign_paired_binary_operation(&mut self, operation, other: Self)` (which for value types can resolve to the non-assign version) - * For the = operator: - * The left hand side will resolve non-Variable expressions, and be effectively - left with something which can form a destructuring of the right hand side. - e.g. `[x.y, z[x.a][12]] = [1, 2]` - * In the rust reference, this is an "Assignee Expression" and is a generalization - of place expressions - * Test cases: -```rust - // These examples should compile: - let mut a = [0; 5]; - let mut b: u32 = 3; - let c; - let out = c = (a[2], _) = (4, 5); - let out = a[1] += 2; - let out = b = 2; - // In the below, a = [0, 5], showing the right side executes first - let mut a = [0; 2]; - let mut b = 0; - a[b] += { b += 1; 5 }; - // This works: - let (x, y); - [x, .., y] = [1, 2, 3, 4]; // x = 1, y = 4 - // This works... - // In other words, the assignee operation is executed incrementally, - // The first assignment arr[0] = 1 occurs before being overwritten by - // the arr[0] = 5 in the second section. - let mut arr = [0; 2]; - (arr[0], arr[{arr[0] = 5; 1}]) = (1, 1); - assert_eq!(arr, [5, 1]); - // This doesn't work - two errors: - // error[E0308]: mismatched types: expected `[{integer}]`, found `[{integer}; 4]` - // error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time - // (it looks like you can't just overwrite array subslices) - let arr = [0; 5]; - arr[1..=4] = [1, 2, 3, 4]; - // This doesn't work. - // error[E0368]: binary assignment operation `+=` cannot be applied to type `({integer}, {integer})` - // https://doc.rust-lang.org/error_codes/E0368.html - let (a, b) = (1, 1); - (a, b) += (1, 2); -``` -* Variable typing (stream / value / object to start with), including an `object` type, like a JS object: - * Objects: - * Backed by an indexmap - * Can be created with `#({ a: x, ... })` - * Can be read with `#(x.hello)` or `#(x["hello"])` - * Debug impl is `#({ hello: [!group! BLAH], ["#world"]: Hi, })` - * They have an input object - * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` - * Can be destructured with `{ hello, world: _, ... }` - * (Until we get custom type support into a syn fork), can be embedded into an output stream as a single token - e.g. `PREINTERPRET_OBJECT_2313` - (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default - stream for the object, or error and suggest fields the user should use instead. - * Have `!zip!` support `{ objects }` - * Values: - * When we parse a `#x` (`@(x = INFER_TOKEN_TREE)`) binding, it tries to parse a stream as a value before interpreting a `[!group! ...]` as a stream. - * Output to final output as unwrapped content +* Add `..` and `.., x` support to the array pattern, like the place expression. +* Objects, like a JS object: + * Backed by an indexmap (or maybe an immutable `IndexMap` wrapping an `im::HashMap` and `im::Vec` or entry orderings) + * Can be created with `#({ a: x, ... })` + * Can be read with `#(x.hello)` or `#(x["hello"])` + * Debug impl is `#({ hello: [!group! BLAH], ["#world"]: Hi, })` + * They have an input object + * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` + * Can be destructured with `{ hello, world: _, ... }` + * (Until we get custom type support into a syn fork), can be embedded into an output stream as a single token - e.g. `PREINTERPRET_OBJECT_2313` + (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default + stream for the object, or error and suggest fields the user should use instead. + * Have `!zip!` support `{ objects }` * Method calls * Mutable methods notes: * They require either: @@ -192,6 +124,14 @@ Inside a transform stream, the following grammar is supported: * `.len()` on stream * `.push(x)` on array * Consider `.map(|| {})` +* Introduce interpreter stack frames + * Merge `GroupedVariable` and `ExpressionBlock` into an `ExplicitExpression`: + * Either `#ident` or `#(...)` or `#{ ... }`... + the latter defines a new variable stack frame, just like Rust + * To avoid confusion (such as below) and teach the user to only include #var where + necessary, only expression _blocks_ are allowed in an expression. + * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal + rust because the inside is a `{ .. }` which defines a new scope. * TRANSFORMERS => PARSERS cont * Re-read the `Parsers Revisited` section below * Manually search for transform and rename to parse in folder names and file. @@ -242,7 +182,6 @@ Inside a transform stream, the following grammar is supported: * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) * `#(x[0..3])` returns a TokenStream * `#(x[0..=3])` returns a TokenStream -* Add `..` and `.., x` support to the array pattern * Consider: * Moving control flow (`for` and `while`) to the expression side? * Dropping lots of the `group` wrappers? diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 2a7e69aa..3687232a 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -117,27 +117,42 @@ impl ExpressionArray { }) } - pub(super) fn handle_index_access( - &self, + pub(super) fn into_indexed( + self, access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult { - Ok(match index { + let index = self.resolve_valid_index(index)?; + let span_range = SpanRange::new_between(self.span_range.start(), access.span()); + Ok(self.items[index].clone().with_span_range(span_range)) + } + + pub(super) fn index_mut( + &mut self, + _access: IndexAccess, + index: ExpressionValue, + ) -> ExecutionResult<&mut ExpressionValue> { + let index = self.resolve_valid_index(index)?; + Ok(&mut self.items[index]) + } + + fn resolve_valid_index(&self, index: ExpressionValue) -> ExecutionResult { + match index { ExpressionValue::Integer(int) => { let span_range = int.span_range; let index = int.expect_usize()?; if index < self.items.len() { - self.items[index].clone().with_span(access.span()) + Ok(index) } else { - return span_range.execution_err(format!( + span_range.execution_err(format!( "Index of {} is out of range of the array length of {}", index, self.items.len() - )); + )) } } - _ => return index.execution_err("The index must be an integer"), - }) + _ => index.execution_err("The index must be an integer"), + } } pub(crate) fn concat_recursive_into( diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 6ed2f5b8..f9f50c60 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -1,100 +1,297 @@ use super::*; -pub(super) struct ExpressionEvaluator<'a, K: Expressionable> { - nodes: &'a [ExpressionNode], - operation_stack: Vec, -} +pub(super) use inner::ExpressionEvaluator; -impl<'a> ExpressionEvaluator<'a, Source> { - pub(super) fn new(nodes: &'a [ExpressionNode]) -> Self { - Self { - nodes, - operation_stack: Vec::new(), - } +/// This is to hide implementation details to protect the abstraction and make it harder to make mistakes. +mod inner { + use super::*; + + pub(in super::super) struct ExpressionEvaluator<'a, K: Expressionable> { + nodes: &'a [ExpressionNode], + stacks: Stacks, } - pub(super) fn evaluate( - mut self, - root: ExpressionNodeId, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let mut next = self.begin_node_evaluation(root, interpreter)?; + impl<'a> ExpressionEvaluator<'a, Source> { + pub(in super::super) fn new(nodes: &'a [ExpressionNode]) -> Self { + Self { + nodes, + stacks: Stacks::new(), + } + } + + pub(in super::super) fn evaluate( + mut self, + root: ExpressionNodeId, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut next_action = NextActionInner::ReadNodeAsValue(root); - loop { - match next { - NextAction::HandleValue(value) => { - let top_of_stack = match self.operation_stack.pop() { + loop { + match self.step(next_action, interpreter)? { + StepResult::Continue(continue_action) => { + next_action = continue_action.0; + } + StepResult::Return(value) => { + return Ok(value); + } + } + } + } + + fn step( + &mut self, + action: NextActionInner, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + Ok(StepResult::Continue(match action { + NextActionInner::ReadNodeAsValue(node) => self.nodes[node.0] + .handle_as_value(interpreter, self.stacks.creator(ReturnMode::Value))?, + NextActionInner::HandleReturnedValue(value) => { + let top_of_stack = match self.stacks.value_stack.pop() { + Some(top) => top, + None => { + debug_assert!(self.stacks.assignment_stack.is_empty(), "Evaluation completed with none-empty assignment stack - there's some bug in the ExpressionEvaluator"); + debug_assert!(self.stacks.place_stack.is_empty(), "Evaluation completed with none-empty place stack - there's some bug in the ExpressionEvaluator"); + return Ok(StepResult::Return(value)); + } + }; + let next_creator = self.stacks.creator(top_of_stack.ultimate_return_mode()); + top_of_stack.handle_value(value, next_creator)? + } + NextActionInner::ReadNodeAsAssignee(node, value) => self.nodes[node.0] + .handle_as_assignee( + self.nodes, + node, + value, + interpreter, + self.stacks.creator(ReturnMode::AssignmentCompletion), + )?, + NextActionInner::HandleAssignmentComplete(assignment_complete) => { + let top_of_stack = match self.stacks.assignment_stack.pop() { Some(top) => top, - None => return Ok(value), + None => unreachable!("Received AssignmentComplete without any assignment stack frames - there's some bug in the ExpressionEvaluator"), }; - next = self.handle_value(top_of_stack, value, interpreter)?; + let next_creator = self.stacks.creator(top_of_stack.ultimate_return_mode()); + top_of_stack.handle_assignment_complete(assignment_complete, next_creator)? } - NextAction::EnterValueNode(next_node) => { - next = self.begin_node_evaluation(next_node, interpreter)?; + NextActionInner::ReadNodeAsPlace(node) => self.nodes[node.0] + .handle_as_place(interpreter, self.stacks.creator(ReturnMode::Place))?, + NextActionInner::HandleReturnedPlace(place) => { + let top_of_stack = match self.stacks.place_stack.pop() { + Some(top) => top, + None => unreachable!("Received Place without any place stack frames - there's some bug in the ExpressionEvaluator"), + }; + let next_creator = self.stacks.creator(top_of_stack.ultimate_return_mode()); + top_of_stack.handle_place(place, next_creator)? } + })) + } + } + + /// See the [rust reference] for a good description of assignee vs place. + /// + /// [rust reference]: https://doc.rust-lang.org/reference/expressions/assignment-expressions.html#assignee-vs-place + pub(super) struct Stacks { + /// The stack of operations which will output a value + value_stack: Vec, + /// The stack of operations which will handle an assignment completion + assignment_stack: Vec, + /// The stack of operations which will output a place + place_stack: Vec, + } + + impl Stacks { + pub(super) fn new() -> Self { + Self { + value_stack: Vec::new(), + assignment_stack: Vec::new(), + place_stack: Vec::new(), + } + } + + fn creator(&mut self, return_mode: ReturnMode) -> ActionCreator { + ActionCreator { + stacks: self, + return_mode, } } } - fn begin_node_evaluation( - &mut self, - node_id: ExpressionNodeId, + pub(super) enum StepResult { + Continue(NextAction), + Return(ExpressionValue), + } + + pub(super) struct NextAction(NextActionInner); + + enum NextActionInner { + /// Enters an expression node to output a value + ReadNodeAsValue(ExpressionNodeId), + HandleReturnedValue(ExpressionValue), + // Enters an expression node for assignment purposes + ReadNodeAsAssignee(ExpressionNodeId, ExpressionValue), + HandleAssignmentComplete(AssignmentCompletion), + // Enters an expression node to output a place + ReadNodeAsPlace(ExpressionNodeId), + HandleReturnedPlace(Place), + } + + impl From for NextAction { + fn from(value: NextActionInner) -> Self { + Self(value) + } + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) enum ReturnMode { + Value, + AssignmentCompletion, + Place, + } + + pub(super) struct ActionCreator<'a> { + stacks: &'a mut Stacks, + return_mode: ReturnMode, + } + + impl ActionCreator<'_> { + pub(super) fn return_value(self, value: ExpressionValue) -> NextAction { + debug_assert_eq!( + self.return_mode, + ReturnMode::Value, + "This handler claimed to ultimately return {:?}, but it returned a value", + self.return_mode + ); + NextActionInner::HandleReturnedValue(value).into() + } + + pub(super) fn read_value_with_handler( + self, + node: ExpressionNodeId, + handler: ValueStackFrame, + ) -> NextAction { + debug_assert_eq!( + handler.ultimate_return_mode(), + self.return_mode, + "Handler is expected to return {:?}, but claims to ultimately return {:?}", + self.return_mode, + handler.ultimate_return_mode() + ); + self.stacks.value_stack.push(handler); + NextActionInner::ReadNodeAsValue(node).into() + } + + pub(super) fn return_place(self, place: Place) -> NextAction { + debug_assert_eq!( + self.return_mode, + ReturnMode::Place, + "This handler claimed to ultimately return {:?}, but it returned a place", + self.return_mode + ); + NextActionInner::HandleReturnedPlace(place).into() + } + + pub(super) fn read_place_with_handler( + self, + node: ExpressionNodeId, + handler: PlaceStackFrame, + ) -> NextAction { + debug_assert_eq!( + handler.ultimate_return_mode(), + self.return_mode, + "Handler is expected to return {:?}, but claims to ultimately return {:?}", + self.return_mode, + handler.ultimate_return_mode() + ); + self.stacks.place_stack.push(handler); + NextActionInner::ReadNodeAsPlace(node).into() + } + + /// The span range should cover from the start of the assignee to the end of the value + pub(super) fn return_assignment_completion(self, span_range: SpanRange) -> NextAction { + debug_assert_eq!(self.return_mode, ReturnMode::AssignmentCompletion, "This handler claimed to ultimately return {:?}, but it returned an assignment completion", self.return_mode); + NextActionInner::HandleAssignmentComplete(AssignmentCompletion { span_range }).into() + } + + pub(super) fn handle_assignment_and_return_to( + self, + node: ExpressionNodeId, + value: ExpressionValue, + handler: AssignmentStackFrame, + ) -> NextAction { + debug_assert_eq!( + handler.ultimate_return_mode(), + self.return_mode, + "Handler is expected to return {:?}, but claims to ultimately return {:?}", + self.return_mode, + handler.ultimate_return_mode() + ); + self.stacks.assignment_stack.push(handler); + NextActionInner::ReadNodeAsAssignee(node, value).into() + } + } +} + +use inner::*; + +impl ExpressionNode { + fn handle_as_value( + &self, interpreter: &mut Interpreter, + next: ActionCreator, ) -> ExecutionResult { - Ok(match &self.nodes[node_id.0] { + Ok(match self { ExpressionNode::Leaf(leaf) => { - NextAction::HandleValue(Source::evaluate_leaf(leaf, interpreter)?) + next.return_value(Source::evaluate_leaf(leaf, interpreter)?) } - ExpressionNode::Grouped { delim_span, inner } => { - self.operation_stack.push(EvaluationStackFrame::Group { + ExpressionNode::Grouped { delim_span, inner } => next.read_value_with_handler( + *inner, + ValueStackFrame::Group { span: delim_span.join(), - }); - NextAction::EnterValueNode(*inner) - } - ExpressionNode::Array { delim_span, items } => ArrayStackFrame { + }, + ), + ExpressionNode::Array { delim_span, items } => ArrayValueStackFrame { span: delim_span.join(), unevaluated_items: items.clone(), evaluated_items: Vec::with_capacity(items.len()), } - .next(&mut self.operation_stack), - ExpressionNode::UnaryOperation { operation, input } => { - self.operation_stack - .push(EvaluationStackFrame::UnaryOperation { - operation: operation.clone(), - }); - NextAction::EnterValueNode(*input) - } + .next(next), + ExpressionNode::UnaryOperation { operation, input } => next.read_value_with_handler( + *input, + ValueStackFrame::UnaryOperation { + operation: operation.clone(), + }, + ), ExpressionNode::BinaryOperation { operation, left_input, right_input, - } => { - self.operation_stack - .push(EvaluationStackFrame::BinaryOperation { - operation: operation.clone(), - state: BinaryPath::OnLeftBranch { - right: *right_input, - }, - }); - NextAction::EnterValueNode(*left_input) - } - ExpressionNode::Property { node, access } => { - self.operation_stack.push(EvaluationStackFrame::Property { + } => next.read_value_with_handler( + *left_input, + ValueStackFrame::BinaryOperation { + operation: operation.clone(), + state: BinaryPath::OnLeftBranch { + right: *right_input, + }, + }, + ), + ExpressionNode::Property { node, access } => next.read_value_with_handler( + *node, + ValueStackFrame::Property { access: access.clone(), - }); - NextAction::EnterValueNode(*node) - } + }, + ), ExpressionNode::Index { node, access, index, - } => { - self.operation_stack.push(EvaluationStackFrame::Index { - access: access.clone(), + } => next.read_value_with_handler( + *node, + ValueStackFrame::Index { + access: *access, state: IndexPath::OnObjectBranch { index: *index }, - }); - NextAction::EnterValueNode(*node) - } + }, + ), ExpressionNode::Range { left, range_limits, @@ -104,122 +301,246 @@ impl<'a> ExpressionEvaluator<'a, Source> { (None, None) => match range_limits { syn::RangeLimits::HalfOpen(token) => { let inner = ExpressionRangeInner::RangeFull { token: *token }; - NextAction::HandleValue(inner.to_value(token.span_range())) + next.return_value(inner.to_value(token.span_range())) } syn::RangeLimits::Closed(_) => { unreachable!("A closed range should have been given a right in continue_range(..)") } }, - (None, Some(right)) => { - self.operation_stack.push(EvaluationStackFrame::Range { + (None, Some(right)) => next.read_value_with_handler( + *right, + ValueStackFrame::Range { range_limits: *range_limits, state: RangePath::OnRightBranch { left: None }, - }); - NextAction::EnterValueNode(*right) - } - (Some(left), right) => { - self.operation_stack.push(EvaluationStackFrame::Range { + }, + ), + (Some(left), right) => next.read_value_with_handler( + *left, + ValueStackFrame::Range { range_limits: *range_limits, state: RangePath::OnLeftBranch { right: *right }, - }); - NextAction::EnterValueNode(*left) - } + }, + ), } } ExpressionNode::Assignment { assignee, equals_token, value, - } => { - self.operation_stack - .push(EvaluationStackFrame::AssignmentValue { - assignee: *assignee, - equals_token: *equals_token, - }); - NextAction::EnterValueNode(*value) - } + } => next.read_value_with_handler( + *value, + ValueStackFrame::HandleValueForAssignment { + assignee: *assignee, + equals_token: *equals_token, + }, + ), ExpressionNode::CompoundAssignment { place, operation, value, - } => { - self.operation_stack - .push(EvaluationStackFrame::CompoundAssignmentValue { - place: *place, - operation: *operation, - }); - NextAction::EnterValueNode(*value) + } => next.read_value_with_handler( + *value, + ValueStackFrame::HandleValueForCompoundAssignment { + place: *place, + operation: *operation, + }, + ), + }) + } + + fn handle_as_assignee( + &self, + nodes: &[ExpressionNode], + self_node_id: ExpressionNodeId, + value: ExpressionValue, + _: &mut Interpreter, + next: ActionCreator, + ) -> ExecutionResult { + Ok(match self { + ExpressionNode::Leaf(SourceExpressionLeaf::Variable(_)) + | ExpressionNode::Leaf(SourceExpressionLeaf::Discarded(_)) + | ExpressionNode::Index { .. } + | ExpressionNode::Property { .. } => { + next.read_place_with_handler(self_node_id, PlaceStackFrame::Assignment { value }) + } + ExpressionNode::Array { + delim_span, + items: assignee_item_node_ids, + } => ArrayAssigneeStackFrame::new( + nodes, + delim_span.join(), + assignee_item_node_ids, + value, + )? + .handle_next(next), + ExpressionNode::Grouped { inner, .. } => { + next.handle_assignment_and_return_to(*inner, value, AssignmentStackFrame::Grouped) + } + other => { + return other + .operator_span_range() + .execution_err("This type of expression is not supported as an assignee"); + } + }) + } + + fn handle_as_place( + &self, + interpreter: &mut Interpreter, + next: ActionCreator, + ) -> ExecutionResult { + Ok(match self { + ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => next.return_place( + Place::MutableReference(variable.reference(interpreter)?.into_mut()?), + ), + ExpressionNode::Leaf(SourceExpressionLeaf::Discarded(token)) => { + next.return_place(Place::Discarded(*token)) + } + ExpressionNode::Index { + node, + access, + index, + } => next.read_place_with_handler( + *node, + PlaceStackFrame::Indexed { + access: *access, + index: *index, + }, + ), + ExpressionNode::Property { access, .. } => { + return access.execution_err("TODO: Not yet supported!") + } + ExpressionNode::Grouped { inner, .. } => { + next.read_place_with_handler(*inner, PlaceStackFrame::Grouped) + } + other => { + return other + .operator_span_range() + .execution_err("This type of expression is not supported as an assignee"); } }) } +} + +/// Stack frames which need to receive a value to continue their execution. +enum ValueStackFrame { + Group { + span: Span, + }, + Array(ArrayValueStackFrame), + UnaryOperation { + operation: UnaryOperation, + }, + BinaryOperation { + operation: BinaryOperation, + state: BinaryPath, + }, + Property { + access: PropertyAccess, + }, + Index { + access: IndexAccess, + state: IndexPath, + }, + Range { + range_limits: syn::RangeLimits, + state: RangePath, + }, + HandleValueForAssignment { + assignee: ExpressionNodeId, + equals_token: Token![=], + }, + HandleValueForCompoundAssignment { + place: ExpressionNodeId, + operation: CompoundAssignmentOperation, + }, + ResolveIndexedPlace { + place: Place, + access: IndexAccess, + }, +} + +impl ValueStackFrame { + fn ultimate_return_mode(&self) -> ReturnMode { + match self { + ValueStackFrame::Group { .. } => ReturnMode::Value, + ValueStackFrame::Array(_) => ReturnMode::Value, + ValueStackFrame::UnaryOperation { .. } => ReturnMode::Value, + ValueStackFrame::BinaryOperation { .. } => ReturnMode::Value, + ValueStackFrame::Property { .. } => ReturnMode::Value, + ValueStackFrame::Index { .. } => ReturnMode::Value, + ValueStackFrame::Range { .. } => ReturnMode::Value, + ValueStackFrame::HandleValueForAssignment { .. } => ReturnMode::Value, + ValueStackFrame::HandleValueForCompoundAssignment { .. } => ReturnMode::Value, + ValueStackFrame::ResolveIndexedPlace { .. } => ReturnMode::Place, + } + } fn handle_value( - &mut self, - top_of_stack: EvaluationStackFrame, + self, value: ExpressionValue, - interpreter: &mut Interpreter, + next: ActionCreator, ) -> ExecutionResult { - Ok(match top_of_stack { - EvaluationStackFrame::Group { span } => NextAction::HandleValue(value.with_span(span)), - EvaluationStackFrame::UnaryOperation { operation } => { - NextAction::HandleValue(operation.evaluate(value)?) + Ok(match self { + ValueStackFrame::Group { span } => next.return_value(value.with_span(span)), + ValueStackFrame::UnaryOperation { operation } => { + next.return_value(operation.evaluate(value)?) } - EvaluationStackFrame::Array(mut array) => { + ValueStackFrame::Array(mut array) => { array.evaluated_items.push(value); - array.next(&mut self.operation_stack) + array.next(next) } - EvaluationStackFrame::BinaryOperation { operation, state } => match state { + ValueStackFrame::BinaryOperation { operation, state } => match state { BinaryPath::OnLeftBranch { right } => { if let Some(result) = operation.lazy_evaluate(&value)? { - NextAction::HandleValue(result) + next.return_value(result) } else { - self.operation_stack - .push(EvaluationStackFrame::BinaryOperation { + next.read_value_with_handler( + right, + ValueStackFrame::BinaryOperation { operation, state: BinaryPath::OnRightBranch { left: value }, - }); - NextAction::EnterValueNode(right) + }, + ) } } BinaryPath::OnRightBranch { left } => { - let result = operation.evaluate(left, value)?; - NextAction::HandleValue(result) + next.return_value(operation.evaluate(left, value)?) } }, - EvaluationStackFrame::Property { access } => { - let result = value.handle_property_access(access)?; - NextAction::HandleValue(result) + ValueStackFrame::Property { access } => { + next.return_value(value.handle_property_access(access)?) } - EvaluationStackFrame::Index { access, state } => match state { - IndexPath::OnObjectBranch { index } => { - self.operation_stack.push(EvaluationStackFrame::Index { + ValueStackFrame::Index { access, state } => match state { + IndexPath::OnObjectBranch { index } => next.read_value_with_handler( + index, + ValueStackFrame::Index { access, state: IndexPath::OnIndexBranch { object: value }, - }); - NextAction::EnterValueNode(index) - } + }, + ), IndexPath::OnIndexBranch { object } => { - let result = object.handle_index_access(access, value)?; - NextAction::HandleValue(result) + next.return_value(object.into_indexed(access, value)?) } }, - EvaluationStackFrame::Range { + ValueStackFrame::Range { range_limits, state, } => match (state, range_limits) { - (RangePath::OnLeftBranch { right: Some(right) }, range_limits) => { - self.operation_stack.push(EvaluationStackFrame::Range { - range_limits, - state: RangePath::OnRightBranch { left: Some(value) }, - }); - NextAction::EnterValueNode(right) - } + (RangePath::OnLeftBranch { right: Some(right) }, range_limits) => next + .read_value_with_handler( + right, + ValueStackFrame::Range { + range_limits, + state: RangePath::OnRightBranch { left: Some(value) }, + }, + ), (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::HalfOpen(token)) => { let inner = ExpressionRangeInner::RangeFrom { start_inclusive: value, token, }; - NextAction::HandleValue(inner.to_value(token.span_range())) + next.return_value(inner.to_value(token.span_range())) } (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::Closed(_)) => { unreachable!( @@ -235,7 +556,7 @@ impl<'a> ExpressionEvaluator<'a, Source> { token, end_exclusive: value, }; - NextAction::HandleValue(inner.to_value(token.span_range())) + next.return_value(inner.to_value(token.span_range())) } ( RangePath::OnRightBranch { left: Some(left) }, @@ -246,183 +567,67 @@ impl<'a> ExpressionEvaluator<'a, Source> { token, end_inclusive: value, }; - NextAction::HandleValue(inner.to_value(token.span_range())) + next.return_value(inner.to_value(token.span_range())) } (RangePath::OnRightBranch { left: None }, syn::RangeLimits::HalfOpen(token)) => { let inner = ExpressionRangeInner::RangeTo { token, end_exclusive: value, }; - NextAction::HandleValue(inner.to_value(token.span_range())) + next.return_value(inner.to_value(token.span_range())) } (RangePath::OnRightBranch { left: None }, syn::RangeLimits::Closed(token)) => { let inner = ExpressionRangeInner::RangeToInclusive { token, end_inclusive: value, }; - NextAction::HandleValue(inner.to_value(token.span_range())) + next.return_value(inner.to_value(token.span_range())) } }, - EvaluationStackFrame::AssignmentValue { + ValueStackFrame::HandleValueForAssignment { assignee, equals_token, - } => { - // TODO: This should be replaced with setting a stack frame for AssignmentResolution - self.handle_assignment(assignee, equals_token, value, interpreter)? - } - EvaluationStackFrame::CompoundAssignmentValue { place, operation } => { - // TODO: This should be replaced with setting a stack frame for CompoundAssignmentResolution - self.handle_compound_assignment(place, operation, value, interpreter)? - } - }) - } - - fn handle_assignment( - &mut self, - assignee: ExpressionNodeId, - equals_token: Token![=], - value: ExpressionValue, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - // TODO: When we add place resolution, we likely wish to make use of the execution stack. - let assignee = self.resolve_assignee_or_place(assignee, interpreter)?; - Ok(match assignee { - Assignee::Place(place) => { - let span_range = - SpanRange::new_between(place.span_range().start(), value.span_range().end()); - place.set(value)?; - NextAction::HandleValue(ExpressionValue::None(span_range)) - } - Assignee::CompoundWip => { - return equals_token.execution_err("Compound assignment is not yet supported") - } - }) - } - - fn handle_compound_assignment( - &mut self, - place: ExpressionNodeId, - operation: CompoundAssignmentOperation, - value: ExpressionValue, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - // TODO: When we add place resolution, we likely wish to make use of the execution stack. - let assignee = self.resolve_assignee_or_place(place, interpreter)?; - Ok(match assignee { - Assignee::Place(place) => { - let span_range = - SpanRange::new_between(place.span_range().start(), value.span_range().end()); - // TODO - replace with handling a compound operation for better performance - // of e.g. arrays or streams - let left = place.get_value_cloned()?.clone(); - place.set(operation.to_binary().evaluate(left, value)?)?; - NextAction::HandleValue(ExpressionValue::None(span_range)) - } - Assignee::CompoundWip => { - return operation - .execution_err("Compound values are not supported for operation assignment") + } => next.handle_assignment_and_return_to( + assignee, + value, + AssignmentStackFrame::AssignmentRoot { equals_token }, + ), + ValueStackFrame::HandleValueForCompoundAssignment { place, operation } => next + .read_place_with_handler( + place, + PlaceStackFrame::CompoundAssignmentRoot { operation, value }, + ), + ValueStackFrame::ResolveIndexedPlace { place, access } => { + next.return_place(match place { + Place::MutableReference(reference) => { + Place::MutableReference(reference.resolve_indexed(access, value)?) + } + Place::Discarded(_) => { + return access.execution_err("Cannot index into a discarded value"); + } + }) } }) } - - /// See the [rust reference] for a good description of assignee vs place. - /// - /// [rust reference]: https://doc.rust-lang.org/reference/expressions/assignment-expressions.html#assignee-vs-place - fn resolve_assignee_or_place( - &self, - mut assignee: ExpressionNodeId, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let resolved = loop { - match &self.nodes[assignee.0] { - ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => { - break Assignee::Place(variable.reference(interpreter)?); - } - ExpressionNode::Index { .. } => { - todo!() - } - ExpressionNode::Property { .. } => { - todo!() - } - ExpressionNode::Array { .. } => { - break Assignee::CompoundWip; - } - ExpressionNode::Grouped { inner, .. } => { - assignee = *inner; - continue; - } - other => { - return other - .operator_span_range() - .execution_err("This type of expression is not supported as an assignee"); - } - }; - }; - Ok(resolved) - } -} - -enum Assignee { - Place(VariableReference), - CompoundWip, // To come -} - -enum NextAction { - HandleValue(ExpressionValue), - EnterValueNode(ExpressionNodeId), -} - -enum EvaluationStackFrame { - Group { - span: Span, - }, - Array(ArrayStackFrame), - UnaryOperation { - operation: UnaryOperation, - }, - BinaryOperation { - operation: BinaryOperation, - state: BinaryPath, - }, - Property { - access: PropertyAccess, - }, - Index { - access: IndexAccess, - state: IndexPath, - }, - Range { - range_limits: syn::RangeLimits, - state: RangePath, - }, - AssignmentValue { - assignee: ExpressionNodeId, - equals_token: Token![=], - }, - CompoundAssignmentValue { - place: ExpressionNodeId, - operation: CompoundAssignmentOperation, - }, } -struct ArrayStackFrame { +struct ArrayValueStackFrame { span: Span, unevaluated_items: Vec, evaluated_items: Vec, } -impl ArrayStackFrame { - fn next(self, operation_stack: &mut Vec) -> NextAction { +impl ArrayValueStackFrame { + fn next(self, action_creator: ActionCreator) -> NextAction { match self .unevaluated_items .get(self.evaluated_items.len()) .cloned() { Some(next) => { - operation_stack.push(EvaluationStackFrame::Array(self)); - NextAction::EnterValueNode(next) + action_creator.read_value_with_handler(next, ValueStackFrame::Array(self)) } - None => NextAction::HandleValue(ExpressionValue::Array(ExpressionArray { + None => action_creator.return_value(ExpressionValue::Array(ExpressionArray { items: self.evaluated_items, span_range: self.span.span_range(), })), @@ -444,3 +649,197 @@ enum RangePath { OnLeftBranch { right: Option }, OnRightBranch { left: Option }, } + +enum AssignmentStackFrame { + /// An instruction to return a `None` value to the Value stack + AssignmentRoot { + #[allow(unused)] + equals_token: Token![=], + }, + Grouped, + Array(ArrayAssigneeStackFrame), +} + +impl AssignmentStackFrame { + fn ultimate_return_mode(&self) -> ReturnMode { + match self { + AssignmentStackFrame::AssignmentRoot { .. } => ReturnMode::Value, + AssignmentStackFrame::Grouped { .. } => ReturnMode::AssignmentCompletion, + AssignmentStackFrame::Array { .. } => ReturnMode::AssignmentCompletion, + } + } + + fn handle_assignment_complete( + self, + completion: AssignmentCompletion, + next: ActionCreator, + ) -> ExecutionResult { + let AssignmentCompletion { span_range } = completion; + Ok(match self { + AssignmentStackFrame::AssignmentRoot { .. } => { + next.return_value(ExpressionValue::None(span_range)) + } + AssignmentStackFrame::Grouped => next.return_assignment_completion(span_range), + AssignmentStackFrame::Array(array) => array.handle_next(next), + }) + } +} + +struct ArrayAssigneeStackFrame { + span_range: SpanRange, + assignee_stack: Vec<(ExpressionNodeId, ExpressionValue)>, +} + +impl ArrayAssigneeStackFrame { + fn new( + nodes: &[ExpressionNode], + assignee_span: Span, + assignee_item_node_ids: &[ExpressionNodeId], + value: ExpressionValue, + ) -> ExecutionResult { + let value_items = value.expect_array("The assignee of an array place")?; + let span_range = SpanRange::new_between(assignee_span, value_items.span_range.end()); + let mut has_seen_dot_dot = false; + let mut prefix_assignees = Vec::new(); + let mut suffix_assignees = Vec::new(); + for node in assignee_item_node_ids.iter() { + match nodes[node.0] { + ExpressionNode::Range { + left: None, + range_limits: syn::RangeLimits::HalfOpen(_), + right: None, + } => { + if has_seen_dot_dot { + return assignee_span + .execution_err("Only one .. is allowed in an array assignee"); + } + has_seen_dot_dot = true; + } + _ => { + if has_seen_dot_dot { + suffix_assignees.push(*node); + } else { + prefix_assignees.push(*node); + } + } + } + } + let mut assignee_pairs: Vec<_> = if has_seen_dot_dot { + if prefix_assignees.len() + suffix_assignees.len() > value_items.items.len() { + return assignee_span.execution_err( + "The number of assignees exceeds the number of items in the array", + ); + } + let discarded_count = + value_items.items.len() - prefix_assignees.len() - suffix_assignees.len(); + let assignees = prefix_assignees + .into_iter() + .map(Some) + .chain(std::iter::repeat(None).take(discarded_count)) + .chain(suffix_assignees.into_iter().map(Some)); + + assignees + .zip(value_items.items) + .filter_map(|(assignee, value)| Some((assignee?, value))) + .collect() + } else { + if prefix_assignees.len() != value_items.items.len() { + return assignee_span.execution_err( + "The number of assignees does not equal the number of items in the array", + ); + } + prefix_assignees + .into_iter() + .zip(value_items.items) + .collect() + }; + Ok(Self { + span_range, + assignee_stack: { + assignee_pairs.reverse(); + assignee_pairs + }, + }) + } + + fn handle_next(mut self, next: ActionCreator) -> NextAction { + match self.assignee_stack.pop() { + Some((node, value)) => { + next.handle_assignment_and_return_to(node, value, AssignmentStackFrame::Array(self)) + } + None => next.return_assignment_completion(self.span_range), + } + } +} + +struct AssignmentCompletion { + span_range: SpanRange, +} + +enum PlaceStackFrame { + Assignment { + value: ExpressionValue, + }, + CompoundAssignmentRoot { + operation: CompoundAssignmentOperation, + value: ExpressionValue, + }, + Grouped, + Indexed { + access: IndexAccess, + index: ExpressionNodeId, + }, +} + +impl PlaceStackFrame { + fn ultimate_return_mode(&self) -> ReturnMode { + match self { + PlaceStackFrame::Assignment { .. } => ReturnMode::AssignmentCompletion, + PlaceStackFrame::CompoundAssignmentRoot { .. } => ReturnMode::Value, + PlaceStackFrame::Grouped { .. } => ReturnMode::Place, + PlaceStackFrame::Indexed { .. } => ReturnMode::Place, + } + } + + fn handle_place(self, place: Place, next: ActionCreator) -> ExecutionResult { + Ok(match self { + PlaceStackFrame::Assignment { value } => { + let span_range = match place { + Place::MutableReference(mut variable) => { + let span_range = + SpanRange::new_between(variable.span_range(), value.span_range()); + variable.set(value); + span_range + } + Place::Discarded(token) => token.span_range(), + }; + next.return_assignment_completion(span_range) + } + PlaceStackFrame::CompoundAssignmentRoot { operation, value } => { + let span_range = match place { + Place::MutableReference(mut variable) => { + let span_range = + SpanRange::new_between(variable.span_range(), value.span_range()); + // TODO - replace with handling a compound operation for better performance + // of e.g. arrays or streams + let left = variable.get_value_cloned(); + variable.set(operation.to_binary().evaluate(left, value)?); + span_range + } + Place::Discarded(token) => token.span_range(), + }; + next.return_value(ExpressionValue::None(span_range)) + } + PlaceStackFrame::Grouped => next.return_place(place), + PlaceStackFrame::Indexed { access, index } => next.read_value_with_handler( + index, + ValueStackFrame::ResolveIndexedPlace { place, access }, + ), + }) + } +} + +enum Place { + MutableReference(MutableReference), + Discarded(Token![_]), +} diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 203e651b..5a78f762 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -30,6 +30,7 @@ impl InterpretToValue for &SourceExpression { pub(super) enum SourceExpressionLeaf { Command(Command), Variable(VariableIdentifier), + Discarded(Token![_]), ExpressionBlock(ExpressionBlock), Value(ExpressionValue), } @@ -39,6 +40,7 @@ impl HasSpanRange for SourceExpressionLeaf { match self { SourceExpressionLeaf::Command(command) => command.span_range(), SourceExpressionLeaf::Variable(variable) => variable.span_range(), + SourceExpressionLeaf::Discarded(token) => token.span_range(), SourceExpressionLeaf::ExpressionBlock(block) => block.span_range(), SourceExpressionLeaf::Value(value) => value.span_range(), } @@ -96,11 +98,16 @@ impl Expressionable for Source { UnaryAtom::PrefixUnaryOperation(input.parse()?) } } - SourcePeekMatch::Ident(_) => match input.try_parse_or_revert() { - Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( - ExpressionBoolean::for_litbool(bool), - ))), - Err(_) => UnaryAtom::Leaf(Self::Leaf::Variable(input.parse()?)), + SourcePeekMatch::Ident(_) => { + if input.peek(Token![_]) { + return Ok(UnaryAtom::Leaf(Self::Leaf::Discarded(input.parse()?))); + } + match input.try_parse_or_revert() { + Ok(bool) => UnaryAtom::Leaf(Self::Leaf::Value(ExpressionValue::Boolean( + ExpressionBoolean::for_litbool(bool), + ))), + Err(_) => UnaryAtom::Leaf(Self::Leaf::Variable(input.parse()?)), + } }, SourcePeekMatch::Literal(_) => { let value = ExpressionValue::for_syn_lit(input.parse()?); @@ -196,6 +203,9 @@ impl Expressionable for Source { SourceExpressionLeaf::Command(command) => { command.clone().interpret_to_value(interpreter)? } + SourceExpressionLeaf::Discarded(token) => { + return token.execution_err("This cannot be used in a value expression"); + } SourceExpressionLeaf::Variable(variable_path) => { variable_path.interpret_to_value(interpreter)? } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index b3488b9f..6016a0c9 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -1,5 +1,5 @@ use super::*; - + /// ## Overview /// /// Expression parsing is quite complicated, but this is possibly slightly more complicated in some ways than it needs to be. @@ -7,7 +7,7 @@ use super::*; /// and to avoid recursion by making use of an explicit stack frame approach with [`ExpressionStackFrame`]s. /// /// This allows parsing of very long expressions (e.g. a sum of 1 million terms) without hitting recursion limits. -/// +/// /// ## Intuition /// /// You can think of these frames in two ways - as: @@ -15,7 +15,7 @@ use super::*; /// (b) the specifics of a parent operator, which can allow judging if/when an introduced expression with a possible /// extension should either bind to its parent (ignoring the extension for now, and retrying the extension with /// its parent) or bind to the extension itself. -/// +/// /// ## Examples /// /// See the rust doc on the [`ExpressionStackFrame`] for further details. @@ -374,6 +374,11 @@ enum OperatorPrecendence { NonTerminalComma, /// = += -= *= /= %= &= |= ^= <<= >>= Assign, + // [PREINTERPRET ADDITION] + // By giving assign extensions slightly higher priority than existing assign + // frames, this effectively makes them right-associative, so that: + // a = b = c parses as a = (b = c) instead of (a = b) = c + AssignExtension, // .. ..= Range, // || @@ -682,8 +687,8 @@ impl NodeExtension { NodeExtension::Index { .. } => OperatorPrecendence::Unambiguous, NodeExtension::Range(_) => OperatorPrecendence::Range, NodeExtension::EndOfStream => OperatorPrecendence::MIN, - NodeExtension::AssignmentOperation(_) => OperatorPrecendence::Assign, - NodeExtension::CompoundAssignmentOperation(_) => OperatorPrecendence::Assign, + NodeExtension::AssignmentOperation(_) => OperatorPrecendence::AssignExtension, + NodeExtension::CompoundAssignmentOperation(_) => OperatorPrecendence::AssignExtension, NodeExtension::NoValidExtensionForCurrentParent => OperatorPrecendence::MIN, } } diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 09281a03..81a5ebc8 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -131,16 +131,18 @@ pub(super) enum FloatKind { pub(super) struct UntypedFloat( /// The span of the literal is ignored, and will be set when converted to an output. LitFloat, + SpanRange, ); pub(super) type FallbackFloat = f64; impl UntypedFloat { pub(super) fn new_from_lit_float(lit_float: LitFloat) -> Self { - Self(lit_float) + let span_range = lit_float.span().span_range(); + Self(lit_float, span_range) } pub(super) fn new_from_literal(literal: Literal) -> Self { - Self(syn::LitFloat::from(literal)) + Self::new_from_lit_float(literal.into()) } pub(super) fn handle_unary_operation( @@ -248,7 +250,7 @@ impl UntypedFloat { fn parse_fallback(&self) -> ExecutionResult { self.0.base10_digits().parse().map_err(|err| { - self.0.span().execution_error(format!( + self.1.execution_error(format!( "Could not parse as the default inferred type {}: {}", core::any::type_name::(), err @@ -262,7 +264,7 @@ impl UntypedFloat { N::Err: core::fmt::Display, { self.0.base10_digits().parse().map_err(|err| { - self.0.span().execution_error(format!( + self.1.execution_error(format!( "Could not parse as {}: {}", core::any::type_name::(), err @@ -282,7 +284,8 @@ impl HasValueType for UntypedFloat { } impl ToExpressionValue for UntypedFloat { - fn to_value(self, span_range: SpanRange) -> ExpressionValue { + fn to_value(mut self, span_range: SpanRange) -> ExpressionValue { + self.1 = span_range; ExpressionValue::Float(ExpressionFloat { value: ExpressionFloatValue::Untyped(self), span_range, diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index a9cf31e8..0c2c5980 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -257,16 +257,18 @@ impl HasValueType for UntypedInteger { pub(crate) struct UntypedInteger( /// The span of the literal is ignored, and will be set when converted to an output. syn::LitInt, + SpanRange, ); pub(crate) type FallbackInteger = i128; impl UntypedInteger { pub(super) fn new_from_lit_int(lit_int: LitInt) -> Self { - Self(lit_int) + let span_range = lit_int.span().span_range(); + Self(lit_int, span_range) } pub(super) fn new_from_literal(literal: Literal) -> Self { - Self(syn::LitInt::from(literal)) + Self::new_from_lit_int(literal.into()) } pub(super) fn handle_unary_operation( @@ -434,7 +436,7 @@ impl UntypedInteger { pub(super) fn parse_fallback(&self) -> ExecutionResult { self.0.base10_digits().parse().map_err(|err| { - self.0.span().execution_error(format!( + self.1.execution_error(format!( "Could not parse as the default inferred type {}: {}", core::any::type_name::(), err @@ -448,7 +450,7 @@ impl UntypedInteger { N::Err: core::fmt::Display, { self.0.base10_digits().parse().map_err(|err| { - self.0.span().execution_error(format!( + self.1.execution_error(format!( "Could not parse as {}: {}", core::any::type_name::(), err @@ -462,7 +464,8 @@ impl UntypedInteger { } impl ToExpressionValue for UntypedInteger { - fn to_value(self, span_range: SpanRange) -> ExpressionValue { + fn to_value(mut self, span_range: SpanRange) -> ExpressionValue { + self.1 = span_range; ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::Untyped(self), span_range, diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 6a16242f..d86a1dde 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -17,6 +17,7 @@ mod value; pub(crate) use expression::*; pub(crate) use expression_block::*; pub(crate) use iterator::*; +pub(crate) use operations::*; pub(crate) use stream::*; pub(crate) use value::*; @@ -29,6 +30,5 @@ use evaluation::*; use expression_parsing::*; use float::*; use integer::*; -use operations::*; use range::*; use string::*; diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 509b3137..60371fc1 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -668,8 +668,8 @@ impl HasSpanRange for PropertyAccess { } } -#[derive(Clone)] -pub(super) struct IndexAccess { +#[derive(Copy, Clone)] +pub(crate) struct IndexAccess { pub(super) brackets: Brackets, } diff --git a/src/expressions/value.rs b/src/expressions/value.rs index a1b53a53..e25eec9d 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -379,13 +379,20 @@ impl ExpressionValue { } } - pub(super) fn handle_index_access( - self, + pub(super) fn into_indexed(self, access: IndexAccess, index: Self) -> ExecutionResult { + match self { + ExpressionValue::Array(array) => array.into_indexed(access, index), + other => access.execution_err(format!("Cannot index into a {}", other.value_type())), + } + } + + pub(crate) fn index_mut( + &mut self, access: IndexAccess, index: Self, - ) -> ExecutionResult { + ) -> ExecutionResult<&mut Self> { match self { - ExpressionValue::Array(array) => array.handle_index_access(access, index), + ExpressionValue::Array(array) => array.index_mut(access, index), other => access.execution_err(format!("Cannot index into a {}", other.value_type())), } } diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index cad5d943..0187f65c 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -88,8 +88,11 @@ impl SpanRange { } } - pub(crate) fn new_between(start: Span, end: Span) -> Self { - Self { start, end } + pub(crate) fn new_between(start: impl HasSpanRange, end: impl HasSpanRange) -> Self { + Self { + start: start.span_range().start, + end: end.span_range().end, + } } fn create_error(&self, message: impl std::fmt::Display) -> syn::Error { @@ -295,6 +298,7 @@ macro_rules! impl_auto_span_range { impl_auto_span_range! { syn::BinOp, syn::UnOp, + syn::token::Underscore, syn::token::Dot, syn::token::DotDot, syn::token::DotDotEq, diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index e5348362..29ab80ab 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -1,6 +1,6 @@ pub(crate) use core::iter; pub(crate) use core::marker::PhantomData; -pub(crate) use core::ops::DerefMut; +pub(crate) use core::ops::{Deref, DerefMut}; pub(crate) use proc_macro2::extra::*; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 67d92b0b..e54744de 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -97,7 +97,7 @@ impl NoOutputCommandDefinition for SetCommand { let variable_data = variable.reference(interpreter)?; content.interpret_into( interpreter, - variable_data.get_value_stream_mut()?.deref_mut(), + variable_data.into_mut()?.into_stream()?.value_mut(), )?; } SetArguments::SetVariablesEmpty { variables } => { diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 0d6951e8..8d301848 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -30,32 +30,141 @@ impl VariableReference { .with_span_range(self.variable_span_range)) } - pub(crate) fn get_value_mut(&self) -> ExecutionResult> { - self.data.try_borrow_mut().map_err(|_| { - self.execution_error( - "The variable cannot be modified if it is already currently being modified", - ) + pub(crate) fn into_mut(self) -> ExecutionResult> { + MutableReference::new(self) + } +} + +impl HasSpanRange for VariableReference { + fn span_range(&self) -> SpanRange { + self.variable_span_range + } +} + +pub(crate) struct MutRcRefCell { + /// This is actually a reference to the contents of the RefCell + /// but we store it using `unsafe` as `'static`, and use unsafe blocks + /// to ensure it's dropped first. + /// + /// SAFETY: This *must* appear before `pointed_at` so that it's dropped first. + ref_mut: RefMut<'static, U>, + pointed_at: Rc>, +} + +impl MutRcRefCell { + fn new(pointed_at: Rc>) -> Result { + let ref_mut = pointed_at.try_borrow_mut()?; + Ok(Self { + // SAFETY: We must ensure that this lifetime lives as long as the + // reference to pointed_at (i.e. the RefCell). + // This is guaranteed by the fact that the only time we drop the RefCell + // is when we drop the MutRcRefCell, and we ensure that the RefMut is dropped first. + ref_mut: unsafe { std::mem::transmute::, RefMut<'static, T>>(ref_mut) }, + pointed_at, + }) + } + + fn try_map( + self, + f: impl FnOnce(&mut T) -> Result<&mut U, E>, + ) -> Result, E> { + let mut error = None; + let outcome = RefMut::filter_map(self.ref_mut, |inner| match f(inner) { + Ok(value) => Some(value), + Err(e) => { + error = Some(e); + None + } + }); + match outcome { + Ok(ref_mut) => Ok(MutRcRefCell { + ref_mut, + pointed_at: self.pointed_at, + }), + Err(_) => Err(error.unwrap()), + } + } +} + +impl DerefMut for MutRcRefCell { + fn deref_mut(&mut self) -> &mut U { + &mut self.ref_mut + } +} + +impl Deref for MutRcRefCell { + type Target = U; + fn deref(&self) -> &U { + &self.ref_mut + } +} + +pub(crate) struct MutableReference { + mut_cell: MutRcRefCell, + span_range: SpanRange, +} + +impl MutableReference { + fn new(reference: VariableReference) -> ExecutionResult { + Ok(Self { + mut_cell: MutRcRefCell::new(reference.data).map_err(|_| { + reference.variable_span_range.execution_error( + "The variable cannot be modified as it is already being modified", + ) + })?, + span_range: reference.variable_span_range, + }) + } + + // Gets the cloned expression value, setting the span range appropriately + pub(crate) fn get_value_cloned(&self) -> ExpressionValue { + self.mut_cell + .deref() + .clone() + .with_span_range(self.span_range) + } + + pub(crate) fn into_stream(self) -> ExecutionResult> { + let stream = self.mut_cell.try_map(|value| match value { + ExpressionValue::Stream(stream) => Ok(&mut stream.value), + _ => self + .span_range + .execution_err("The variable is not a stream"), + })?; + Ok(MutableReference { + mut_cell: stream, + span_range: self.span_range, }) } - pub(crate) fn get_value_stream_mut(&self) -> ExecutionResult> { - let mut_guard = self.get_value_mut()?; - RefMut::filter_map(mut_guard, |mut_guard| match mut_guard { - ExpressionValue::Stream(stream) => Some(&mut stream.value), - _ => None, + pub(crate) fn resolve_indexed( + self, + access: IndexAccess, + index: ExpressionValue, + ) -> ExecutionResult { + let indexed = self + .mut_cell + .try_map(|value| value.index_mut(access, index))?; + Ok(Self { + mut_cell: indexed, + span_range: SpanRange::new_between(self.span_range.start(), access.span()), }) - .map_err(|_| self.execution_error("The variable is not a stream")) } - pub(crate) fn set(&self, content: impl ToExpressionValue) -> ExecutionResult<()> { - *self.get_value_mut()? = content.to_value(self.variable_span_range); - Ok(()) + pub(crate) fn set(&mut self, content: impl ToExpressionValue) { + *self.mut_cell = content.to_value(self.span_range); } } -impl HasSpanRange for VariableReference { +impl MutableReference { + pub(crate) fn value_mut(&mut self) -> &mut T { + &mut self.mut_cell + } +} + +impl HasSpanRange for MutableReference { fn span_range(&self) -> SpanRange { - self.variable_span_range + self.span_range } } diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index e30f2fdf..6b2bb4fb 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -256,7 +256,7 @@ impl HandleTransformation for ExplicitTransformStream { content.handle_transform( input, interpreter, - reference.get_value_stream_mut()?.deref_mut(), + reference.into_mut()?.into_stream()?.value_mut(), )?; } ExplicitTransformStreamArguments::Discard { content, .. } => { diff --git a/src/transformation/variable_binding.rs b/src/transformation/variable_binding.rs index c70d1981..119798d6 100644 --- a/src/transformation/variable_binding.rs +++ b/src/transformation/variable_binding.rs @@ -189,25 +189,29 @@ impl HandleTransformation for VariableBinding { let reference = self.reference(interpreter)?; input .parse::()? - .push_as_token_tree(reference.get_value_stream_mut()?.deref_mut()); + .push_as_token_tree(reference.into_mut()?.into_stream()?.value_mut()); } VariableBinding::GroupedAppendFlattened { .. } => { let reference = self.reference(interpreter)?; input .parse::()? - .flatten_into(reference.get_value_stream_mut()?.deref_mut()); + .flatten_into(reference.into_mut()?.into_stream()?.value_mut()); } VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { let reference = self.reference(interpreter)?; - reference.get_value_stream_mut()?.push_grouped( - |inner| until.handle_parse_into(input, inner), - Delimiter::None, - marker.span, - )?; + reference + .into_mut()? + .into_stream()? + .value_mut() + .push_grouped( + |inner| until.handle_parse_into(input, inner), + Delimiter::None, + marker.span, + )?; } VariableBinding::FlattenedAppendFlattened { until, .. } => { let reference = self.reference(interpreter)?; - until.handle_parse_into(input, reference.get_value_stream_mut()?.deref_mut())?; + until.handle_parse_into(input, reference.into_mut()?.into_stream()?.value_mut())?; } } Ok(()) diff --git a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr index 05749f29..738eacd7 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_extend_it_again.stderr @@ -1,4 +1,4 @@ -error: The variable cannot be modified if it is already currently being modified +error: The variable cannot be modified as it is already being modified --> tests/compilation_failures/core/extend_variable_and_then_extend_it_again.rs:6:42 | 6 | [!set! #variable += World [!set! #variable += !]] diff --git a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr index 5e395d4a..793c8290 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_set_it.stderr @@ -1,4 +1,4 @@ -error: The variable cannot be modified if it is already currently being modified +error: The variable cannot be modified as it is already being modified --> tests/compilation_failures/core/extend_variable_and_then_set_it.rs:6:37 | 6 | [!set! #variable += World #(variable = [!stream! Hello2])] diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.rs b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.rs new file mode 100644 index 00000000..3ee71560 --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #([_, _, _] = [1, 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr new file mode 100644 index 00000000..c9a2eb3f --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr @@ -0,0 +1,5 @@ +error: The number of assignees does not equal the number of items in the array + --> tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.rs:5:11 + | +5 | #([_, _, _] = [1, 2]) + | ^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.rs b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.rs new file mode 100644 index 00000000..fde742e8 --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #([_] = [1, 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr new file mode 100644 index 00000000..12665bb2 --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr @@ -0,0 +1,5 @@ +error: The number of assignees does not equal the number of items in the array + --> tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.rs:5:11 + | +5 | #([_] = [1, 2]) + | ^^^ diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.rs b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.rs new file mode 100644 index 00000000..4afec6f3 --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #([_, _, .., _] = [1, 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr new file mode 100644 index 00000000..6afc8e15 --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr @@ -0,0 +1,5 @@ +error: The number of assignees exceeds the number of items in the array + --> tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.rs:5:11 + | +5 | #([_, _, .., _] = [1, 2]) + | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.rs b/tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.rs new file mode 100644 index 00000000..559db0a6 --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #([_, .., _, .., _] = [1, 2, 3, 4]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.stderr b/tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.stderr new file mode 100644 index 00000000..6d6761be --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.stderr @@ -0,0 +1,5 @@ +error: Only one .. is allowed in an array assignee + --> tests/compilation_failures/expressions/array_place_destructure_multiple_dot_dots.rs:5:11 + | +5 | #([_, .., _, .., _] = [1, 2, 3, 4]) + | ^^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.rs b/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.rs new file mode 100644 index 00000000..e6c1fa0a --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let arr = [0, 1]; arr[arr[1]] = 3) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr b/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr new file mode 100644 index 00000000..467936ed --- /dev/null +++ b/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr @@ -0,0 +1,5 @@ +error: The variable cannot be read if it is currently being modified + --> tests/compilation_failures/expressions/array_place_destructure_multiple_muts.rs:5:33 + | +5 | #(let arr = [0, 1]; arr[arr[1]] = 3) + | ^^^ diff --git a/tests/compilation_failures/expressions/discard_in_value_position.rs b/tests/compilation_failures/expressions/discard_in_value_position.rs new file mode 100644 index 00000000..845bf0d3 --- /dev/null +++ b/tests/compilation_failures/expressions/discard_in_value_position.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(_) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/discard_in_value_position.stderr b/tests/compilation_failures/expressions/discard_in_value_position.stderr new file mode 100644 index 00000000..4acb8839 --- /dev/null +++ b/tests/compilation_failures/expressions/discard_in_value_position.stderr @@ -0,0 +1,5 @@ +error: This cannot be used in a value expression + --> tests/compilation_failures/expressions/discard_in_value_position.rs:5:11 + | +5 | #(_) + | ^ diff --git a/tests/compilation_failures/expressions/index_into_discarded_place.rs b/tests/compilation_failures/expressions/index_into_discarded_place.rs new file mode 100644 index 00000000..85e1d62b --- /dev/null +++ b/tests/compilation_failures/expressions/index_into_discarded_place.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(_[0] = 10) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/index_into_discarded_place.stderr b/tests/compilation_failures/expressions/index_into_discarded_place.stderr new file mode 100644 index 00000000..c6da3ce2 --- /dev/null +++ b/tests/compilation_failures/expressions/index_into_discarded_place.stderr @@ -0,0 +1,5 @@ +error: Cannot index into a discarded value + --> tests/compilation_failures/expressions/index_into_discarded_place.rs:5:12 + | +5 | #(_[0] = 10) + | ^^^ diff --git a/tests/expressions.rs b/tests/expressions.rs index a7b83f5a..aba19297 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -229,4 +229,98 @@ fn test_indexing() { preinterpret_assert_eq!( #(let x = [1, 2, 3]; x[x[0] + x[x[1] - 1] - 1]), 3 ); + // And setting indices... + preinterpret_assert_eq!( + #( + let x = [0, 0, 0]; + x[0] = 2; + (x[1 + 1]) = x[0]; + x as debug + ), + "[2, 0, 2]" + ); + // And array destructuring + preinterpret_assert_eq!( + #( + let a = 0; let b = 0; let c = 0; + let x = [1, 2, 3, 4, 5]; + [a, b, _, _, c] = x; + [a, b, c] as debug + ), + "[1, 2, 5]" + ); + preinterpret_assert_eq!( + #( + let a = 0; let b = 0; let c = 0; + let x = [1, 2, 3, 4, 5]; + [a, b, c, ..] = x; + [a, b, c] as debug + ), + "[1, 2, 3]" + ); + preinterpret_assert_eq!( + #( + let a = 0; let b = 0; let c = 0; + let x = [1, 2, 3, 4, 5]; + [.., a, b] = x; + [a, b, c] as debug + ), + "[4, 5, 0]" + ); + preinterpret_assert_eq!( + #( + let a = 0; let b = 0; let c = 0; + let x = [1, 2, 3, 4, 5]; + [a, .., b, c] = x; + [a, b, c] as debug + ), + "[1, 4, 5]" + ); + // Nested places + preinterpret_assert_eq!( + #( + let out = [[0, 0], 0]; + let a = 0; let b = 0; let c = 0; + [out[1], .., out[0][0], out[0][1]] = [1, 2, 3, 4, 5]; + out as debug + ), + "[[4, 5], 1]" + ); + // Misc + preinterpret_assert_eq!( + #( + let a = [0, 0, 0, 0, 0]; + let b = 3; + let c = 0; + let _ = c = [a[2], _] = [4, 5]; + let _ = a[1] += 2; + let _ = b = 2; + [a, b, c] as debug + ), + "[[0, 2, 4, 0, 0], 2, None]" + ); + // This test demonstrates that the right side executes first. + // This aligns with the rust behaviour. + preinterpret_assert_eq!( + #( + let a = [0, 0]; + let b = 0; + a[b] += #(b += 1; 5); + a as debug + ), + "[0, 5]" + ); + // This test demonstrates that the assignee operation is executed + // incrementally, to align with the rust behaviour. + preinterpret_assert_eq!( + #( + let arr = [0, 0]; + let arr2 = [0, 0]; + // The first assignment arr[0] = 1 occurs before being overwritten + // by the arr[0] = 5 in the second index. + [arr[0], arr2[#(arr[0] = 5; 1)]] = [1, 1]; + arr[0] + ), + 5 + ); } From 360a8025c062d98d04880a6ad0397565547d5089 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 18 Feb 2025 01:19:56 +0000 Subject: [PATCH 102/126] refactor: Move a few files around --- src/interpretation/interpreter.rs | 260 ++++++++++++------------------ src/misc/mod.rs | 2 + src/misc/mut_rc_ref_cell.rs | 63 ++++++++ 3 files changed, 166 insertions(+), 159 deletions(-) create mode 100644 src/misc/mut_rc_ref_cell.rs diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 8d301848..c40b0fc7 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -9,165 +9,6 @@ pub(crate) struct Interpreter { variable_data: VariableData, } -#[derive(Clone)] -pub(crate) struct VariableReference { - data: Rc>, - variable_span_range: SpanRange, -} - -impl VariableReference { - pub(crate) fn get_value_ref(&self) -> ExecutionResult> { - self.data.try_borrow().map_err(|_| { - self.execution_error("The variable cannot be read if it is currently being modified") - }) - } - - // Gets the cloned expression value, setting the span range appropriately - pub(crate) fn get_value_cloned(&self) -> ExecutionResult { - Ok(self - .get_value_ref()? - .clone() - .with_span_range(self.variable_span_range)) - } - - pub(crate) fn into_mut(self) -> ExecutionResult> { - MutableReference::new(self) - } -} - -impl HasSpanRange for VariableReference { - fn span_range(&self) -> SpanRange { - self.variable_span_range - } -} - -pub(crate) struct MutRcRefCell { - /// This is actually a reference to the contents of the RefCell - /// but we store it using `unsafe` as `'static`, and use unsafe blocks - /// to ensure it's dropped first. - /// - /// SAFETY: This *must* appear before `pointed_at` so that it's dropped first. - ref_mut: RefMut<'static, U>, - pointed_at: Rc>, -} - -impl MutRcRefCell { - fn new(pointed_at: Rc>) -> Result { - let ref_mut = pointed_at.try_borrow_mut()?; - Ok(Self { - // SAFETY: We must ensure that this lifetime lives as long as the - // reference to pointed_at (i.e. the RefCell). - // This is guaranteed by the fact that the only time we drop the RefCell - // is when we drop the MutRcRefCell, and we ensure that the RefMut is dropped first. - ref_mut: unsafe { std::mem::transmute::, RefMut<'static, T>>(ref_mut) }, - pointed_at, - }) - } - - fn try_map( - self, - f: impl FnOnce(&mut T) -> Result<&mut U, E>, - ) -> Result, E> { - let mut error = None; - let outcome = RefMut::filter_map(self.ref_mut, |inner| match f(inner) { - Ok(value) => Some(value), - Err(e) => { - error = Some(e); - None - } - }); - match outcome { - Ok(ref_mut) => Ok(MutRcRefCell { - ref_mut, - pointed_at: self.pointed_at, - }), - Err(_) => Err(error.unwrap()), - } - } -} - -impl DerefMut for MutRcRefCell { - fn deref_mut(&mut self) -> &mut U { - &mut self.ref_mut - } -} - -impl Deref for MutRcRefCell { - type Target = U; - fn deref(&self) -> &U { - &self.ref_mut - } -} - -pub(crate) struct MutableReference { - mut_cell: MutRcRefCell, - span_range: SpanRange, -} - -impl MutableReference { - fn new(reference: VariableReference) -> ExecutionResult { - Ok(Self { - mut_cell: MutRcRefCell::new(reference.data).map_err(|_| { - reference.variable_span_range.execution_error( - "The variable cannot be modified as it is already being modified", - ) - })?, - span_range: reference.variable_span_range, - }) - } - - // Gets the cloned expression value, setting the span range appropriately - pub(crate) fn get_value_cloned(&self) -> ExpressionValue { - self.mut_cell - .deref() - .clone() - .with_span_range(self.span_range) - } - - pub(crate) fn into_stream(self) -> ExecutionResult> { - let stream = self.mut_cell.try_map(|value| match value { - ExpressionValue::Stream(stream) => Ok(&mut stream.value), - _ => self - .span_range - .execution_err("The variable is not a stream"), - })?; - Ok(MutableReference { - mut_cell: stream, - span_range: self.span_range, - }) - } - - pub(crate) fn resolve_indexed( - self, - access: IndexAccess, - index: ExpressionValue, - ) -> ExecutionResult { - let indexed = self - .mut_cell - .try_map(|value| value.index_mut(access, index))?; - Ok(Self { - mut_cell: indexed, - span_range: SpanRange::new_between(self.span_range.start(), access.span()), - }) - } - - pub(crate) fn set(&mut self, content: impl ToExpressionValue) { - *self.mut_cell = content.to_value(self.span_range); - } -} - -impl MutableReference { - pub(crate) fn value_mut(&mut self) -> &mut T { - &mut self.mut_cell - } -} - -impl HasSpanRange for MutableReference { - fn span_range(&self) -> SpanRange { - self.span_range - } -} - impl Interpreter { pub(crate) fn new() -> Self { Self { @@ -297,3 +138,104 @@ impl Default for InterpreterConfig { } } } + +#[derive(Clone)] +pub(crate) struct VariableReference { + data: Rc>, + variable_span_range: SpanRange, +} + +impl VariableReference { + pub(crate) fn get_value_ref(&self) -> ExecutionResult> { + self.data.try_borrow().map_err(|_| { + self.execution_error("The variable cannot be read if it is currently being modified") + }) + } + + // Gets the cloned expression value, setting the span range appropriately + pub(crate) fn get_value_cloned(&self) -> ExecutionResult { + Ok(self + .get_value_ref()? + .clone() + .with_span_range(self.variable_span_range)) + } + + pub(crate) fn into_mut(self) -> ExecutionResult> { + MutableReference::new(self) + } +} + +impl HasSpanRange for VariableReference { + fn span_range(&self) -> SpanRange { + self.variable_span_range + } +} + +pub(crate) struct MutableReference { + mut_cell: MutRcRefCell, + span_range: SpanRange, +} + +impl MutableReference { + fn new(reference: VariableReference) -> ExecutionResult { + Ok(Self { + mut_cell: MutRcRefCell::new(reference.data).map_err(|_| { + reference.variable_span_range.execution_error( + "The variable cannot be modified as it is already being modified", + ) + })?, + span_range: reference.variable_span_range, + }) + } + + // Gets the cloned expression value, setting the span range appropriately + pub(crate) fn get_value_cloned(&self) -> ExpressionValue { + self.mut_cell + .deref() + .clone() + .with_span_range(self.span_range) + } + + pub(crate) fn into_stream(self) -> ExecutionResult> { + let stream = self.mut_cell.try_map(|value| match value { + ExpressionValue::Stream(stream) => Ok(&mut stream.value), + _ => self + .span_range + .execution_err("The variable is not a stream"), + })?; + Ok(MutableReference { + mut_cell: stream, + span_range: self.span_range, + }) + } + + pub(crate) fn resolve_indexed( + self, + access: IndexAccess, + index: ExpressionValue, + ) -> ExecutionResult { + let indexed = self + .mut_cell + .try_map(|value| value.index_mut(access, index))?; + Ok(Self { + mut_cell: indexed, + span_range: SpanRange::new_between(self.span_range.start(), access.span()), + }) + } + + pub(crate) fn set(&mut self, content: impl ToExpressionValue) { + *self.mut_cell = content.to_value(self.span_range); + } +} + +impl MutableReference { + pub(crate) fn value_mut(&mut self) -> &mut T { + &mut self.mut_cell + } +} + +impl HasSpanRange for MutableReference { + fn span_range(&self) -> SpanRange { + self.span_range + } +} diff --git a/src/misc/mod.rs b/src/misc/mod.rs index 2dbf0503..eae8d3ee 100644 --- a/src/misc/mod.rs +++ b/src/misc/mod.rs @@ -1,10 +1,12 @@ mod errors; mod field_inputs; +mod mut_rc_ref_cell; mod parse_traits; mod string_conversion; pub(crate) use errors::*; pub(crate) use field_inputs::*; +pub(crate) use mut_rc_ref_cell::*; pub(crate) use parse_traits::*; pub(crate) use string_conversion::*; diff --git a/src/misc/mut_rc_ref_cell.rs b/src/misc/mut_rc_ref_cell.rs new file mode 100644 index 00000000..8c7f1e4b --- /dev/null +++ b/src/misc/mut_rc_ref_cell.rs @@ -0,0 +1,63 @@ +use crate::internal_prelude::*; +use std::cell::*; +use std::rc::Rc; + +pub(crate) struct MutRcRefCell { + /// This is actually a reference to the contents of the RefCell + /// but we store it using `unsafe` as `'static`, and use unsafe blocks + /// to ensure it's dropped first. + /// + /// SAFETY: This *must* appear before `pointed_at` so that it's dropped first. + ref_mut: RefMut<'static, U>, + pointed_at: Rc>, +} + +impl MutRcRefCell { + pub(crate) fn new(pointed_at: Rc>) -> Result { + let ref_mut = pointed_at.try_borrow_mut()?; + Ok(Self { + // SAFETY: We must ensure that this lifetime lives as long as the + // reference to pointed_at (i.e. the RefCell). + // This is guaranteed by the fact that the only time we drop the RefCell + // is when we drop the MutRcRefCell, and we ensure that the RefMut is dropped first. + ref_mut: unsafe { std::mem::transmute::, RefMut<'static, T>>(ref_mut) }, + pointed_at, + }) + } +} + +impl MutRcRefCell { + pub(crate) fn try_map( + self, + f: impl FnOnce(&mut U) -> Result<&mut V, E>, + ) -> Result, E> { + let mut error = None; + let outcome = RefMut::filter_map(self.ref_mut, |inner| match f(inner) { + Ok(value) => Some(value), + Err(e) => { + error = Some(e); + None + } + }); + match outcome { + Ok(ref_mut) => Ok(MutRcRefCell { + ref_mut, + pointed_at: self.pointed_at, + }), + Err(_) => Err(error.unwrap()), + } + } +} + +impl DerefMut for MutRcRefCell { + fn deref_mut(&mut self) -> &mut U { + &mut self.ref_mut + } +} + +impl Deref for MutRcRefCell { + type Target = U; + fn deref(&self) -> &U { + &self.ref_mut + } +} From 5ecd8017213088c99d5ecb9dffc9a31346012649 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 18 Feb 2025 12:02:19 +0000 Subject: [PATCH 103/126] feature: Support range indices for arrays --- CHANGELOG.md | 10 ++----- src/expressions/array.rs | 62 ++++++++++++++++++++++++++++------------ src/expressions/range.rs | 32 +++++++++++++++++++++ tests/expressions.rs | 49 ++++++++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efb94a79..2cf44b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,21 +96,17 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* Support `#(x[..])` syntax for indexing arrays at read time (streams to follow in a separate task below after parsers are updated) - * `#(x[0..3])` returns an array - * `#(x[0..=3])` returns an array * Add `..` and `.., x` support to the array pattern, like the place expression. +* More efficient `+=` etc * Objects, like a JS object: * Backed by an indexmap (or maybe an immutable `IndexMap` wrapping an `im::HashMap` and `im::Vec` or entry orderings) * Can be created with `#({ a: x, ... })` * Can be read with `#(x.hello)` or `#(x["hello"])` * Debug impl is `#({ hello: [!group! BLAH], ["#world"]: Hi, })` - * They have an input object * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` + * Commands have an input object * Can be destructured with `{ hello, world: _, ... }` - * (Until we get custom type support into a syn fork), can be embedded into an output stream as a single token - e.g. `PREINTERPRET_OBJECT_2313` - (The value can be looked up via a weak reference in the interpreter (as a central location), and the stream owning a reference to it to stop it being dropped). The final conversion to tokens can look up the object in the interpreter, and use its `stream()` function to either output the default - stream for the object, or error and suggest fields the user should use instead. + * Throws if output to a stream * Have `!zip!` support `{ objects }` * Method calls * Mutable methods notes: diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 3687232a..767df12b 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -118,13 +118,23 @@ impl ExpressionArray { } pub(super) fn into_indexed( - self, + mut self, access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult { - let index = self.resolve_valid_index(index)?; - let span_range = SpanRange::new_between(self.span_range.start(), access.span()); - Ok(self.items[index].clone().with_span_range(span_range)) + let span_range = SpanRange::new_between(self.span_range, access); + Ok(match index { + ExpressionValue::Integer(integer) => { + let index = self.resolve_valid_index_from_integer(integer, false)?; + self.items[index].clone().with_span_range(span_range) + } + ExpressionValue::Range(range) => { + let range = range.resolve_to_index_range(&self)?; + let new_items: Vec<_> = self.items.drain(range).collect(); + new_items.to_value(span_range) + } + _ => return index.execution_err("The index must be an integer or a range"), + }) } pub(super) fn index_mut( @@ -132,29 +142,43 @@ impl ExpressionArray { _access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult<&mut ExpressionValue> { - let index = self.resolve_valid_index(index)?; + let index = self.resolve_valid_index(index, false)?; Ok(&mut self.items[index]) } - fn resolve_valid_index(&self, index: ExpressionValue) -> ExecutionResult { + pub(super) fn resolve_valid_index(&self, index: ExpressionValue, is_exclusive: bool) -> ExecutionResult { match index { - ExpressionValue::Integer(int) => { - let span_range = int.span_range; - let index = int.expect_usize()?; - if index < self.items.len() { - Ok(index) - } else { - span_range.execution_err(format!( - "Index of {} is out of range of the array length of {}", - index, - self.items.len() - )) - } - } + ExpressionValue::Integer(int) => self.resolve_valid_index_from_integer(int, is_exclusive), _ => index.execution_err("The index must be an integer"), } } + fn resolve_valid_index_from_integer(&self, integer: ExpressionInteger, is_exclusive: bool) -> ExecutionResult { + let span_range = integer.span_range; + let index = integer.expect_usize()?; + if is_exclusive { + if index <= self.items.len() { + Ok(index) + } else { + span_range.execution_err(format!( + "Exclusive index of {} must be less than or equal to the array length of {}", + index, + self.items.len() + )) + } + } else { + if index < self.items.len() { + Ok(index) + } else { + span_range.execution_err(format!( + "Inclusive index of {} must be less than the array length of {}", + index, + self.items.len() + )) + } + } + } + pub(crate) fn concat_recursive_into( self, output: &mut String, diff --git a/src/expressions/range.rs b/src/expressions/range.rs index 32322bf9..cf4d8223 100644 --- a/src/expressions/range.rs +++ b/src/expressions/range.rs @@ -54,6 +54,38 @@ impl ExpressionRange { } Ok(()) } + + pub(crate) fn resolve_to_index_range(self, array: &ExpressionArray) -> ExecutionResult> { + let mut start = 0; + let mut end = array.items.len(); + Ok(match *self.inner { + ExpressionRangeInner::Range { start_inclusive, end_exclusive, .. } => { + start = array.resolve_valid_index(start_inclusive, false)?; + end = array.resolve_valid_index(end_exclusive, true)?; + start..end + }, + ExpressionRangeInner::RangeFrom { start_inclusive, .. } => { + start = array.resolve_valid_index(start_inclusive, false)?; + start..array.items.len() + }, + ExpressionRangeInner::RangeTo { end_exclusive, .. } => { + end = array.resolve_valid_index(end_exclusive, true)?; + start..end + }, + ExpressionRangeInner::RangeFull { .. } => start..end, + ExpressionRangeInner::RangeInclusive { start_inclusive, end_inclusive, .. } => { + start = array.resolve_valid_index(start_inclusive, false)?; + // +1 is safe because it must be < array length. + end = array.resolve_valid_index(end_inclusive, false)? + 1; + start..end + }, + ExpressionRangeInner::RangeToInclusive { end_inclusive, .. } => { + // +1 is safe because it must be < array length. + end = array.resolve_valid_index(end_inclusive, false)? + 1; + start..end + }, + }) + } } impl HasSpanRange for ExpressionRange { diff --git a/tests/expressions.rs b/tests/expressions.rs index aba19297..0413cb9b 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -222,7 +222,7 @@ fn test_range() { } #[test] -fn test_indexing() { +fn test_array_indexing() { preinterpret_assert_eq!( #(let x = [1, 2, 3]; x[1]), 2 ); @@ -239,6 +239,53 @@ fn test_indexing() { ), "[2, 0, 2]" ); + // And ranges... + preinterpret_assert_eq!( + #( + let x = [1, 2, 3, 4, 5]; + x[..] as debug + ), + "[1, 2, 3, 4, 5]" + ); + preinterpret_assert_eq!( + #( + let x = [1, 2, 3, 4, 5]; + x[0..0] as debug + ), + "[]" + ); + preinterpret_assert_eq!( + #( + let x = [1, 2, 3, 4, 5]; + x[2..=2] as debug + ), + "[3]" + ); + preinterpret_assert_eq!( + #( + let x = [1, 2, 3, 4, 5]; + x[..=2] as debug + ), + "[1, 2, 3]" + ); + preinterpret_assert_eq!( + #( + let x = [1, 2, 3, 4, 5]; + x[..4] as debug + ), + "[1, 2, 3, 4]" + ); + preinterpret_assert_eq!( + #( + let x = [1, 2, 3, 4, 5]; + x[2..] as debug + ), + "[3, 4, 5]" + ); +} + +#[test] +fn test_array_destructurings() { // And array destructuring preinterpret_assert_eq!( #( From 388d355fd6d13b597a193d18523c40e25b7c58c9 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 18 Feb 2025 13:59:23 +0000 Subject: [PATCH 104/126] feature: Support .. in array patterns --- CHANGELOG.md | 3 +- src/expressions/array.rs | 32 ++++--- src/expressions/evaluation.rs | 37 ++++---- src/expressions/expression_block.rs | 16 +--- src/expressions/range.rs | 31 ++++-- src/transformation/destructuring.rs | 90 ------------------ src/transformation/mod.rs | 4 +- src/transformation/patterns.rs | 140 ++++++++++++++++++++++++++++ tests/expressions.rs | 44 ++++++++- 9 files changed, 252 insertions(+), 145 deletions(-) delete mode 100644 src/transformation/destructuring.rs create mode 100644 src/transformation/patterns.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cf44b78..cad5f30e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,7 +96,6 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* Add `..` and `.., x` support to the array pattern, like the place expression. * More efficient `+=` etc * Objects, like a JS object: * Backed by an indexmap (or maybe an immutable `IndexMap` wrapping an `im::HashMap` and `im::Vec` or entry orderings) @@ -174,7 +173,7 @@ Inside a transform stream, the following grammar is supported: handle_separator?: { }, // Default is to not output the separator }] ``` -* Support `#(x[..])` syntax for indexing streams +* Support `#(x[..])` syntax for indexing streams, like with arrays * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) * `#(x[0..3])` returns a TokenStream * `#(x[0..=3])` returns a TokenStream diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 767df12b..2719f89d 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -146,14 +146,24 @@ impl ExpressionArray { Ok(&mut self.items[index]) } - pub(super) fn resolve_valid_index(&self, index: ExpressionValue, is_exclusive: bool) -> ExecutionResult { + pub(super) fn resolve_valid_index( + &self, + index: ExpressionValue, + is_exclusive: bool, + ) -> ExecutionResult { match index { - ExpressionValue::Integer(int) => self.resolve_valid_index_from_integer(int, is_exclusive), + ExpressionValue::Integer(int) => { + self.resolve_valid_index_from_integer(int, is_exclusive) + } _ => index.execution_err("The index must be an integer"), } } - fn resolve_valid_index_from_integer(&self, integer: ExpressionInteger, is_exclusive: bool) -> ExecutionResult { + fn resolve_valid_index_from_integer( + &self, + integer: ExpressionInteger, + is_exclusive: bool, + ) -> ExecutionResult { let span_range = integer.span_range; let index = integer.expect_usize()?; if is_exclusive { @@ -166,16 +176,14 @@ impl ExpressionArray { self.items.len() )) } + } else if index < self.items.len() { + Ok(index) } else { - if index < self.items.len() { - Ok(index) - } else { - span_range.execution_err(format!( - "Inclusive index of {} must be less than the array length of {}", - index, - self.items.len() - )) - } + span_range.execution_err(format!( + "Inclusive index of {} must be less than the array length of {}", + index, + self.items.len() + )) } } diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index f9f50c60..081e5a2f 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -691,14 +691,15 @@ struct ArrayAssigneeStackFrame { } impl ArrayAssigneeStackFrame { + /// See also `ArrayPattern` in `patterns.rs` fn new( nodes: &[ExpressionNode], assignee_span: Span, assignee_item_node_ids: &[ExpressionNodeId], value: ExpressionValue, ) -> ExecutionResult { - let value_items = value.expect_array("The assignee of an array place")?; - let span_range = SpanRange::new_between(assignee_span, value_items.span_range.end()); + let array = value.expect_array("The assignee of an array place")?; + let span_range = SpanRange::new_between(assignee_span, array.span_range.end()); let mut has_seen_dot_dot = false; let mut prefix_assignees = Vec::new(); let mut suffix_assignees = Vec::new(); @@ -724,14 +725,19 @@ impl ArrayAssigneeStackFrame { } } } + + let array_length = array.items.len(); + let mut assignee_pairs: Vec<_> = if has_seen_dot_dot { - if prefix_assignees.len() + suffix_assignees.len() > value_items.items.len() { - return assignee_span.execution_err( - "The number of assignees exceeds the number of items in the array", - ); + let total_assignees = prefix_assignees.len() + suffix_assignees.len(); + if total_assignees > array_length { + return assignee_span.execution_err(format!( + "The array has {} items, but the assignee expected at least {}", + array_length, total_assignees, + )); } let discarded_count = - value_items.items.len() - prefix_assignees.len() - suffix_assignees.len(); + array.items.len() - prefix_assignees.len() - suffix_assignees.len(); let assignees = prefix_assignees .into_iter() .map(Some) @@ -739,19 +745,18 @@ impl ArrayAssigneeStackFrame { .chain(suffix_assignees.into_iter().map(Some)); assignees - .zip(value_items.items) + .zip(array.items) .filter_map(|(assignee, value)| Some((assignee?, value))) .collect() } else { - if prefix_assignees.len() != value_items.items.len() { - return assignee_span.execution_err( - "The number of assignees does not equal the number of items in the array", - ); + let total_assignees = prefix_assignees.len(); + if total_assignees != array_length { + return assignee_span.execution_err(format!( + "The array has {} items, but the assignee expected {}", + array_length, total_assignees, + )); } - prefix_assignees - .into_iter() - .zip(value_items.items) - .collect() + prefix_assignees.into_iter().zip(array.items).collect() }; Ok(Self { span_range, diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index 1c9bbc93..c69fca4e 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -130,18 +130,10 @@ impl InterpretToValue for &Statement { } } -/// In a rust expression, assignments are allowed in the middle of an expression. -/// -/// But the following is rather hard to parse in a streaming manner, -/// due to ambiguity and right-associativity of = -/// ```rust,ignore -/// let a; -/// let b; -/// // When the = (4,) is revealed, the `(b,)` changes from a value to a destructuring -/// let out = a = (b,) = (4,); -/// // When the += 2 is revealed, b changes from a value to a place -/// let out = b += 2; -/// ``` +/// Note a `let x = ...;` is very different to `x = ...;` inside an expression. +/// In the former, `x` is a pattern, and any identifiers creates new variable/bindings. +/// In the latter, `x` is a place expression, and identifiers can be either place references or +/// values, e.g. `a.x[y[0]][3] = ...` has `y[0]` evaluated as a value. #[derive(Clone)] pub(crate) struct LetStatement { let_token: Token![let], diff --git a/src/expressions/range.rs b/src/expressions/range.rs index cf4d8223..b07291e6 100644 --- a/src/expressions/range.rs +++ b/src/expressions/range.rs @@ -55,35 +55,48 @@ impl ExpressionRange { Ok(()) } - pub(crate) fn resolve_to_index_range(self, array: &ExpressionArray) -> ExecutionResult> { + pub(crate) fn resolve_to_index_range( + self, + array: &ExpressionArray, + ) -> ExecutionResult> { let mut start = 0; let mut end = array.items.len(); Ok(match *self.inner { - ExpressionRangeInner::Range { start_inclusive, end_exclusive, .. } => { + ExpressionRangeInner::Range { + start_inclusive, + end_exclusive, + .. + } => { start = array.resolve_valid_index(start_inclusive, false)?; end = array.resolve_valid_index(end_exclusive, true)?; start..end - }, - ExpressionRangeInner::RangeFrom { start_inclusive, .. } => { + } + ExpressionRangeInner::RangeFrom { + start_inclusive, .. + } => { start = array.resolve_valid_index(start_inclusive, false)?; start..array.items.len() - }, + } ExpressionRangeInner::RangeTo { end_exclusive, .. } => { end = array.resolve_valid_index(end_exclusive, true)?; start..end - }, + } ExpressionRangeInner::RangeFull { .. } => start..end, - ExpressionRangeInner::RangeInclusive { start_inclusive, end_inclusive, .. } => { + ExpressionRangeInner::RangeInclusive { + start_inclusive, + end_inclusive, + .. + } => { start = array.resolve_valid_index(start_inclusive, false)?; // +1 is safe because it must be < array length. end = array.resolve_valid_index(end_inclusive, false)? + 1; start..end - }, + } ExpressionRangeInner::RangeToInclusive { end_inclusive, .. } => { // +1 is safe because it must be < array length. end = array.resolve_valid_index(end_inclusive, false)? + 1; start..end - }, + } }) } } diff --git a/src/transformation/destructuring.rs b/src/transformation/destructuring.rs deleted file mode 100644 index 67a2f130..00000000 --- a/src/transformation/destructuring.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::internal_prelude::*; - -pub(crate) trait HandleDestructure { - fn handle_destructure( - &self, - interpreter: &mut Interpreter, - value: ExpressionValue, - ) -> ExecutionResult<()>; -} - -#[derive(Clone)] -pub(crate) enum Pattern { - Variable(VariablePattern), - Array(ArrayPattern), - Stream(ExplicitTransformStream), - #[allow(unused)] - Discarded(Token![_]), -} - -impl Parse for Pattern { - fn parse(input: ParseStream) -> ParseResult { - let lookahead = input.lookahead1(); - if lookahead.peek(syn::Ident) { - Ok(Pattern::Variable(input.parse()?)) - } else if lookahead.peek(syn::token::Bracket) { - Ok(Pattern::Array(input.parse()?)) - } else if lookahead.peek(Token![@]) { - Ok(Pattern::Stream(input.parse()?)) - } else if lookahead.peek(Token![_]) { - Ok(Pattern::Discarded(input.parse()?)) - } else if input.peek(Token![#]) { - return input.parse_err("Use `var` instead of `#var` in a destructuring"); - } else { - Err(lookahead.error().into()) - } - } -} - -impl HandleDestructure for Pattern { - fn handle_destructure( - &self, - interpreter: &mut Interpreter, - value: ExpressionValue, - ) -> ExecutionResult<()> { - match self { - Pattern::Variable(variable) => variable.handle_destructure(interpreter, value), - Pattern::Array(array) => array.handle_destructure(interpreter, value), - Pattern::Stream(stream) => stream.handle_destructure(interpreter, value), - Pattern::Discarded(_) => Ok(()), - } - } -} - -#[derive(Clone)] -pub struct ArrayPattern { - #[allow(unused)] - brackets: Brackets, - items: Punctuated, -} - -impl Parse for ArrayPattern { - fn parse(input: ParseStream) -> ParseResult { - let (brackets, inner) = input.parse_brackets()?; - Ok(Self { - brackets, - items: inner.parse_terminated()?, - }) - } -} - -impl HandleDestructure for ArrayPattern { - fn handle_destructure( - &self, - interpreter: &mut Interpreter, - value: ExpressionValue, - ) -> ExecutionResult<()> { - let array = value.expect_array("The destructure source")?; - if array.items.len() != self.items.len() { - return array.execution_err(format!( - "The array has {} items, but the destructuring expected {}", - array.items.len(), - self.items.len() - )); - } - for (value, destructuring) in array.items.into_iter().zip(&self.items) { - destructuring.handle_destructure(interpreter, value)?; - } - Ok(()) - } -} diff --git a/src/transformation/mod.rs b/src/transformation/mod.rs index 3dd9203d..6ad5d561 100644 --- a/src/transformation/mod.rs +++ b/src/transformation/mod.rs @@ -1,18 +1,18 @@ -mod destructuring; mod exact_stream; mod fields; mod parse_utilities; +mod patterns; mod transform_stream; mod transformation_traits; mod transformer; mod transformers; mod variable_binding; -pub(crate) use destructuring::*; pub(crate) use exact_stream::*; #[allow(unused)] pub(crate) use fields::*; pub(crate) use parse_utilities::*; +pub(crate) use patterns::*; pub(crate) use transform_stream::*; pub(crate) use transformation_traits::*; pub(crate) use transformer::*; diff --git a/src/transformation/patterns.rs b/src/transformation/patterns.rs new file mode 100644 index 00000000..085193de --- /dev/null +++ b/src/transformation/patterns.rs @@ -0,0 +1,140 @@ +use crate::internal_prelude::*; + +pub(crate) trait HandleDestructure { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()>; +} + +#[derive(Clone)] +pub(crate) enum Pattern { + Variable(VariablePattern), + Array(ArrayPattern), + Stream(ExplicitTransformStream), + DotDot(Token![..]), + #[allow(unused)] + Discarded(Token![_]), +} + +impl Parse for Pattern { + fn parse(input: ParseStream) -> ParseResult { + let lookahead = input.lookahead1(); + if lookahead.peek(syn::Ident) { + Ok(Pattern::Variable(input.parse()?)) + } else if lookahead.peek(syn::token::Bracket) { + Ok(Pattern::Array(input.parse()?)) + } else if lookahead.peek(Token![@]) { + Ok(Pattern::Stream(input.parse()?)) + } else if lookahead.peek(Token![_]) { + Ok(Pattern::Discarded(input.parse()?)) + } else if lookahead.peek(Token![..]) { + Ok(Pattern::DotDot(input.parse()?)) + } else if input.peek(Token![#]) { + return input.parse_err("Use `var` instead of `#var` in a destructuring"); + } else { + Err(lookahead.error().into()) + } + } +} + +impl HandleDestructure for Pattern { + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + match self { + Pattern::Variable(variable) => variable.handle_destructure(interpreter, value), + Pattern::Array(array) => array.handle_destructure(interpreter, value), + Pattern::Stream(stream) => stream.handle_destructure(interpreter, value), + Pattern::DotDot(token) => token.execution_err("This cannot be used here"), + Pattern::Discarded(_) => Ok(()), + } + } +} + +#[derive(Clone)] +pub struct ArrayPattern { + #[allow(unused)] + brackets: Brackets, + items: Punctuated, +} + +impl Parse for ArrayPattern { + fn parse(input: ParseStream) -> ParseResult { + let (brackets, inner) = input.parse_brackets()?; + Ok(Self { + brackets, + items: inner.parse_terminated()?, + }) + } +} + +impl HandleDestructure for ArrayPattern { + /// See also `ArrayAssigneeStackFrame` in `evaluation.rs` + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + let array = value.expect_array("The value destructured with an array pattern")?; + let mut has_seen_dot_dot = false; + let mut prefix_assignees = Vec::new(); + let mut suffix_assignees = Vec::new(); + for pattern in self.items.iter() { + match pattern { + Pattern::DotDot(dot_dot) => { + if has_seen_dot_dot { + return dot_dot.execution_err("Only one .. is allowed in an array pattern"); + } + has_seen_dot_dot = true; + } + _ => { + if has_seen_dot_dot { + suffix_assignees.push(pattern); + } else { + prefix_assignees.push(pattern); + } + } + } + } + let array_length = array.items.len(); + + let assignee_pairs: Vec<_> = if has_seen_dot_dot { + let total_assignees = prefix_assignees.len() + suffix_assignees.len(); + if total_assignees > array_length { + return self.brackets.execution_err(format!( + "The array has {} items, but the pattern expected at least {}", + array_length, total_assignees, + )); + } + let discarded_count = + array.items.len() - prefix_assignees.len() - suffix_assignees.len(); + let assignees = prefix_assignees + .into_iter() + .map(Some) + .chain(std::iter::repeat(None).take(discarded_count)) + .chain(suffix_assignees.into_iter().map(Some)); + + assignees + .zip(array.items) + .filter_map(|(assignee, value)| Some((assignee?, value))) + .collect() + } else { + let total_assignees = prefix_assignees.len(); + if total_assignees != array_length { + return self.brackets.execution_err(format!( + "The array has {} items, but the pattern expected {}", + array_length, total_assignees, + )); + } + prefix_assignees.into_iter().zip(array.items).collect() + }; + for (pattern, value) in assignee_pairs { + pattern.handle_destructure(interpreter, value)?; + } + Ok(()) + } +} diff --git a/tests/expressions.rs b/tests/expressions.rs index 0413cb9b..809d33fd 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -239,7 +239,7 @@ fn test_array_indexing() { ), "[2, 0, 2]" ); - // And ranges... + // And ranges in value position preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; @@ -285,7 +285,7 @@ fn test_array_indexing() { } #[test] -fn test_array_destructurings() { +fn test_array_place_destructurings() { // And array destructuring preinterpret_assert_eq!( #( @@ -371,3 +371,43 @@ fn test_array_destructurings() { 5 ); } + +#[test] +fn test_array_pattern_destructurings() { + // And array destructuring + preinterpret_assert_eq!( + #( + let [a, b, _, _, c] = [1, 2, 3, 4, 5]; + [a, b, c] as debug + ), + "[1, 2, 5]" + ); + preinterpret_assert_eq!( + #( + let [a, b, c, ..] = [1, 2, 3, 4, 5]; + [a, b, c] as debug + ), + "[1, 2, 3]" + ); + preinterpret_assert_eq!( + #( + let [.., a, b] = [1, 2, 3, 4, 5]; + [a, b] as debug + ), + "[4, 5]" + ); + preinterpret_assert_eq!( + #( + let [a, .., b, c] = [1, 2, 3, 4, 5]; + [a, b, c] as debug + ), + "[1, 4, 5]" + ); + preinterpret_assert_eq!( + #( + let [a, .., b, c] = [[1, "a"], 2, 3, 4, 5]; + [a, b, c] as debug + ), + r#"[[1, "a"], 4, 5]"# + ); +} From 0610fcb959536dc80bd46bce73291f12686fb092 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 18 Feb 2025 14:27:49 +0000 Subject: [PATCH 105/126] feature: += for arrays and streams is more efficient --- CHANGELOG.md | 1 - src/expressions/evaluation.rs | 7 ++--- src/expressions/value.rs | 29 +++++++++++++++++++ src/interpretation/interpreter.rs | 8 ----- ...lace_destructure_element_mismatch_1.stderr | 2 +- ...lace_destructure_element_mismatch_2.stderr | 2 +- ...lace_destructure_element_mismatch_3.stderr | 2 +- 7 files changed, 35 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cad5f30e..5c826953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,7 +96,6 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* More efficient `+=` etc * Objects, like a JS object: * Backed by an indexmap (or maybe an immutable `IndexMap` wrapping an `im::HashMap` and `im::Vec` or entry orderings) * Can be created with `#({ a: x, ... })` diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 081e5a2f..db8a438d 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -825,10 +825,9 @@ impl PlaceStackFrame { Place::MutableReference(mut variable) => { let span_range = SpanRange::new_between(variable.span_range(), value.span_range()); - // TODO - replace with handling a compound operation for better performance - // of e.g. arrays or streams - let left = variable.get_value_cloned(); - variable.set(operation.to_binary().evaluate(left, value)?); + variable + .value_mut() + .handle_compound_assignment(&operation, value, span_range)?; span_range } Place::Discarded(token) => token.span_range(), diff --git a/src/expressions/value.rs b/src/expressions/value.rs index e25eec9d..b26c6322 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -347,6 +347,35 @@ impl ExpressionValue { } } + pub(super) fn handle_compound_assignment( + &mut self, + operation: &CompoundAssignmentOperation, + right: Self, + source_span_range: SpanRange, + ) -> ExecutionResult<()> { + match (self, operation) { + (ExpressionValue::Stream(left_mut), CompoundAssignmentOperation::Add(_)) => { + let right = right.expect_stream("The target of += on a stream")?; + right.value.append_into(&mut left_mut.value); + left_mut.span_range = source_span_range; + } + (ExpressionValue::Array(left_mut), CompoundAssignmentOperation::Add(_)) => { + let mut right = right.expect_array("The target of += on an array")?; + left_mut.items.append(&mut right.items); + left_mut.span_range = source_span_range; + } + (left_mut, operation) => { + // Fallback to just clone and use the normal operator + let left = left_mut.clone(); + *left_mut = operation + .to_binary() + .evaluate(left, right)? + .with_span_range(source_span_range); + } + } + Ok(()) + } + pub(super) fn handle_integer_binary_operation( self, right: ExpressionInteger, diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index c40b0fc7..e94a4a69 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -188,14 +188,6 @@ impl MutableReference { }) } - // Gets the cloned expression value, setting the span range appropriately - pub(crate) fn get_value_cloned(&self) -> ExpressionValue { - self.mut_cell - .deref() - .clone() - .with_span_range(self.span_range) - } - pub(crate) fn into_stream(self) -> ExecutionResult> { let stream = self.mut_cell.try_map(|value| match value { ExpressionValue::Stream(stream) => Ok(&mut stream.value), diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr index c9a2eb3f..788fe926 100644 --- a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.stderr @@ -1,4 +1,4 @@ -error: The number of assignees does not equal the number of items in the array +error: The array has 2 items, but the assignee expected 3 --> tests/compilation_failures/expressions/array_place_destructure_element_mismatch_1.rs:5:11 | 5 | #([_, _, _] = [1, 2]) diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr index 12665bb2..e472871e 100644 --- a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.stderr @@ -1,4 +1,4 @@ -error: The number of assignees does not equal the number of items in the array +error: The array has 2 items, but the assignee expected 1 --> tests/compilation_failures/expressions/array_place_destructure_element_mismatch_2.rs:5:11 | 5 | #([_] = [1, 2]) diff --git a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr index 6afc8e15..5e279625 100644 --- a/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr +++ b/tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.stderr @@ -1,4 +1,4 @@ -error: The number of assignees exceeds the number of items in the array +error: The array has 2 items, but the assignee expected at least 3 --> tests/compilation_failures/expressions/array_place_destructure_element_mismatch_3.rs:5:11 | 5 | #([_, _, .., _] = [1, 2]) From 18b45643028a0653487ce25549f3ab50cc81d687 Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 18 Feb 2025 16:38:07 +0000 Subject: [PATCH 106/126] refactor: Remove `Deref` impl from `ParseStream` --- CHANGELOG.md | 27 ++++++++++----------- src/misc/parse_traits.rs | 52 +++++++++++++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c826953..afae9a13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,19 +106,16 @@ Inside a transform stream, the following grammar is supported: * Can be destructured with `{ hello, world: _, ... }` * Throws if output to a stream * Have `!zip!` support `{ objects }` -* Method calls - * Mutable methods notes: - * They require either: - * Reference semantics (e.g. using `Rc>` inside Object, Array) with explicit cloning - * AND/OR Place semantics (e.g. each type supports being either a value or a reference to a path, so that an operator e.g. `+` can mutate) - * I think we want reference semantics for object and array anyway. Unclear for stream. - * Ideally we'd arrange it so that `x += ["Hello"] + ["World]` would append Hello and World; but `x += (["Hello"] + ["World])` would behave differently. - * I think that means that `+=` becomes an operator inside an expression, and its LHS is a `PlaceValue` (`PlaceValue::Stream` or `PlaceValue::Array`) +* Method calls on values + * Mutable params notes: + * For now we can assume all params are values/clones, no mutable references * Also add support for methods (for e.g. exposing functions on syn objects). * `.len()` on stream * `.push(x)` on array - * Consider `.map(|| {})` + * Consider `.map(|| {})` * Introduce interpreter stack frames + * Design the data model => is it some kind of linked list of frames? + * Add a new frame inside loops * Merge `GroupedVariable` and `ExpressionBlock` into an `ExplicitExpression`: * Either `#ident` or `#(...)` or `#{ ... }`... the latter defines a new variable stack frame, just like Rust @@ -137,8 +134,7 @@ Inside a transform stream, the following grammar is supported: * `@(..)*` returns an array of some output of it's inner thing * But things don't output so what does that mean?? * I think in practice it will be quite an annoying restriction anyway - * OLD: Support `@[x = ...]` and `@[let x = ...]` for individual parsers. - * CHANGE OF THOUGHT: instead, support `#(x = @IDENT)` where `#` blocks inside parsers can embed @parsers and consume from a parse stream. + * Support `#(x = @IDENT)` where `#` blocks inside parsers can embed @parsers and consume from a parse stream. * This makes it kinda like a `Parse` implementation code block. * This lets us just support variable definitions in expression statements. * Don't support `@(x = ...)` - instead we can have `#(x = @[STREAM ...])` @@ -260,9 +256,12 @@ Inside a transform stream, the following grammar is supported: // * There is still an issue with "what happens to mutated state when a repetition is not possible?" // ... this is also true in a case statement... We need some way to rollback in these cases: // => Easier - Use of efficient-ish immutable data structures, e.g. ImmutableList, Copy-on-write leaf types etc -// ... so that we can clone them cheaply (e.g. https://crates.io/crates/im-rc) -// => Middle (best?) - Any changes to variables outside the scope of a given refutable parser => it's a fatal error -// to parse further until that scope is closed. (this needs to handle nested repetitions) +// ... so that we can clone them cheaply (e.g. https://crates.io/crates/im-rc) or even just +// some manually written tries/cons-lists +// => CoW - We do some kind of copy-on-write - but it still give O(N^2) if we're reverting occasional array.push(..)es +// [BEST]? => Middle - Any changes to variables outside the scope of a given refutable parser => it's a fatal error +// to parse further until that scope is closed. (this needs to handle nested repetitions). +// [EASIEST?] OR conversely - at reversion time, we check there have been no changes to variables below that depth in the stack tree (say by recording a "lowest_stack_touched: usize"), and panic if so; and tell people to use `return` instead; or move state changes to the end. // => Hardest - Some kind of storing of diffs / layered stores without any O(N^2) behaviours // // EXAMPLE (Updated, 17th February) diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 970884c3..409bb39f 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -1,5 +1,3 @@ -use std::ops::Deref; - use crate::internal_prelude::*; // Parsing of source code tokens @@ -259,7 +257,7 @@ impl<'a, K> ParseBuffer<'a, K> { } pub(crate) fn parse_ident_matching(&self, content: &str) -> ParseResult { - Ok(self.step(|cursor| { + Ok(self.inner.step(|cursor| { cursor .ident_matching(content) .ok_or_else(|| cursor.span().error(format!("expected {}", content))) @@ -271,7 +269,7 @@ impl<'a, K> ParseBuffer<'a, K> { } pub(crate) fn parse_punct_matching(&self, punct: char) -> ParseResult { - Ok(self.step(|cursor| { + Ok(self.inner.step(|cursor| { cursor .punct_matching(punct) .ok_or_else(|| cursor.span().error(format!("expected {}", punct))) @@ -283,7 +281,7 @@ impl<'a, K> ParseBuffer<'a, K> { } pub(crate) fn parse_literal_matching(&self, content: &str) -> ParseResult { - Ok(self.step(|cursor| { + Ok(self.inner.step(|cursor| { cursor .literal_matching(content) .ok_or_else(|| cursor.span().error(format!("expected {}", content))) @@ -292,7 +290,7 @@ impl<'a, K> ParseBuffer<'a, K> { pub(crate) fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)> { use syn::parse::discouraged::AnyDelimiter; - let (delimiter, delim_span, parse_buffer) = self.parse_any_delimiter()?; + let (delimiter, delim_span, parse_buffer) = self.inner.parse_any_delimiter()?; Ok((delimiter, delim_span, parse_buffer.into())) } @@ -347,6 +345,13 @@ impl<'a, K> ParseBuffer<'a, K> { Ok((TransparentDelimiters { delim_span }, inner)) } + pub(crate) fn advance_to(&self, fork: &Self) { + self.inner.advance_to(&fork.inner) + } + + // ERRORS + // ====== + pub(crate) fn parse_err(&self, message: impl std::fmt::Display) -> ParseResult { Err(self.parse_error(message)) } @@ -354,12 +359,37 @@ impl<'a, K> ParseBuffer<'a, K> { pub(crate) fn parse_error(&self, message: impl std::fmt::Display) -> ParseError { self.span().parse_error(message) } -} -impl<'a, K> Deref for ParseBuffer<'a, K> { - type Target = SynParseBuffer<'a>; + // Pass-throughs to SynParseBuffer + // =============================== + + pub(crate) fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub(crate) fn cursor(&self) -> syn::buffer::Cursor<'a> { + self.inner.cursor() + } + + pub(crate) fn span(&self) -> Span { + self.inner.span() + } + + pub(crate) fn peek(&self, token: impl syn::parse::Peek) -> bool { + self.inner.peek(token) + } + + pub(crate) fn peek2(&self, token: impl syn::parse::Peek) -> bool { + self.inner.peek2(token) + } + + #[allow(unused)] + pub(crate) fn peek3(&self, token: impl syn::parse::Peek) -> bool { + self.inner.peek3(token) + } - fn deref(&self) -> &Self::Target { - &self.inner + /// End with `Err(lookahead.error())?` + pub(crate) fn lookahead1(&self) -> syn::parse::Lookahead1<'a> { + self.inner.lookahead1() } } From 4cfbd83ffc9a8556b474254804f86caa93417a6c Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 19 Feb 2025 01:29:26 +0000 Subject: [PATCH 107/126] feature: Basic object support --- CHANGELOG.md | 20 +- src/expressions/evaluation.rs | 129 ++++++++++-- src/expressions/expression.rs | 53 +++-- src/expressions/expression_parsing.rs | 190 ++++++++++++++---- src/expressions/mod.rs | 2 + src/expressions/object.rs | 174 ++++++++++++++++ src/expressions/operations.rs | 4 +- src/expressions/value.rs | 51 ++++- src/internal_prelude.rs | 1 + src/interpretation/interpreter.rs | 9 + src/interpretation/variable.rs | 2 +- .../expressions/braces.rs | 7 - .../expressions/braces.stderr | 5 - .../expressions/inner_braces.rs | 7 - .../expressions/inner_braces.stderr | 5 - tests/expressions.rs | 33 +++ 16 files changed, 585 insertions(+), 107 deletions(-) create mode 100644 src/expressions/object.rs delete mode 100644 tests/compilation_failures/expressions/braces.rs delete mode 100644 tests/compilation_failures/expressions/braces.stderr delete mode 100644 tests/compilation_failures/expressions/inner_braces.rs delete mode 100644 tests/compilation_failures/expressions/inner_braces.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index afae9a13..f2a2d89d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,15 +96,13 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* Objects, like a JS object: - * Backed by an indexmap (or maybe an immutable `IndexMap` wrapping an `im::HashMap` and `im::Vec` or entry orderings) - * Can be created with `#({ a: x, ... })` - * Can be read with `#(x.hello)` or `#(x["hello"])` - * Debug impl is `#({ hello: [!group! BLAH], ["#world"]: Hi, })` - * Fields can be read/written to with `#(x.hello)` or `#(x.hello.world)` +* Objects continuation: + * Can be place-destructured and pattern-destructured with `{ hello, world: _ }` (note - like JS, fields don't need to be complete!) + * Add error tests: + * Throws if object is output to a stream + * Throws if field names re-used + * Invalid object syntax * Commands have an input object - * Can be destructured with `{ hello, world: _, ... }` - * Throws if output to a stream * Have `!zip!` support `{ objects }` * Method calls on values * Mutable params notes: @@ -172,8 +170,10 @@ Inside a transform stream, the following grammar is supported: * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) * `#(x[0..3])` returns a TokenStream * `#(x[0..=3])` returns a TokenStream +* Add `LiteralPattern` (wrapping a `Literal`) +* Add `Eq` support on composite types and streams * Consider: - * Moving control flow (`for` and `while`) to the expression side? + * Moving control flow (`for` and `while`) to the expression side? Possibly with an `output` auto-variable with `output += [!stream! ...]` * Dropping lots of the `group` wrappers? * If any types should have reference semantics instead of clone/value semantics? * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty @@ -256,7 +256,7 @@ Inside a transform stream, the following grammar is supported: // * There is still an issue with "what happens to mutated state when a repetition is not possible?" // ... this is also true in a case statement... We need some way to rollback in these cases: // => Easier - Use of efficient-ish immutable data structures, e.g. ImmutableList, Copy-on-write leaf types etc -// ... so that we can clone them cheaply (e.g. https://crates.io/crates/im-rc) or even just +// ... so that we can clone them cheaply (e.g. https://github.com/orium/rpds) or even just // some manually written tries/cons-lists // => CoW - We do some kind of copy-on-write - but it still give O(N^2) if we're reverting occasional array.push(..)es // [BEST]? => Middle - Any changes to variables outside the scope of a given refutable parser => it's a fatal error diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index db8a438d..0f189b3a 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -250,12 +250,19 @@ impl ExpressionNode { span: delim_span.join(), }, ), - ExpressionNode::Array { delim_span, items } => ArrayValueStackFrame { - span: delim_span.join(), + ExpressionNode::Array { brackets, items } => ArrayValueStackFrame { + span: brackets.join(), unevaluated_items: items.clone(), evaluated_items: Vec::with_capacity(items.len()), } .next(next), + ExpressionNode::Object { braces, entries } => ObjectValueStackFrame { + span: braces.join(), + unevaluated_entries: entries.clone(), + evaluated_entries: BTreeMap::new(), + pending: None, + } + .next(next)?, ExpressionNode::UnaryOperation { operation, input } => next.read_value_with_handler( *input, ValueStackFrame::UnaryOperation { @@ -364,15 +371,12 @@ impl ExpressionNode { next.read_place_with_handler(self_node_id, PlaceStackFrame::Assignment { value }) } ExpressionNode::Array { - delim_span, + brackets, items: assignee_item_node_ids, - } => ArrayAssigneeStackFrame::new( - nodes, - delim_span.join(), - assignee_item_node_ids, - value, - )? - .handle_next(next), + } => { + ArrayAssigneeStackFrame::new(nodes, brackets.join(), assignee_item_node_ids, value)? + .handle_next(next) + } ExpressionNode::Grouped { inner, .. } => { next.handle_assignment_and_return_to(*inner, value, AssignmentStackFrame::Grouped) } @@ -407,9 +411,12 @@ impl ExpressionNode { index: *index, }, ), - ExpressionNode::Property { access, .. } => { - return access.execution_err("TODO: Not yet supported!") - } + ExpressionNode::Property { node, access, .. } => next.read_place_with_handler( + *node, + PlaceStackFrame::PropertyAccess { + access: access.clone(), + }, + ), ExpressionNode::Grouped { inner, .. } => { next.read_place_with_handler(*inner, PlaceStackFrame::Grouped) } @@ -428,6 +435,7 @@ enum ValueStackFrame { span: Span, }, Array(ArrayValueStackFrame), + Object(ObjectValueStackFrame), UnaryOperation { operation: UnaryOperation, }, @@ -465,6 +473,7 @@ impl ValueStackFrame { match self { ValueStackFrame::Group { .. } => ReturnMode::Value, ValueStackFrame::Array(_) => ReturnMode::Value, + ValueStackFrame::Object(_) => ReturnMode::Value, ValueStackFrame::UnaryOperation { .. } => ReturnMode::Value, ValueStackFrame::BinaryOperation { .. } => ReturnMode::Value, ValueStackFrame::Property { .. } => ReturnMode::Value, @@ -490,6 +499,7 @@ impl ValueStackFrame { array.evaluated_items.push(value); array.next(next) } + ValueStackFrame::Object(object) => object.handle_value(value, next)?, ValueStackFrame::BinaryOperation { operation, state } => match state { BinaryPath::OnLeftBranch { right } => { if let Some(result) = operation.lazy_evaluate(&value)? { @@ -508,9 +518,7 @@ impl ValueStackFrame { next.return_value(operation.evaluate(left, value)?) } }, - ValueStackFrame::Property { access } => { - next.return_value(value.handle_property_access(access)?) - } + ValueStackFrame::Property { access } => next.return_value(value.into_property(access)?), ValueStackFrame::Index { access, state } => match state { IndexPath::OnObjectBranch { index } => next.read_value_with_handler( index, @@ -635,6 +643,82 @@ impl ArrayValueStackFrame { } } +struct ObjectValueStackFrame { + span: Span, + pending: Option, + unevaluated_entries: Vec<(ObjectKey, ExpressionNodeId)>, + evaluated_entries: BTreeMap, +} + +impl ObjectValueStackFrame { + fn handle_value( + mut self, + value: ExpressionValue, + next: ActionCreator, + ) -> ExecutionResult { + let pending = self.pending.take(); + Ok(match pending { + Some(PendingEntryPath::OnIndexKeyBranch { + brackets, + value_node, + }) => { + let key = value.expect_string("An object key")?.value; + if self.evaluated_entries.contains_key(&key) { + return brackets.execution_err(format!("The key {} has already been set", key)); + } + self.pending = Some(PendingEntryPath::OnValueBranch { key }); + next.read_value_with_handler(value_node, ValueStackFrame::Object(self)) + } + Some(PendingEntryPath::OnValueBranch { key }) => { + self.evaluated_entries.insert(key, value); + self.next(next)? + } + None => { + unreachable!("Should not receive a value without a pending handler set") + } + }) + } + + fn next(mut self, action_creator: ActionCreator) -> ExecutionResult { + Ok( + match self + .unevaluated_entries + .get(self.evaluated_entries.len()) + .cloned() + { + Some((ObjectKey::Identifier(ident), node)) => { + let key = ident.to_string(); + if self.evaluated_entries.contains_key(&key) { + return ident + .execution_err(format!("The key {} has already been set", key)); + } + self.pending = Some(PendingEntryPath::OnValueBranch { key }); + action_creator.read_value_with_handler(node, ValueStackFrame::Object(self)) + } + Some((ObjectKey::Indexed { brackets, index }, value_node)) => { + self.pending = Some(PendingEntryPath::OnIndexKeyBranch { + brackets, + value_node, + }); + action_creator.read_value_with_handler(index, ValueStackFrame::Object(self)) + } + None => action_creator + .return_value(self.evaluated_entries.to_value(self.span.span_range())), + }, + ) + } +} + +enum PendingEntryPath { + OnIndexKeyBranch { + brackets: Brackets, + value_node: ExpressionNodeId, + }, + OnValueBranch { + key: String, + }, +} + enum BinaryPath { OnLeftBranch { right: ExpressionNodeId }, OnRightBranch { left: ExpressionValue }, @@ -790,6 +874,9 @@ enum PlaceStackFrame { value: ExpressionValue, }, Grouped, + PropertyAccess { + access: PropertyAccess, + }, Indexed { access: IndexAccess, index: ExpressionNodeId, @@ -802,6 +889,7 @@ impl PlaceStackFrame { PlaceStackFrame::Assignment { .. } => ReturnMode::AssignmentCompletion, PlaceStackFrame::CompoundAssignmentRoot { .. } => ReturnMode::Value, PlaceStackFrame::Grouped { .. } => ReturnMode::Place, + PlaceStackFrame::PropertyAccess { .. } => ReturnMode::Place, PlaceStackFrame::Indexed { .. } => ReturnMode::Place, } } @@ -834,6 +922,15 @@ impl PlaceStackFrame { }; next.return_value(ExpressionValue::None(span_range)) } + PlaceStackFrame::PropertyAccess { access } => match place { + Place::MutableReference(reference) => { + next.return_place(Place::MutableReference(reference.resolve_property(access)?)) + } + Place::Discarded(underscore) => { + return underscore + .execution_err("Cannot access the property of a discarded value") + } + }, PlaceStackFrame::Grouped => next.return_place(place), PlaceStackFrame::Indexed { access, index } => next.read_value_with_handler( index, diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 5a78f762..75682ee0 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -79,17 +79,15 @@ impl Expressionable for Source { UnaryAtom::Group(delim_span) } SourcePeekMatch::Group(Delimiter::Brace) => { - return input.parse_err("Braces { ... } are not supported in an expression") + let (_, delim_span) = input.parse_and_enter_group()?; + UnaryAtom::Object(Braces { delim_span }) } SourcePeekMatch::Group(Delimiter::Bracket) => { // This could be handled as parsing a vector of SourceExpressions, // but it's more efficient to handle nested vectors as a single expression // in the expression parser let (_, delim_span) = input.parse_and_enter_group()?; - UnaryAtom::Array { - delim_span, - is_empty: input.is_current_empty(), - } + UnaryAtom::Array(Brackets { delim_span }) } SourcePeekMatch::Punct(punct) => { if punct.as_char() == '.' { @@ -139,6 +137,17 @@ impl Expressionable for Source { return Ok(NodeExtension::NonTerminalArrayComma); } } + ExpressionStackFrame::Object { + state: ObjectStackFrameState::EntryValue { .. }, + .. + } => { + input.parse::()?; + if input.is_current_empty() { + return Ok(NodeExtension::EndOfStream); + } else { + return Ok(NodeExtension::NonTerminalObjectValueComma); + } + } ExpressionStackFrame::Group { .. } => { return input.parse_err("Commas are only permitted inside preinterpret arrays []. Preinterpret arrays [a, b] can be used as a drop-in replacement for rust tuples (a, b).") } @@ -179,9 +188,16 @@ impl Expressionable for Source { match parent_stack_frame { ExpressionStackFrame::Root => Ok(NodeExtension::NoValidExtensionForCurrentParent), ExpressionStackFrame::Group { .. } => input.parse_err("Expected ) or operator"), - ExpressionStackFrame::Array { .. } | ExpressionStackFrame::IncompleteIndex { .. } => { - input.parse_err("Expected comma, ], or operator") - } + ExpressionStackFrame::Array { .. } => input.parse_err("Expected comma, ], or operator"), + ExpressionStackFrame::IncompleteIndex { .. } + | ExpressionStackFrame::Object { + state: ObjectStackFrameState::EntryIndex { .. }, + .. + } => input.parse_err("Expected ], or operator"), + ExpressionStackFrame::Object { + state: ObjectStackFrameState::EntryValue { .. }, + .. + } => input.parse_err("Expected comma, }, or operator"), // e.g. I've just matched the true in !true or false || true, // and I want to see if there's an extension (e.g. a cast). // There's nothing matching, so we fall through to an EndOfFrame @@ -217,6 +233,12 @@ impl Expressionable for Source { } } +impl Parse for Expression { + fn parse(input: ParseStream) -> ParseResult { + ExpressionParser::parse(input) + } +} + // Generic // ======= @@ -234,12 +256,6 @@ impl Clone for Expression { } } -impl Parse for Expression { - fn parse(input: ParseStream) -> ParseResult { - ExpressionParser::parse(input) - } -} - #[derive(Clone, Copy, PartialEq, Eq)] pub(super) struct ExpressionNodeId(pub(super) usize); @@ -250,9 +266,13 @@ pub(super) enum ExpressionNode { inner: ExpressionNodeId, }, Array { - delim_span: DelimSpan, + brackets: Brackets, items: Vec, }, + Object { + braces: Braces, + entries: Vec<(ObjectKey, ExpressionNodeId)>, + }, UnaryOperation { operation: UnaryOperation, input: ExpressionNodeId, @@ -293,7 +313,8 @@ impl ExpressionNode { match self { ExpressionNode::Leaf(leaf) => leaf.span_range(), ExpressionNode::Grouped { delim_span, .. } => delim_span.span_range(), - ExpressionNode::Array { delim_span, .. } => delim_span.span_range(), + ExpressionNode::Array { brackets, .. } => brackets.span_range(), + ExpressionNode::Object { braces, .. } => braces.span_range(), ExpressionNode::Property { access, .. } => access.span_range(), ExpressionNode::Index { access, .. } => access.span_range(), ExpressionNode::UnaryOperation { operation, .. } => operation.span_range(), diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 6016a0c9..0359eb4a 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -1,3 +1,5 @@ +use syn::token; + use super::*; /// ## Overview @@ -26,8 +28,8 @@ pub(super) struct ExpressionParser<'a, K: Expressionable> { kind: PhantomData, } -impl<'a, K: Expressionable> ExpressionParser<'a, K> { - pub(super) fn parse(input: ParseStream<'a, K>) -> ParseResult> { +impl<'a> ExpressionParser<'a, Source> { + pub(super) fn parse(input: ParseStream<'a, Source>) -> ParseResult> { Self { streams: ParseStreamStack::new(input), nodes: ExpressionNodes::new(), @@ -37,16 +39,16 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { .run() } - fn run(mut self) -> ParseResult> { + fn run(mut self) -> ParseResult> { let mut work_item = self.push_stack_frame(ExpressionStackFrame::Root); loop { work_item = match work_item { WorkItem::RequireUnaryAtom => { - let unary_atom = K::parse_unary_atom(&mut self.streams)?; + let unary_atom = Source::parse_unary_atom(&mut self.streams)?; self.extend_with_unary_atom(unary_atom)? } WorkItem::TryParseAndApplyExtension { node } => { - let extension = K::parse_extension( + let extension = Source::parse_extension( &mut self.streams, self.expression_stack.last().unwrap(), )?; @@ -65,31 +67,29 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } } - fn extend_with_unary_atom(&mut self, unary_atom: UnaryAtom) -> ParseResult { + fn extend_with_unary_atom(&mut self, unary_atom: UnaryAtom) -> ParseResult { Ok(match unary_atom { UnaryAtom::Leaf(leaf) => self.add_leaf(leaf), UnaryAtom::Group(delim_span) => { self.push_stack_frame(ExpressionStackFrame::Group { delim_span }) } - UnaryAtom::Array { - delim_span, - is_empty: true, - } => { - self.streams.exit_group(); - WorkItem::TryParseAndApplyExtension { - node: self.nodes.add_node(ExpressionNode::Array { - delim_span, + UnaryAtom::Array(brackets) => { + if self.streams.is_current_empty() { + self.streams.exit_group(); + WorkItem::TryParseAndApplyExtension { + node: self.nodes.add_node(ExpressionNode::Array { + brackets, + items: Vec::new(), + }), + } + } else { + self.push_stack_frame(ExpressionStackFrame::Array { + brackets, items: Vec::new(), - }), + }) } } - UnaryAtom::Array { - delim_span, - is_empty: false, - } => self.push_stack_frame(ExpressionStackFrame::Array { - delim_span, - items: Vec::new(), - }), + UnaryAtom::Object(braces) => self.continue_object(braces, Vec::new())?, UnaryAtom::PrefixUnaryOperation(operation) => { self.push_stack_frame(ExpressionStackFrame::IncompleteUnaryPrefixOperation { operation, @@ -130,10 +130,25 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { ExpressionStackFrame::Array { items, .. } => { items.push(node); } - _ => unreachable!("CommaOperator is only returned under an Array parent."), + _ => unreachable!( + "NonTerminalArrayComma is only returned under an Array parent." + ), } WorkItem::RequireUnaryAtom } + NodeExtension::NonTerminalObjectValueComma => { + match self.expression_stack.pop().unwrap() { + ExpressionStackFrame::Object { + braces, + state: ObjectStackFrameState::EntryValue(key, _), + mut complete_entries, + } => { + complete_entries.push((key, node)); + self.continue_object(braces, complete_entries)? + } + _ => unreachable!("NonTerminalObjectValueComma is only returned under an Object EntryValue parent."), + } + } NodeExtension::Property(access) => WorkItem::TryParseAndApplyExtension { node: self .nodes @@ -187,7 +202,7 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } ExpressionStackFrame::Array { mut items, - delim_span, + brackets, } => { assert!(matches!(extension, NodeExtension::EndOfStream)); items.push(node); @@ -195,9 +210,43 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { WorkItem::TryParseAndApplyExtension { node: self .nodes - .add_node(ExpressionNode::Array { delim_span, items }), + .add_node(ExpressionNode::Array { brackets, items }), } } + ExpressionStackFrame::Object { + braces, + complete_entries, + state: ObjectStackFrameState::EntryIndex(brackets), + } => { + assert!(matches!(extension, NodeExtension::EndOfStream)); + self.streams.exit_group(); + let colon = self.streams.parse()?; + self.expression_stack.push(ExpressionStackFrame::Object { + braces, + complete_entries, + state: ObjectStackFrameState::EntryValue( + ObjectKey::Indexed { + brackets, + index: node, + }, + colon, + ), + }); + WorkItem::RequireUnaryAtom + } + ExpressionStackFrame::Object { + braces, + complete_entries: mut entries, + state: ObjectStackFrameState::EntryValue(key, _), + } => { + assert!(matches!(extension, NodeExtension::EndOfStream)); + self.streams.exit_group(); + entries.push((key, node)); + let node = self + .nodes + .add_node(ExpressionNode::Object { braces, entries }); + WorkItem::TryParseAndApplyExtension { node } + } ExpressionStackFrame::IncompleteUnaryPrefixOperation { operation } => { let node = self.nodes.add_node(ExpressionNode::UnaryOperation { operation: operation.into(), @@ -284,7 +333,7 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { let can_parse_unary_atom = { let forked = self.streams.fork_current(); let mut forked_stack = ParseStreamStack::new(&forked); - K::parse_unary_atom(&mut forked_stack).is_ok() + Source::parse_unary_atom(&mut forked_stack).is_ok() }; if can_parse_unary_atom { // A unary atom can be parsed so let's attempt to complete the range with it @@ -300,7 +349,56 @@ impl<'a, K: Expressionable> ExpressionParser<'a, K> { } } - fn add_leaf(&mut self, leaf: K::Leaf) -> WorkItem { + fn continue_object( + &mut self, + braces: Braces, + mut complete_entries: Vec<(ObjectKey, ExpressionNodeId)>, + ) -> ParseResult { + let state = loop { + if self.streams.is_current_empty() { + self.streams.exit_group(); + let node = self.nodes.add_node(ExpressionNode::Object { + braces, + entries: complete_entries, + }); + return Ok(WorkItem::TryParseAndApplyExtension { node }); + } else if self.streams.peek(syn::Ident) { + let key: Ident = self.streams.parse()?; + + if self.streams.is_current_empty() { + // Fall through + } else if self.streams.peek(token::Comma) { + self.streams.parse::()?; + // Fall through + } else { + let colon = self.streams.parse()?; + break ObjectStackFrameState::EntryValue(ObjectKey::Identifier(key), colon); + } + + let node = + self.nodes + .add_node(ExpressionNode::Leaf(SourceExpressionLeaf::Variable( + VariableIdentifier { ident: key.clone() }, + ))); + complete_entries.push((ObjectKey::Identifier(key), node)); + continue; + } else if self.streams.peek(token::Bracket) { + let (_, delim_span) = self.streams.parse_and_enter_group()?; + break ObjectStackFrameState::EntryIndex(Brackets { delim_span }); + } else { + return self + .streams + .parse_err("Expected an identifier or indexed key"); + } + }; + Ok(self.push_stack_frame(ExpressionStackFrame::Object { + braces, + complete_entries, + state, + })) + } + + fn add_leaf(&mut self, leaf: SourceExpressionLeaf) -> WorkItem { let node = self.nodes.add_node(ExpressionNode::Leaf(leaf)); WorkItem::TryParseAndApplyExtension { node } } @@ -562,9 +660,17 @@ pub(super) enum ExpressionStackFrame { /// * When the array is opened, we add its inside to the parse stream stack /// * When the array is closed, we pop it from the parse stream stack Array { - delim_span: DelimSpan, + brackets: Brackets, items: Vec, }, + /// A marker for an object literal. + /// * When the object is opened, we add its inside to the parse stream stack + /// * When the object is closed, we pop it from the parse stream stack + Object { + braces: Braces, + complete_entries: Vec<(ObjectKey, ExpressionNodeId)>, + state: ObjectStackFrameState, + }, /// An incomplete unary prefix operation /// NB: unary postfix operations such as `as` casting go straight to ExtendableNode IncompleteUnaryPrefixOperation { operation: PrefixUnaryOperation }, @@ -601,12 +707,28 @@ pub(super) enum ExpressionStackFrame { }, } +#[derive(Clone)] +pub(super) enum ObjectKey { + Identifier(Ident), + Indexed { + brackets: Brackets, + index: ExpressionNodeId, + }, +} + +pub(super) enum ObjectStackFrameState { + EntryIndex(Brackets), + #[allow(unused)] + EntryValue(ObjectKey, Token![:]), +} + impl ExpressionStackFrame { fn precedence_to_bind_to_child(&self) -> OperatorPrecendence { match self { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::Object { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteIndex { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteRange { .. } => OperatorPrecendence::Range, ExpressionStackFrame::IncompleteAssignment { .. } => OperatorPrecendence::Assign, @@ -656,10 +778,8 @@ enum WorkItem { pub(super) enum UnaryAtom { Leaf(K::Leaf), Group(DelimSpan), - Array { - delim_span: DelimSpan, - is_empty: bool, - }, + Array(Brackets), + Object(Braces), PrefixUnaryOperation(PrefixUnaryOperation), Range(syn::RangeLimits), } @@ -668,6 +788,7 @@ pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), NonTerminalArrayComma, + NonTerminalObjectValueComma, Property(PropertyAccess), Index(IndexAccess), Range(syn::RangeLimits), @@ -683,6 +804,7 @@ impl NodeExtension { NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), NodeExtension::NonTerminalArrayComma => OperatorPrecendence::NonTerminalComma, + NodeExtension::NonTerminalObjectValueComma => OperatorPrecendence::NonTerminalComma, NodeExtension::Property { .. } => OperatorPrecendence::Unambiguous, NodeExtension::Index { .. } => OperatorPrecendence::Unambiguous, NodeExtension::Range(_) => OperatorPrecendence::Range, @@ -707,8 +829,8 @@ impl NodeExtension { | NodeExtension::EndOfStream) => { WorkItem::TryApplyAlreadyParsedExtension { node, extension } } - NodeExtension::NonTerminalArrayComma => { - unreachable!("Array comma is only possible on array parent") + NodeExtension::NonTerminalArrayComma | NodeExtension::NonTerminalObjectValueComma => { + unreachable!("Commas is only possible on array or object parent") } NodeExtension::NoValidExtensionForCurrentParent => { // We have to reparse in case the extension is valid for the new parent. diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index d86a1dde..b3a1a0d2 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -8,6 +8,7 @@ mod expression_parsing; mod float; mod integer; mod iterator; +mod object; mod operations; mod range; mod stream; @@ -17,6 +18,7 @@ mod value; pub(crate) use expression::*; pub(crate) use expression_block::*; pub(crate) use iterator::*; +pub(crate) use object::*; pub(crate) use operations::*; pub(crate) use stream::*; pub(crate) use value::*; diff --git a/src/expressions/object.rs b/src/expressions/object.rs new file mode 100644 index 00000000..d859f009 --- /dev/null +++ b/src/expressions/object.rs @@ -0,0 +1,174 @@ +use super::*; + +#[derive(Clone)] +pub(crate) struct ExpressionObject { + // TODO: Convert to an IndexMap + pub(crate) entries: BTreeMap, + /// The span range that generated this value. + /// For a complex expression, the start span is the most left part + /// of the expression, and the end span is the most right part. + pub(crate) span_range: SpanRange, +} + +impl ExpressionObject { + pub(super) fn handle_unary_operation( + self, + operation: OutputSpanned, + ) -> ExecutionResult { + Ok(match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { + return operation.unsupported(self) + } + UnaryOperation::Cast { target, .. } => match target { + CastTarget::DebugString => { + operation.output(self.entries).into_debug_string_value()? + } + CastTarget::String + | CastTarget::Stream + | CastTarget::Group + | CastTarget::Boolean + | CastTarget::Char + | CastTarget::Integer(_) + | CastTarget::Float(_) => { + return operation.unsupported(self); + } + }, + }) + } + + pub(super) fn handle_integer_binary_operation( + self, + _right: ExpressionInteger, + operation: OutputSpanned, + ) -> ExecutionResult { + operation.unsupported(self) + } + + pub(super) fn handle_paired_binary_operation( + self, + _rhs: Self, + operation: OutputSpanned, + ) -> ExecutionResult { + operation.unsupported(self) + } + + pub(super) fn into_indexed( + self, + access: IndexAccess, + index: ExpressionValue, + ) -> ExecutionResult { + let span_range = SpanRange::new_between(self.span_range, access.span_range()); + let key = index.expect_string("An object key")?.value; + Ok(self.into_entry_or_none(&key, span_range)) + } + + pub(super) fn into_property(self, access: PropertyAccess) -> ExecutionResult { + let span_range = SpanRange::new_between(self.span_range, access.span_range()); + let key = access.property.to_string(); + Ok(self.into_entry_or_none(&key, span_range)) + } + + fn into_entry_or_none(mut self, key: &str, span_range: SpanRange) -> ExpressionValue { + match self.entries.remove(key) { + Some(value) => value.with_span_range(span_range), + None => ExpressionValue::None(span_range), + } + } + + pub(super) fn index_mut( + &mut self, + _access: IndexAccess, + index: ExpressionValue, + ) -> ExecutionResult<&mut ExpressionValue> { + let key = index.expect_string("An object key")?.value; + Ok(self.mut_entry_or_create(key)) + } + + pub(super) fn property_mut( + &mut self, + access: PropertyAccess, + ) -> ExecutionResult<&mut ExpressionValue> { + Ok(self.mut_entry_or_create(access.property.to_string())) + } + + fn mut_entry_or_create(&mut self, key: String) -> &mut ExpressionValue { + use std::collections::btree_map::*; + match self.entries.entry(key) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => entry.insert(ExpressionValue::None(self.span_range)), + } + } + + pub(crate) fn concat_recursive_into( + self, + output: &mut String, + behaviour: &ConcatBehaviour, + ) -> ExecutionResult<()> { + if behaviour.output_array_structure { + if self.entries.is_empty() { + output.push_str("{}"); + return Ok(()); + } + output.push('{'); + if behaviour.add_space_between_token_trees { + output.push(' '); + } + } + let mut is_first = true; + for (key, value) in self.entries { + if !is_first && behaviour.output_array_structure { + output.push(','); + } + if !is_first && behaviour.add_space_between_token_trees { + output.push(' '); + } + if syn::parse_str::(&key).is_ok() { + output.push_str(&key); + } else { + output.push_str("[\""); + output.push_str(&key); + output.push_str("\"]"); + } + output.push(':'); + if behaviour.add_space_between_token_trees { + output.push(' '); + } + value.concat_recursive_into(output, behaviour)?; + is_first = false; + } + if behaviour.output_array_structure { + if behaviour.add_space_between_token_trees { + output.push(' '); + } + output.push('}'); + } + Ok(()) + } +} + +impl HasSpanRange for ExpressionObject { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl HasValueType for ExpressionObject { + fn value_type(&self) -> &'static str { + self.entries.value_type() + } +} + +impl HasValueType for BTreeMap { + fn value_type(&self) -> &'static str { + "object" + } +} + +impl ToExpressionValue for BTreeMap { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::Object(ExpressionObject { + entries: self, + span_range, + }) + } +} diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 60371fc1..7a3269b6 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -657,9 +657,9 @@ impl HasSpanRange for CompoundAssignmentOperation { } #[derive(Clone)] -pub(super) struct PropertyAccess { +pub(crate) struct PropertyAccess { pub(super) dot: Token![.], - pub(super) property: Ident, + pub(crate) property: Ident, } impl HasSpanRange for PropertyAccess { diff --git a/src/expressions/value.rs b/src/expressions/value.rs index b26c6322..cd3b8777 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -12,6 +12,7 @@ pub(crate) enum ExpressionValue { // as a value rather than a stream, and give it better error messages UnsupportedLiteral(UnsupportedLiteral), Array(ExpressionArray), + Object(ExpressionObject), Stream(ExpressionStream), Range(ExpressionRange), Iterator(ExpressionIterator), @@ -235,6 +236,9 @@ impl ExpressionValue { (ExpressionValue::Array(left), ExpressionValue::Array(right)) => { ExpressionValuePair::ArrayPair(left, right) } + (ExpressionValue::Object(left), ExpressionValue::Object(right)) => { + ExpressionValuePair::ObjectPair(left, right) + } (ExpressionValue::Stream(left), ExpressionValue::Stream(right)) => { ExpressionValuePair::StreamPair(left, right) } @@ -331,7 +335,13 @@ impl ExpressionValue { operation: OutputSpanned, ) -> ExecutionResult { match self { - ExpressionValue::None(_) => operation.unsupported(self), + ExpressionValue::None(_) => match operation.operation { + UnaryOperation::Cast { + target: CastTarget::DebugString, + .. + } => ExpressionValue::None(operation.output_span_range).into_debug_string_value(), + _ => operation.unsupported(self), + }, ExpressionValue::Integer(value) => value.handle_unary_operation(operation), ExpressionValue::Float(value) => value.handle_unary_operation(operation), ExpressionValue::Boolean(value) => value.handle_unary_operation(operation), @@ -339,6 +349,7 @@ impl ExpressionValue { ExpressionValue::Char(value) => value.handle_unary_operation(operation), ExpressionValue::Stream(value) => value.handle_unary_operation(operation), ExpressionValue::Array(value) => value.handle_unary_operation(operation), + ExpressionValue::Object(value) => value.handle_unary_operation(operation), ExpressionValue::Range(range) => { ExpressionIterator::new_for_range(range)?.handle_unary_operation(operation) } @@ -400,6 +411,9 @@ impl ExpressionValue { ExpressionValue::Array(value) => { value.handle_integer_binary_operation(right, operation) } + ExpressionValue::Object(value) => { + value.handle_integer_binary_operation(right, operation) + } ExpressionValue::Stream(value) => { value.handle_integer_binary_operation(right, operation) } @@ -411,6 +425,7 @@ impl ExpressionValue { pub(super) fn into_indexed(self, access: IndexAccess, index: Self) -> ExecutionResult { match self { ExpressionValue::Array(array) => array.into_indexed(access, index), + ExpressionValue::Object(object) => object.into_indexed(access, index), other => access.execution_err(format!("Cannot index into a {}", other.value_type())), } } @@ -422,12 +437,29 @@ impl ExpressionValue { ) -> ExecutionResult<&mut Self> { match self { ExpressionValue::Array(array) => array.index_mut(access, index), + ExpressionValue::Object(object) => object.index_mut(access, index), other => access.execution_err(format!("Cannot index into a {}", other.value_type())), } } - pub(super) fn handle_property_access(self, access: PropertyAccess) -> ExecutionResult { - access.execution_err("Fields are not supported") + pub(crate) fn into_property(self, access: PropertyAccess) -> ExecutionResult { + match self { + ExpressionValue::Object(object) => object.into_property(access), + other => access.execution_err(format!( + "Cannot access properties on a {}", + other.value_type() + )), + } + } + + pub(crate) fn property_mut(&mut self, access: PropertyAccess) -> ExecutionResult<&mut Self> { + match self { + ExpressionValue::Object(object) => object.property_mut(access), + other => access.execution_err(format!( + "Cannot access properties on a {}", + other.value_type() + )), + } } fn span_range_mut(&mut self) -> &mut SpanRange { @@ -440,6 +472,7 @@ impl ExpressionValue { Self::Char(value) => &mut value.span_range, Self::UnsupportedLiteral(value) => &mut value.span_range, Self::Array(value) => &mut value.span_range, + Self::Object(value) => &mut value.span_range, Self::Stream(value) => &mut value.span_range, Self::Iterator(value) => &mut value.span_range, Self::Range(value) => &mut value.span_range, @@ -511,6 +544,9 @@ impl ExpressionValue { Self::UnsupportedLiteral(literal) => { output.extend_raw_tokens(literal.lit.to_token_stream()) } + Self::Object(_) => { + return self.execution_err("Objects cannot be output to a stream"); + } Self::Array(array) => { if behaviour.should_output_arrays() { array.output_grouped_items_to(output)? @@ -569,6 +605,9 @@ impl ExpressionValue { ExpressionValue::Array(array) => { array.concat_recursive_into(output, behaviour)?; } + ExpressionValue::Object(object) => { + object.concat_recursive_into(output, behaviour)?; + } ExpressionValue::Iterator(iterator) => { iterator.concat_recursive_into(output, behaviour)?; } @@ -628,7 +667,7 @@ pub(crate) enum Grouping { impl HasValueType for ExpressionValue { fn value_type(&self) -> &'static str { match self { - Self::None { .. } => "none", + Self::None { .. } => "none value", Self::Integer(value) => value.value_type(), Self::Float(value) => value.value_type(), Self::Boolean(value) => value.value_type(), @@ -636,6 +675,7 @@ impl HasValueType for ExpressionValue { Self::Char(value) => value.value_type(), Self::UnsupportedLiteral(value) => value.value_type(), Self::Array(value) => value.value_type(), + Self::Object(value) => value.value_type(), Self::Stream(value) => value.value_type(), Self::Iterator(value) => value.value_type(), Self::Range(value) => value.value_type(), @@ -654,6 +694,7 @@ impl HasSpanRange for ExpressionValue { ExpressionValue::Char(char) => char.span_range, ExpressionValue::UnsupportedLiteral(lit) => lit.span_range, ExpressionValue::Array(array) => array.span_range, + ExpressionValue::Object(object) => object.span_range, ExpressionValue::Stream(stream) => stream.span_range, ExpressionValue::Iterator(iterator) => iterator.span_range, ExpressionValue::Range(iterator) => iterator.span_range, @@ -684,6 +725,7 @@ pub(super) enum ExpressionValuePair { StringPair(ExpressionString, ExpressionString), CharPair(ExpressionChar, ExpressionChar), ArrayPair(ExpressionArray, ExpressionArray), + ObjectPair(ExpressionObject, ExpressionObject), StreamPair(ExpressionStream, ExpressionStream), } @@ -699,6 +741,7 @@ impl ExpressionValuePair { Self::StringPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::CharPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::ArrayPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), + Self::ObjectPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), Self::StreamPair(lhs, rhs) => lhs.handle_paired_binary_operation(rhs, operation), } } diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 29ab80ab..0c754a2a 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -4,6 +4,7 @@ pub(crate) use core::ops::{Deref, DerefMut}; pub(crate) use proc_macro2::extra::*; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; +pub(crate) use std::collections::BTreeMap; pub(crate) use std::{collections::HashMap, str::FromStr}; pub(crate) use syn::buffer::Cursor; pub(crate) use syn::ext::IdentExt as SynIdentExt; diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index e94a4a69..5918496a 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -215,6 +215,15 @@ impl MutableReference { }) } + pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { + let span_range = SpanRange::new_between(self.span_range.start(), access.span_range()); + let indexed = self.mut_cell.try_map(|value| value.property_mut(access))?; + Ok(Self { + mut_cell: indexed, + span_range, + }) + } + pub(crate) fn set(&mut self, content: impl ToExpressionValue) { *self.mut_cell = content.to_value(self.span_range); } diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index cc49b762..84295cb7 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -213,7 +213,7 @@ impl core::fmt::Display for FlattenedVariable { // An identifier for a variable path in an expression #[derive(Clone)] pub(crate) struct VariableIdentifier { - ident: Ident, + pub(crate) ident: Ident, } impl Parse for VariableIdentifier { diff --git a/tests/compilation_failures/expressions/braces.rs b/tests/compilation_failures/expressions/braces.rs deleted file mode 100644 index a4712e16..00000000 --- a/tests/compilation_failures/expressions/braces.rs +++ /dev/null @@ -1,7 +0,0 @@ -use preinterpret::*; - -fn main() { - let _ = preinterpret!{ - #({ true }) - }; -} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/braces.stderr b/tests/compilation_failures/expressions/braces.stderr deleted file mode 100644 index e59b2e83..00000000 --- a/tests/compilation_failures/expressions/braces.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Braces { ... } are not supported in an expression - --> tests/compilation_failures/expressions/braces.rs:5:11 - | -5 | #({ true }) - | ^ diff --git a/tests/compilation_failures/expressions/inner_braces.rs b/tests/compilation_failures/expressions/inner_braces.rs deleted file mode 100644 index 58b6efa4..00000000 --- a/tests/compilation_failures/expressions/inner_braces.rs +++ /dev/null @@ -1,7 +0,0 @@ -use preinterpret::*; - -fn main() { - let _ = preinterpret!{ - #(({ true })) - }; -} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/inner_braces.stderr b/tests/compilation_failures/expressions/inner_braces.stderr deleted file mode 100644 index 820014a6..00000000 --- a/tests/compilation_failures/expressions/inner_braces.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Braces { ... } are not supported in an expression - --> tests/compilation_failures/expressions/inner_braces.rs:5:12 - | -5 | #(({ true })) - | ^ diff --git a/tests/expressions.rs b/tests/expressions.rs index 809d33fd..c5c4a2e6 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -411,3 +411,36 @@ fn test_array_pattern_destructurings() { r#"[[1, "a"], 4, 5]"# ); } + +#[test] +fn test_objects() { + preinterpret_assert_eq!( + #( + let a = {}; + let b = "Hello"; + let x = { a, hello: 1, ["world"]: 2, b }; + x["x y z"] = 4; + x.y = 5; + x as debug + ), + r#"{ a: {}, b: "Hello", hello: 1, world: 2, ["x y z"]: 4, y: 5 }"# + ); + preinterpret_assert_eq!( + #( + { prop1: 1 }["prop1"] as debug + ), + r#"1"# + ); + preinterpret_assert_eq!( + #( + { prop1: 1 }["prop2"] as debug + ), + r#"None"# + ); + preinterpret_assert_eq!( + #( + { prop1: 1 }.prop1 as debug + ), + r#"1"# + ); +} From 3c53e1f52cba0dfce8e1bab575163dcc60013038 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 19 Feb 2025 19:12:58 +0000 Subject: [PATCH 108/126] feature: Objects can be destructured --- CHANGELOG.md | 12 +- src/expressions/evaluation.rs | 136 +++++++++++++++--- src/expressions/expression_parsing.rs | 21 +-- src/expressions/object.rs | 7 +- src/expressions/operations.rs | 2 +- src/expressions/value.rs | 47 ++++-- src/internal_prelude.rs | 6 +- src/interpretation/variable.rs | 2 +- src/transformation/patterns.rs | 113 +++++++++++++++ ..._pattern_destructure_element_mismatch_1.rs | 7 + ...tern_destructure_element_mismatch_1.stderr | 5 + ..._pattern_destructure_element_mismatch_2.rs | 7 + ...tern_destructure_element_mismatch_2.stderr | 5 + ..._pattern_destructure_element_mismatch_3.rs | 7 + ...tern_destructure_element_mismatch_3.stderr | 5 + ...y_pattern_destructure_multiple_dot_dots.rs | 7 + ...ttern_destructure_multiple_dot_dots.stderr | 5 + .../cannot_output_object_to_stream.rs | 7 + .../cannot_output_object_to_stream.stderr | 5 + .../expressions/object_block_confusion.rs | 7 + .../expressions/object_block_confusion.stderr | 5 + .../expressions/object_field_duplication.rs | 7 + .../object_field_duplication.stderr | 5 + .../expressions/object_incorrect_comma.rs | 8 ++ .../expressions/object_incorrect_comma.stderr | 5 + ...ct_pattern_destructuring_repeated_field.rs | 7 + ...attern_destructuring_repeated_field.stderr | 5 + ...object_pattern_destructuring_wrong_type.rs | 7 + ...ct_pattern_destructuring_wrong_type.stderr | 5 + ...ject_place_destructuring_repeated_field.rs | 11 ++ ..._place_destructuring_repeated_field.stderr | 5 + .../object_place_destructuring_wrong_type.rs | 11 ++ ...ject_place_destructuring_wrong_type.stderr | 5 + ...structure_with_ident_flattened_variable.rs | 2 +- ...cture_with_ident_flattened_variable.stderr | 9 +- ...ructure_with_literal_flattened_variable.rs | 2 +- ...ure_with_literal_flattened_variable.stderr | 9 +- ...structure_with_punct_flattened_variable.rs | 2 +- ...cture_with_punct_flattened_variable.stderr | 9 +- tests/expressions.rs | 20 ++- 40 files changed, 481 insertions(+), 71 deletions(-) create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.rs create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.stderr create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.rs create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.stderr create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.rs create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.stderr create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.rs create mode 100644 tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.stderr create mode 100644 tests/compilation_failures/expressions/cannot_output_object_to_stream.rs create mode 100644 tests/compilation_failures/expressions/cannot_output_object_to_stream.stderr create mode 100644 tests/compilation_failures/expressions/object_block_confusion.rs create mode 100644 tests/compilation_failures/expressions/object_block_confusion.stderr create mode 100644 tests/compilation_failures/expressions/object_field_duplication.rs create mode 100644 tests/compilation_failures/expressions/object_field_duplication.stderr create mode 100644 tests/compilation_failures/expressions/object_incorrect_comma.rs create mode 100644 tests/compilation_failures/expressions/object_incorrect_comma.stderr create mode 100644 tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.rs create mode 100644 tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.stderr create mode 100644 tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.rs create mode 100644 tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.stderr create mode 100644 tests/compilation_failures/expressions/object_place_destructuring_repeated_field.rs create mode 100644 tests/compilation_failures/expressions/object_place_destructuring_repeated_field.stderr create mode 100644 tests/compilation_failures/expressions/object_place_destructuring_wrong_type.rs create mode 100644 tests/compilation_failures/expressions/object_place_destructuring_wrong_type.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index f2a2d89d..7d9143d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,13 +97,10 @@ Inside a transform stream, the following grammar is supported: ### To come * Objects continuation: - * Can be place-destructured and pattern-destructured with `{ hello, world: _ }` (note - like JS, fields don't need to be complete!) - * Add error tests: - * Throws if object is output to a stream - * Throws if field names re-used - * Invalid object syntax * Commands have an input object + * We may wish to store idents with the values so we can complain about invalid names? * Have `!zip!` support `{ objects }` +* Support `let x;` * Method calls on values * Mutable params notes: * For now we can assume all params are values/clones, no mutable references @@ -111,6 +108,8 @@ Inside a transform stream, the following grammar is supported: * `.len()` on stream * `.push(x)` on array * Consider `.map(|| {})` +* Reference equality for streams, objects and arrays + * And new `.clone()` method * Introduce interpreter stack frames * Design the data model => is it some kind of linked list of frames? * Add a new frame inside loops @@ -119,8 +118,7 @@ Inside a transform stream, the following grammar is supported: the latter defines a new variable stack frame, just like Rust * To avoid confusion (such as below) and teach the user to only include #var where necessary, only expression _blocks_ are allowed in an expression. - * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal - rust because the inside is a `{ .. }` which defines a new scope. + * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal rust because the inside is a `{ .. }` which defines a new scope. * TRANSFORMERS => PARSERS cont * Re-read the `Parsers Revisited` section below * Manually search for transform and rename to parse in folder names and file. diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 0f189b3a..ee2a9913 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -256,12 +256,12 @@ impl ExpressionNode { evaluated_items: Vec::with_capacity(items.len()), } .next(next), - ExpressionNode::Object { braces, entries } => ObjectValueStackFrame { + ExpressionNode::Object { braces, entries } => Box::new(ObjectValueStackFrame { span: braces.join(), unevaluated_entries: entries.clone(), evaluated_entries: BTreeMap::new(), pending: None, - } + }) .next(next)?, ExpressionNode::UnaryOperation { operation, input } => next.read_value_with_handler( *input, @@ -377,13 +377,19 @@ impl ExpressionNode { ArrayAssigneeStackFrame::new(nodes, brackets.join(), assignee_item_node_ids, value)? .handle_next(next) } + ExpressionNode::Object { braces, entries } => Box::new(ObjectAssigneeStackFrame::new( + braces.join(), + entries, + value, + )?) + .handle_next(next)?, ExpressionNode::Grouped { inner, .. } => { next.handle_assignment_and_return_to(*inner, value, AssignmentStackFrame::Grouped) } other => { return other .operator_span_range() - .execution_err("This type of expression is not supported as an assignee"); + .execution_err("This type of expression is not supported as an assignee. You may wish to use `_` to ignore the value."); } }) } @@ -423,7 +429,7 @@ impl ExpressionNode { other => { return other .operator_span_range() - .execution_err("This type of expression is not supported as an assignee"); + .execution_err("This type of expression is not supported as here. You may wish to use `_` to ignore the value."); } }) } @@ -435,7 +441,7 @@ enum ValueStackFrame { span: Span, }, Array(ArrayValueStackFrame), - Object(ObjectValueStackFrame), + Object(Box), UnaryOperation { operation: UnaryOperation, }, @@ -466,6 +472,11 @@ enum ValueStackFrame { place: Place, access: IndexAccess, }, + ResolveObjectIndexForAssignment { + object: Box, + assignee_node: ExpressionNodeId, + access: IndexAccess, + }, } impl ValueStackFrame { @@ -482,6 +493,9 @@ impl ValueStackFrame { ValueStackFrame::HandleValueForAssignment { .. } => ReturnMode::Value, ValueStackFrame::HandleValueForCompoundAssignment { .. } => ReturnMode::Value, ValueStackFrame::ResolveIndexedPlace { .. } => ReturnMode::Place, + ValueStackFrame::ResolveObjectIndexForAssignment { .. } => { + ReturnMode::AssignmentCompletion + } } } @@ -615,6 +629,11 @@ impl ValueStackFrame { } }) } + ValueStackFrame::ResolveObjectIndexForAssignment { + object, + assignee_node, + access, + } => object.handle_index_value(access, value, assignee_node, next)?, }) } } @@ -652,19 +671,16 @@ struct ObjectValueStackFrame { impl ObjectValueStackFrame { fn handle_value( - mut self, + mut self: Box, value: ExpressionValue, next: ActionCreator, ) -> ExecutionResult { let pending = self.pending.take(); Ok(match pending { - Some(PendingEntryPath::OnIndexKeyBranch { - brackets, - value_node, - }) => { + Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }) => { let key = value.expect_string("An object key")?.value; if self.evaluated_entries.contains_key(&key) { - return brackets.execution_err(format!("The key {} has already been set", key)); + return access.execution_err(format!("The key {} has already been set", key)); } self.pending = Some(PendingEntryPath::OnValueBranch { key }); next.read_value_with_handler(value_node, ValueStackFrame::Object(self)) @@ -679,7 +695,7 @@ impl ObjectValueStackFrame { }) } - fn next(mut self, action_creator: ActionCreator) -> ExecutionResult { + fn next(mut self: Box, action_creator: ActionCreator) -> ExecutionResult { Ok( match self .unevaluated_entries @@ -695,11 +711,8 @@ impl ObjectValueStackFrame { self.pending = Some(PendingEntryPath::OnValueBranch { key }); action_creator.read_value_with_handler(node, ValueStackFrame::Object(self)) } - Some((ObjectKey::Indexed { brackets, index }, value_node)) => { - self.pending = Some(PendingEntryPath::OnIndexKeyBranch { - brackets, - value_node, - }); + Some((ObjectKey::Indexed { access, index }, value_node)) => { + self.pending = Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }); action_creator.read_value_with_handler(index, ValueStackFrame::Object(self)) } None => action_creator @@ -711,7 +724,7 @@ impl ObjectValueStackFrame { enum PendingEntryPath { OnIndexKeyBranch { - brackets: Brackets, + access: IndexAccess, value_node: ExpressionNodeId, }, OnValueBranch { @@ -742,6 +755,7 @@ enum AssignmentStackFrame { }, Grouped, Array(ArrayAssigneeStackFrame), + Object(Box), } impl AssignmentStackFrame { @@ -750,6 +764,7 @@ impl AssignmentStackFrame { AssignmentStackFrame::AssignmentRoot { .. } => ReturnMode::Value, AssignmentStackFrame::Grouped { .. } => ReturnMode::AssignmentCompletion, AssignmentStackFrame::Array { .. } => ReturnMode::AssignmentCompletion, + AssignmentStackFrame::Object { .. } => ReturnMode::AssignmentCompletion, } } @@ -765,6 +780,7 @@ impl AssignmentStackFrame { } AssignmentStackFrame::Grouped => next.return_assignment_completion(span_range), AssignmentStackFrame::Array(array) => array.handle_next(next), + AssignmentStackFrame::Object(object) => object.handle_next(next)?, }) } } @@ -782,7 +798,7 @@ impl ArrayAssigneeStackFrame { assignee_item_node_ids: &[ExpressionNodeId], value: ExpressionValue, ) -> ExecutionResult { - let array = value.expect_array("The assignee of an array place")?; + let array = value.expect_array("The value destructured as an array")?; let span_range = SpanRange::new_between(assignee_span, array.span_range.end()); let mut has_seen_dot_dot = false; let mut prefix_assignees = Vec::new(); @@ -861,6 +877,88 @@ impl ArrayAssigneeStackFrame { } } +struct ObjectAssigneeStackFrame { + span_range: SpanRange, + entries: BTreeMap, + already_used_keys: HashSet, + unresolved_stack: Vec<(ObjectKey, ExpressionNodeId)>, +} + +impl ObjectAssigneeStackFrame { + /// See also `ObjectPattern` in `patterns.rs` + fn new( + assignee_span: Span, + assignee_pairs: &[(ObjectKey, ExpressionNodeId)], + value: ExpressionValue, + ) -> ExecutionResult { + let object = value.expect_object("The value destructured as an object")?; + let span_range = SpanRange::new_between(assignee_span, object.span_range.end()); + + Ok(Self { + span_range, + entries: object.entries, + already_used_keys: HashSet::with_capacity(assignee_pairs.len()), + unresolved_stack: assignee_pairs.iter().rev().cloned().collect(), + }) + } + + fn handle_index_value( + mut self: Box, + access: IndexAccess, + index: ExpressionValue, + assignee_node: ExpressionNodeId, + next: ActionCreator, + ) -> ExecutionResult { + let key = index.expect_string("An object key")?.value; + let value = self.resolve_value(key, access.span_range())?; + Ok(next.handle_assignment_and_return_to( + assignee_node, + value, + AssignmentStackFrame::Object(self), + )) + } + + fn handle_next(mut self: Box, next: ActionCreator) -> ExecutionResult { + Ok(match self.unresolved_stack.pop() { + Some((ObjectKey::Identifier(ident), assignee_node)) => { + let key = ident.to_string(); + let value = self.resolve_value(key, ident.span_range())?; + next.handle_assignment_and_return_to( + assignee_node, + value, + AssignmentStackFrame::Object(self), + ) + } + Some((ObjectKey::Indexed { index, access }, assignee_node)) => next + .read_value_with_handler( + index, + ValueStackFrame::ResolveObjectIndexForAssignment { + object: self, + assignee_node, + access, + }, + ), + None => next.return_assignment_completion(self.span_range), + }) + } + + fn resolve_value( + &mut self, + key: String, + span_range: SpanRange, + ) -> ExecutionResult { + if self.already_used_keys.contains(&key) { + return span_range.execution_err(format!("The key `{}` has already used", key)); + } + let value = self + .entries + .remove(&key) + .unwrap_or_else(|| ExpressionValue::None(span_range)); + self.already_used_keys.insert(key); + Ok(value) + } +} + struct AssignmentCompletion { span_range: SpanRange, } diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 0359eb4a..4d1838e2 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -216,7 +216,7 @@ impl<'a> ExpressionParser<'a, Source> { ExpressionStackFrame::Object { braces, complete_entries, - state: ObjectStackFrameState::EntryIndex(brackets), + state: ObjectStackFrameState::EntryIndex(access), } => { assert!(matches!(extension, NodeExtension::EndOfStream)); self.streams.exit_group(); @@ -226,7 +226,7 @@ impl<'a> ExpressionParser<'a, Source> { complete_entries, state: ObjectStackFrameState::EntryValue( ObjectKey::Indexed { - brackets, + access, index: node, }, colon, @@ -354,6 +354,7 @@ impl<'a> ExpressionParser<'a, Source> { braces: Braces, mut complete_entries: Vec<(ObjectKey, ExpressionNodeId)>, ) -> ParseResult { + const ERROR_MESSAGE: &str = r##"Expected an object entry (`field,` `field: ..,` or `["field"]: ..,`). If you meant to start a new block, use #{ ... } instead."##; let state = loop { if self.streams.is_current_empty() { self.streams.exit_group(); @@ -370,9 +371,11 @@ impl<'a> ExpressionParser<'a, Source> { } else if self.streams.peek(token::Comma) { self.streams.parse::()?; // Fall through - } else { + } else if self.streams.peek(token::Colon) { let colon = self.streams.parse()?; break ObjectStackFrameState::EntryValue(ObjectKey::Identifier(key), colon); + } else { + return self.streams.parse_err(ERROR_MESSAGE); } let node = @@ -384,11 +387,11 @@ impl<'a> ExpressionParser<'a, Source> { continue; } else if self.streams.peek(token::Bracket) { let (_, delim_span) = self.streams.parse_and_enter_group()?; - break ObjectStackFrameState::EntryIndex(Brackets { delim_span }); + break ObjectStackFrameState::EntryIndex(IndexAccess { + brackets: Brackets { delim_span }, + }); } else { - return self - .streams - .parse_err("Expected an identifier or indexed key"); + return self.streams.parse_err(ERROR_MESSAGE); } }; Ok(self.push_stack_frame(ExpressionStackFrame::Object { @@ -711,13 +714,13 @@ pub(super) enum ExpressionStackFrame { pub(super) enum ObjectKey { Identifier(Ident), Indexed { - brackets: Brackets, + access: IndexAccess, index: ExpressionNodeId, }, } pub(super) enum ObjectStackFrameState { - EntryIndex(Brackets), + EntryIndex(IndexAccess), #[allow(unused)] EntryValue(ObjectKey, Token![:]), } diff --git a/src/expressions/object.rs b/src/expressions/object.rs index d859f009..cf87f400 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -2,7 +2,6 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionObject { - // TODO: Convert to an IndexMap pub(crate) entries: BTreeMap, /// The span range that generated this value. /// For a complex expression, the start span is the most left part @@ -125,9 +124,9 @@ impl ExpressionObject { if syn::parse_str::(&key).is_ok() { output.push_str(&key); } else { - output.push_str("[\""); - output.push_str(&key); - output.push_str("\"]"); + output.push('['); + output.push_str(format!("{:?}", key).as_str()); + output.push(']'); } output.push(':'); if behaviour.add_space_between_token_trees { diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index 7a3269b6..dcd410ec 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -670,7 +670,7 @@ impl HasSpanRange for PropertyAccess { #[derive(Copy, Clone)] pub(crate) struct IndexAccess { - pub(super) brackets: Brackets, + pub(crate) brackets: Brackets, } impl HasSpan for IndexAccess { diff --git a/src/expressions/value.rs b/src/expressions/value.rs index cd3b8777..21c3b2b8 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -259,9 +259,9 @@ impl ExpressionValue { match self { ExpressionValue::Boolean(value) => Ok(value), other => other.execution_err(format!( - "{} must be a boolean, but it is a {}", + "{} must be a boolean, but it is {}", place_descriptor, - other.value_type(), + other.articled_value_type(), )), } } @@ -273,9 +273,9 @@ impl ExpressionValue { match self { ExpressionValue::Integer(value) => Ok(value), other => other.execution_err(format!( - "{} must be an integer, but it is a {}", + "{} must be an integer, but it is {}", place_descriptor, - other.value_type(), + other.articled_value_type(), )), } } @@ -284,9 +284,9 @@ impl ExpressionValue { match self { ExpressionValue::String(value) => Ok(value), other => other.execution_err(format!( - "{} must be a string, but it is a {}", + "{} must be a string, but it is {}", place_descriptor, - other.value_type(), + other.articled_value_type(), )), } } @@ -295,9 +295,20 @@ impl ExpressionValue { match self { ExpressionValue::Array(value) => Ok(value), other => other.execution_err(format!( - "{} must be an array, but it is a {}", + "{} must be an array, but it is {}", place_descriptor, - other.value_type(), + other.articled_value_type(), + )), + } + } + + pub(crate) fn expect_object(self, place_descriptor: &str) -> ExecutionResult { + match self { + ExpressionValue::Object(value) => Ok(value), + other => other.execution_err(format!( + "{} must be an object, but it is {}", + place_descriptor, + other.articled_value_type(), )), } } @@ -306,9 +317,9 @@ impl ExpressionValue { match self { ExpressionValue::Stream(value) => Ok(value), other => other.execution_err(format!( - "{} must be a stream, but it is a {}", + "{} must be a stream, but it is {}", place_descriptor, - other.value_type(), + other.articled_value_type(), )), } } @@ -323,9 +334,9 @@ impl ExpressionValue { ExpressionValue::Iterator(value) => Ok(value), ExpressionValue::Range(value) => Ok(ExpressionIterator::new_for_range(value)?), other => other.execution_err(format!( - "{} must be iterable (an array, stream, range or iterator), but it is a {}", + "{} must be iterable (an array, stream, range or iterator), but it is {}", place_descriptor, - other.value_type(), + other.articled_value_type(), )), } } @@ -704,6 +715,18 @@ impl HasSpanRange for ExpressionValue { pub(super) trait HasValueType { fn value_type(&self) -> &'static str; + + fn articled_value_type(&self) -> String { + let value_type = self.value_type(); + if value_type.is_empty() { + return value_type.to_string(); + } + let first_char = value_type.chars().next().unwrap(); + match first_char { + 'a' | 'e' | 'i' | 'o' | 'u' => format!("an {}", value_type), + _ => value_type.to_string(), + } + } } #[derive(Clone)] diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 0c754a2a..8f01f1d6 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -4,8 +4,10 @@ pub(crate) use core::ops::{Deref, DerefMut}; pub(crate) use proc_macro2::extra::*; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; -pub(crate) use std::collections::BTreeMap; -pub(crate) use std::{collections::HashMap, str::FromStr}; +pub(crate) use std::{ + collections::{BTreeMap, HashMap, HashSet}, + str::FromStr, +}; pub(crate) use syn::buffer::Cursor; pub(crate) use syn::ext::IdentExt as SynIdentExt; pub(crate) use syn::parse::{ diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 84295cb7..79a1b47d 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -249,7 +249,7 @@ impl InterpretToValue for &VariableIdentifier { #[derive(Clone)] pub(crate) struct VariablePattern { - name: Ident, + pub(crate) name: Ident, } impl Parse for VariablePattern { diff --git a/src/transformation/patterns.rs b/src/transformation/patterns.rs index 085193de..d568f26d 100644 --- a/src/transformation/patterns.rs +++ b/src/transformation/patterns.rs @@ -12,6 +12,7 @@ pub(crate) trait HandleDestructure { pub(crate) enum Pattern { Variable(VariablePattern), Array(ArrayPattern), + Object(ObjectPattern), Stream(ExplicitTransformStream), DotDot(Token![..]), #[allow(unused)] @@ -25,6 +26,8 @@ impl Parse for Pattern { Ok(Pattern::Variable(input.parse()?)) } else if lookahead.peek(syn::token::Bracket) { Ok(Pattern::Array(input.parse()?)) + } else if lookahead.peek(syn::token::Brace) { + Ok(Pattern::Object(input.parse()?)) } else if lookahead.peek(Token![@]) { Ok(Pattern::Stream(input.parse()?)) } else if lookahead.peek(Token![_]) { @@ -48,6 +51,7 @@ impl HandleDestructure for Pattern { match self { Pattern::Variable(variable) => variable.handle_destructure(interpreter, value), Pattern::Array(array) => array.handle_destructure(interpreter, value), + Pattern::Object(object) => object.handle_destructure(interpreter, value), Pattern::Stream(stream) => stream.handle_destructure(interpreter, value), Pattern::DotDot(token) => token.execution_err("This cannot be used here"), Pattern::Discarded(_) => Ok(()), @@ -138,3 +142,112 @@ impl HandleDestructure for ArrayPattern { Ok(()) } } + +#[derive(Clone)] +pub struct ObjectPattern { + #[allow(unused)] + braces: Braces, + entries: Punctuated, +} + +impl Parse for ObjectPattern { + fn parse(input: ParseStream) -> ParseResult { + let (braces, inner) = input.parse_braces()?; + Ok(Self { + braces, + entries: inner.parse_terminated()?, + }) + } +} + +impl HandleDestructure for ObjectPattern { + /// See also `ObjectAssigneeStackFrame` in `evaluation.rs` + fn handle_destructure( + &self, + interpreter: &mut Interpreter, + value: ExpressionValue, + ) -> ExecutionResult<()> { + let object = value.expect_object("The value destructured with an object pattern")?; + let mut value_map = object.entries; + let mut already_used_keys = HashSet::with_capacity(self.entries.len()); + for entry in self.entries.iter() { + let (key, key_span, pattern) = match entry { + ObjectEntry::KeyOnly { field, pattern } => { + (field.to_string(), field.span(), pattern) + } + ObjectEntry::KeyValue { field, pattern, .. } => { + (field.to_string(), field.span(), pattern) + } + ObjectEntry::IndexValue { + access, + key, + pattern, + .. + } => (key.value(), access.span(), pattern), + }; + if already_used_keys.contains(&key) { + return key_span.execution_err(format!("The key `{}` has already used", key)); + } + let value = value_map + .remove(&key) + .unwrap_or_else(|| ExpressionValue::None(key_span.span_range())); + already_used_keys.insert(key); + pattern.handle_destructure(interpreter, value)?; + } + Ok(()) + } +} + +#[derive(Clone)] +enum ObjectEntry { + KeyOnly { + field: Ident, + pattern: Pattern, + }, + KeyValue { + field: Ident, + _colon: Token![:], + pattern: Pattern, + }, + IndexValue { + access: IndexAccess, + key: syn::LitStr, + _colon: Token![:], + pattern: Pattern, + }, +} + +impl Parse for ObjectEntry { + fn parse(input: ParseStream) -> ParseResult { + if input.peek(syn::Ident) { + let field = input.parse()?; + if input.peek(Token![:]) { + Ok(ObjectEntry::KeyValue { + field, + _colon: input.parse()?, + pattern: input.parse()?, + }) + } else if input.peek(Token![,]) || input.is_empty() { + let pattern = Pattern::Variable(VariablePattern { + name: field.clone(), + }); + Ok(ObjectEntry::KeyOnly { field, pattern }) + } else { + input.parse_err("Expected `:` or `,`") + } + } else if input.peek(syn::token::Bracket) { + let (access, key) = { + let (brackets, inner) = input.parse_brackets()?; + (IndexAccess { brackets }, inner.parse()?) + }; + Ok(ObjectEntry::IndexValue { + access, + key, + _colon: input.parse()?, + pattern: input.parse()?, + }) + } else { + input.parse_err("Expected `property: ` or `[\"property\"]: `") + } + } +} diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.rs b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.rs new file mode 100644 index 00000000..ffc29e16 --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let [_, _, _] = [1, 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.stderr b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.stderr new file mode 100644 index 00000000..2a86b522 --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.stderr @@ -0,0 +1,5 @@ +error: The array has 2 items, but the pattern expected 3 + --> tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_1.rs:5:15 + | +5 | #(let [_, _, _] = [1, 2]) + | ^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.rs b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.rs new file mode 100644 index 00000000..b10f9848 --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let [_] = [1, 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.stderr b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.stderr new file mode 100644 index 00000000..552f5916 --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.stderr @@ -0,0 +1,5 @@ +error: The array has 2 items, but the pattern expected 1 + --> tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_2.rs:5:15 + | +5 | #(let [_] = [1, 2]) + | ^^^ diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.rs b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.rs new file mode 100644 index 00000000..d03f3c4f --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let [_, _, .., _] = [1, 2]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.stderr b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.stderr new file mode 100644 index 00000000..873bd7d4 --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.stderr @@ -0,0 +1,5 @@ +error: The array has 2 items, but the pattern expected at least 3 + --> tests/compilation_failures/expressions/array_pattern_destructure_element_mismatch_3.rs:5:15 + | +5 | #(let [_, _, .., _] = [1, 2]) + | ^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.rs b/tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.rs new file mode 100644 index 00000000..35ef4683 --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let [_, .., _, .., _] = [1, 2, 3, 4]) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.stderr b/tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.stderr new file mode 100644 index 00000000..7569b8ee --- /dev/null +++ b/tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.stderr @@ -0,0 +1,5 @@ +error: Only one .. is allowed in an array pattern + --> tests/compilation_failures/expressions/array_pattern_destructure_multiple_dot_dots.rs:5:26 + | +5 | #(let [_, .., _, .., _] = [1, 2, 3, 4]) + | ^^ diff --git a/tests/compilation_failures/expressions/cannot_output_object_to_stream.rs b/tests/compilation_failures/expressions/cannot_output_object_to_stream.rs new file mode 100644 index 00000000..e2fc037d --- /dev/null +++ b/tests/compilation_failures/expressions/cannot_output_object_to_stream.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #({ hello: "world" }) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/cannot_output_object_to_stream.stderr b/tests/compilation_failures/expressions/cannot_output_object_to_stream.stderr new file mode 100644 index 00000000..38bdb2fc --- /dev/null +++ b/tests/compilation_failures/expressions/cannot_output_object_to_stream.stderr @@ -0,0 +1,5 @@ +error: Objects cannot be output to a stream + --> tests/compilation_failures/expressions/cannot_output_object_to_stream.rs:5:11 + | +5 | #({ hello: "world" }) + | ^^^^^^^^^^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/object_block_confusion.rs b/tests/compilation_failures/expressions/object_block_confusion.rs new file mode 100644 index 00000000..c5b1a96c --- /dev/null +++ b/tests/compilation_failures/expressions/object_block_confusion.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let x = { 1 + 1 + 1 }) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_block_confusion.stderr b/tests/compilation_failures/expressions/object_block_confusion.stderr new file mode 100644 index 00000000..eb1662f0 --- /dev/null +++ b/tests/compilation_failures/expressions/object_block_confusion.stderr @@ -0,0 +1,5 @@ +error: Expected an object entry (`field,` `field: ..,` or `["field"]: ..,`). If you meant to start a new block, use #{ ... } instead. + --> tests/compilation_failures/expressions/object_block_confusion.rs:5:21 + | +5 | #(let x = { 1 + 1 + 1 }) + | ^ diff --git a/tests/compilation_failures/expressions/object_field_duplication.rs b/tests/compilation_failures/expressions/object_field_duplication.rs new file mode 100644 index 00000000..fe915557 --- /dev/null +++ b/tests/compilation_failures/expressions/object_field_duplication.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #({ hello: "world", ["hello"]: "world_2" }) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_field_duplication.stderr b/tests/compilation_failures/expressions/object_field_duplication.stderr new file mode 100644 index 00000000..632455b3 --- /dev/null +++ b/tests/compilation_failures/expressions/object_field_duplication.stderr @@ -0,0 +1,5 @@ +error: The key hello has already been set + --> tests/compilation_failures/expressions/object_field_duplication.rs:5:29 + | +5 | #({ hello: "world", ["hello"]: "world_2" }) + | ^^^^^^^^^ diff --git a/tests/compilation_failures/expressions/object_incorrect_comma.rs b/tests/compilation_failures/expressions/object_incorrect_comma.rs new file mode 100644 index 00000000..9dc192eb --- /dev/null +++ b/tests/compilation_failures/expressions/object_incorrect_comma.rs @@ -0,0 +1,8 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + let a = 0; + #({ a; b: 1 }) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_incorrect_comma.stderr b/tests/compilation_failures/expressions/object_incorrect_comma.stderr new file mode 100644 index 00000000..84263409 --- /dev/null +++ b/tests/compilation_failures/expressions/object_incorrect_comma.stderr @@ -0,0 +1,5 @@ +error: Expected an object entry (`field,` `field: ..,` or `["field"]: ..,`). If you meant to start a new block, use #{ ... } instead. + --> tests/compilation_failures/expressions/object_incorrect_comma.rs:6:14 + | +6 | #({ a; b: 1 }) + | ^ diff --git a/tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.rs b/tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.rs new file mode 100644 index 00000000..83d17569 --- /dev/null +++ b/tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let { x, x } = { x: 1 }) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.stderr b/tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.stderr new file mode 100644 index 00000000..333ab2ce --- /dev/null +++ b/tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.stderr @@ -0,0 +1,5 @@ +error: The key `x` has already used + --> tests/compilation_failures/expressions/object_pattern_destructuring_repeated_field.rs:5:20 + | +5 | #(let { x, x } = { x: 1 }) + | ^ diff --git a/tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.rs b/tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.rs new file mode 100644 index 00000000..f61105a9 --- /dev/null +++ b/tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let { x } = 0;) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.stderr b/tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.stderr new file mode 100644 index 00000000..745b31c5 --- /dev/null +++ b/tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.stderr @@ -0,0 +1,5 @@ +error: The value destructured with an object pattern must be an object, but it is an untyped integer + --> tests/compilation_failures/expressions/object_pattern_destructuring_wrong_type.rs:5:23 + | +5 | #(let { x } = 0;) + | ^ diff --git a/tests/compilation_failures/expressions/object_place_destructuring_repeated_field.rs b/tests/compilation_failures/expressions/object_place_destructuring_repeated_field.rs new file mode 100644 index 00000000..c569e426 --- /dev/null +++ b/tests/compilation_failures/expressions/object_place_destructuring_repeated_field.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #( + let x = 0; + { x, x } = { x: 1 }; + x + ) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_place_destructuring_repeated_field.stderr b/tests/compilation_failures/expressions/object_place_destructuring_repeated_field.stderr new file mode 100644 index 00000000..4ae8a188 --- /dev/null +++ b/tests/compilation_failures/expressions/object_place_destructuring_repeated_field.stderr @@ -0,0 +1,5 @@ +error: The key `x` has already used + --> tests/compilation_failures/expressions/object_place_destructuring_repeated_field.rs:7:18 + | +7 | { x, x } = { x: 1 }; + | ^ diff --git a/tests/compilation_failures/expressions/object_place_destructuring_wrong_type.rs b/tests/compilation_failures/expressions/object_place_destructuring_wrong_type.rs new file mode 100644 index 00000000..9b2d32fc --- /dev/null +++ b/tests/compilation_failures/expressions/object_place_destructuring_wrong_type.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #( + let x = 0; + { x } = [x]; + x + ) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/object_place_destructuring_wrong_type.stderr b/tests/compilation_failures/expressions/object_place_destructuring_wrong_type.stderr new file mode 100644 index 00000000..435db860 --- /dev/null +++ b/tests/compilation_failures/expressions/object_place_destructuring_wrong_type.stderr @@ -0,0 +1,5 @@ +error: The value destructured as an object must be an object, but it is an array + --> tests/compilation_failures/expressions/object_place_destructuring_wrong_type.rs:7:21 + | +7 | { x } = [x]; + | ^^^ diff --git a/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs index 487613f4..46c13177 100644 --- a/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs +++ b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs @@ -1,5 +1,5 @@ use preinterpret::*; fn main() { - preinterpret!([!let! (!ident! #..x) = Hello]); + preinterpret!([!let! @(#..x = @IDENT) = Hello]); } diff --git a/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr index bd845f2b..ce302ffb 100644 --- a/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr +++ b/tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.stderr @@ -1,5 +1,6 @@ -error: Expected ( - --> tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs:4:43 +error: Expected #variable + Occurred whilst parsing [!let! ...] - Expected [!let! = ...] + --> tests/compilation_failures/transforming/destructure_with_ident_flattened_variable.rs:4:28 | -4 | preinterpret!([!let! (!ident! #..x) = Hello]); - | ^^^^^ +4 | preinterpret!([!let! @(#..x = @IDENT) = Hello]); + | ^ diff --git a/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs index daf6bf81..bb13d834 100644 --- a/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs +++ b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs @@ -1,5 +1,5 @@ use preinterpret::*; fn main() { - preinterpret!([!let! (!literal! #..x) = "Hello"]); + preinterpret!([!let! @(#..x = @LITERAL) = "Hello"]); } diff --git a/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr index 6cc089f9..f114e8dd 100644 --- a/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr +++ b/tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.stderr @@ -1,5 +1,6 @@ -error: Expected ( - --> tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs:4:45 +error: Expected #variable + Occurred whilst parsing [!let! ...] - Expected [!let! = ...] + --> tests/compilation_failures/transforming/destructure_with_literal_flattened_variable.rs:4:28 | -4 | preinterpret!([!let! (!literal! #..x) = "Hello"]); - | ^^^^^^^ +4 | preinterpret!([!let! @(#..x = @LITERAL) = "Hello"]); + | ^ diff --git a/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs index 0c69d7ac..89341d19 100644 --- a/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs +++ b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs @@ -1,5 +1,5 @@ use preinterpret::*; fn main() { - preinterpret!([!let! (!punct! #..x) = @]); + preinterpret!([!let! @(#..x = @PUNCT) = @]); } diff --git a/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr index 23f2ef3e..66a8a803 100644 --- a/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr +++ b/tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.stderr @@ -1,5 +1,6 @@ -error: Expected ( - --> tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs:4:43 +error: Expected #variable + Occurred whilst parsing [!let! ...] - Expected [!let! = ...] + --> tests/compilation_failures/transforming/destructure_with_punct_flattened_variable.rs:4:28 | -4 | preinterpret!([!let! (!punct! #..x) = @]); - | ^ +4 | preinterpret!([!let! @(#..x = @PUNCT) = @]); + | ^ diff --git a/tests/expressions.rs b/tests/expressions.rs index c5c4a2e6..4314efd6 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -420,10 +420,11 @@ fn test_objects() { let b = "Hello"; let x = { a, hello: 1, ["world"]: 2, b }; x["x y z"] = 4; + x["z\" test"] = {}; x.y = 5; x as debug ), - r#"{ a: {}, b: "Hello", hello: 1, world: 2, ["x y z"]: 4, y: 5 }"# + r#"{ a: {}, b: "Hello", hello: 1, world: 2, ["x y z"]: 4, y: 5, ["z\" test"]: {} }"# ); preinterpret_assert_eq!( #( @@ -443,4 +444,21 @@ fn test_objects() { ), r#"1"# ); + preinterpret_assert_eq!( + #( + let a = 0; + let b = 0; + let z = 0; + { a, y: [_, b], z } = { a: 1, y: [5, 7] }; + { a, b, z } as debug + ), + r#"{ a: 1, b: 7, z: None }"# + ); + preinterpret_assert_eq!( + #( + let { a, y: [_, b], ["c"]: c, [r#"two "words"#]: x, z } = { a: 1, y: [5, 7], ["two \"words"]: {}, }; + { a, b, c, x, z } as debug + ), + r#"{ a: 1, b: 7, c: None, x: {}, z: None }"# + ); } From 7211dc08546a1f5a04584854df6a6d39b53aa1fc Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 20 Feb 2025 12:30:18 +0000 Subject: [PATCH 109/126] refactor: Commands take object inputs --- CHANGELOG.md | 3 +- src/expressions/evaluation.rs | 35 ++-- src/expressions/object.rs | 194 ++++++++++++++++-- src/expressions/value.rs | 4 + src/internal_prelude.rs | 1 + src/interpretation/commands/core_commands.rs | 39 ++-- src/interpretation/commands/token_commands.rs | 151 +++++--------- src/interpretation/interpreter.rs | 1 + src/misc/field_inputs.rs | 172 ++++++++-------- src/transformation/patterns.rs | 1 + .../complex/nested.stderr | 7 +- .../core/error_invalid_structure.stderr | 7 +- tests/tokens.rs | 35 ---- 13 files changed, 373 insertions(+), 277 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9143d8..b837b9cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,8 +97,6 @@ Inside a transform stream, the following grammar is supported: ### To come * Objects continuation: - * Commands have an input object - * We may wish to store idents with the values so we can complain about invalid names? * Have `!zip!` support `{ objects }` * Support `let x;` * Method calls on values @@ -174,6 +172,7 @@ Inside a transform stream, the following grammar is supported: * Moving control flow (`for` and `while`) to the expression side? Possibly with an `output` auto-variable with `output += [!stream! ...]` * Dropping lots of the `group` wrappers? * If any types should have reference semantics instead of clone/value semantics? + * Supporting anonymous functions, and support them in e.g. `intersperse` * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` * Adding `!define_command!` diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index ee2a9913..294d65f7 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -666,7 +666,7 @@ struct ObjectValueStackFrame { span: Span, pending: Option, unevaluated_entries: Vec<(ObjectKey, ExpressionNodeId)>, - evaluated_entries: BTreeMap, + evaluated_entries: BTreeMap, } impl ObjectValueStackFrame { @@ -682,11 +682,15 @@ impl ObjectValueStackFrame { if self.evaluated_entries.contains_key(&key) { return access.execution_err(format!("The key {} has already been set", key)); } - self.pending = Some(PendingEntryPath::OnValueBranch { key }); + self.pending = Some(PendingEntryPath::OnValueBranch { + key, + key_span: access.span(), + }); next.read_value_with_handler(value_node, ValueStackFrame::Object(self)) } - Some(PendingEntryPath::OnValueBranch { key }) => { - self.evaluated_entries.insert(key, value); + Some(PendingEntryPath::OnValueBranch { key, key_span }) => { + let entry = ObjectEntry { key_span, value }; + self.evaluated_entries.insert(key, entry); self.next(next)? } None => { @@ -708,7 +712,10 @@ impl ObjectValueStackFrame { return ident .execution_err(format!("The key {} has already been set", key)); } - self.pending = Some(PendingEntryPath::OnValueBranch { key }); + self.pending = Some(PendingEntryPath::OnValueBranch { + key, + key_span: ident.span(), + }); action_creator.read_value_with_handler(node, ValueStackFrame::Object(self)) } Some((ObjectKey::Indexed { access, index }, value_node)) => { @@ -729,6 +736,7 @@ enum PendingEntryPath { }, OnValueBranch { key: String, + key_span: Span, }, } @@ -879,7 +887,7 @@ impl ArrayAssigneeStackFrame { struct ObjectAssigneeStackFrame { span_range: SpanRange, - entries: BTreeMap, + entries: BTreeMap, already_used_keys: HashSet, unresolved_stack: Vec<(ObjectKey, ExpressionNodeId)>, } @@ -910,7 +918,7 @@ impl ObjectAssigneeStackFrame { next: ActionCreator, ) -> ExecutionResult { let key = index.expect_string("An object key")?.value; - let value = self.resolve_value(key, access.span_range())?; + let value = self.resolve_value(key, access.span())?; Ok(next.handle_assignment_and_return_to( assignee_node, value, @@ -922,7 +930,7 @@ impl ObjectAssigneeStackFrame { Ok(match self.unresolved_stack.pop() { Some((ObjectKey::Identifier(ident), assignee_node)) => { let key = ident.to_string(); - let value = self.resolve_value(key, ident.span_range())?; + let value = self.resolve_value(key, ident.span())?; next.handle_assignment_and_return_to( assignee_node, value, @@ -942,18 +950,15 @@ impl ObjectAssigneeStackFrame { }) } - fn resolve_value( - &mut self, - key: String, - span_range: SpanRange, - ) -> ExecutionResult { + fn resolve_value(&mut self, key: String, key_span: Span) -> ExecutionResult { if self.already_used_keys.contains(&key) { - return span_range.execution_err(format!("The key `{}` has already used", key)); + return key_span.execution_err(format!("The key `{}` has already used", key)); } let value = self .entries .remove(&key) - .unwrap_or_else(|| ExpressionValue::None(span_range)); + .map(|entry| entry.value) + .unwrap_or_else(|| ExpressionValue::None(key_span.span_range())); self.already_used_keys.insert(key); Ok(value) } diff --git a/src/expressions/object.rs b/src/expressions/object.rs index cf87f400..d50bba65 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -2,13 +2,20 @@ use super::*; #[derive(Clone)] pub(crate) struct ExpressionObject { - pub(crate) entries: BTreeMap, + pub(crate) entries: BTreeMap, /// The span range that generated this value. /// For a complex expression, the start span is the most left part /// of the expression, and the end span is the most right part. pub(crate) span_range: SpanRange, } +#[derive(Clone)] +pub(crate) struct ObjectEntry { + #[allow(unused)] + pub(crate) key_span: Span, + pub(crate) value: ExpressionValue, +} + impl ExpressionObject { pub(super) fn handle_unary_operation( self, @@ -52,49 +59,76 @@ impl ExpressionObject { } pub(super) fn into_indexed( - self, + mut self, access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult { let span_range = SpanRange::new_between(self.span_range, access.span_range()); let key = index.expect_string("An object key")?.value; - Ok(self.into_entry_or_none(&key, span_range)) + Ok(self.remove_or_none(&key, span_range)) } - pub(super) fn into_property(self, access: PropertyAccess) -> ExecutionResult { + pub(super) fn into_property( + mut self, + access: PropertyAccess, + ) -> ExecutionResult { let span_range = SpanRange::new_between(self.span_range, access.span_range()); let key = access.property.to_string(); - Ok(self.into_entry_or_none(&key, span_range)) + Ok(self.remove_or_none(&key, span_range)) } - fn into_entry_or_none(mut self, key: &str, span_range: SpanRange) -> ExpressionValue { + pub(crate) fn remove_or_none(&mut self, key: &str, span_range: SpanRange) -> ExpressionValue { match self.entries.remove(key) { - Some(value) => value.with_span_range(span_range), + Some(entry) => entry.value.with_span_range(span_range), None => ExpressionValue::None(span_range), } } + pub(crate) fn remove_no_none( + &mut self, + key: &str, + span_range: SpanRange, + ) -> Option { + match self.entries.remove(key) { + Some(entry) => { + if entry.value.is_none() { + None + } else { + Some(entry.value.with_span_range(span_range)) + } + } + None => None, + } + } + pub(super) fn index_mut( &mut self, - _access: IndexAccess, + access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult<&mut ExpressionValue> { let key = index.expect_string("An object key")?.value; - Ok(self.mut_entry_or_create(key)) + Ok(self.mut_entry_or_create(key, access.span())) } pub(super) fn property_mut( &mut self, access: PropertyAccess, ) -> ExecutionResult<&mut ExpressionValue> { - Ok(self.mut_entry_or_create(access.property.to_string())) + Ok(self.mut_entry_or_create(access.property.to_string(), access.property.span())) } - fn mut_entry_or_create(&mut self, key: String) -> &mut ExpressionValue { + fn mut_entry_or_create(&mut self, key: String, key_span: Span) -> &mut ExpressionValue { use std::collections::btree_map::*; match self.entries.entry(key) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => entry.insert(ExpressionValue::None(self.span_range)), + Entry::Occupied(entry) => &mut entry.into_mut().value, + Entry::Vacant(entry) => { + &mut entry + .insert(ObjectEntry { + key_span, + value: ExpressionValue::None(key_span.span_range()), + }) + .value + } } } @@ -114,7 +148,7 @@ impl ExpressionObject { } } let mut is_first = true; - for (key, value) in self.entries { + for (key, entry) in self.entries { if !is_first && behaviour.output_array_structure { output.push(','); } @@ -132,7 +166,7 @@ impl ExpressionObject { if behaviour.add_space_between_token_trees { output.push(' '); } - value.concat_recursive_into(output, behaviour)?; + entry.value.concat_recursive_into(output, behaviour)?; is_first = false; } if behaviour.output_array_structure { @@ -143,6 +177,53 @@ impl ExpressionObject { } Ok(()) } + + // TODO: Make ObjectValidation a trait, and have it implemented by the static + // define_field_inputs! macro + pub(crate) fn validate(&self, validation: &impl ObjectValidate) -> ExecutionResult<()> { + let mut missing_fields = Vec::new(); + for (field_name, _) in validation.required_fields() { + match self.entries.get(field_name) { + None + | Some(ObjectEntry { + value: ExpressionValue::None { .. }, + .. + }) => { + missing_fields.push(field_name); + } + _ => {} + } + } + let mut unexpected_fields = Vec::new(); + if !validation.allow_other_fields() { + let allowed_fields = validation.all_fields_set(); + for field_name in self.entries.keys() { + if !allowed_fields.contains(field_name) { + unexpected_fields.push(field_name.as_str()); + } + } + } + if missing_fields.is_empty() && unexpected_fields.is_empty() { + return Ok(()); + } + let mut error_message = String::new(); + error_message.push_str("Expected:\n"); + error_message.push_str(&validation.describe_object()); + + if !missing_fields.is_empty() { + error_message.push('\n'); + error_message.push_str("The following required field/s are missing: "); + error_message.push_str(&missing_fields.join(", ")); + } + + if !unexpected_fields.is_empty() { + error_message.push('\n'); + error_message.push_str("The following field/s are unexpected: "); + error_message.push_str(&unexpected_fields.join(", ")); + } + + self.span_range().execution_err(error_message) + } } impl HasSpanRange for ExpressionObject { @@ -157,13 +238,13 @@ impl HasValueType for ExpressionObject { } } -impl HasValueType for BTreeMap { +impl HasValueType for BTreeMap { fn value_type(&self) -> &'static str { "object" } } -impl ToExpressionValue for BTreeMap { +impl ToExpressionValue for BTreeMap { fn to_value(self, span_range: SpanRange) -> ExpressionValue { ExpressionValue::Object(ExpressionObject { entries: self, @@ -171,3 +252,82 @@ impl ToExpressionValue for BTreeMap { }) } } + +#[allow(unused)] +pub(crate) struct ObjectValidation { + // Should ideally be an indexmap + fields: Vec<(String, FieldDefinition)>, + allow_other_fields: bool, +} + +pub(crate) trait ObjectValidate { + fn all_fields(&self) -> Vec<(&str, &FieldDefinition)>; + fn allow_other_fields(&self) -> bool; + + fn required_fields(&self) -> Vec<(&str, &FieldDefinition)> { + self.all_fields() + .into_iter() + .filter_map(|(key, field_definition)| { + if field_definition.required { + Some((key, field_definition)) + } else { + None + } + }) + .collect() + } + + fn all_fields_set(&self) -> HashSet { + self.all_fields() + .into_iter() + .map(|(key, _)| key.to_string()) + .collect() + } + + fn describe_object(&self) -> String { + use std::fmt::Write; + let mut buffer = String::new(); + buffer.write_str("{\n").unwrap(); + for (key, definition) in self.all_fields() { + if let Some(description) = &definition.description { + writeln!(buffer, " // {}", description).unwrap(); + } + if definition.required { + writeln!(buffer, " {}: {},", key, definition.example).unwrap(); + } else { + writeln!(buffer, " {}?: {},", key, definition.example).unwrap(); + } + } + buffer.write_str("}").unwrap(); + buffer + } +} + +impl ObjectValidate for ObjectValidation { + fn all_fields(&self) -> Vec<(&str, &FieldDefinition)> { + self.fields + .iter() + .map(|(key, value)| (key.as_str(), value)) + .collect() + } + + fn allow_other_fields(&self) -> bool { + self.allow_other_fields + } +} + +impl ObjectValidate for &[(&'static str, FieldDefinition)] { + fn all_fields(&self) -> Vec<(&str, &FieldDefinition)> { + self.iter().map(|(k, v)| (*k, v)).collect() + } + + fn allow_other_fields(&self) -> bool { + false + } +} + +pub(crate) struct FieldDefinition { + pub(crate) required: bool, + pub(crate) description: Option>, + pub(crate) example: Cow<'static, str>, +} diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 21c3b2b8..2a7ec79e 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -248,6 +248,10 @@ impl ExpressionValue { }) } + pub(crate) fn is_none(&self) -> bool { + matches!(self, ExpressionValue::None(_)) + } + pub(crate) fn into_integer(self) -> Option { match self { ExpressionValue::Integer(value) => Some(value), diff --git a/src/internal_prelude.rs b/src/internal_prelude.rs index 8f01f1d6..dafebc32 100644 --- a/src/internal_prelude.rs +++ b/src/internal_prelude.rs @@ -5,6 +5,7 @@ pub(crate) use proc_macro2::extra::*; pub(crate) use proc_macro2::*; pub(crate) use quote::ToTokens; pub(crate) use std::{ + borrow::Cow, collections::{BTreeMap, HashMap, HashSet}, str::FromStr, }; diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index e54744de..5622fe3a 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -227,18 +227,18 @@ impl StreamCommandDefinition for ReinterpretCommand { #[derive(Clone)] pub(crate) struct SettingsCommand { - inputs: SettingsInputs, + inputs: SourceSettingsInputs, } impl CommandType for SettingsCommand { type OutputKind = OutputKindNone; } -define_field_inputs! { - SettingsInputs { +define_object_arguments! { + SourceSettingsInputs => SettingsInputs { required: {}, optional: { - iteration_limit: SourceExpression = DEFAULT_ITERATION_LIMIT ("The new iteration limit"), + iteration_limit: DEFAULT_ITERATION_LIMIT_STR ("The new iteration limit"), } } } @@ -253,9 +253,9 @@ impl NoOutputCommandDefinition for SettingsCommand { } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { - if let Some(limit) = self.inputs.iteration_limit { + let inputs = self.inputs.interpret_to_value(interpreter)?; + if let Some(limit) = inputs.iteration_limit { let limit = limit - .interpret_to_value(interpreter)? .expect_integer("The iteration limit")? .expect_usize()?; interpreter.set_iteration_limit(Some(limit)); @@ -275,17 +275,17 @@ impl CommandType for ErrorCommand { #[derive(Clone)] enum EitherErrorInput { - Fields(ErrorInputs), + Fields(SourceErrorInputs), JustMessage(SourceStream), } -define_field_inputs! { - ErrorInputs { +define_object_arguments! { + SourceErrorInputs => ErrorInputs { required: { - message: SourceExpression = r#""...""# ("The error message to display"), + message: r#""...""# ("The error message to display"), }, optional: { - spans: SourceExpression = "[!stream! $abc]" ("An optional [token stream], to determine where to show the error message"), + spans: "[!stream! $abc]" ("An optional [token stream], to determine where to show the error message"), } } } @@ -310,14 +310,16 @@ impl NoOutputCommandDefinition for ErrorCommand { }, format!( "Expected [!error! \"Expected X, found: \" #world] or [!error! {}]", - ErrorInputs::fields_description() + SourceErrorInputs::describe_object() ), ) } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult<()> { let fields = match self.inputs { - EitherErrorInput::Fields(error_inputs) => error_inputs, + EitherErrorInput::Fields(error_inputs) => { + error_inputs.interpret_to_value(interpreter)? + } EitherErrorInput::JustMessage(stream) => { let error_message = stream .interpret_to_new_stream(interpreter)? @@ -326,18 +328,11 @@ impl NoOutputCommandDefinition for ErrorCommand { } }; - let error_message = fields - .message - .interpret_to_value(interpreter)? - .expect_string("Error message")? - .value; + let error_message = fields.message.expect_string("Error message")?.value; let error_span = match fields.spans { Some(spans) => { - let error_span_stream = spans - .interpret_to_value(interpreter)? - .expect_stream("The error spans")? - .value; + let error_span_stream = spans.expect_stream("The error spans")?.value; // Consider the case where preinterpret embeds in a declarative macro, and we have // an error like this: diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index 1bc0ca92..d83e4270 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -85,22 +85,22 @@ impl StreamCommandDefinition for GroupCommand { #[derive(Clone)] pub(crate) struct IntersperseCommand { span: Span, - inputs: IntersperseInputs, + inputs: SourceIntersperseInputs, } impl CommandType for IntersperseCommand { type OutputKind = OutputKindValue; } -define_field_inputs! { - IntersperseInputs { +define_object_arguments! { + SourceIntersperseInputs => IntersperseInputs { required: { - items: SourceExpression = r#"["Hello", "World"]"# ("An array or stream (by coerced token-tree) to intersperse"), - separator: SourceExpression = "[!stream! ,]" ("The value to add between each item"), + items: r#"["Hello", "World"]"# ("An array or stream (by coerced token-tree) to intersperse"), + separator: "[!stream! ,]" ("The value to add between each item"), }, optional: { - add_trailing: SourceExpression = "false" ("Whether to add the separator after the last item (default: false)"), - final_separator: SourceExpression = "[!stream! or]" ("Define a different final separator (default: same as normal separator)"), + add_trailing: "false" ("Whether to add the separator after the last item (default: false)"), + final_separator: "[!stream! or]" ("Define a different final separator (default: same as normal separator)"), } } } @@ -116,19 +116,11 @@ impl ValueCommandDefinition for IntersperseCommand { } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let items = self - .inputs - .items - .interpret_to_value(interpreter)? - .expect_any_iterator("The items")?; + let inputs = self.inputs.interpret_to_value(interpreter)?; + let items = inputs.items.expect_any_iterator("The items")?; let output_span_range = self.span.span_range(); - let add_trailing = match self.inputs.add_trailing { - Some(add_trailing) => { - add_trailing - .interpret_to_value(interpreter)? - .expect_bool("This parameter")? - .value - } + let add_trailing = match inputs.add_trailing { + Some(add_trailing) => add_trailing.expect_bool("This parameter")?.value, None => false, }; @@ -142,8 +134,8 @@ impl ValueCommandDefinition for IntersperseCommand { }; let mut appender = SeparatorAppender { - separator: &self.inputs.separator, - final_separator: self.inputs.final_separator.as_ref(), + separator: inputs.separator, + final_separator: inputs.final_separator, add_trailing, }; @@ -157,11 +149,11 @@ impl ValueCommandDefinition for IntersperseCommand { } else { RemainingItemCount::ExactlyOne }; - appender.add_separator(interpreter, remaining, &mut output)?; + appender.add_separator(remaining, &mut output)?; this_item = next_item; } None => { - appender.add_separator(interpreter, RemainingItemCount::None, &mut output)?; + appender.add_separator(RemainingItemCount::None, &mut output)?; break; } } @@ -171,28 +163,23 @@ impl ValueCommandDefinition for IntersperseCommand { } } -struct SeparatorAppender<'a> { - separator: &'a SourceExpression, - final_separator: Option<&'a SourceExpression>, +struct SeparatorAppender { + separator: ExpressionValue, + final_separator: Option, add_trailing: bool, } -impl SeparatorAppender<'_> { +impl SeparatorAppender { fn add_separator( &mut self, - interpreter: &mut Interpreter, remaining: RemainingItemCount, output: &mut Vec, ) -> ExecutionResult<()> { match self.separator(remaining) { - TrailingSeparator::Normal => { - output.push(self.separator.interpret_to_value(interpreter)?) - } + TrailingSeparator::Normal => output.push(self.separator.clone()), TrailingSeparator::Final => match self.final_separator.take() { - Some(final_separator) => { - output.push(final_separator.interpret_to_value(interpreter)?) - } - None => output.push(self.separator.interpret_to_value(interpreter)?), + Some(final_separator) => output.push(final_separator), + None => output.push(self.separator.clone()), }, TrailingSeparator::None => {} } @@ -234,23 +221,23 @@ enum TrailingSeparator { #[derive(Clone)] pub(crate) struct SplitCommand { - inputs: SplitInputs, + inputs: SourceSplitInputs, } impl CommandType for SplitCommand { type OutputKind = OutputKindValue; } -define_field_inputs! { - SplitInputs { +define_object_arguments! { + SourceSplitInputs => SplitInputs { required: { - stream: SourceExpression = "[!stream! ...] or #var" ("The stream-valued expression to split"), - separator: SourceExpression = "[!stream! ::]" ("The token/s to split if they match"), + stream: "[!stream! ...] or #var" ("The stream-valued expression to split"), + separator: "[!stream! ::]" ("The token/s to split if they match"), }, optional: { - drop_empty_start: SourceExpression = "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), - drop_empty_middle: SourceExpression = "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), - drop_empty_end: SourceExpression = "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), + drop_empty_start: "false" ("If true, a leading separator does not yield in an empty item at the start (default: false)"), + drop_empty_middle: "false" ("If true, adjacent separators do not yield an empty item between them (default: false)"), + drop_empty_end: "true" ("If true, a trailing separator does not yield an empty item at the end (default: true)"), } } } @@ -265,44 +252,21 @@ impl ValueCommandDefinition for SplitCommand { } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let stream = self - .inputs - .stream - .interpret_to_value(interpreter)? - .expect_stream("The stream input")?; - - let separator = self - .inputs - .separator - .interpret_to_value(interpreter)? - .expect_stream("The separator")? - .value; - - let drop_empty_start = match self.inputs.drop_empty_start { - Some(value) => { - value - .interpret_to_value(interpreter)? - .expect_bool("This parameter")? - .value - } + let inputs = self.inputs.interpret_to_value(interpreter)?; + let stream = inputs.stream.expect_stream("The stream input")?; + + let separator = inputs.separator.expect_stream("The separator")?.value; + + let drop_empty_start = match inputs.drop_empty_start { + Some(value) => value.expect_bool("This parameter")?.value, None => false, }; - let drop_empty_middle = match self.inputs.drop_empty_middle { - Some(value) => { - value - .interpret_to_value(interpreter)? - .expect_bool("This parameter")? - .value - } + let drop_empty_middle = match inputs.drop_empty_middle { + Some(value) => value.expect_bool("This parameter")?.value, None => false, }; - let drop_empty_end = match self.inputs.drop_empty_end { - Some(value) => { - value - .interpret_to_value(interpreter)? - .expect_bool("This parameter")? - .value - } + let drop_empty_end = match inputs.drop_empty_end { + Some(value) => value.expect_bool("This parameter")?.value, None => true, }; @@ -415,17 +379,17 @@ pub(crate) struct ZipCommand { #[derive(Clone)] enum EitherZipInput { - Fields(ZipInputs), + Fields(SourceZipInputs), JustStream(SourceExpression), } -define_field_inputs! { - ZipInputs { +define_object_arguments! { + SourceZipInputs => ZipInputs { required: { - streams: SourceExpression = r#"[[!stream! Hello Goodbye] ["World", "Friend"]]"# ("An array of arrays/iterators/streams to zip together."), + streams: r#"[[!stream! Hello Goodbye] ["World", "Friend"]]"# ("An array of arrays/iterators/streams to zip together."), }, optional: { - error_on_length_mismatch: SourceExpression = "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), + error_on_length_mismatch: "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), } } } @@ -452,19 +416,23 @@ impl ValueCommandDefinition for ZipCommand { }, format!( "Expected [!zip! [... An array of iterables ...]] or [!zip! {}]", - ZipInputs::fields_description() + SourceZipInputs::describe_object() ), ) } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { let (streams, error_on_length_mismatch) = match self.inputs { - EitherZipInput::Fields(inputs) => (inputs.streams, inputs.error_on_length_mismatch), - EitherZipInput::JustStream(streams) => (streams, None), + EitherZipInput::Fields(inputs) => { + let inputs = inputs.interpret_to_value(interpreter)?; + (inputs.streams, inputs.error_on_length_mismatch) + } + EitherZipInput::JustStream(streams) => { + let streams = streams.interpret_to_value(interpreter)?; + (streams, None) + } }; - let streams = streams - .interpret_to_value(interpreter)? - .expect_array("The zip input")?; + let streams = streams.expect_array("The zip input")?; let output_span_range = streams.span_range; let mut output = Vec::new(); let mut iterators = streams @@ -474,12 +442,7 @@ impl ValueCommandDefinition for ZipCommand { .collect::, _>>()?; let error_on_length_mismatch = match error_on_length_mismatch { - Some(value) => { - value - .interpret_to_value(interpreter)? - .expect_bool("This parameter")? - .value - } + Some(value) => value.expect_bool("This parameter")?.value, None => true, }; diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 5918496a..4034cd2c 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -130,6 +130,7 @@ pub(crate) struct InterpreterConfig { } pub(crate) const DEFAULT_ITERATION_LIMIT: usize = 1000; +pub(crate) const DEFAULT_ITERATION_LIMIT_STR: &str = "1000"; impl Default for InterpreterConfig { fn default() -> Self { diff --git a/src/misc/field_inputs.rs b/src/misc/field_inputs.rs index c76197b5..bfffabcb 100644 --- a/src/misc/field_inputs.rs +++ b/src/misc/field_inputs.rs @@ -1,122 +1,122 @@ -macro_rules! define_field_inputs { +macro_rules! define_object_arguments { ( - $inputs_type:ident { + $source:ident => $validated:ident { required: { $( - $required_field:ident: $required_type:ty = $required_example:tt $(($required_description:literal))? + $required_field:ident: $required_example:tt $(($required_description:literal))? ),* $(,)? }$(,)? optional: { $( - $optional_field:ident: $optional_type:ty = $optional_example:tt $(($optional_description:literal))? + $optional_field:ident: $optional_example:tt $(($optional_description:literal))? ),* $(,)? }$(,)? } ) => { #[derive(Clone)] - struct $inputs_type { - $( - $required_field: $required_type, - )* + struct $source { + inner: SourceExpression, + } - $( - $optional_field: Option<$optional_type>, - )* + impl ArgumentsContent for $source { + fn error_message() -> String { + format!("Expected: {}", Self::describe_object()) + } } - impl Parse for $inputs_type { + impl Parse for $source { fn parse(input: ParseStream) -> ParseResult { - $( - let mut $required_field: Option<$required_type> = None; - )* - $( - let mut $optional_field: Option<$optional_type> = None; - )* - - let (braces, content) = input.parse_braces()?; - - while !content.is_empty() { - let ident = content.parse_any_ident()?; - content.parse::()?; - match ident.to_string().as_str() { - $( - stringify!($required_field) => { - if $required_field.is_some() { - return ident.parse_err("duplicate field"); - } - $required_field = Some(content.parse()?); - } - )* - $( - stringify!($optional_field) => { - if $optional_field.is_some() { - return ident.parse_err("duplicate field"); - } - $optional_field = Some(content.parse()?); - } - )* - _ => return ident.parse_err("unexpected field"), - } - if !content.is_empty() { - content.parse::()?; - } - } - - #[allow(unused_mut)] - let mut missing_fields: Vec = vec![]; - - $( - if $required_field.is_none() { - missing_fields.push(stringify!($required_field).to_string()); - } - )* + Ok(Self { inner: input.parse()? }) + } + } - if !missing_fields.is_empty() { - return braces.parse_err(format!( - "required fields are missing: {}", - missing_fields.join(", ") - )); - } + impl InterpretToValue for &$source { + type OutputValue = $validated; - $( - let $required_field = $required_field.unwrap(); - )* - - Ok(Self { + fn interpret_to_value( + self, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut object = self.inner.interpret_to_value(interpreter)? + .expect_object("The arguments")?; + object.validate(&$source::validation())?; + let span_range = object.span_range; + Ok($validated { $( - $required_field, + $required_field: object.remove_or_none(stringify!($required_field), span_range), )* $( - $optional_field, + $optional_field: object.remove_no_none(stringify!($optional_field), span_range), )* }) } } - impl ArgumentsContent for $inputs_type { - fn error_message() -> String { - format!("Expected: {}", Self::fields_description()) - } + struct $validated { + $( + $required_field: ExpressionValue, + )* + $( + $optional_field: Option, + )* } - impl $inputs_type { - fn fields_description() -> String { - use std::fmt::Write; - let mut buffer = String::new(); - buffer.write_str("{\n").unwrap(); + impl $source { + fn describe_object() -> String { + Self::validation().describe_object() + } + + fn validation() -> &'static [(&'static str, FieldDefinition)] { + &Self::VALIDATION + } + + const VALIDATION: [ + (&'static str, FieldDefinition); + count_fields!($($required_field)* $($optional_field)*) + ] = [ $( - $(writeln!(buffer, " // {}", $required_description).unwrap();)? - writeln!(buffer, " {}: {},", stringify!($required_field), $required_example).unwrap(); + ( + stringify!($required_field), + FieldDefinition { + required: true, + description: optional_else!{ + { $(Some(std::borrow::Cow::Borrowed($required_description)))? } + { None } + }, + example: std::borrow::Cow::Borrowed($required_example), + }, + ), )* $( - $(writeln!(buffer, " // {}", $optional_description).unwrap();)? - writeln!(buffer, " {}?: {},", stringify!($optional_field), $optional_example).unwrap(); + ( + stringify!($optional_field), + FieldDefinition { + required: false, + description: optional_else!{ + { $(Some(std::borrow::Cow::Borrowed($optional_description)))? } + { None } + }, + example: std::borrow::Cow::Borrowed($optional_example), + }, + ), )* - buffer.write_str("}").unwrap(); - buffer - } + ]; } }; } -pub(crate) use define_field_inputs; +pub(crate) use define_object_arguments; + +macro_rules! count_fields { + ($head:tt $($tail:tt)*) => { 1 + count_fields!($($tail)*)}; + () => { 0 } +} + +pub(crate) use count_fields; + +macro_rules! optional_else { + ({$($content:tt)+} {$($else:tt)*}) => { $($content)+ }; + ({} {$($else:tt)*}) => { $($else)* } +} + +pub(crate) use optional_else; diff --git a/src/transformation/patterns.rs b/src/transformation/patterns.rs index d568f26d..3871ac53 100644 --- a/src/transformation/patterns.rs +++ b/src/transformation/patterns.rs @@ -190,6 +190,7 @@ impl HandleDestructure for ObjectPattern { } let value = value_map .remove(&key) + .map(|entry| entry.value) .unwrap_or_else(|| ExpressionValue::None(key_span.span_range())); already_used_keys.insert(key); pattern.handle_destructure(interpreter, value)?; diff --git a/tests/compilation_failures/complex/nested.stderr b/tests/compilation_failures/complex/nested.stderr index 0c6b8cac..2d08d99a 100644 --- a/tests/compilation_failures/complex/nested.stderr +++ b/tests/compilation_failures/complex/nested.stderr @@ -1,10 +1,11 @@ -error: required fields are missing: message - Occurred whilst parsing [!error! ...] - Expected [!error! "Expected X, found: " #world] or [!error! { +error: Expected: + { // The error message to display message: "...", // An optional [token stream], to determine where to show the error message spans?: [!stream! $abc], - }] + } + The following required field/s are missing: message --> tests/compilation_failures/complex/nested.rs:7:26 | 7 | [!error! { diff --git a/tests/compilation_failures/core/error_invalid_structure.stderr b/tests/compilation_failures/core/error_invalid_structure.stderr index 851a6f35..aa2d9809 100644 --- a/tests/compilation_failures/core/error_invalid_structure.stderr +++ b/tests/compilation_failures/core/error_invalid_structure.stderr @@ -1,10 +1,11 @@ -error: required fields are missing: message - Occurred whilst parsing [!error! ...] - Expected [!error! "Expected X, found: " #world] or [!error! { +error: Expected: + { // The error message to display message: "...", // An optional [token stream], to determine where to show the error message spans?: [!stream! $abc], - }] + } + The following required field/s are missing: message --> tests/compilation_failures/core/error_invalid_structure.rs:4:28 | 4 | preinterpret!([!error! { }]); diff --git a/tests/tokens.rs b/tests/tokens.rs index 27d98800..dcca9ebf 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -140,41 +140,6 @@ fn test_intersperse() { #[test] fn complex_cases_for_intersperse_and_input_types() { - // Normal separator is not interpreted if it is unneeded - preinterpret_assert_eq!( - #([!intersperse! { - items: [!stream!], - separator: [!error! { message: "FAIL" }], - add_trailing: true, - }] as stream as string), - "" - ); - // Final separator is not interpreted if it is unneeded - preinterpret_assert_eq!( - #([!intersperse! { - items: [!stream!], - separator: [!stream!], - final_separator: [!error! { message: "FAIL" }], - add_trailing: true, - }] as stream as string), - "" - ); - // The separator is interpreted each time it is included - preinterpret_assert_eq!( - #( - let i = 0; - [!intersperse! { - items: [!stream! A B C D E F G], - separator: #( - let output = [!stream! (#i)]; - i += 1; - output - ), - add_trailing: true, - }] as string - ), - "A(0)B(1)C(2)D(3)E(4)F(5)G(6)", - ); // Variable containing stream be used for items preinterpret_assert_eq!({ [!set! #items = 0 1 2 3] From ed266b4444e11970536e962b1442f9f576121b69 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 20 Feb 2025 14:51:17 +0000 Subject: [PATCH 110/126] feature: Zip also supports objects --- src/interpretation/command.rs | 1 + src/interpretation/commands/token_commands.rs | 259 ++++++++++++------ .../zip_different_length_streams.stderr | 2 +- .../tokens/zip_no_input.stderr | 7 +- tests/tokens.rs | 23 +- 5 files changed, 193 insertions(+), 99 deletions(-) diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index fa2593f7..128e3a75 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -389,6 +389,7 @@ define_command_enums! { SplitCommand, CommaSplitCommand, ZipCommand, + ZipTruncatedCommand, // Destructuring Commands ParseCommand, diff --git a/src/interpretation/commands/token_commands.rs b/src/interpretation/commands/token_commands.rs index d83e4270..d3c9da3f 100644 --- a/src/interpretation/commands/token_commands.rs +++ b/src/interpretation/commands/token_commands.rs @@ -374,118 +374,213 @@ impl ValueCommandDefinition for CommaSplitCommand { #[derive(Clone)] pub(crate) struct ZipCommand { - inputs: EitherZipInput, + inputs: SourceExpression, } -#[derive(Clone)] -enum EitherZipInput { - Fields(SourceZipInputs), - JustStream(SourceExpression), +impl CommandType for ZipCommand { + type OutputKind = OutputKindValue; } -define_object_arguments! { - SourceZipInputs => ZipInputs { - required: { - streams: r#"[[!stream! Hello Goodbye] ["World", "Friend"]]"# ("An array of arrays/iterators/streams to zip together."), - }, - optional: { - error_on_length_mismatch: "true" ("If false, uses shortest stream length, if true, errors on unequal length. Defaults to true."), - } +impl ValueCommandDefinition for ZipCommand { + const COMMAND_NAME: &'static str = "zip"; + + fn parse(arguments: CommandArguments) -> ParseResult { + arguments.fully_parse_or_error( + |input| { + Ok(Self { + inputs: input.parse()?, + }) + }, + "Expected [!zip! [, , ..]] or [!zip! {{ x: , y: , .. }}] for iterable values of the same length. If you instead want to permit different lengths and truncate to the shortest, use `!zip_truncated!` instead.", + ) + } + + fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { + zip_inner( + self.inputs.interpret_to_value(interpreter)?, + interpreter, + true, + ) } } -impl CommandType for ZipCommand { +#[derive(Clone)] +pub(crate) struct ZipTruncatedCommand { + inputs: SourceExpression, +} + +impl CommandType for ZipTruncatedCommand { type OutputKind = OutputKindValue; } -impl ValueCommandDefinition for ZipCommand { - const COMMAND_NAME: &'static str = "zip"; +impl ValueCommandDefinition for ZipTruncatedCommand { + const COMMAND_NAME: &'static str = "zip_truncated"; fn parse(arguments: CommandArguments) -> ParseResult { arguments.fully_parse_or_error( |input| { - if input.peek(syn::token::Brace) { - Ok(Self { - inputs: EitherZipInput::Fields(input.parse()?), - }) - } else { - Ok(Self { - inputs: EitherZipInput::JustStream(input.parse()?), - }) - } + Ok(Self { + inputs: input.parse()?, + }) }, - format!( - "Expected [!zip! [... An array of iterables ...]] or [!zip! {}]", - SourceZipInputs::describe_object() - ), + "Expected [!zip_truncated! [, , ..]] or [!zip_truncated! {{ x: , y: , .. }}] for iterable values of possible different lengths (the shortest length will be used). If you want to ensure the lengths are equal, use `!zip!` instead", ) } fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let (streams, error_on_length_mismatch) = match self.inputs { - EitherZipInput::Fields(inputs) => { - let inputs = inputs.interpret_to_value(interpreter)?; - (inputs.streams, inputs.error_on_length_mismatch) + zip_inner( + self.inputs.interpret_to_value(interpreter)?, + interpreter, + false, + ) + } +} + +fn zip_inner( + iterators: ExpressionValue, + interpreter: &mut Interpreter, + error_on_length_mismatch: bool, +) -> ExecutionResult { + let output_span_range = iterators.span_range(); + let mut iterators = ZipIterators::from_value(iterators)?; + let mut output = Vec::new(); + + if iterators.len() == 0 { + return Ok(output.to_value(output_span_range)); + } + + let (min_iterator_min_length, max_iterator_max_length) = iterators.size_hint_range(); + + if error_on_length_mismatch && Some(min_iterator_min_length) != max_iterator_max_length { + return output_span_range.execution_err(format!( + "The iterables have different lengths. The lengths vary from {} to {}. To truncate to the shortest, use `!zip_truncated!` instead of `!zip!", + min_iterator_min_length, + match max_iterator_max_length { + Some(max_max) => max_max.to_string(), + None => "unbounded".to_string(), + }, + )); + } + + iterators.zip_into( + min_iterator_min_length, + interpreter, + output_span_range, + &mut output, + )?; + + Ok(output.to_value(output_span_range)) +} + +enum ZipIterators { + Array(Vec), + Object(Vec<(String, Span, ExpressionIterator)>), +} + +impl ZipIterators { + fn from_value(value: ExpressionValue) -> ExecutionResult { + Ok(match value { + ExpressionValue::Object(object) => { + let span_range = object.span_range; + let entries = object + .entries + .into_iter() + .take(101) + .map(|(k, v)| -> ExecutionResult<_> { + Ok(( + k, + v.key_span, + v.value.expect_any_iterator("A zip iterator")?, + )) + }) + .collect::, _>>()?; + if entries.len() == 101 { + return span_range.execution_err("A maximum of 100 iterators are allowed"); + } + ZipIterators::Object(entries) } - EitherZipInput::JustStream(streams) => { - let streams = streams.interpret_to_value(interpreter)?; - (streams, None) + other => { + let span_range = other.span_range(); + let iterator = other.expect_any_iterator("") + .map_err(|_| span_range.execution_error("Expected an object with iterable values, an array of iterables, or some other iterator of iterables."))?; + let vec = iterator + .take(101) + .map(|x| x.expect_any_iterator("A zip iterator")) + .collect::, _>>()?; + if vec.len() == 101 { + return span_range.execution_err("A maximum of 100 iterators are allowed"); + } + ZipIterators::Array(vec) } - }; - let streams = streams.expect_array("The zip input")?; - let output_span_range = streams.span_range; - let mut output = Vec::new(); - let mut iterators = streams - .items - .into_iter() - .map(|x| x.expect_any_iterator("A zip input")) - .collect::, _>>()?; + }) + } - let error_on_length_mismatch = match error_on_length_mismatch { - Some(value) => value.expect_bool("This parameter")?.value, - None => true, + /// Panics if called on an empty list of iterators + fn size_hint_range(&self) -> (usize, Option) { + let size_hints: Vec<_> = match self { + ZipIterators::Array(inner) => inner.iter().map(|x| x.size_hint()).collect(), + ZipIterators::Object(inner) => inner.iter().map(|x| x.2.size_hint()).collect(), }; + let min_min = size_hints.iter().map(|s| s.0).min().unwrap(); + let max_max = size_hints + .iter() + .map(|sh| sh.1) + .max_by(|a, b| match (a, b) { + (None, None) => core::cmp::Ordering::Equal, + (None, Some(_)) => core::cmp::Ordering::Greater, + (Some(_), None) => core::cmp::Ordering::Less, + (Some(a), Some(b)) => a.cmp(b), + }) + .unwrap(); + (min_min, max_max) + } - if iterators.is_empty() { - return Ok(output.to_value(output_span_range)); - } - - let min_stream_length = iterators.iter().map(|x| x.size_hint().0).min().unwrap(); - - if error_on_length_mismatch { - let max_stream_length = iterators - .iter() - .map(|x| x.size_hint().1) - .max_by(|a, b| match (a, b) { - (None, None) => core::cmp::Ordering::Equal, - (None, Some(_)) => core::cmp::Ordering::Greater, - (Some(_), None) => core::cmp::Ordering::Less, - (Some(a), Some(b)) => a.cmp(b), - }) - .unwrap(); - if Some(min_stream_length) != max_stream_length { - return output_span_range.execution_err(format!( - "Streams have different lengths and zip's error_on_length_mismatch is true. The lengths vary from {} to {}", - min_stream_length, - match max_stream_length { - Some(max_stream_length) => max_stream_length.to_string(), - None => "unbounded".to_string(), - }, - )); - } + fn len(&self) -> usize { + match self { + ZipIterators::Array(inner) => inner.len(), + ZipIterators::Object(inner) => inner.len(), } + } + /// Panics if count is larger than the minimum length of any iterator + fn zip_into( + &mut self, + count: usize, + interpreter: &mut Interpreter, + output_span_range: SpanRange, + output: &mut Vec, + ) -> ExecutionResult<()> { let mut counter = interpreter.start_iteration_counter(&output_span_range); - for _ in 0..min_stream_length { - counter.increment_and_check()?; - let mut inner = Vec::with_capacity(iterators.len()); - for iter in iterators.iter_mut() { - inner.push(iter.next().unwrap()); + match self { + ZipIterators::Array(iterators) => { + for _ in 0..count { + counter.increment_and_check()?; + let mut inner = Vec::with_capacity(iterators.len()); + for iter in iterators.iter_mut() { + inner.push(iter.next().unwrap()); + } + output.push(inner.to_value(output_span_range)); + } + } + ZipIterators::Object(iterators) => { + for _ in 0..count { + counter.increment_and_check()?; + let mut inner = BTreeMap::new(); + for (key, key_span, iter) in iterators.iter_mut() { + inner.insert( + key.clone(), + ObjectEntry { + key_span: *key_span, + value: iter.next().unwrap(), + }, + ); + } + output.push(inner.to_value(output_span_range)); + } } - output.push(inner.to_value(output_span_range)); } - Ok(output.to_value(output_span_range)) + Ok(()) } } diff --git a/tests/compilation_failures/tokens/zip_different_length_streams.stderr b/tests/compilation_failures/tokens/zip_different_length_streams.stderr index 757cc110..671b7a37 100644 --- a/tests/compilation_failures/tokens/zip_different_length_streams.stderr +++ b/tests/compilation_failures/tokens/zip_different_length_streams.stderr @@ -1,4 +1,4 @@ -error: Streams have different lengths and zip's error_on_length_mismatch is true. The lengths vary from 3 to 4 +error: The iterables have different lengths. The lengths vary from 3 to 4. To truncate to the shortest, use `!zip_truncated!` instead of `!zip! --> tests/compilation_failures/tokens/zip_different_length_streams.rs:5:16 | 5 | [!zip! [["A", "B", "C"], [1, 2, 3, 4]]] diff --git a/tests/compilation_failures/tokens/zip_no_input.stderr b/tests/compilation_failures/tokens/zip_no_input.stderr index 9dd7214c..de103dd5 100644 --- a/tests/compilation_failures/tokens/zip_no_input.stderr +++ b/tests/compilation_failures/tokens/zip_no_input.stderr @@ -1,10 +1,5 @@ error: Expected an expression - Occurred whilst parsing [!zip! ...] - Expected [!zip! [... An array of iterables ...]] or [!zip! { - // An array of arrays/iterators/streams to zip together. - streams: [[!stream! Hello Goodbye] ["World", "Friend"]], - // If false, uses shortest stream length, if true, errors on unequal length. Defaults to true. - error_on_length_mismatch?: true, - }] + Occurred whilst parsing [!zip! ...] - Expected [!zip! [, , ..]] or [!zip! {{ x: , y: , .. }}] for iterable values of the same length. If you instead want to permit different lengths and truncate to the shortest, use `!zip_truncated!` instead. --> tests/compilation_failures/tokens/zip_no_input.rs:5:15 | 5 | [!zip!] diff --git a/tests/tokens.rs b/tests/tokens.rs index dcca9ebf..cbbcaa94 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -327,9 +327,7 @@ fn test_zip() { [!set! #countries = "France" "Germany" "Italy"] [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] [!set! #capitals = "Paris" "Berlin" "Rome"] - [!debug! [!zip! { - streams: [countries, flags, capitals], - }]] + [!debug! [!zip! [countries, flags, capitals]]] }, r#"[["France", "🇫🇷", "Paris"], ["Germany", "🇩🇪", "Berlin"], ["Italy", "🇮🇹", "Rome"]]"#, ); @@ -337,10 +335,7 @@ fn test_zip() { { [!set! #longer = A B C D] [!set! #shorter = 1 2 3] - [!debug! [!zip! { - streams: [longer, shorter], - error_on_length_mismatch: false, - }]] + [!debug! [!zip_truncated! [longer, shorter]]] }, r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); @@ -348,9 +343,7 @@ fn test_zip() { { [!set! #letters = A B C] #(let numbers = [1, 2, 3]) - [!debug! [!zip! { - streams: [letters, numbers], - }]] + [!debug! [!zip! [letters, numbers]]] }, r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); @@ -363,7 +356,17 @@ fn test_zip() { }, r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); + preinterpret_assert_eq!( + { + [!set! #letters = A B C] + #(let numbers = [1, 2, 3]) + #(let letter = [letters, numbers]) + [!debug! [!zip! { number: numbers, letter: letters }]] + }, + r#"[{ letter: [!stream! A], number: 1 }, { letter: [!stream! B], number: 2 }, { letter: [!stream! C], number: 3 }]"#, + ); preinterpret_assert_eq!([!debug![!zip![]]], r#"[]"#,); + preinterpret_assert_eq!([!debug![!zip! {}]], r#"[]"#,); } #[test] From 8718e5f922dbf44c5706b278d09dcc5b5b6157fd Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 4 Mar 2025 20:51:35 +0000 Subject: [PATCH 111/126] tweak: Support `let x;` syntax. --- CHANGELOG.md | 6 +---- src/expressions/expression_block.rs | 39 ++++++++++++++++++++++------- tests/expressions.rs | 6 ++--- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b837b9cd..ca08d9ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,9 +96,6 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* Objects continuation: - * Have `!zip!` support `{ objects }` -* Support `let x;` * Method calls on values * Mutable params notes: * For now we can assume all params are values/clones, no mutable references @@ -114,8 +111,7 @@ Inside a transform stream, the following grammar is supported: * Merge `GroupedVariable` and `ExpressionBlock` into an `ExplicitExpression`: * Either `#ident` or `#(...)` or `#{ ... }`... the latter defines a new variable stack frame, just like Rust - * To avoid confusion (such as below) and teach the user to only include #var where - necessary, only expression _blocks_ are allowed in an expression. + * To avoid confusion (such as below) and teach the user to only include #var where necessary, only expression _blocks_ are allowed in an expression. * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal rust because the inside is a `{ .. }` which defines a new scope. * TRANSFORMERS => PARSERS cont * Re-read the `Parsers Revisited` section below diff --git a/src/expressions/expression_block.rs b/src/expressions/expression_block.rs index c69fca4e..12bd9005 100644 --- a/src/expressions/expression_block.rs +++ b/src/expressions/expression_block.rs @@ -138,6 +138,11 @@ impl InterpretToValue for &Statement { pub(crate) struct LetStatement { let_token: Token![let], pattern: Pattern, + assignment: Option, +} + +#[derive(Clone)] +struct LetStatementAssignment { #[allow(unused)] equals: Token![=], expression: SourceExpression, @@ -145,12 +150,26 @@ pub(crate) struct LetStatement { impl Parse for LetStatement { fn parse(input: ParseStream) -> ParseResult { - Ok(Self { - let_token: input.parse()?, - pattern: input.parse()?, - equals: input.parse()?, - expression: input.parse()?, - }) + let let_token = input.parse()?; + let pattern = input.parse()?; + if input.peek(Token![=]) { + Ok(Self { + let_token, + pattern, + assignment: Some(LetStatementAssignment { + equals: input.parse()?, + expression: input.parse()?, + }), + }) + } else if input.is_empty() || input.peek(Token![;]) { + Ok(Self { + let_token, + pattern, + assignment: None, + }) + } else { + input.parse_err("Expected = or ;") + } } } @@ -164,10 +183,12 @@ impl InterpretToValue for &LetStatement { let LetStatement { let_token, pattern, - expression, - .. + assignment, } = self; - let value = expression.interpret_to_value(interpreter)?; + let value = match assignment { + Some(assignment) => assignment.expression.interpret_to_value(interpreter)?, + None => ExpressionValue::None(let_token.span.span_range()), + }; let mut span_range = value.span_range(); pattern.handle_destructure(interpreter, value)?; span_range.set_start(let_token.span); diff --git a/tests/expressions.rs b/tests/expressions.rs index 4314efd6..3497e8f8 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -446,9 +446,9 @@ fn test_objects() { ); preinterpret_assert_eq!( #( - let a = 0; - let b = 0; - let z = 0; + let a; + let b; + let z; { a, y: [_, b], z } = { a: 1, y: [5, 7] }; { a, b, z } as debug ), From b6d0988f7b1664fe7f9ef796753ca362aa34a5aa Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 15 Aug 2025 19:15:50 +0100 Subject: [PATCH 112/126] docs: Update CHANGELOG with better plan --- CHANGELOG.md | 281 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 176 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca08d9ea..0242e8a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ * Core commands: * `[!error! ...]` to output a compile error. - * `[!set! #x += ...]` to performantly add extra characters to the stream. + * `[!set! #x += ...]` to performantly add extra characters to a variable's stream. * `[!set! _ = ...]` interprets its arguments but then ignores any outputs. * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. * `[!stream! ...]` can be used to just output its interpreted contents. It's useful to create a stream value inside an expression. @@ -102,49 +102,58 @@ Inside a transform stream, the following grammar is supported: * Also add support for methods (for e.g. exposing functions on syn objects). * `.len()` on stream * `.push(x)` on array - * Consider `.map(|| {})` -* Reference equality for streams, objects and arrays - * And new `.clone()` method + * `.push(x)` on token stream +* Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write +* No clone required for testing equality of streams, objects and arrays + * Some kind of reference support + * Add new `.clone()` method * Introduce interpreter stack frames + * Read the `REVERSION` comment in the `Parsers Revisited` section below to consider approaches, which will work with possibly needing + to revert state if a parser fails. * Design the data model => is it some kind of linked list of frames? - * Add a new frame inside loops + * If we introduce functions in future, then a reference is a closure, in _lexical_ scope, which is different from stack frames. + * We probably don't want to allow closures to start with. + * Maybe we simply pass in a lexical parent frame when resolving variable references? + * Possibly with some local cache of parent references, so we don't need to re-discover them if they're used multiple times in a loop + e.g. `x` in `#(let x = 0; {{{ loop { x++; if x < 10 { break }} }}})` + * Add a new frame inside loops, or wherever we see `{ ... }` * Merge `GroupedVariable` and `ExpressionBlock` into an `ExplicitExpression`: * Either `#ident` or `#(...)` or `#{ ... }`... the latter defines a new variable stack frame, just like Rust * To avoid confusion (such as below) and teach the user to only include #var where necessary, only expression _blocks_ are allowed in an expression. * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal rust because the inside is a `{ .. }` which defines a new scope. + * The `#()` syntax can be used in certain places (such as parse streams) +* Introduce `~(...)` and `r~(...)` streams instead of `[!stream! ...]` and `[!raw! ...]` +* Support for/while/loop/break/continue inside expressions + * And allow them to start with an expression block with `{}` or `#{}` or a stream block with `~{}` + QUESTION: Do we require either `#{}` or `~{}`? Maybe! That way we can give a helpful error message. + * ... and remove the control flow commands `[!if! ...]` / `[!while! ...]` / `[!break! ...]` / `[!for! ...]`, `[!while! ...]` and `[!loop! ...]` * TRANSFORMERS => PARSERS cont - * Re-read the `Parsers Revisited` section below * Manually search for transform and rename to parse in folder names and file. * Parsers no longer output to a stream. + * See all of `Parsers Revisited` below! * We let `#(let x)` INSIDE a `@(...)` bind to the same scope as its surroundings... Now some repetition like `@{..}*` needs to introduce its own scope. ==> Annoyingly, a `@(..)*` has to really push/output to an array internally to have good developer experience; which means we need to solve the "relatively performant staged/reverted interpreter state" problem regardless; and putting an artificial limitation on conditional parse streams to not do that doesn't really work. TBC - needs more consideration. - ==> Maybe there is an alternative such as: - * `@(..)*` returns an array of some output of it's inner thing - * But things don't output so what does that mean?? - * I think in practice it will be quite an annoying restriction anyway - * Support `#(x = @IDENT)` where `#` blocks inside parsers can embed @parsers and consume from a parse stream. - * This makes it kinda like a `Parse` implementation code block. - * This lets us just support variable definitions in expression statements. * Don't support `@(x = ...)` - instead we can have `#(x = @[STREAM ...])` - * This can capture the original tokens by using `let forked = input.fork()` - and then `let end_cursor = input.end();` and then consuming `TokenTree`s - from `forked` until `forked.cursor >= end_cursor` (making use of the - PartialEq implementation) - * Revisit `parse` + * This can capture the original tokens by using `let forked = input.fork()` and then `let end_cursor = input.end();` and then consuming `TokenTree`s + from `forked` until `forked.cursor >= end_cursor` (making use of the PartialEq implementation) + * Scrap `[!set!]` in favour of `#(x = ..)` and `#(x += ..)` * Scrap `#>>x` etc in favour of `#(a.push(@[XXX]))` - * Scrap `[!let!]` in favour of `#(let = #x)` + * Scrap `[!let!]` in favour of `#(let = x)` +* Implement the following named parsers * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. * `@INFER_TOKEN_TREE` - Infers values, falls back to Stream * `@[ANY_GROUP ...]` * `@REST` + * `@[FORK @{ ...parser... }]` the parser block creates a `commit=false` variable, if this is set to `commit=true` then it commits the fork. + * `@[PEEK ...]` which does a `@[FORK ...]` internally * `@[FIELDS { ... }]` and `@[SUBFIELDS { ... }]` * Add ability to add scope to interpreter state (see `Parsers Revisited`) and can then add: * `@[OPTIONAL ...]` and `@(...)?` * `@[REPEATED { ... }]` (see below) - * `@[UNTIL @{...}]` - takes an explicit parse block or parser. Reverts any state change + * `@[UNTIL @{...}]` - takes an explicit parse block or parser. Reverts any state change if it doesn't match. * `[!match! ...]` command * `@[MATCH { ... }]` (with `#..x` as a catch-all) with optional arms... * `#(..)?`, `#(..)+`, `#(..),+`, `#(..)*`, `#(..),*` @@ -172,13 +181,13 @@ Inside a transform stream, the following grammar is supported: * Adding all of these: https://veykril.github.io/tlborm/decl-macros/minutiae/fragment-specifiers.html#ty * Adding `preinterpret::macro` * Adding `!define_command!` - * Adding `!define_transformer!` + * Adding `!define_parser!` * Whether `preinterpret` should start in expression mode? => Or whether to have `preinterpet::stream` / `preinterpret::run` / `preinterpret::define_macro` options? => Maybe `preinterpret::preinterpret` is marked as deprecated; starts in `stream` mode, and enables `[!set!]`? * `[!is_set! #x]` * Have UntypedInteger have an inner representation of either i128 or literal (and same with float) -* Maybe add a `@[REINTERPRET ..]` transformer. +* Maybe add a `@[REINTERPRET ..]` parser. * CastTarget expansion: * Add `as iterator` and uncomment the test at the end of `test_range()` * Support a CastTarget of `array` using `into_iterator()`. @@ -201,117 +210,179 @@ Inside a transform stream, the following grammar is supported: ### Parsers Revisited ```rust // PROPOSAL (subject to the object proposal above): -// * Four modes: -// * Output stream mode -// * Outputs stream, does not parse -// * Can embed [!commands! ...] and #() -// * #([...]) gets appended to the stream -// * #( ... ) = Expression mode. -// * One or more statements, terminated by ; (except maybe the last) +// * Various modes... +// * EXPRESSION MODE +// * OPENING SYNTAX: +// * #{ ... } or #'name { ... } with a new lexical variable scope +// * #( ... ) with no new scope (TBC if we need both - probably) +// * CONTEXT: +// * Is *not* stream-based - it consists of expressions rather than a stream +// * It *may* have a contextual input parse stream +// * One or more statements, terminated by ; (except maybe the last, which is returned) // * Idents mean variables. -// * `let #x; let _ = ; ; if {} else {}` +// * `let x; let _ = ; ; if {} else {}` // * Error not to discard a non-None value with `let _ = ` -// * If embedded into a parse stream, it's allowed to parse by embedding parsers, e.g. @[STREAM ...] -// * Only last statement can (optionally) output, with a value model of: +// * If it has an input parse stream, it's allowed to parse by embedding parsers, e.g. @TOKEN_TREE +// * Only the last statement can (optionally) output, with a value model of: // * Leaf(Bool | Int | Float | String) // * Object // * Stream // * None +// * (In future we could add breaking from labelled block: https://blog.rust-lang.org/2022/11/03/Rust-1.65.0/#break-from-labeled-blocks) // * The actual type is only known at evaluation time. // * #var is equivalent to #(var) -// * Named parser @XXX or @[XXX] or @[XXX ] -// * Can parse; has no output. -// * XXX is: -// * A named destructurer (possibly taking further input) -// * An { .. } object destructurer -// * A @(...) transformer stream (output = input) -// * @() or @{} = Parse block -// * Expression/command outputs are parsed exactly. -// * Can return a value with `#(return X)` -// * (`@[STREAM ...]` is broadly equivalent, but does output the input stream) -// * Its only output is an input stream reference -// * ...and only if it's redirected inside a @[x = $(...)] -// * [!command! ...] -// * Can output but does not parse. -// * Every named parser has a typed output, which is: -// * EITHER its input token stream (for simple matchers, e.g. @IDENT) -// * OR an #output OBJECT with at least two properties: -// * input => all matched characters (a slice reference which can be dropped...) -// (it might only be possible to performantly capture this after the syn fork) -// * output_to => A lazy function, used to handle the output when #x is in the final output... -// likely `input` or an error depending on the case. -// * into_iterator -// * ... other properties, depending on the TRANSFORMER: -// * e.g. a Rust ITEM might have quite a few (mostly lazy) +// * OUTPUT STREAM MODE +// * OPENING SYNTAX: ~{ ... } or ~( ... ) or raw stream literal with r~( ... ) +// * SIDENOTES: I'd have liked to use $, but it clashes with $() repetition syntax inside macro rules +// * And % might appear in real code as 1 % (2 + 3) +// * But ~ seems to not be used much, and looks a bit like an s. So good enough? +// * CONTEXT: +// * Can make use of a parent output stream, by concatenating into it. +// * Is stream-based: Anything embedded to it can have their outputs concatenated onto the output stream +// * Does *not* have an input parse stream +// * Can embed [!commands! ...], #() and #{ ... }, but not parsers +// * The output from #([...]) gets appended to the stream; ideally by passing it an optional output stream +// * It has a corresponding pattern: +// * SYNTAX: +// * ~#( .. ) to start in expression mode OR +// * ~@( .. ) ~@XXX or ~@[XXX ...] to start with a parser e.g. ~@REST can be used to parse any stream opaquely OR +// * r~( .. ) to match a raw stream (it doesn't have a mode!) +// * Can be used where patterns can be used, in a let expression or a match arm, e.g. +// * let ~#( let x = @IDENT ) = r~(...) +// * ~#( ... ) => { ... } +// * CONTEXT: +// * It implicitly creates a parse-stream for its contents +// * It would never have an output stream +// * It is the only syntax which can create a parse stream, other than the [!parse! ..] command +// * NAMED PARSER +// * OPENING SYNTAX: @XXX or @[XXX] or @[XXX ] +// * CONTEXT: +// * Is typically *not* stream-based - it may has its own arguments, which are typically a pseudo-object `{ }` +// * It has an input parse stream +// * Every named parser @XXX has a typed output, which is: +// * EITHER its input token stream (for simple matchers, e.g. @IDENT) +// * OR an #output OBJECT with at least two properties: +// * input => all matched characters (a slice reference which can be dropped...) +// (it might only be possible to performantly capture this after the syn fork) +// THIS NOTE MIGHT BE RELEVANT, BUT I'M WRITING THIS AFTER 6 MONTHS LACKING CONTEXT: +// This can capture the original tokens by using `let forked = input.fork()` +// and then `let end_cursor = input.end();` and then consuming `TokenTree`s +// from `forked` until `forked.cursor >= end_cursor` (making use of the +// PartialEq implementation) +// * output_to => A lazy function, used to handle the output when #x is embedded into a stream output... +// likely `input` or an error depending on the case. +// * into_iterator +// * ... other properties, depending on the parsee: +// * e.g. a Rust ITEM might have quite a few (mostly lazy) +// * PARSE-STREAM MODE +// * OPENING SYNTAX: +// * @{ ... } or @'block_name { ... } (as a possibly named block) -- both have variable scoping semantics +// * We also support various repetitions such as: @{ ... }? and @{ ... }+ and @{ ... },+ - discussed in next section in more detail +// * We likely don't want/need an un-scoped parse stream +// * CONTEXT: +// * It is stream-based: Any outputs of items in the stream are converted to an exact parse matcher +// * Does have an input parse stream which it can mutate +// * Expression/command outputs are converted into an exact parse matcher +// * RETURN VALUE: +// * Parse blocks can return a value with `#(return X)` or `#(return 'block_name X)` else return None +// * By contrast, `@[STREAM ...]` works very similarly, but outputs its input stream +// * COMMAND +// * OPENING SYNTAX: [!command! ...] +// * CONTEXT: +// * Takes an optional output stream from its parent (for efficiency, if it returns a stream it can instead concat to that) +// * May or may not be stream-based depending on the command... +// * Does *not* have an input parse stream (I think? Maybe it's needed. TBC) +// * Lots of commands can/should probably become functions +// * A flexible utility for different kinds of functionality +// // * Repetitions (including optional), e.g. @IDENT,* or @[IDENT ...],* or @{...},* // * Output their contents in a `Repeated` type, with an `items` property, and an into_iterator implementation? -// * Transform streams can't be repeated, only transform blocks (because they create a new interpreter frame) -// * There is still an issue with "what happens to mutated state when a repetition is not possible?" -// ... this is also true in a case statement... We need some way to rollback in these cases: -// => Easier - Use of efficient-ish immutable data structures, e.g. ImmutableList, Copy-on-write leaf types etc -// ... so that we can clone them cheaply (e.g. https://github.com/orium/rpds) or even just -// some manually written tries/cons-lists -// => CoW - We do some kind of copy-on-write - but it still give O(N^2) if we're reverting occasional array.push(..)es -// [BEST]? => Middle - Any changes to variables outside the scope of a given refutable parser => it's a fatal error +// * Unscoped parse streams can't be repeated, only parse blocks (because they create a new interpreter frame) +// * If we don't move our cursor forward in a loop iteration, it's an error -- this prevents @{}* or @{x?}* causing an infinite loop +// * We do *not* support backtracking. +// * This avoids most kind of catastrophic backtracking explosions, e.g. (x+x+)+y) +// * Things such as @IDENT+ @IDENT will not match `hello world` - this is OK I think +// * Instead, we suggest people to use a match statement or something +// * There is still an issue of REVERSION - "what happens to external state mutated this iteration repetition when a repetition is not possible?" +// * This appears in lots of places: +// * In ? or * blocks where an error isn't fatal, but should continue +// * In match arms, considering various stream parsers, some of which might fail after mutating state +// * In nested * blocks, with different levels of reversion +// * But it's only a problem with state lexically captured from a parent block +// * We need some way to handle these cases: +// (A) Immutable structures - We use efficient-ish immutable data structures, e.g. ImmutableList, Copy-on-write leaf types etc +// ... so that we can clone them cheaply (e.g. https://github.com/orium/rpds) or even just some manually written tries/cons-lists +// (B) Use CoW/extensions - But naively it still give O(N^2) if we're reverting occasional array.push(..)es +// We could possibly add in some tweaks to avoid clones in some special cases (e.g. array.push and array.pop which don't touch a prefix at the start of a frame) +// (C) Error on mutate - Any changes to variables outside the scope of a given refutable parser => it's a fatal error // to parse further until that scope is closed. (this needs to handle nested repetitions). -// [EASIEST?] OR conversely - at reversion time, we check there have been no changes to variables below that depth in the stack tree (say by recording a "lowest_stack_touched: usize"), and panic if so; and tell people to use `return` instead; or move state changes to the end. -// => Hardest - Some kind of storing of diffs / layered stores without any O(N^2) behaviours +// (D) Error on revert - at reversion time, we check there have been no changes to variables below that depth in the stack tree +// (say by recording a "lowest_stack_touched: usize"), and panic if so; and tell people to use `return` instead; or move state changes to the end. // -// EXAMPLE (Updated, 17th February) -[!parse! { - input: [...], - parser: #( - let parsed = @{ +// ==> D might be easiest, most flexible AND most performant... But it might not cover enough use cases. +// ... but I think advising people to only mutate lexical-closure'd state at the end, when a parse is confirmed, is reasonably... +// +// +// QUESTION: +// - What mode do we start in, in v2? +// => Possibly expression mode, which could convert to an output with ~{ ... } +// +// ========= +// EXAMPLES +// ========= + +// EXAMPLE WITH !parse! COMMAND +#( + let parsed = [!parse! { + input: r~( + impl A for X, impl B for Y + ), + parser: @{ // This @{ .. }, returns a `Repeated` type with an `into_iterator` over its items impl #(let the_trait = @IDENT) for #(let the_type = @IDENT) #(return { the_trait, the_type }) },* - ), -}] -// EXAMPLE IN PATTERN POSITION + }]; + + // Assuming we are writing into an output stream (e.g. outputting from the macro), + // we can auto-optimize - the last command of the expression block gets to write directly to the stream, + // and by extension, each block of the for-loop gets to write directly to the stream + for { the_trait, the_type } in parsed ~{ + impl #the_trait for #the_type {} + } +) + +// EXAMPLE WITH STREAM-PARSING-PATTERN #( - let [!parser! // Takes an expression, which could include a #(...) - let parsed = @( + let ~#( // Defines a stream pattern, starting in expression mode, and can be used to bind variables + let parsed = @{ #(let item = {}) impl #(item.the_trait = @IDENT) for #(item.the_type = @IDENT) #(return item) - )* - ] = [...] -) + } + ) = r~(...) -[!for! { the_trait, the_type } in parsed { - impl #the_trait for #the_type {} -}] -// Example inlined: -[!for! { the_trait, the_type } in [!parse! { - input: [...], - parser: @( // This is implicitly an expression, which returns a `Repeated` type with an `into_iterator` over its items. - impl #(let the_trait = @IDENT) for #(let the_type = @IDENT) - #(return { the_trait, the_type }) - ),* -}] { - impl #the_trait for #the_type {} -}] + for { the_trait, the_type } in parsed ~{ + impl #the_trait for #the_type {} + } +) -// Example pre-defined: -[!parser! @IMPL_ITEM { - @(impl); - let the_trait = @IDENT; - @(for); - let the_type = @IDENT; - return { the_trait, the_type }; +// Example pre-defined with no arguments: +[!define_parser! @IMPL_ITEM @{ // Can either start as #{ ... } or @{ ... } + impl #(let the_trait = @IDENT) for #(let the_type = @IDENT) + #(return { the_trait, the_type }) }] -[!for! { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] { +for { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] ~{ impl #the_trait for #the_type {} -}] +} -// Example pre-defined 2: -[!parser! @[IMPL_ITEM /* argument parse stream */] { +// Example pre-defined 2 with arguments: +[!define_parser! @[IMPL_ITEM /* arguments named-parser or parse-stream */] { @(impl #(let the_trait = @IDENT) for #(let the_type = @IDENT)); { the_trait, the_type } }] -[!for! { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] { +for { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] ~{ impl #the_trait for #the_type {} -}] +} ``` * Pushed to 0.4: From c7027a9604913c589d5c8527b3e59a08bd15ca69 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 00:23:28 +0100 Subject: [PATCH 113/126] feature: Added method call support to expression parsing --- CHANGELOG.md | 14 +-- src/expressions/evaluation.rs | 58 +++++++++++ src/expressions/expression.rs | 49 +++++---- src/expressions/expression_parsing.rs | 139 ++++++++++++++++++-------- src/expressions/operations.rs | 13 +++ src/expressions/value.rs | 30 ++++++ tests/expressions.rs | 12 +++ 7 files changed, 249 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0242e8a7..55367174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,13 +96,15 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* Method calls on values - * Mutable params notes: - * For now we can assume all params are values/clones, no mutable references +* Method calls continues * Also add support for methods (for e.g. exposing functions on syn objects). - * `.len()` on stream - * `.push(x)` on array - * `.push(x)` on token stream + * Add support for &mut methods like `push(..)`... how? unclear. + * Maybe the method set-up is derived by calling a method on &ExpressionValue + which then tells us whether we want/need self to be a reference, mutable reference, or owned value + * Maybe parameters should be some cow-like enum for Value or (variable) Reference? + * For now we can assume all non-self parameters are effectively read-only references, no mutability + * Then add `.push(x)` on array and stream + * Improve support to make it easier to add methods * Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write * No clone required for testing equality of streams, objects and arrays * Some kind of reference support diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 294d65f7..30d8a083 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -288,6 +288,13 @@ impl ExpressionNode { access: access.clone(), }, ), + ExpressionNode::MethodCall { node, method, parameters } => next.read_value_with_handler( + *node, + ValueStackFrame::MethodCall(MethodCallStackFrame::CallerPath { + method: method.clone(), + parameters: parameters.clone(), + }), + ), ExpressionNode::Index { node, access, @@ -452,6 +459,7 @@ enum ValueStackFrame { Property { access: PropertyAccess, }, + MethodCall(MethodCallStackFrame), Index { access: IndexAccess, state: IndexPath, @@ -488,6 +496,7 @@ impl ValueStackFrame { ValueStackFrame::UnaryOperation { .. } => ReturnMode::Value, ValueStackFrame::BinaryOperation { .. } => ReturnMode::Value, ValueStackFrame::Property { .. } => ReturnMode::Value, + ValueStackFrame::MethodCall { .. } => ReturnMode::Value, ValueStackFrame::Index { .. } => ReturnMode::Value, ValueStackFrame::Range { .. } => ReturnMode::Value, ValueStackFrame::HandleValueForAssignment { .. } => ReturnMode::Value, @@ -533,6 +542,7 @@ impl ValueStackFrame { } }, ValueStackFrame::Property { access } => next.return_value(value.into_property(access)?), + ValueStackFrame::MethodCall(call) => call.handle_value(value, next)?, ValueStackFrame::Index { access, state } => match state { IndexPath::OnObjectBranch { index } => next.read_value_with_handler( index, @@ -638,6 +648,54 @@ impl ValueStackFrame { } } +enum MethodCallStackFrame { + CallerPath { + method: MethodAccess, + parameters: Vec, + }, + ArgumentsPath { + caller: ExpressionValue, + method: MethodAccess, + unevaluated_parameters: Vec, + evaluated_parameters: Vec, + }, +} + +impl MethodCallStackFrame { + fn handle_value(self, value: ExpressionValue, action_creator: ActionCreator) -> ExecutionResult { + let (caller, method, unevaluated_parameters, evaluated_parameters) = match self { + MethodCallStackFrame::CallerPath { + method, + parameters, + } => (value, method, parameters, Vec::new()), + MethodCallStackFrame::ArgumentsPath { + caller, + method, + unevaluated_parameters, + mut evaluated_parameters, + } => { + evaluated_parameters.push(value); + (caller, method, unevaluated_parameters, evaluated_parameters) + } + }; + let next_action = match unevaluated_parameters + .get(evaluated_parameters.len()) + .cloned() + { + Some(next) => { + action_creator.read_value_with_handler(next, ValueStackFrame::MethodCall(MethodCallStackFrame::ArgumentsPath { + caller, + method, + unevaluated_parameters, + evaluated_parameters, + })) + } + None => action_creator.return_value(caller.call_method(method, evaluated_parameters)?), + }; + Ok(next_action) + } +} + struct ArrayValueStackFrame { span: Span, unevaluated_items: Vec, diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 75682ee0..040e03dc 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -1,3 +1,5 @@ +use syn::token; + use super::*; // Source @@ -129,23 +131,17 @@ impl Expressionable for Source { } SourcePeekMatch::Punct(punct) if punct.as_char() == ',' => { match parent_stack_frame { - ExpressionStackFrame::Array { .. } => { - input.parse::()?; - if input.is_current_empty() { - return Ok(NodeExtension::EndOfStream); - } else { - return Ok(NodeExtension::NonTerminalArrayComma); - } - } - ExpressionStackFrame::Object { + ExpressionStackFrame::NonEmptyArray { .. } + | ExpressionStackFrame::NonEmptyMethodCallParametersList { .. } + | ExpressionStackFrame::NonEmptyObject { state: ObjectStackFrameState::EntryValue { .. }, .. } => { input.parse::()?; if input.is_current_empty() { - return Ok(NodeExtension::EndOfStream); + return Ok(NodeExtension::EndOfStreamOrGroup); } else { - return Ok(NodeExtension::NonTerminalObjectValueComma); + return Ok(NodeExtension::NonTerminalComma); } } ExpressionStackFrame::Group { .. } => { @@ -157,9 +153,19 @@ impl Expressionable for Source { } SourcePeekMatch::Punct(punct) => { if punct.as_char() == '.' && input.peek2(syn::Ident) { + let dot = input.parse()?; + let ident = input.parse()?; + if input.peek(token::Paren) { + let (_, delim_span) = input.parse_and_enter_group()?; + return Ok(NodeExtension::MethodCall(MethodAccess { + dot, + method: ident, + parentheses: Parentheses { delim_span }, + })); + } return Ok(NodeExtension::Property(PropertyAccess { - dot: input.parse()?, - property: input.parse()?, + dot, + property: ident, })); } if let Ok(operation) = input.try_parse_or_revert() { @@ -180,7 +186,7 @@ impl Expressionable for Source { UnaryOperation::for_cast_operation(input.parse()?, input.parse_any_ident()?)?; return Ok(NodeExtension::PostfixOperation(cast_operation)); } - SourcePeekMatch::End => return Ok(NodeExtension::EndOfStream), + SourcePeekMatch::End => return Ok(NodeExtension::EndOfStreamOrGroup), _ => {} }; // We are not at the end of the stream, but the tokens which follow are @@ -188,16 +194,19 @@ impl Expressionable for Source { match parent_stack_frame { ExpressionStackFrame::Root => Ok(NodeExtension::NoValidExtensionForCurrentParent), ExpressionStackFrame::Group { .. } => input.parse_err("Expected ) or operator"), - ExpressionStackFrame::Array { .. } => input.parse_err("Expected comma, ], or operator"), + ExpressionStackFrame::NonEmptyArray { .. } => input.parse_err("Expected comma, ], or operator"), ExpressionStackFrame::IncompleteIndex { .. } - | ExpressionStackFrame::Object { + | ExpressionStackFrame::NonEmptyObject { state: ObjectStackFrameState::EntryIndex { .. }, .. } => input.parse_err("Expected ], or operator"), - ExpressionStackFrame::Object { + ExpressionStackFrame::NonEmptyObject { state: ObjectStackFrameState::EntryValue { .. }, .. } => input.parse_err("Expected comma, }, or operator"), + ExpressionStackFrame::NonEmptyMethodCallParametersList { .. } => { + input.parse_err("Expected comma, ) or operator") + } // e.g. I've just matched the true in !true or false || true, // and I want to see if there's an extension (e.g. a cast). // There's nothing matching, so we fall through to an EndOfFrame @@ -286,6 +295,11 @@ pub(super) enum ExpressionNode { node: ExpressionNodeId, access: PropertyAccess, }, + MethodCall { + node: ExpressionNodeId, + method: MethodAccess, + parameters: Vec, + }, Index { node: ExpressionNodeId, access: IndexAccess, @@ -315,6 +329,7 @@ impl ExpressionNode { ExpressionNode::Grouped { delim_span, .. } => delim_span.span_range(), ExpressionNode::Array { brackets, .. } => brackets.span_range(), ExpressionNode::Object { braces, .. } => braces.span_range(), + ExpressionNode::MethodCall { method, .. } => method.span_range(), ExpressionNode::Property { access, .. } => access.span_range(), ExpressionNode::Index { access, .. } => access.span_range(), ExpressionNode::UnaryOperation { operation, .. } => operation.span_range(), diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index 4d1838e2..c908a2bb 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -83,7 +83,7 @@ impl<'a> ExpressionParser<'a, Source> { }), } } else { - self.push_stack_frame(ExpressionStackFrame::Array { + self.push_stack_frame(ExpressionStackFrame::NonEmptyArray { brackets, items: Vec::new(), }) @@ -125,28 +125,38 @@ impl<'a> ExpressionParser<'a, Source> { operation, }) } - NodeExtension::NonTerminalArrayComma => { - match self.expression_stack.last_mut().unwrap() { - ExpressionStackFrame::Array { items, .. } => { + NodeExtension::NonTerminalComma => { + let next_item = match self.expression_stack.last_mut().unwrap() { + ExpressionStackFrame::NonEmptyMethodCallParametersList { parameters, .. } => { + parameters.push(node); + Some(WorkItem::RequireUnaryAtom) + } + ExpressionStackFrame::NonEmptyArray { items, .. } => { items.push(node); + Some(WorkItem::RequireUnaryAtom) + } + ExpressionStackFrame::NonEmptyObject { .. } => { + None // Placeholder to indicate separate handling below } _ => unreachable!( - "NonTerminalArrayComma is only returned under an Array parent." + "NonTerminalComma is only returned under an Array, Object or MethodCallParametersList parent." ), - } - WorkItem::RequireUnaryAtom - } - NodeExtension::NonTerminalObjectValueComma => { - match self.expression_stack.pop().unwrap() { - ExpressionStackFrame::Object { - braces, - state: ObjectStackFrameState::EntryValue(key, _), - mut complete_entries, - } => { - complete_entries.push((key, node)); - self.continue_object(braces, complete_entries)? + }; + match next_item { + Some(item) => item, + None => { // Indicates object + match self.expression_stack.pop().unwrap() { + ExpressionStackFrame::NonEmptyObject { + braces, + state: ObjectStackFrameState::EntryValue(key, _), + mut complete_entries, + } => { + complete_entries.push((key, node)); + self.continue_object(braces, complete_entries)? + } + _ => unreachable!("This None code path is only reachable under an object"), + } } - _ => unreachable!("NonTerminalObjectValueComma is only returned under an Object EntryValue parent."), } } NodeExtension::Property(access) => WorkItem::TryParseAndApplyExtension { @@ -154,6 +164,23 @@ impl<'a> ExpressionParser<'a, Source> { .nodes .add_node(ExpressionNode::Property { node, access }), }, + NodeExtension::MethodCall(method) => { + if self.streams.is_current_empty() { + self.streams.exit_group(); + let node = self.nodes.add_node(ExpressionNode::MethodCall { + node, + method, + parameters: Vec::new(), + }); + WorkItem::TryParseAndApplyExtension { node } + } else { + self.push_stack_frame(ExpressionStackFrame::NonEmptyMethodCallParametersList { + node, + method, + parameters: Vec::new(), + }) + } + } NodeExtension::Index(access) => { self.push_stack_frame(ExpressionStackFrame::IncompleteIndex { node, access }) } @@ -173,7 +200,7 @@ impl<'a> ExpressionParser<'a, Source> { operation, }) } - NodeExtension::EndOfStream | NodeExtension::NoValidExtensionForCurrentParent => { + NodeExtension::EndOfStreamOrGroup | NodeExtension::NoValidExtensionForCurrentParent => { unreachable!("Not possible, as these have minimum precedence") } }) @@ -185,13 +212,13 @@ impl<'a> ExpressionParser<'a, Source> { ExpressionStackFrame::Root => { assert!(matches!( extension, - NodeExtension::EndOfStream + NodeExtension::EndOfStreamOrGroup | NodeExtension::NoValidExtensionForCurrentParent )); WorkItem::Finished { root: node } } ExpressionStackFrame::Group { delim_span } => { - assert!(matches!(extension, NodeExtension::EndOfStream)); + assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); self.streams.exit_group(); WorkItem::TryParseAndApplyExtension { node: self.nodes.add_node(ExpressionNode::Grouped { @@ -200,11 +227,11 @@ impl<'a> ExpressionParser<'a, Source> { }), } } - ExpressionStackFrame::Array { + ExpressionStackFrame::NonEmptyArray { mut items, brackets, } => { - assert!(matches!(extension, NodeExtension::EndOfStream)); + assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); items.push(node); self.streams.exit_group(); WorkItem::TryParseAndApplyExtension { @@ -213,15 +240,30 @@ impl<'a> ExpressionParser<'a, Source> { .add_node(ExpressionNode::Array { brackets, items }), } } - ExpressionStackFrame::Object { + ExpressionStackFrame::NonEmptyMethodCallParametersList { + node: source, + mut parameters, + method, + } => { + assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); + parameters.push(node); + self.streams.exit_group(); + let node = self.nodes.add_node(ExpressionNode::MethodCall { + node: source, + method, + parameters, + }); + WorkItem::TryParseAndApplyExtension { node } + } + ExpressionStackFrame::NonEmptyObject { braces, complete_entries, state: ObjectStackFrameState::EntryIndex(access), } => { - assert!(matches!(extension, NodeExtension::EndOfStream)); + assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); self.streams.exit_group(); let colon = self.streams.parse()?; - self.expression_stack.push(ExpressionStackFrame::Object { + self.expression_stack.push(ExpressionStackFrame::NonEmptyObject { braces, complete_entries, state: ObjectStackFrameState::EntryValue( @@ -234,12 +276,12 @@ impl<'a> ExpressionParser<'a, Source> { }); WorkItem::RequireUnaryAtom } - ExpressionStackFrame::Object { + ExpressionStackFrame::NonEmptyObject { braces, complete_entries: mut entries, state: ObjectStackFrameState::EntryValue(key, _), } => { - assert!(matches!(extension, NodeExtension::EndOfStream)); + assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); self.streams.exit_group(); entries.push((key, node)); let node = self @@ -266,7 +308,7 @@ impl<'a> ExpressionParser<'a, Source> { node: source, access, } => { - assert!(matches!(extension, NodeExtension::EndOfStream)); + assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); self.streams.exit_group(); let node = self.nodes.add_node(ExpressionNode::Index { node: source, @@ -394,7 +436,7 @@ impl<'a> ExpressionParser<'a, Source> { return self.streams.parse_err(ERROR_MESSAGE); } }; - Ok(self.push_stack_frame(ExpressionStackFrame::Object { + Ok(self.push_stack_frame(ExpressionStackFrame::NonEmptyObject { braces, complete_entries, state, @@ -662,18 +704,26 @@ pub(super) enum ExpressionStackFrame { /// A marker for the bracketed array. /// * When the array is opened, we add its inside to the parse stream stack /// * When the array is closed, we pop it from the parse stream stack - Array { + NonEmptyArray { brackets: Brackets, items: Vec, }, /// A marker for an object literal. /// * When the object is opened, we add its inside to the parse stream stack /// * When the object is closed, we pop it from the parse stream stack - Object { + NonEmptyObject { braces: Braces, complete_entries: Vec<(ObjectKey, ExpressionNodeId)>, state: ObjectStackFrameState, }, + /// A method call with a possibly incomplete list of parameters. + /// * When the method parameters list is opened, we add its inside to the parse stream stack + /// * When the method parameters list is closed, we pop it from the parse stream stack + NonEmptyMethodCallParametersList { + node: ExpressionNodeId, + method: MethodAccess, + parameters: Vec, + }, /// An incomplete unary prefix operation /// NB: unary postfix operations such as `as` casting go straight to ExtendableNode IncompleteUnaryPrefixOperation { operation: PrefixUnaryOperation }, @@ -730,8 +780,9 @@ impl ExpressionStackFrame { match self { ExpressionStackFrame::Root => OperatorPrecendence::MIN, ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, - ExpressionStackFrame::Array { .. } => OperatorPrecendence::MIN, - ExpressionStackFrame::Object { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::NonEmptyArray { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::NonEmptyObject { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::NonEmptyMethodCallParametersList { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteIndex { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteRange { .. } => OperatorPrecendence::Range, ExpressionStackFrame::IncompleteAssignment { .. } => OperatorPrecendence::Assign, @@ -790,14 +841,15 @@ pub(super) enum UnaryAtom { pub(super) enum NodeExtension { PostfixOperation(UnaryOperation), BinaryOperation(BinaryOperation), - NonTerminalArrayComma, - NonTerminalObjectValueComma, + /// Used under Arrays, Objects, and Method Parameter List parents + NonTerminalComma, Property(PropertyAccess), + MethodCall(MethodAccess), Index(IndexAccess), Range(syn::RangeLimits), AssignmentOperation(Token![=]), CompoundAssignmentOperation(CompoundAssignmentOperation), - EndOfStream, + EndOfStreamOrGroup, NoValidExtensionForCurrentParent, } @@ -806,12 +858,12 @@ impl NodeExtension { match self { NodeExtension::PostfixOperation(op) => OperatorPrecendence::of_unary_operation(op), NodeExtension::BinaryOperation(op) => OperatorPrecendence::of_binary_operation(op), - NodeExtension::NonTerminalArrayComma => OperatorPrecendence::NonTerminalComma, - NodeExtension::NonTerminalObjectValueComma => OperatorPrecendence::NonTerminalComma, + NodeExtension::NonTerminalComma => OperatorPrecendence::NonTerminalComma, NodeExtension::Property { .. } => OperatorPrecendence::Unambiguous, + NodeExtension::MethodCall { .. } => OperatorPrecendence::Unambiguous, NodeExtension::Index { .. } => OperatorPrecendence::Unambiguous, NodeExtension::Range(_) => OperatorPrecendence::Range, - NodeExtension::EndOfStream => OperatorPrecendence::MIN, + NodeExtension::EndOfStreamOrGroup => OperatorPrecendence::MIN, NodeExtension::AssignmentOperation(_) => OperatorPrecendence::AssignExtension, NodeExtension::CompoundAssignmentOperation(_) => OperatorPrecendence::AssignExtension, NodeExtension::NoValidExtensionForCurrentParent => OperatorPrecendence::MIN, @@ -825,15 +877,16 @@ impl NodeExtension { extension @ (NodeExtension::PostfixOperation { .. } | NodeExtension::BinaryOperation { .. } | NodeExtension::Property { .. } + | NodeExtension::MethodCall { .. } | NodeExtension::Index { .. } | NodeExtension::Range { .. } | NodeExtension::AssignmentOperation { .. } | NodeExtension::CompoundAssignmentOperation { .. } - | NodeExtension::EndOfStream) => { + | NodeExtension::EndOfStreamOrGroup) => { WorkItem::TryApplyAlreadyParsedExtension { node, extension } } - NodeExtension::NonTerminalArrayComma | NodeExtension::NonTerminalObjectValueComma => { - unreachable!("Commas is only possible on array or object parent") + NodeExtension::NonTerminalComma => { + unreachable!("Comma is only possible on method parameter list or array or object parent") } NodeExtension::NoValidExtensionForCurrentParent => { // We have to reparse in case the extension is valid for the new parent. diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index dcd410ec..d584dc25 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -668,6 +668,19 @@ impl HasSpanRange for PropertyAccess { } } +#[derive(Clone)] +pub(crate) struct MethodAccess { + pub(super) dot: Token![.], + pub(crate) method: Ident, + pub(crate) parentheses: Parentheses, +} + +impl HasSpanRange for MethodAccess { + fn span_range(&self) -> SpanRange { + SpanRange::new_between(self.dot.span, self.parentheses.span()) + } +} + #[derive(Copy, Clone)] pub(crate) struct IndexAccess { pub(crate) brackets: Brackets, diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 2a7ec79e..f7bbcc0a 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -457,6 +457,36 @@ impl ExpressionValue { } } + pub(crate) fn call_method(self, method: MethodAccess, parameters: Vec) -> ExecutionResult { + // TODO: Make this more extensible / usable + match self { + ExpressionValue::Array(array) => match method.method.to_string().as_str() { + "len" => { + match parameters.as_slice() { + [] => {} + _ => return method.execution_err(format!("The `len` method does not take parameters")), + } + let len = array.items.len(); + return Ok(len.to_value(method.span_range())) + } + _ => {} + } + ExpressionValue::Stream(stream) => match method.method.to_string().as_str() { + "len" => { + match parameters.as_slice() { + [] => {} + _ => return method.execution_err(format!("The `len` method does not take parameters")), + } + let len = stream.value.len(); + return Ok(len.to_value(method.span_range())) + } + _ => {} + } + _ => {}, + } + method.execution_err("Methods are not currently supported") + } + pub(crate) fn into_property(self, access: PropertyAccess) -> ExecutionResult { match self { ExpressionValue::Object(object) => object.into_property(access), diff --git a/tests/expressions.rs b/tests/expressions.rs index 3497e8f8..fbf10c2e 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -462,3 +462,15 @@ fn test_objects() { r#"{ a: 1, b: 7, c: None, x: {}, z: None }"# ); } + + +#[test] +fn test_method_calls() { + preinterpret_assert_eq!( + #( + let x = [1, 2, 3]; + x.len() + [!stream! "Hello" world].len() + ), + 2 + 3 + ); +} \ No newline at end of file From bbbe671b0c6a4430a4463fcea53859c5baa5dfb7 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 00:44:34 +0100 Subject: [PATCH 114/126] Add parsing support for method calls --- CHANGELOG.md | 13 +++++-------- src/expressions/evaluation.rs | 7 +++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55367174..9526d4bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,15 +96,13 @@ Inside a transform stream, the following grammar is supported: * `@(inner = ...) [!stream! #inner]` - wraps the output in a transparent group ### To come -* Method calls continues - * Also add support for methods (for e.g. exposing functions on syn objects). +* Method calls continued * Add support for &mut methods like `push(..)`... how? unclear. - * Maybe the method set-up is derived by calling a method on &ExpressionValue - which then tells us whether we want/need self to be a reference, mutable reference, or owned value - * Maybe parameters should be some cow-like enum for Value or (variable) Reference? - * For now we can assume all non-self parameters are effectively read-only references, no mutability + * See comment in `evaluation.rs` `handle_as_value` above `ExpressionNode::MethodCall` + * Possibly parameters should be some cow-like enum for Value or (variable) Reference? * Then add `.push(x)` on array and stream - * Improve support to make it easier to add methods + * Scrap `#>>x` etc in favour of `#(a.push(@XXX))` + * Also improve support to make it easier to add methods on built-in types or (in future) user-designed types * Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write * No clone required for testing equality of streams, objects and arrays * Some kind of reference support @@ -141,7 +139,6 @@ Inside a transform stream, the following grammar is supported: * This can capture the original tokens by using `let forked = input.fork()` and then `let end_cursor = input.end();` and then consuming `TokenTree`s from `forked` until `forked.cursor >= end_cursor` (making use of the PartialEq implementation) * Scrap `[!set!]` in favour of `#(x = ..)` and `#(x += ..)` - * Scrap `#>>x` etc in favour of `#(a.push(@[XXX]))` * Scrap `[!let!]` in favour of `#(let = x)` * Implement the following named parsers * `@TOKEN_TREE` diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 30d8a083..2a91317b 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -132,6 +132,7 @@ mod inner { ReadNodeAsAssignee(ExpressionNodeId, ExpressionValue), HandleAssignmentComplete(AssignmentCompletion), // Enters an expression node to output a place + // A place can be thought of as a mutable reference for e.g. a += operation ReadNodeAsPlace(ExpressionNodeId), HandleReturnedPlace(Place), } @@ -288,6 +289,12 @@ impl ExpressionNode { access: access.clone(), }, ), + // TODO - we need to instead: + // * Resolve the expression value type of the node (e.g. from a & reference to the node) + // * For that value type, the method name and arguments, determine if the given method call: + // * Takes a self, &self or &mut self... + // * Whether each parameter is owned, & or &mut + // * Read the node and its parameters, and execute the method call ExpressionNode::MethodCall { node, method, parameters } => next.read_value_with_handler( *node, ValueStackFrame::MethodCall(MethodCallStackFrame::CallerPath { From d9a9ebef08d3e0f5cf2da64e15f0964507db63c4 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 13:21:01 +0100 Subject: [PATCH 115/126] fix: Fix warnings and compiler errors --- .github/workflows/ci.yml | 2 +- src/expressions/evaluation.rs | 34 +++++++++------ src/expressions/expression.rs | 4 +- src/expressions/expression_parsing.rs | 59 ++++++++++++++++----------- src/expressions/value.rs | 34 +++++++++------ src/interpretation/interpreter.rs | 2 +- src/misc/parse_traits.rs | 16 ++++---- src/transformation/patterns.rs | 2 +- tests/expressions.rs | 3 +- 9 files changed, 94 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b60bacb9..bb670873 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ permissions: contents: read env: - RUSTFLAGS: -Dwarnings + RUSTFLAGS: -Dwarnings # Deny warnings - i.e. warnings cause build errors jobs: test: diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs index 2a91317b..bb5ab361 100644 --- a/src/expressions/evaluation.rs +++ b/src/expressions/evaluation.rs @@ -109,7 +109,7 @@ mod inner { } } - fn creator(&mut self, return_mode: ReturnMode) -> ActionCreator { + fn creator(&mut self, return_mode: ReturnMode) -> ActionCreator<'_> { ActionCreator { stacks: self, return_mode, @@ -295,7 +295,11 @@ impl ExpressionNode { // * Takes a self, &self or &mut self... // * Whether each parameter is owned, & or &mut // * Read the node and its parameters, and execute the method call - ExpressionNode::MethodCall { node, method, parameters } => next.read_value_with_handler( + ExpressionNode::MethodCall { + node, + method, + parameters, + } => next.read_value_with_handler( *node, ValueStackFrame::MethodCall(MethodCallStackFrame::CallerPath { method: method.clone(), @@ -669,12 +673,15 @@ enum MethodCallStackFrame { } impl MethodCallStackFrame { - fn handle_value(self, value: ExpressionValue, action_creator: ActionCreator) -> ExecutionResult { + fn handle_value( + self, + value: ExpressionValue, + action_creator: ActionCreator, + ) -> ExecutionResult { let (caller, method, unevaluated_parameters, evaluated_parameters) = match self { - MethodCallStackFrame::CallerPath { - method, - parameters, - } => (value, method, parameters, Vec::new()), + MethodCallStackFrame::CallerPath { method, parameters } => { + (value, method, parameters, Vec::new()) + } MethodCallStackFrame::ArgumentsPath { caller, method, @@ -689,14 +696,15 @@ impl MethodCallStackFrame { .get(evaluated_parameters.len()) .cloned() { - Some(next) => { - action_creator.read_value_with_handler(next, ValueStackFrame::MethodCall(MethodCallStackFrame::ArgumentsPath { + Some(next) => action_creator.read_value_with_handler( + next, + ValueStackFrame::MethodCall(MethodCallStackFrame::ArgumentsPath { caller, method, unevaluated_parameters, evaluated_parameters, - })) - } + }), + ), None => action_creator.return_value(caller.call_method(method, evaluated_parameters)?), }; Ok(next_action) @@ -835,7 +843,7 @@ impl AssignmentStackFrame { fn ultimate_return_mode(&self) -> ReturnMode { match self { AssignmentStackFrame::AssignmentRoot { .. } => ReturnMode::Value, - AssignmentStackFrame::Grouped { .. } => ReturnMode::AssignmentCompletion, + AssignmentStackFrame::Grouped => ReturnMode::AssignmentCompletion, AssignmentStackFrame::Array { .. } => ReturnMode::AssignmentCompletion, AssignmentStackFrame::Object { .. } => ReturnMode::AssignmentCompletion, } @@ -1056,7 +1064,7 @@ impl PlaceStackFrame { match self { PlaceStackFrame::Assignment { .. } => ReturnMode::AssignmentCompletion, PlaceStackFrame::CompoundAssignmentRoot { .. } => ReturnMode::Value, - PlaceStackFrame::Grouped { .. } => ReturnMode::Place, + PlaceStackFrame::Grouped => ReturnMode::Place, PlaceStackFrame::PropertyAccess { .. } => ReturnMode::Place, PlaceStackFrame::Indexed { .. } => ReturnMode::Place, } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 040e03dc..91d0bebe 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -194,7 +194,9 @@ impl Expressionable for Source { match parent_stack_frame { ExpressionStackFrame::Root => Ok(NodeExtension::NoValidExtensionForCurrentParent), ExpressionStackFrame::Group { .. } => input.parse_err("Expected ) or operator"), - ExpressionStackFrame::NonEmptyArray { .. } => input.parse_err("Expected comma, ], or operator"), + ExpressionStackFrame::NonEmptyArray { .. } => { + input.parse_err("Expected comma, ], or operator") + } ExpressionStackFrame::IncompleteIndex { .. } | ExpressionStackFrame::NonEmptyObject { state: ObjectStackFrameState::EntryIndex { .. }, diff --git a/src/expressions/expression_parsing.rs b/src/expressions/expression_parsing.rs index c908a2bb..aa817349 100644 --- a/src/expressions/expression_parsing.rs +++ b/src/expressions/expression_parsing.rs @@ -144,17 +144,20 @@ impl<'a> ExpressionParser<'a, Source> { }; match next_item { Some(item) => item, - None => { // Indicates object + None => { + // Indicates object match self.expression_stack.pop().unwrap() { ExpressionStackFrame::NonEmptyObject { - braces, - state: ObjectStackFrameState::EntryValue(key, _), - mut complete_entries, + braces, + state: ObjectStackFrameState::EntryValue(key, _), + mut complete_entries, } => { complete_entries.push((key, node)); self.continue_object(braces, complete_entries)? } - _ => unreachable!("This None code path is only reachable under an object"), + _ => unreachable!( + "This None code path is only reachable under an object" + ), } } } @@ -174,11 +177,13 @@ impl<'a> ExpressionParser<'a, Source> { }); WorkItem::TryParseAndApplyExtension { node } } else { - self.push_stack_frame(ExpressionStackFrame::NonEmptyMethodCallParametersList { - node, - method, - parameters: Vec::new(), - }) + self.push_stack_frame( + ExpressionStackFrame::NonEmptyMethodCallParametersList { + node, + method, + parameters: Vec::new(), + }, + ) } } NodeExtension::Index(access) => { @@ -200,7 +205,8 @@ impl<'a> ExpressionParser<'a, Source> { operation, }) } - NodeExtension::EndOfStreamOrGroup | NodeExtension::NoValidExtensionForCurrentParent => { + NodeExtension::EndOfStreamOrGroup + | NodeExtension::NoValidExtensionForCurrentParent => { unreachable!("Not possible, as these have minimum precedence") } }) @@ -263,17 +269,18 @@ impl<'a> ExpressionParser<'a, Source> { assert!(matches!(extension, NodeExtension::EndOfStreamOrGroup)); self.streams.exit_group(); let colon = self.streams.parse()?; - self.expression_stack.push(ExpressionStackFrame::NonEmptyObject { - braces, - complete_entries, - state: ObjectStackFrameState::EntryValue( - ObjectKey::Indexed { - access, - index: node, - }, - colon, - ), - }); + self.expression_stack + .push(ExpressionStackFrame::NonEmptyObject { + braces, + complete_entries, + state: ObjectStackFrameState::EntryValue( + ObjectKey::Indexed { + access, + index: node, + }, + colon, + ), + }); WorkItem::RequireUnaryAtom } ExpressionStackFrame::NonEmptyObject { @@ -782,7 +789,9 @@ impl ExpressionStackFrame { ExpressionStackFrame::Group { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::NonEmptyArray { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::NonEmptyObject { .. } => OperatorPrecendence::MIN, - ExpressionStackFrame::NonEmptyMethodCallParametersList { .. } => OperatorPrecendence::MIN, + ExpressionStackFrame::NonEmptyMethodCallParametersList { .. } => { + OperatorPrecendence::MIN + } ExpressionStackFrame::IncompleteIndex { .. } => OperatorPrecendence::MIN, ExpressionStackFrame::IncompleteRange { .. } => OperatorPrecendence::Range, ExpressionStackFrame::IncompleteAssignment { .. } => OperatorPrecendence::Assign, @@ -886,7 +895,9 @@ impl NodeExtension { WorkItem::TryApplyAlreadyParsedExtension { node, extension } } NodeExtension::NonTerminalComma => { - unreachable!("Comma is only possible on method parameter list or array or object parent") + unreachable!( + "Comma is only possible on method parameter list or array or object parent" + ) } NodeExtension::NoValidExtensionForCurrentParent => { // We have to reparse in case the extension is valid for the new parent. diff --git a/src/expressions/value.rs b/src/expressions/value.rs index f7bbcc0a..b1a712e8 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -457,32 +457,42 @@ impl ExpressionValue { } } - pub(crate) fn call_method(self, method: MethodAccess, parameters: Vec) -> ExecutionResult { + pub(crate) fn call_method( + self, + method: MethodAccess, + parameters: Vec, + ) -> ExecutionResult { // TODO: Make this more extensible / usable match self { - ExpressionValue::Array(array) => match method.method.to_string().as_str() { - "len" => { + ExpressionValue::Array(array) => { + if method.method.to_string().as_str() == "len" { match parameters.as_slice() { [] => {} - _ => return method.execution_err(format!("The `len` method does not take parameters")), + _ => { + return method.execution_err( + "The `len` method does not take parameters".to_string(), + ) + } } let len = array.items.len(); - return Ok(len.to_value(method.span_range())) + return Ok(len.to_value(method.span_range())); } - _ => {} } - ExpressionValue::Stream(stream) => match method.method.to_string().as_str() { - "len" => { + ExpressionValue::Stream(stream) => { + if method.method.to_string().as_str() == "len" { match parameters.as_slice() { [] => {} - _ => return method.execution_err(format!("The `len` method does not take parameters")), + _ => { + return method.execution_err( + "The `len` method does not take parameters".to_string(), + ) + } } let len = stream.value.len(); - return Ok(len.to_value(method.span_range())) + return Ok(len.to_value(method.span_range())); } - _ => {} } - _ => {}, + _ => {} } method.execution_err("Methods are not currently supported") } diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 4034cd2c..f1614bba 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -147,7 +147,7 @@ pub(crate) struct VariableReference { } impl VariableReference { - pub(crate) fn get_value_ref(&self) -> ExecutionResult> { + pub(crate) fn get_value_ref(&self) -> ExecutionResult> { self.data.try_borrow().map_err(|_| { self.execution_error("The variable cannot be read if it is currently being modified") }) diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index 409bb39f..d5117345 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -288,7 +288,9 @@ impl<'a, K> ParseBuffer<'a, K> { })?) } - pub(crate) fn parse_any_group(&self) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer)> { + pub(crate) fn parse_any_group( + &self, + ) -> ParseResult<(Delimiter, DelimSpan, ParseBuffer<'_, K>)> { use syn::parse::discouraged::AnyDelimiter; let (delimiter, delim_span, parse_buffer) = self.inner.parse_any_delimiter()?; Ok((delimiter, delim_span, parse_buffer.into())) @@ -302,7 +304,7 @@ impl<'a, K> ParseBuffer<'a, K> { &self, matching: impl FnOnce(Delimiter) -> bool, expected_message: impl FnOnce() -> String, - ) -> ParseResult<(DelimSpan, ParseBuffer)> { + ) -> ParseResult<(DelimSpan, ParseBuffer<'_, K>)> { let error_span = match self.parse_any_group() { Ok((delimiter, delim_span, inner)) if matching(delimiter) => { return Ok((delim_span, inner)); @@ -316,31 +318,31 @@ impl<'a, K> ParseBuffer<'a, K> { pub(crate) fn parse_specific_group( &self, expected_delimiter: Delimiter, - ) -> ParseResult<(DelimSpan, ParseBuffer)> { + ) -> ParseResult<(DelimSpan, ParseBuffer<'_, K>)> { self.parse_group_matching( |delimiter| delimiter == expected_delimiter, || format!("Expected {}", expected_delimiter.description_of_open()), ) } - pub(crate) fn parse_braces(&self) -> ParseResult<(Braces, ParseBuffer)> { + pub(crate) fn parse_braces(&self) -> ParseResult<(Braces, ParseBuffer<'_, K>)> { let (delim_span, inner) = self.parse_specific_group(Delimiter::Brace)?; Ok((Braces { delim_span }, inner)) } - pub(crate) fn parse_brackets(&self) -> ParseResult<(Brackets, ParseBuffer)> { + pub(crate) fn parse_brackets(&self) -> ParseResult<(Brackets, ParseBuffer<'_, K>)> { let (delim_span, inner) = self.parse_specific_group(Delimiter::Bracket)?; Ok((Brackets { delim_span }, inner)) } - pub(crate) fn parse_parentheses(&self) -> ParseResult<(Parentheses, ParseBuffer)> { + pub(crate) fn parse_parentheses(&self) -> ParseResult<(Parentheses, ParseBuffer<'_, K>)> { let (delim_span, inner) = self.parse_specific_group(Delimiter::Parenthesis)?; Ok((Parentheses { delim_span }, inner)) } pub(crate) fn parse_transparent_group( &self, - ) -> ParseResult<(TransparentDelimiters, ParseBuffer)> { + ) -> ParseResult<(TransparentDelimiters, ParseBuffer<'_, K>)> { let (delim_span, inner) = self.parse_specific_group(Delimiter::None)?; Ok((TransparentDelimiters { delim_span }, inner)) } diff --git a/src/transformation/patterns.rs b/src/transformation/patterns.rs index 3871ac53..eb4397da 100644 --- a/src/transformation/patterns.rs +++ b/src/transformation/patterns.rs @@ -35,7 +35,7 @@ impl Parse for Pattern { } else if lookahead.peek(Token![..]) { Ok(Pattern::DotDot(input.parse()?)) } else if input.peek(Token![#]) { - return input.parse_err("Use `var` instead of `#var` in a destructuring"); + input.parse_err("Use `var` instead of `#var` in a destructuring") } else { Err(lookahead.error().into()) } diff --git a/tests/expressions.rs b/tests/expressions.rs index fbf10c2e..e7d6f888 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -463,7 +463,6 @@ fn test_objects() { ); } - #[test] fn test_method_calls() { preinterpret_assert_eq!( @@ -473,4 +472,4 @@ fn test_method_calls() { ), 2 + 3 ); -} \ No newline at end of file +} From ebcf6df93fb857be93b325ab65bddd5b182f2e0a Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 13:39:18 +0100 Subject: [PATCH 116/126] fix: Fix right alignment change in error messages --- Cargo.toml | 2 +- .../control_flow/error_after_continue.stderr | 8 ++++---- .../compilation_failures/core/error_no_fields.stderr | 10 +++++----- tests/compilation_failures/core/error_no_span.stderr | 12 ++++++------ .../core/error_span_repeat.stderr | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7cdd6213..5a99a9b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,4 +25,4 @@ syn = { version = "2.0.98", default-features = false, features = ["parsing", "de quote = { version = "1.0.38", default-features = false } [dev-dependencies] -trybuild = { version = "1.0.103", features = ["diff"] } +trybuild = { version = "1.0.110", features = ["diff"] } diff --git a/tests/compilation_failures/control_flow/error_after_continue.stderr b/tests/compilation_failures/control_flow/error_after_continue.stderr index b98f5442..7042f559 100644 --- a/tests/compilation_failures/control_flow/error_after_continue.stderr +++ b/tests/compilation_failures/control_flow/error_after_continue.stderr @@ -1,10 +1,10 @@ error: And now we error --> tests/compilation_failures/control_flow/error_after_continue.rs:4:5 | -4 | / preinterpret!( -5 | | #(let x = 0) -6 | | [!while! true { -7 | | #(x += 1) + 4 | / preinterpret!( + 5 | | #(let x = 0) + 6 | | [!while! true { + 7 | | #(x += 1) ... | 17 | | }] 18 | | ); diff --git a/tests/compilation_failures/core/error_no_fields.stderr b/tests/compilation_failures/core/error_no_fields.stderr index 332f8824..47a5d66d 100644 --- a/tests/compilation_failures/core/error_no_fields.stderr +++ b/tests/compilation_failures/core/error_no_fields.stderr @@ -1,12 +1,12 @@ error: Expected 102 to equal 64 --> tests/compilation_failures/core/error_no_fields.rs:4:44 | -4 | ($input1:literal, $input2:literal) => {preinterpret!{ + 4 | ($input1:literal, $input2:literal) => {preinterpret!{ | ____________________________________________^ -5 | | [!if! ($input1 != $input2) { -6 | | [!error! "Expected " $input1 " to equal " $input2] -7 | | }] -8 | | }}; + 5 | | [!if! ($input1 != $input2) { + 6 | | [!error! "Expected " $input1 " to equal " $input2] + 7 | | }] + 8 | | }}; | |_____^ ... 12 | assert_literals_eq!(102, 64); diff --git a/tests/compilation_failures/core/error_no_span.stderr b/tests/compilation_failures/core/error_no_span.stderr index f29628b2..648d1188 100644 --- a/tests/compilation_failures/core/error_no_span.stderr +++ b/tests/compilation_failures/core/error_no_span.stderr @@ -1,13 +1,13 @@ error: Expected 102 to equal 64 --> tests/compilation_failures/core/error_no_span.rs:4:47 | -4 | ($input1:literal and $input2:literal) => {preinterpret!{ + 4 | ($input1:literal and $input2:literal) => {preinterpret!{ | _______________________________________________^ -5 | | [!if! ($input1 != $input2) { -6 | | [!error! { -7 | | message: [!string! "Expected " $input1 " to equal " $input2], -8 | | }] -9 | | }] + 5 | | [!if! ($input1 != $input2) { + 6 | | [!error! { + 7 | | message: [!string! "Expected " $input1 " to equal " $input2], + 8 | | }] + 9 | | }] 10 | | }}; | |_____^ ... diff --git a/tests/compilation_failures/core/error_span_repeat.stderr b/tests/compilation_failures/core/error_span_repeat.stderr index 93939f2f..46307d33 100644 --- a/tests/compilation_failures/core/error_span_repeat.stderr +++ b/tests/compilation_failures/core/error_span_repeat.stderr @@ -1,7 +1,7 @@ error: Cannot infer common type from stream != untyped integer. Consider using `as` to cast the operands to matching types. --> tests/compilation_failures/core/error_span_repeat.rs:6:28 | -6 | [!if! input_length != 3 { + 6 | [!if! input_length != 3 { | ^^ ... 16 | assert_input_length_of_3!(42 101 666 1024); From d347bc277edbb63ec2a405df07a3fc174ca23c8d Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 13:51:40 +0100 Subject: [PATCH 117/126] fix: Disable UI tests on beta and nightly --- .../control_flow/error_after_continue.stderr | 8 ++++---- .../compilation_failures/core/error_no_fields.stderr | 10 +++++----- tests/compilation_failures/core/error_no_span.stderr | 12 ++++++------ .../core/error_span_repeat.stderr | 2 +- tests/complex.rs | 4 ++++ tests/control_flow.rs | 4 ++++ tests/core.rs | 4 ++++ tests/expressions.rs | 4 ++++ tests/helpers/prelude.rs | 10 ++++++++++ tests/tokens.rs | 4 ++++ tests/transforming.rs | 2 +- 11 files changed, 47 insertions(+), 17 deletions(-) diff --git a/tests/compilation_failures/control_flow/error_after_continue.stderr b/tests/compilation_failures/control_flow/error_after_continue.stderr index 7042f559..b98f5442 100644 --- a/tests/compilation_failures/control_flow/error_after_continue.stderr +++ b/tests/compilation_failures/control_flow/error_after_continue.stderr @@ -1,10 +1,10 @@ error: And now we error --> tests/compilation_failures/control_flow/error_after_continue.rs:4:5 | - 4 | / preinterpret!( - 5 | | #(let x = 0) - 6 | | [!while! true { - 7 | | #(x += 1) +4 | / preinterpret!( +5 | | #(let x = 0) +6 | | [!while! true { +7 | | #(x += 1) ... | 17 | | }] 18 | | ); diff --git a/tests/compilation_failures/core/error_no_fields.stderr b/tests/compilation_failures/core/error_no_fields.stderr index 47a5d66d..332f8824 100644 --- a/tests/compilation_failures/core/error_no_fields.stderr +++ b/tests/compilation_failures/core/error_no_fields.stderr @@ -1,12 +1,12 @@ error: Expected 102 to equal 64 --> tests/compilation_failures/core/error_no_fields.rs:4:44 | - 4 | ($input1:literal, $input2:literal) => {preinterpret!{ +4 | ($input1:literal, $input2:literal) => {preinterpret!{ | ____________________________________________^ - 5 | | [!if! ($input1 != $input2) { - 6 | | [!error! "Expected " $input1 " to equal " $input2] - 7 | | }] - 8 | | }}; +5 | | [!if! ($input1 != $input2) { +6 | | [!error! "Expected " $input1 " to equal " $input2] +7 | | }] +8 | | }}; | |_____^ ... 12 | assert_literals_eq!(102, 64); diff --git a/tests/compilation_failures/core/error_no_span.stderr b/tests/compilation_failures/core/error_no_span.stderr index 648d1188..f29628b2 100644 --- a/tests/compilation_failures/core/error_no_span.stderr +++ b/tests/compilation_failures/core/error_no_span.stderr @@ -1,13 +1,13 @@ error: Expected 102 to equal 64 --> tests/compilation_failures/core/error_no_span.rs:4:47 | - 4 | ($input1:literal and $input2:literal) => {preinterpret!{ +4 | ($input1:literal and $input2:literal) => {preinterpret!{ | _______________________________________________^ - 5 | | [!if! ($input1 != $input2) { - 6 | | [!error! { - 7 | | message: [!string! "Expected " $input1 " to equal " $input2], - 8 | | }] - 9 | | }] +5 | | [!if! ($input1 != $input2) { +6 | | [!error! { +7 | | message: [!string! "Expected " $input1 " to equal " $input2], +8 | | }] +9 | | }] 10 | | }}; | |_____^ ... diff --git a/tests/compilation_failures/core/error_span_repeat.stderr b/tests/compilation_failures/core/error_span_repeat.stderr index 46307d33..93939f2f 100644 --- a/tests/compilation_failures/core/error_span_repeat.stderr +++ b/tests/compilation_failures/core/error_span_repeat.stderr @@ -1,7 +1,7 @@ error: Cannot infer common type from stream != untyped integer. Consider using `as` to cast the operands to matching types. --> tests/compilation_failures/core/error_span_repeat.rs:6:28 | - 6 | [!if! input_length != 3 { +6 | [!if! input_length != 3 { | ^^ ... 16 | assert_input_length_of_3!(42 101 666 1024); diff --git a/tests/complex.rs b/tests/complex.rs index c968e17c..f8c4398c 100644 --- a/tests/complex.rs +++ b/tests/complex.rs @@ -18,6 +18,10 @@ preinterpret! { #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_complex_compilation_failures() { + if !should_run_ui_tests() { + // Some of the outputs are different on nightly, so don't test these + return; + } let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/complex/*.rs"); } diff --git a/tests/control_flow.rs b/tests/control_flow.rs index 820dc08d..277e62c5 100644 --- a/tests/control_flow.rs +++ b/tests/control_flow.rs @@ -7,6 +7,10 @@ use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_control_flow_compilation_failures() { + if !should_run_ui_tests() { + // Some of the outputs are different on nightly, so don't test these + return; + } let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/control_flow/*.rs"); } diff --git a/tests/core.rs b/tests/core.rs index 80c4b009..467804de 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -7,6 +7,10 @@ use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_core_compilation_failures() { + if !should_run_ui_tests() { + // Some of the outputs are different on nightly, so don't test these + return; + } let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/core/*.rs"); } diff --git a/tests/expressions.rs b/tests/expressions.rs index e7d6f888..f18b6446 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -5,6 +5,10 @@ use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_expression_compilation_failures() { + if !should_run_ui_tests() { + // Some of the outputs are different on nightly, so don't test these + return; + } let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/expressions/*.rs"); } diff --git a/tests/helpers/prelude.rs b/tests/helpers/prelude.rs index 5c90d782..425881e8 100644 --- a/tests/helpers/prelude.rs +++ b/tests/helpers/prelude.rs @@ -3,6 +3,16 @@ #![allow(unused_imports, unused_macros)] pub use preinterpret::*; +pub(crate) fn should_run_ui_tests() -> bool { + // Nightly has different outputs for some tests + // And as of https://github.com/rust-lang/rust/pull/144609 both beta and nightly do + // So we only run these tests on stable or in local developer environments. + match option_env!("TEST_RUST_MODE") { + Some("nightly") | Some("beta") => false, + _ => true, // Default case: run the tests + } +} + macro_rules! preinterpret_assert_eq { (#($($input:tt)*), $($output:tt)*) => { assert_eq!(preinterpret!(#($($input)*)), $($output)*); diff --git a/tests/tokens.rs b/tests/tokens.rs index cbbcaa94..82e152ae 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -5,6 +5,10 @@ use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_tokens_compilation_failures() { + if !should_run_ui_tests() { + // Some of the outputs are different on nightly, so don't test these + return; + } let t = trybuild::TestCases::new(); t.compile_fail("tests/compilation_failures/tokens/*.rs"); } diff --git a/tests/transforming.rs b/tests/transforming.rs index 29995d47..1b267db4 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -5,7 +5,7 @@ use prelude::*; #[test] #[cfg_attr(miri, ignore = "incompatible with miri")] fn test_transfoming_compilation_failures() { - if option_env!("TEST_RUST_MODE") == Some("nightly") { + if !should_run_ui_tests() { // Some of the outputs are different on nightly, so don't test these return; } From 23706a6558b912f0e1ff8a1fafb4c9efb6d41923 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 13:56:43 +0100 Subject: [PATCH 118/126] tweak: Fix dead code error --- tests/helpers/prelude.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/prelude.rs b/tests/helpers/prelude.rs index 425881e8..d68733dc 100644 --- a/tests/helpers/prelude.rs +++ b/tests/helpers/prelude.rs @@ -3,6 +3,7 @@ #![allow(unused_imports, unused_macros)] pub use preinterpret::*; +#[allow(dead_code)] // This is used only when ui tests are running pub(crate) fn should_run_ui_tests() -> bool { // Nightly has different outputs for some tests // And as of https://github.com/rust-lang/rust/pull/144609 both beta and nightly do From f88c6149c9e0adba387e4150af96c70db9f89772 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 16 Aug 2025 13:57:45 +0100 Subject: [PATCH 119/126] fix: Fix style --- tests/helpers/prelude.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/prelude.rs b/tests/helpers/prelude.rs index d68733dc..41836121 100644 --- a/tests/helpers/prelude.rs +++ b/tests/helpers/prelude.rs @@ -10,7 +10,7 @@ pub(crate) fn should_run_ui_tests() -> bool { // So we only run these tests on stable or in local developer environments. match option_env!("TEST_RUST_MODE") { Some("nightly") | Some("beta") => false, - _ => true, // Default case: run the tests + _ => true, // Default case: run the tests } } From e060669be0eb11f891a84255eeb0735441f68d2e Mon Sep 17 00:00:00 2001 From: David Edey Date: Tue, 2 Sep 2025 00:49:06 +0100 Subject: [PATCH 120/126] refactor: Much improved expression evaluation readability --- CHANGELOG.md | 23 +- src/expressions/array.rs | 17 +- src/expressions/evaluation.rs | 1122 ----------------- .../evaluation/assignment_frames.rs | 335 +++++ src/expressions/evaluation/evaluator.rs | 434 +++++++ src/expressions/evaluation/mod.rs | 13 + src/expressions/evaluation/node_conversion.rs | 144 +++ src/expressions/evaluation/place_frames.rs | 192 +++ src/expressions/evaluation/type_resolution.rs | 40 + src/expressions/evaluation/value_frames.rs | 797 ++++++++++++ src/expressions/integer.rs | 6 +- src/expressions/object.rs | 28 +- src/expressions/range.rs | 4 +- src/expressions/value.rs | 51 +- src/interpretation/commands/core_commands.rs | 2 +- src/interpretation/interpreter.rs | 141 ++- src/misc/mut_rc_ref_cell.rs | 73 +- src/transformation/transform_stream.rs | 2 +- src/transformation/variable_binding.rs | 20 +- .../index_into_discarded_place.stderr | 6 +- 20 files changed, 2269 insertions(+), 1181 deletions(-) delete mode 100644 src/expressions/evaluation.rs create mode 100644 src/expressions/evaluation/assignment_frames.rs create mode 100644 src/expressions/evaluation/evaluator.rs create mode 100644 src/expressions/evaluation/mod.rs create mode 100644 src/expressions/evaluation/node_conversion.rs create mode 100644 src/expressions/evaluation/place_frames.rs create mode 100644 src/expressions/evaluation/type_resolution.rs create mode 100644 src/expressions/evaluation/value_frames.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9526d4bf..58c03a74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,10 +97,25 @@ Inside a transform stream, the following grammar is supported: ### To come * Method calls continued - * Add support for &mut methods like `push(..)`... how? unclear. - * See comment in `evaluation.rs` `handle_as_value` above `ExpressionNode::MethodCall` - * Possibly parameters should be some cow-like enum for Value or (variable) Reference? - * Then add `.push(x)` on array and stream + * Add support for &mut methods like `push(..)`... + * WIP: Finish the late binding / TODOs in the evaluation module + * WIP: Add in basic method resolution, even if just with hardcoded strings for now! + * CHALLENGE: Given a.b(c,d,e) we need to resolve: + * What method / code to use + * Whether a, c, d and e should be resolved as &, &mut or owned + * SOLUTION: + * We change `evaluation.rs` to start by: + * Resolving a value's reference path as a `MutRef | SharedRef` at the same time (basically taking a `MutRef` if we can)... We probably want value resolution to take a `ValueOwnership::Any|Owned|MutRef|SharedRef` to guide this process. + * We'll need to back-propogate through places, so will also need to support `SharedRef` + in `Place` and have a `PlaceOwnership::Any|MutRef|SharedRef` + => e.g. if we're resolving `x.add(arr[3])` and want to read `arr[3]` as a shared reference. + => or if we're resolve `arr[3].to_string()` we want to read `arr[3]` as `LateBound` and then resolve to shared. + * Existing resolutions likely convert to an `Owned`, possibly via a clone... Although some of these can be fixed too. + * When resolving a method call, we start by requesting a `PermittedValueKind::Any` for `a` and then getting a `&a` from the`Owned | MutRef | SharedRef`; and alongside the name `b`, and possibly # of arguments, we resolve a method definition (or error) + * The method definition tells us whether we need `a` to be `Owned | MutRef | SharedRef`, and similarly for each argument. And the type of `&a` should tell us whether + it is copy (i.e. can be transparently cloned from a ref to an owned if needed) + * We can then resolve the correct value for each argument, and execute the method. + * Then we can add `.push(x)` on array and stream * Scrap `#>>x` etc in favour of `#(a.push(@XXX))` * Also improve support to make it easier to add methods on built-in types or (in future) user-designed types * Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 2719f89d..0e03eaf0 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -120,7 +120,7 @@ impl ExpressionArray { pub(super) fn into_indexed( mut self, access: IndexAccess, - index: ExpressionValue, + index: &ExpressionValue, ) -> ExecutionResult { let span_range = SpanRange::new_between(self.span_range, access); Ok(match index { @@ -140,15 +140,24 @@ impl ExpressionArray { pub(super) fn index_mut( &mut self, _access: IndexAccess, - index: ExpressionValue, + index: &ExpressionValue, ) -> ExecutionResult<&mut ExpressionValue> { let index = self.resolve_valid_index(index, false)?; Ok(&mut self.items[index]) } + pub(super) fn index_ref( + &self, + _access: IndexAccess, + index: &ExpressionValue, + ) -> ExecutionResult<&ExpressionValue> { + let index = self.resolve_valid_index(index, false)?; + Ok(&self.items[index]) + } + pub(super) fn resolve_valid_index( &self, - index: ExpressionValue, + index: &ExpressionValue, is_exclusive: bool, ) -> ExecutionResult { match index { @@ -161,7 +170,7 @@ impl ExpressionArray { fn resolve_valid_index_from_integer( &self, - integer: ExpressionInteger, + integer: &ExpressionInteger, is_exclusive: bool, ) -> ExecutionResult { let span_range = integer.span_range; diff --git a/src/expressions/evaluation.rs b/src/expressions/evaluation.rs deleted file mode 100644 index bb5ab361..00000000 --- a/src/expressions/evaluation.rs +++ /dev/null @@ -1,1122 +0,0 @@ -use super::*; - -pub(super) use inner::ExpressionEvaluator; - -/// This is to hide implementation details to protect the abstraction and make it harder to make mistakes. -mod inner { - use super::*; - - pub(in super::super) struct ExpressionEvaluator<'a, K: Expressionable> { - nodes: &'a [ExpressionNode], - stacks: Stacks, - } - - impl<'a> ExpressionEvaluator<'a, Source> { - pub(in super::super) fn new(nodes: &'a [ExpressionNode]) -> Self { - Self { - nodes, - stacks: Stacks::new(), - } - } - - pub(in super::super) fn evaluate( - mut self, - root: ExpressionNodeId, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - let mut next_action = NextActionInner::ReadNodeAsValue(root); - - loop { - match self.step(next_action, interpreter)? { - StepResult::Continue(continue_action) => { - next_action = continue_action.0; - } - StepResult::Return(value) => { - return Ok(value); - } - } - } - } - - fn step( - &mut self, - action: NextActionInner, - interpreter: &mut Interpreter, - ) -> ExecutionResult { - Ok(StepResult::Continue(match action { - NextActionInner::ReadNodeAsValue(node) => self.nodes[node.0] - .handle_as_value(interpreter, self.stacks.creator(ReturnMode::Value))?, - NextActionInner::HandleReturnedValue(value) => { - let top_of_stack = match self.stacks.value_stack.pop() { - Some(top) => top, - None => { - debug_assert!(self.stacks.assignment_stack.is_empty(), "Evaluation completed with none-empty assignment stack - there's some bug in the ExpressionEvaluator"); - debug_assert!(self.stacks.place_stack.is_empty(), "Evaluation completed with none-empty place stack - there's some bug in the ExpressionEvaluator"); - return Ok(StepResult::Return(value)); - } - }; - let next_creator = self.stacks.creator(top_of_stack.ultimate_return_mode()); - top_of_stack.handle_value(value, next_creator)? - } - NextActionInner::ReadNodeAsAssignee(node, value) => self.nodes[node.0] - .handle_as_assignee( - self.nodes, - node, - value, - interpreter, - self.stacks.creator(ReturnMode::AssignmentCompletion), - )?, - NextActionInner::HandleAssignmentComplete(assignment_complete) => { - let top_of_stack = match self.stacks.assignment_stack.pop() { - Some(top) => top, - None => unreachable!("Received AssignmentComplete without any assignment stack frames - there's some bug in the ExpressionEvaluator"), - }; - let next_creator = self.stacks.creator(top_of_stack.ultimate_return_mode()); - top_of_stack.handle_assignment_complete(assignment_complete, next_creator)? - } - NextActionInner::ReadNodeAsPlace(node) => self.nodes[node.0] - .handle_as_place(interpreter, self.stacks.creator(ReturnMode::Place))?, - NextActionInner::HandleReturnedPlace(place) => { - let top_of_stack = match self.stacks.place_stack.pop() { - Some(top) => top, - None => unreachable!("Received Place without any place stack frames - there's some bug in the ExpressionEvaluator"), - }; - let next_creator = self.stacks.creator(top_of_stack.ultimate_return_mode()); - top_of_stack.handle_place(place, next_creator)? - } - })) - } - } - - /// See the [rust reference] for a good description of assignee vs place. - /// - /// [rust reference]: https://doc.rust-lang.org/reference/expressions/assignment-expressions.html#assignee-vs-place - pub(super) struct Stacks { - /// The stack of operations which will output a value - value_stack: Vec, - /// The stack of operations which will handle an assignment completion - assignment_stack: Vec, - /// The stack of operations which will output a place - place_stack: Vec, - } - - impl Stacks { - pub(super) fn new() -> Self { - Self { - value_stack: Vec::new(), - assignment_stack: Vec::new(), - place_stack: Vec::new(), - } - } - - fn creator(&mut self, return_mode: ReturnMode) -> ActionCreator<'_> { - ActionCreator { - stacks: self, - return_mode, - } - } - } - - pub(super) enum StepResult { - Continue(NextAction), - Return(ExpressionValue), - } - - pub(super) struct NextAction(NextActionInner); - - enum NextActionInner { - /// Enters an expression node to output a value - ReadNodeAsValue(ExpressionNodeId), - HandleReturnedValue(ExpressionValue), - // Enters an expression node for assignment purposes - ReadNodeAsAssignee(ExpressionNodeId, ExpressionValue), - HandleAssignmentComplete(AssignmentCompletion), - // Enters an expression node to output a place - // A place can be thought of as a mutable reference for e.g. a += operation - ReadNodeAsPlace(ExpressionNodeId), - HandleReturnedPlace(Place), - } - - impl From for NextAction { - fn from(value: NextActionInner) -> Self { - Self(value) - } - } - - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub(super) enum ReturnMode { - Value, - AssignmentCompletion, - Place, - } - - pub(super) struct ActionCreator<'a> { - stacks: &'a mut Stacks, - return_mode: ReturnMode, - } - - impl ActionCreator<'_> { - pub(super) fn return_value(self, value: ExpressionValue) -> NextAction { - debug_assert_eq!( - self.return_mode, - ReturnMode::Value, - "This handler claimed to ultimately return {:?}, but it returned a value", - self.return_mode - ); - NextActionInner::HandleReturnedValue(value).into() - } - - pub(super) fn read_value_with_handler( - self, - node: ExpressionNodeId, - handler: ValueStackFrame, - ) -> NextAction { - debug_assert_eq!( - handler.ultimate_return_mode(), - self.return_mode, - "Handler is expected to return {:?}, but claims to ultimately return {:?}", - self.return_mode, - handler.ultimate_return_mode() - ); - self.stacks.value_stack.push(handler); - NextActionInner::ReadNodeAsValue(node).into() - } - - pub(super) fn return_place(self, place: Place) -> NextAction { - debug_assert_eq!( - self.return_mode, - ReturnMode::Place, - "This handler claimed to ultimately return {:?}, but it returned a place", - self.return_mode - ); - NextActionInner::HandleReturnedPlace(place).into() - } - - pub(super) fn read_place_with_handler( - self, - node: ExpressionNodeId, - handler: PlaceStackFrame, - ) -> NextAction { - debug_assert_eq!( - handler.ultimate_return_mode(), - self.return_mode, - "Handler is expected to return {:?}, but claims to ultimately return {:?}", - self.return_mode, - handler.ultimate_return_mode() - ); - self.stacks.place_stack.push(handler); - NextActionInner::ReadNodeAsPlace(node).into() - } - - /// The span range should cover from the start of the assignee to the end of the value - pub(super) fn return_assignment_completion(self, span_range: SpanRange) -> NextAction { - debug_assert_eq!(self.return_mode, ReturnMode::AssignmentCompletion, "This handler claimed to ultimately return {:?}, but it returned an assignment completion", self.return_mode); - NextActionInner::HandleAssignmentComplete(AssignmentCompletion { span_range }).into() - } - - pub(super) fn handle_assignment_and_return_to( - self, - node: ExpressionNodeId, - value: ExpressionValue, - handler: AssignmentStackFrame, - ) -> NextAction { - debug_assert_eq!( - handler.ultimate_return_mode(), - self.return_mode, - "Handler is expected to return {:?}, but claims to ultimately return {:?}", - self.return_mode, - handler.ultimate_return_mode() - ); - self.stacks.assignment_stack.push(handler); - NextActionInner::ReadNodeAsAssignee(node, value).into() - } - } -} - -use inner::*; - -impl ExpressionNode { - fn handle_as_value( - &self, - interpreter: &mut Interpreter, - next: ActionCreator, - ) -> ExecutionResult { - Ok(match self { - ExpressionNode::Leaf(leaf) => { - next.return_value(Source::evaluate_leaf(leaf, interpreter)?) - } - ExpressionNode::Grouped { delim_span, inner } => next.read_value_with_handler( - *inner, - ValueStackFrame::Group { - span: delim_span.join(), - }, - ), - ExpressionNode::Array { brackets, items } => ArrayValueStackFrame { - span: brackets.join(), - unevaluated_items: items.clone(), - evaluated_items: Vec::with_capacity(items.len()), - } - .next(next), - ExpressionNode::Object { braces, entries } => Box::new(ObjectValueStackFrame { - span: braces.join(), - unevaluated_entries: entries.clone(), - evaluated_entries: BTreeMap::new(), - pending: None, - }) - .next(next)?, - ExpressionNode::UnaryOperation { operation, input } => next.read_value_with_handler( - *input, - ValueStackFrame::UnaryOperation { - operation: operation.clone(), - }, - ), - ExpressionNode::BinaryOperation { - operation, - left_input, - right_input, - } => next.read_value_with_handler( - *left_input, - ValueStackFrame::BinaryOperation { - operation: operation.clone(), - state: BinaryPath::OnLeftBranch { - right: *right_input, - }, - }, - ), - ExpressionNode::Property { node, access } => next.read_value_with_handler( - *node, - ValueStackFrame::Property { - access: access.clone(), - }, - ), - // TODO - we need to instead: - // * Resolve the expression value type of the node (e.g. from a & reference to the node) - // * For that value type, the method name and arguments, determine if the given method call: - // * Takes a self, &self or &mut self... - // * Whether each parameter is owned, & or &mut - // * Read the node and its parameters, and execute the method call - ExpressionNode::MethodCall { - node, - method, - parameters, - } => next.read_value_with_handler( - *node, - ValueStackFrame::MethodCall(MethodCallStackFrame::CallerPath { - method: method.clone(), - parameters: parameters.clone(), - }), - ), - ExpressionNode::Index { - node, - access, - index, - } => next.read_value_with_handler( - *node, - ValueStackFrame::Index { - access: *access, - state: IndexPath::OnObjectBranch { index: *index }, - }, - ), - ExpressionNode::Range { - left, - range_limits, - right, - } => { - match (left, right) { - (None, None) => match range_limits { - syn::RangeLimits::HalfOpen(token) => { - let inner = ExpressionRangeInner::RangeFull { token: *token }; - next.return_value(inner.to_value(token.span_range())) - } - syn::RangeLimits::Closed(_) => { - unreachable!("A closed range should have been given a right in continue_range(..)") - } - }, - (None, Some(right)) => next.read_value_with_handler( - *right, - ValueStackFrame::Range { - range_limits: *range_limits, - state: RangePath::OnRightBranch { left: None }, - }, - ), - (Some(left), right) => next.read_value_with_handler( - *left, - ValueStackFrame::Range { - range_limits: *range_limits, - state: RangePath::OnLeftBranch { right: *right }, - }, - ), - } - } - ExpressionNode::Assignment { - assignee, - equals_token, - value, - } => next.read_value_with_handler( - *value, - ValueStackFrame::HandleValueForAssignment { - assignee: *assignee, - equals_token: *equals_token, - }, - ), - ExpressionNode::CompoundAssignment { - place, - operation, - value, - } => next.read_value_with_handler( - *value, - ValueStackFrame::HandleValueForCompoundAssignment { - place: *place, - operation: *operation, - }, - ), - }) - } - - fn handle_as_assignee( - &self, - nodes: &[ExpressionNode], - self_node_id: ExpressionNodeId, - value: ExpressionValue, - _: &mut Interpreter, - next: ActionCreator, - ) -> ExecutionResult { - Ok(match self { - ExpressionNode::Leaf(SourceExpressionLeaf::Variable(_)) - | ExpressionNode::Leaf(SourceExpressionLeaf::Discarded(_)) - | ExpressionNode::Index { .. } - | ExpressionNode::Property { .. } => { - next.read_place_with_handler(self_node_id, PlaceStackFrame::Assignment { value }) - } - ExpressionNode::Array { - brackets, - items: assignee_item_node_ids, - } => { - ArrayAssigneeStackFrame::new(nodes, brackets.join(), assignee_item_node_ids, value)? - .handle_next(next) - } - ExpressionNode::Object { braces, entries } => Box::new(ObjectAssigneeStackFrame::new( - braces.join(), - entries, - value, - )?) - .handle_next(next)?, - ExpressionNode::Grouped { inner, .. } => { - next.handle_assignment_and_return_to(*inner, value, AssignmentStackFrame::Grouped) - } - other => { - return other - .operator_span_range() - .execution_err("This type of expression is not supported as an assignee. You may wish to use `_` to ignore the value."); - } - }) - } - - fn handle_as_place( - &self, - interpreter: &mut Interpreter, - next: ActionCreator, - ) -> ExecutionResult { - Ok(match self { - ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => next.return_place( - Place::MutableReference(variable.reference(interpreter)?.into_mut()?), - ), - ExpressionNode::Leaf(SourceExpressionLeaf::Discarded(token)) => { - next.return_place(Place::Discarded(*token)) - } - ExpressionNode::Index { - node, - access, - index, - } => next.read_place_with_handler( - *node, - PlaceStackFrame::Indexed { - access: *access, - index: *index, - }, - ), - ExpressionNode::Property { node, access, .. } => next.read_place_with_handler( - *node, - PlaceStackFrame::PropertyAccess { - access: access.clone(), - }, - ), - ExpressionNode::Grouped { inner, .. } => { - next.read_place_with_handler(*inner, PlaceStackFrame::Grouped) - } - other => { - return other - .operator_span_range() - .execution_err("This type of expression is not supported as here. You may wish to use `_` to ignore the value."); - } - }) - } -} - -/// Stack frames which need to receive a value to continue their execution. -enum ValueStackFrame { - Group { - span: Span, - }, - Array(ArrayValueStackFrame), - Object(Box), - UnaryOperation { - operation: UnaryOperation, - }, - BinaryOperation { - operation: BinaryOperation, - state: BinaryPath, - }, - Property { - access: PropertyAccess, - }, - MethodCall(MethodCallStackFrame), - Index { - access: IndexAccess, - state: IndexPath, - }, - Range { - range_limits: syn::RangeLimits, - state: RangePath, - }, - HandleValueForAssignment { - assignee: ExpressionNodeId, - equals_token: Token![=], - }, - HandleValueForCompoundAssignment { - place: ExpressionNodeId, - operation: CompoundAssignmentOperation, - }, - ResolveIndexedPlace { - place: Place, - access: IndexAccess, - }, - ResolveObjectIndexForAssignment { - object: Box, - assignee_node: ExpressionNodeId, - access: IndexAccess, - }, -} - -impl ValueStackFrame { - fn ultimate_return_mode(&self) -> ReturnMode { - match self { - ValueStackFrame::Group { .. } => ReturnMode::Value, - ValueStackFrame::Array(_) => ReturnMode::Value, - ValueStackFrame::Object(_) => ReturnMode::Value, - ValueStackFrame::UnaryOperation { .. } => ReturnMode::Value, - ValueStackFrame::BinaryOperation { .. } => ReturnMode::Value, - ValueStackFrame::Property { .. } => ReturnMode::Value, - ValueStackFrame::MethodCall { .. } => ReturnMode::Value, - ValueStackFrame::Index { .. } => ReturnMode::Value, - ValueStackFrame::Range { .. } => ReturnMode::Value, - ValueStackFrame::HandleValueForAssignment { .. } => ReturnMode::Value, - ValueStackFrame::HandleValueForCompoundAssignment { .. } => ReturnMode::Value, - ValueStackFrame::ResolveIndexedPlace { .. } => ReturnMode::Place, - ValueStackFrame::ResolveObjectIndexForAssignment { .. } => { - ReturnMode::AssignmentCompletion - } - } - } - - fn handle_value( - self, - value: ExpressionValue, - next: ActionCreator, - ) -> ExecutionResult { - Ok(match self { - ValueStackFrame::Group { span } => next.return_value(value.with_span(span)), - ValueStackFrame::UnaryOperation { operation } => { - next.return_value(operation.evaluate(value)?) - } - ValueStackFrame::Array(mut array) => { - array.evaluated_items.push(value); - array.next(next) - } - ValueStackFrame::Object(object) => object.handle_value(value, next)?, - ValueStackFrame::BinaryOperation { operation, state } => match state { - BinaryPath::OnLeftBranch { right } => { - if let Some(result) = operation.lazy_evaluate(&value)? { - next.return_value(result) - } else { - next.read_value_with_handler( - right, - ValueStackFrame::BinaryOperation { - operation, - state: BinaryPath::OnRightBranch { left: value }, - }, - ) - } - } - BinaryPath::OnRightBranch { left } => { - next.return_value(operation.evaluate(left, value)?) - } - }, - ValueStackFrame::Property { access } => next.return_value(value.into_property(access)?), - ValueStackFrame::MethodCall(call) => call.handle_value(value, next)?, - ValueStackFrame::Index { access, state } => match state { - IndexPath::OnObjectBranch { index } => next.read_value_with_handler( - index, - ValueStackFrame::Index { - access, - state: IndexPath::OnIndexBranch { object: value }, - }, - ), - IndexPath::OnIndexBranch { object } => { - next.return_value(object.into_indexed(access, value)?) - } - }, - ValueStackFrame::Range { - range_limits, - state, - } => match (state, range_limits) { - (RangePath::OnLeftBranch { right: Some(right) }, range_limits) => next - .read_value_with_handler( - right, - ValueStackFrame::Range { - range_limits, - state: RangePath::OnRightBranch { left: Some(value) }, - }, - ), - (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::HalfOpen(token)) => { - let inner = ExpressionRangeInner::RangeFrom { - start_inclusive: value, - token, - }; - next.return_value(inner.to_value(token.span_range())) - } - (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::Closed(_)) => { - unreachable!( - "A closed range should have been given a right in continue_range(..)" - ) - } - ( - RangePath::OnRightBranch { left: Some(left) }, - syn::RangeLimits::HalfOpen(token), - ) => { - let inner = ExpressionRangeInner::Range { - start_inclusive: left, - token, - end_exclusive: value, - }; - next.return_value(inner.to_value(token.span_range())) - } - ( - RangePath::OnRightBranch { left: Some(left) }, - syn::RangeLimits::Closed(token), - ) => { - let inner = ExpressionRangeInner::RangeInclusive { - start_inclusive: left, - token, - end_inclusive: value, - }; - next.return_value(inner.to_value(token.span_range())) - } - (RangePath::OnRightBranch { left: None }, syn::RangeLimits::HalfOpen(token)) => { - let inner = ExpressionRangeInner::RangeTo { - token, - end_exclusive: value, - }; - next.return_value(inner.to_value(token.span_range())) - } - (RangePath::OnRightBranch { left: None }, syn::RangeLimits::Closed(token)) => { - let inner = ExpressionRangeInner::RangeToInclusive { - token, - end_inclusive: value, - }; - next.return_value(inner.to_value(token.span_range())) - } - }, - ValueStackFrame::HandleValueForAssignment { - assignee, - equals_token, - } => next.handle_assignment_and_return_to( - assignee, - value, - AssignmentStackFrame::AssignmentRoot { equals_token }, - ), - ValueStackFrame::HandleValueForCompoundAssignment { place, operation } => next - .read_place_with_handler( - place, - PlaceStackFrame::CompoundAssignmentRoot { operation, value }, - ), - ValueStackFrame::ResolveIndexedPlace { place, access } => { - next.return_place(match place { - Place::MutableReference(reference) => { - Place::MutableReference(reference.resolve_indexed(access, value)?) - } - Place::Discarded(_) => { - return access.execution_err("Cannot index into a discarded value"); - } - }) - } - ValueStackFrame::ResolveObjectIndexForAssignment { - object, - assignee_node, - access, - } => object.handle_index_value(access, value, assignee_node, next)?, - }) - } -} - -enum MethodCallStackFrame { - CallerPath { - method: MethodAccess, - parameters: Vec, - }, - ArgumentsPath { - caller: ExpressionValue, - method: MethodAccess, - unevaluated_parameters: Vec, - evaluated_parameters: Vec, - }, -} - -impl MethodCallStackFrame { - fn handle_value( - self, - value: ExpressionValue, - action_creator: ActionCreator, - ) -> ExecutionResult { - let (caller, method, unevaluated_parameters, evaluated_parameters) = match self { - MethodCallStackFrame::CallerPath { method, parameters } => { - (value, method, parameters, Vec::new()) - } - MethodCallStackFrame::ArgumentsPath { - caller, - method, - unevaluated_parameters, - mut evaluated_parameters, - } => { - evaluated_parameters.push(value); - (caller, method, unevaluated_parameters, evaluated_parameters) - } - }; - let next_action = match unevaluated_parameters - .get(evaluated_parameters.len()) - .cloned() - { - Some(next) => action_creator.read_value_with_handler( - next, - ValueStackFrame::MethodCall(MethodCallStackFrame::ArgumentsPath { - caller, - method, - unevaluated_parameters, - evaluated_parameters, - }), - ), - None => action_creator.return_value(caller.call_method(method, evaluated_parameters)?), - }; - Ok(next_action) - } -} - -struct ArrayValueStackFrame { - span: Span, - unevaluated_items: Vec, - evaluated_items: Vec, -} - -impl ArrayValueStackFrame { - fn next(self, action_creator: ActionCreator) -> NextAction { - match self - .unevaluated_items - .get(self.evaluated_items.len()) - .cloned() - { - Some(next) => { - action_creator.read_value_with_handler(next, ValueStackFrame::Array(self)) - } - None => action_creator.return_value(ExpressionValue::Array(ExpressionArray { - items: self.evaluated_items, - span_range: self.span.span_range(), - })), - } - } -} - -struct ObjectValueStackFrame { - span: Span, - pending: Option, - unevaluated_entries: Vec<(ObjectKey, ExpressionNodeId)>, - evaluated_entries: BTreeMap, -} - -impl ObjectValueStackFrame { - fn handle_value( - mut self: Box, - value: ExpressionValue, - next: ActionCreator, - ) -> ExecutionResult { - let pending = self.pending.take(); - Ok(match pending { - Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }) => { - let key = value.expect_string("An object key")?.value; - if self.evaluated_entries.contains_key(&key) { - return access.execution_err(format!("The key {} has already been set", key)); - } - self.pending = Some(PendingEntryPath::OnValueBranch { - key, - key_span: access.span(), - }); - next.read_value_with_handler(value_node, ValueStackFrame::Object(self)) - } - Some(PendingEntryPath::OnValueBranch { key, key_span }) => { - let entry = ObjectEntry { key_span, value }; - self.evaluated_entries.insert(key, entry); - self.next(next)? - } - None => { - unreachable!("Should not receive a value without a pending handler set") - } - }) - } - - fn next(mut self: Box, action_creator: ActionCreator) -> ExecutionResult { - Ok( - match self - .unevaluated_entries - .get(self.evaluated_entries.len()) - .cloned() - { - Some((ObjectKey::Identifier(ident), node)) => { - let key = ident.to_string(); - if self.evaluated_entries.contains_key(&key) { - return ident - .execution_err(format!("The key {} has already been set", key)); - } - self.pending = Some(PendingEntryPath::OnValueBranch { - key, - key_span: ident.span(), - }); - action_creator.read_value_with_handler(node, ValueStackFrame::Object(self)) - } - Some((ObjectKey::Indexed { access, index }, value_node)) => { - self.pending = Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }); - action_creator.read_value_with_handler(index, ValueStackFrame::Object(self)) - } - None => action_creator - .return_value(self.evaluated_entries.to_value(self.span.span_range())), - }, - ) - } -} - -enum PendingEntryPath { - OnIndexKeyBranch { - access: IndexAccess, - value_node: ExpressionNodeId, - }, - OnValueBranch { - key: String, - key_span: Span, - }, -} - -enum BinaryPath { - OnLeftBranch { right: ExpressionNodeId }, - OnRightBranch { left: ExpressionValue }, -} - -enum IndexPath { - OnObjectBranch { index: ExpressionNodeId }, - OnIndexBranch { object: ExpressionValue }, -} - -enum RangePath { - OnLeftBranch { right: Option }, - OnRightBranch { left: Option }, -} - -enum AssignmentStackFrame { - /// An instruction to return a `None` value to the Value stack - AssignmentRoot { - #[allow(unused)] - equals_token: Token![=], - }, - Grouped, - Array(ArrayAssigneeStackFrame), - Object(Box), -} - -impl AssignmentStackFrame { - fn ultimate_return_mode(&self) -> ReturnMode { - match self { - AssignmentStackFrame::AssignmentRoot { .. } => ReturnMode::Value, - AssignmentStackFrame::Grouped => ReturnMode::AssignmentCompletion, - AssignmentStackFrame::Array { .. } => ReturnMode::AssignmentCompletion, - AssignmentStackFrame::Object { .. } => ReturnMode::AssignmentCompletion, - } - } - - fn handle_assignment_complete( - self, - completion: AssignmentCompletion, - next: ActionCreator, - ) -> ExecutionResult { - let AssignmentCompletion { span_range } = completion; - Ok(match self { - AssignmentStackFrame::AssignmentRoot { .. } => { - next.return_value(ExpressionValue::None(span_range)) - } - AssignmentStackFrame::Grouped => next.return_assignment_completion(span_range), - AssignmentStackFrame::Array(array) => array.handle_next(next), - AssignmentStackFrame::Object(object) => object.handle_next(next)?, - }) - } -} - -struct ArrayAssigneeStackFrame { - span_range: SpanRange, - assignee_stack: Vec<(ExpressionNodeId, ExpressionValue)>, -} - -impl ArrayAssigneeStackFrame { - /// See also `ArrayPattern` in `patterns.rs` - fn new( - nodes: &[ExpressionNode], - assignee_span: Span, - assignee_item_node_ids: &[ExpressionNodeId], - value: ExpressionValue, - ) -> ExecutionResult { - let array = value.expect_array("The value destructured as an array")?; - let span_range = SpanRange::new_between(assignee_span, array.span_range.end()); - let mut has_seen_dot_dot = false; - let mut prefix_assignees = Vec::new(); - let mut suffix_assignees = Vec::new(); - for node in assignee_item_node_ids.iter() { - match nodes[node.0] { - ExpressionNode::Range { - left: None, - range_limits: syn::RangeLimits::HalfOpen(_), - right: None, - } => { - if has_seen_dot_dot { - return assignee_span - .execution_err("Only one .. is allowed in an array assignee"); - } - has_seen_dot_dot = true; - } - _ => { - if has_seen_dot_dot { - suffix_assignees.push(*node); - } else { - prefix_assignees.push(*node); - } - } - } - } - - let array_length = array.items.len(); - - let mut assignee_pairs: Vec<_> = if has_seen_dot_dot { - let total_assignees = prefix_assignees.len() + suffix_assignees.len(); - if total_assignees > array_length { - return assignee_span.execution_err(format!( - "The array has {} items, but the assignee expected at least {}", - array_length, total_assignees, - )); - } - let discarded_count = - array.items.len() - prefix_assignees.len() - suffix_assignees.len(); - let assignees = prefix_assignees - .into_iter() - .map(Some) - .chain(std::iter::repeat(None).take(discarded_count)) - .chain(suffix_assignees.into_iter().map(Some)); - - assignees - .zip(array.items) - .filter_map(|(assignee, value)| Some((assignee?, value))) - .collect() - } else { - let total_assignees = prefix_assignees.len(); - if total_assignees != array_length { - return assignee_span.execution_err(format!( - "The array has {} items, but the assignee expected {}", - array_length, total_assignees, - )); - } - prefix_assignees.into_iter().zip(array.items).collect() - }; - Ok(Self { - span_range, - assignee_stack: { - assignee_pairs.reverse(); - assignee_pairs - }, - }) - } - - fn handle_next(mut self, next: ActionCreator) -> NextAction { - match self.assignee_stack.pop() { - Some((node, value)) => { - next.handle_assignment_and_return_to(node, value, AssignmentStackFrame::Array(self)) - } - None => next.return_assignment_completion(self.span_range), - } - } -} - -struct ObjectAssigneeStackFrame { - span_range: SpanRange, - entries: BTreeMap, - already_used_keys: HashSet, - unresolved_stack: Vec<(ObjectKey, ExpressionNodeId)>, -} - -impl ObjectAssigneeStackFrame { - /// See also `ObjectPattern` in `patterns.rs` - fn new( - assignee_span: Span, - assignee_pairs: &[(ObjectKey, ExpressionNodeId)], - value: ExpressionValue, - ) -> ExecutionResult { - let object = value.expect_object("The value destructured as an object")?; - let span_range = SpanRange::new_between(assignee_span, object.span_range.end()); - - Ok(Self { - span_range, - entries: object.entries, - already_used_keys: HashSet::with_capacity(assignee_pairs.len()), - unresolved_stack: assignee_pairs.iter().rev().cloned().collect(), - }) - } - - fn handle_index_value( - mut self: Box, - access: IndexAccess, - index: ExpressionValue, - assignee_node: ExpressionNodeId, - next: ActionCreator, - ) -> ExecutionResult { - let key = index.expect_string("An object key")?.value; - let value = self.resolve_value(key, access.span())?; - Ok(next.handle_assignment_and_return_to( - assignee_node, - value, - AssignmentStackFrame::Object(self), - )) - } - - fn handle_next(mut self: Box, next: ActionCreator) -> ExecutionResult { - Ok(match self.unresolved_stack.pop() { - Some((ObjectKey::Identifier(ident), assignee_node)) => { - let key = ident.to_string(); - let value = self.resolve_value(key, ident.span())?; - next.handle_assignment_and_return_to( - assignee_node, - value, - AssignmentStackFrame::Object(self), - ) - } - Some((ObjectKey::Indexed { index, access }, assignee_node)) => next - .read_value_with_handler( - index, - ValueStackFrame::ResolveObjectIndexForAssignment { - object: self, - assignee_node, - access, - }, - ), - None => next.return_assignment_completion(self.span_range), - }) - } - - fn resolve_value(&mut self, key: String, key_span: Span) -> ExecutionResult { - if self.already_used_keys.contains(&key) { - return key_span.execution_err(format!("The key `{}` has already used", key)); - } - let value = self - .entries - .remove(&key) - .map(|entry| entry.value) - .unwrap_or_else(|| ExpressionValue::None(key_span.span_range())); - self.already_used_keys.insert(key); - Ok(value) - } -} - -struct AssignmentCompletion { - span_range: SpanRange, -} - -enum PlaceStackFrame { - Assignment { - value: ExpressionValue, - }, - CompoundAssignmentRoot { - operation: CompoundAssignmentOperation, - value: ExpressionValue, - }, - Grouped, - PropertyAccess { - access: PropertyAccess, - }, - Indexed { - access: IndexAccess, - index: ExpressionNodeId, - }, -} - -impl PlaceStackFrame { - fn ultimate_return_mode(&self) -> ReturnMode { - match self { - PlaceStackFrame::Assignment { .. } => ReturnMode::AssignmentCompletion, - PlaceStackFrame::CompoundAssignmentRoot { .. } => ReturnMode::Value, - PlaceStackFrame::Grouped => ReturnMode::Place, - PlaceStackFrame::PropertyAccess { .. } => ReturnMode::Place, - PlaceStackFrame::Indexed { .. } => ReturnMode::Place, - } - } - - fn handle_place(self, place: Place, next: ActionCreator) -> ExecutionResult { - Ok(match self { - PlaceStackFrame::Assignment { value } => { - let span_range = match place { - Place::MutableReference(mut variable) => { - let span_range = - SpanRange::new_between(variable.span_range(), value.span_range()); - variable.set(value); - span_range - } - Place::Discarded(token) => token.span_range(), - }; - next.return_assignment_completion(span_range) - } - PlaceStackFrame::CompoundAssignmentRoot { operation, value } => { - let span_range = match place { - Place::MutableReference(mut variable) => { - let span_range = - SpanRange::new_between(variable.span_range(), value.span_range()); - variable - .value_mut() - .handle_compound_assignment(&operation, value, span_range)?; - span_range - } - Place::Discarded(token) => token.span_range(), - }; - next.return_value(ExpressionValue::None(span_range)) - } - PlaceStackFrame::PropertyAccess { access } => match place { - Place::MutableReference(reference) => { - next.return_place(Place::MutableReference(reference.resolve_property(access)?)) - } - Place::Discarded(underscore) => { - return underscore - .execution_err("Cannot access the property of a discarded value") - } - }, - PlaceStackFrame::Grouped => next.return_place(place), - PlaceStackFrame::Indexed { access, index } => next.read_value_with_handler( - index, - ValueStackFrame::ResolveIndexedPlace { place, access }, - ), - }) - } -} - -enum Place { - MutableReference(MutableReference), - Discarded(Token![_]), -} diff --git a/src/expressions/evaluation/assignment_frames.rs b/src/expressions/evaluation/assignment_frames.rs new file mode 100644 index 00000000..eab6d06b --- /dev/null +++ b/src/expressions/evaluation/assignment_frames.rs @@ -0,0 +1,335 @@ +use super::*; + +pub(super) struct AssignmentCompletion { + pub(super) span_range: SpanRange, +} + +/// Handlers which return an AssignmentCompletion +pub(super) enum AnyAssignmentFrame { + Place(PlaceAssigner), + Grouped(GroupedAssigner), + Array(ArrayBasedAssigner), + Object(Box), +} + +impl AnyAssignmentFrame { + pub(super) fn handle_item( + self, + context: Context, + item: EvaluationItem, + ) -> ExecutionResult { + match self { + Self::Place(frame) => frame.handle_item(context, item), + Self::Grouped(frame) => frame.handle_item(context, item), + Self::Object(frame) => frame.handle_item(context, item), + Self::Array(frame) => frame.handle_item(context, item), + } + } +} + +struct PrivateUnit; + +pub(super) struct PlaceAssigner { + value: ExpressionValue, +} + +impl PlaceAssigner { + pub(super) fn start( + context: AssignmentContext, + place: ExpressionNodeId, + value: ExpressionValue, + ) -> NextAction { + let frame = Self { value }; + context.handle_node_as_place(frame, place, RequestedPlaceOwnership::MutableReference) + } +} + +impl EvaluationFrame for PlaceAssigner { + type ReturnType = AssignmentType; + + fn into_any(self) -> AnyAssignmentFrame { + AnyAssignmentFrame::Place(self) + } + + fn handle_item( + self, + context: AssignmentContext, + item: EvaluationItem, + ) -> ExecutionResult { + let mut mutable_place = item.expect_mutable_ref(); + let value = self.value; + let span_range = SpanRange::new_between(mutable_place.span_range(), value.span_range()); + mutable_place.set(value); + Ok(context.return_assignment_completion(span_range)) + } +} + +pub(super) struct GroupedAssigner(PrivateUnit); + +impl GroupedAssigner { + pub(super) fn start( + context: AssignmentContext, + inner: ExpressionNodeId, + value: ExpressionValue, + ) -> NextAction { + let frame = Self(PrivateUnit); + context.handle_node_as_assignment(frame, inner, value) + } +} + +impl EvaluationFrame for GroupedAssigner { + type ReturnType = AssignmentType; + + fn into_any(self) -> AnyAssignmentFrame { + AnyAssignmentFrame::Grouped(self) + } + + fn handle_item( + self, + context: AssignmentContext, + item: EvaluationItem, + ) -> ExecutionResult { + let AssignmentCompletion { span_range } = item.expect_assignment_complete(); + Ok(context.return_assignment_completion(span_range)) + } +} + +pub(super) struct ArrayBasedAssigner { + span_range: SpanRange, + assignee_stack: Vec<(ExpressionNodeId, ExpressionValue)>, +} + +impl ArrayBasedAssigner { + pub(super) fn start( + context: AssignmentContext, + nodes: &[ExpressionNode], + brackets: &Brackets, + assignee_item_node_ids: &[ExpressionNodeId], + value: ExpressionValue, + ) -> ExecutionResult { + let frame = Self::new(nodes, brackets.join(), assignee_item_node_ids, value)?; + Ok(frame.handle_next(context)) + } + + /// See also `ArrayPattern` in `patterns.rs` + fn new( + nodes: &[ExpressionNode], + assignee_span: Span, + assignee_item_node_ids: &[ExpressionNodeId], + value: ExpressionValue, + ) -> ExecutionResult { + let array = value.expect_array("The value destructured as an array")?; + let span_range = SpanRange::new_between(assignee_span, array.span_range.end()); + let mut has_seen_dot_dot = false; + let mut prefix_assignees = Vec::new(); + let mut suffix_assignees = Vec::new(); + for node in assignee_item_node_ids.iter() { + match nodes[node.0] { + ExpressionNode::Range { + left: None, + range_limits: syn::RangeLimits::HalfOpen(_), + right: None, + } => { + if has_seen_dot_dot { + return assignee_span + .execution_err("Only one .. is allowed in an array assignee"); + } + has_seen_dot_dot = true; + } + _ => { + if has_seen_dot_dot { + suffix_assignees.push(*node); + } else { + prefix_assignees.push(*node); + } + } + } + } + + let array_length = array.items.len(); + + let mut assignee_pairs: Vec<_> = if has_seen_dot_dot { + let total_assignees = prefix_assignees.len() + suffix_assignees.len(); + if total_assignees > array_length { + return assignee_span.execution_err(format!( + "The array has {} items, but the assignee expected at least {}", + array_length, total_assignees, + )); + } + let discarded_count = + array.items.len() - prefix_assignees.len() - suffix_assignees.len(); + let assignees = prefix_assignees + .into_iter() + .map(Some) + .chain(std::iter::repeat(None).take(discarded_count)) + .chain(suffix_assignees.into_iter().map(Some)); + + assignees + .zip(array.items) + .filter_map(|(assignee, value)| Some((assignee?, value))) + .collect() + } else { + let total_assignees = prefix_assignees.len(); + if total_assignees != array_length { + return assignee_span.execution_err(format!( + "The array has {} items, but the assignee expected {}", + array_length, total_assignees, + )); + } + prefix_assignees.into_iter().zip(array.items).collect() + }; + Ok(Self { + span_range, + assignee_stack: { + assignee_pairs.reverse(); + assignee_pairs + }, + }) + } + + fn handle_next(mut self, context: AssignmentContext) -> NextAction { + match self.assignee_stack.pop() { + Some((node, value)) => context.handle_node_as_assignment(self, node, value), + None => context.return_assignment_completion(self.span_range), + } + } +} + +impl EvaluationFrame for ArrayBasedAssigner { + type ReturnType = AssignmentType; + + fn into_any(self) -> AnyAssignmentFrame { + AnyAssignmentFrame::Array(self) + } + + fn handle_item( + self, + context: AssignmentContext, + item: EvaluationItem, + ) -> ExecutionResult { + let AssignmentCompletion { .. } = item.expect_assignment_complete(); + Ok(self.handle_next(context)) + } +} + +pub(super) struct ObjectBasedAssigner { + span_range: SpanRange, + entries: BTreeMap, + already_used_keys: HashSet, + unresolved_stack: Vec<(ObjectKey, ExpressionNodeId)>, + state: ObjectAssignmentState, +} + +enum ObjectAssignmentState { + WaitingForSubassignment, + ResolvingIndex { + assignee_node: ExpressionNodeId, + access: IndexAccess, + }, +} + +impl ObjectBasedAssigner { + pub(super) fn start( + context: AssignmentContext, + braces: &Braces, + assignee_pairs: &[(ObjectKey, ExpressionNodeId)], + value: ExpressionValue, + ) -> ExecutionResult { + let frame = Box::new(Self::new(braces.join(), assignee_pairs, value)?); + frame.handle_next(context) + } + + fn new( + assignee_span: Span, + assignee_pairs: &[(ObjectKey, ExpressionNodeId)], + value: ExpressionValue, + ) -> ExecutionResult { + let object = value.expect_object("The value destructured as an object")?; + let span_range = SpanRange::new_between(assignee_span, object.span_range.end()); + + Ok(Self { + span_range, + entries: object.entries, + already_used_keys: HashSet::with_capacity(assignee_pairs.len()), + unresolved_stack: assignee_pairs.iter().rev().cloned().collect(), + state: ObjectAssignmentState::WaitingForSubassignment, + }) + } + + fn handle_index_value( + mut self: Box, + context: AssignmentContext, + access: IndexAccess, + index: &ExpressionValue, + assignee_node: ExpressionNodeId, + ) -> ExecutionResult { + let key = index.ref_expect_string("An object key")?.clone().value; + let value = self.resolve_value(key, access.span())?; + Ok(context.handle_node_as_assignment(self, assignee_node, value)) + } + + fn handle_next(mut self: Box, context: AssignmentContext) -> ExecutionResult { + Ok(match self.unresolved_stack.pop() { + Some((ObjectKey::Identifier(ident), assignee_node)) => { + let key = ident.to_string(); + let value = self.resolve_value(key, ident.span())?; + self.state = ObjectAssignmentState::WaitingForSubassignment; + context.handle_node_as_assignment(self, assignee_node, value) + } + Some((ObjectKey::Indexed { index, access }, assignee_node)) => { + self.state = ObjectAssignmentState::ResolvingIndex { + assignee_node, + access, + }; + context.handle_node_as_value( + self, + index, + // This only needs to be read-only, as we are just using it to work out which field/s to assign + RequestedValueOwnership::SharedReference, + ) + } + None => context.return_assignment_completion(self.span_range), + }) + } + + fn resolve_value(&mut self, key: String, key_span: Span) -> ExecutionResult { + if self.already_used_keys.contains(&key) { + return key_span.execution_err(format!("The key `{}` has already used", key)); + } + let value = self + .entries + .remove(&key) + .map(|entry| entry.value) + .unwrap_or_else(|| ExpressionValue::None(key_span.span_range())); + self.already_used_keys.insert(key); + Ok(value) + } +} + +impl EvaluationFrame for Box { + type ReturnType = AssignmentType; + + fn into_any(self) -> AnyAssignmentFrame { + AnyAssignmentFrame::Object(self) + } + + fn handle_item( + self, + context: AssignmentContext, + item: EvaluationItem, + ) -> ExecutionResult { + match self.state { + ObjectAssignmentState::ResolvingIndex { + assignee_node, + access, + } => { + let index_place = item.expect_shared_ref(); + self.handle_index_value(context, access, index_place.as_ref(), assignee_node) + } + ObjectAssignmentState::WaitingForSubassignment => { + let AssignmentCompletion { .. } = item.expect_assignment_complete(); + self.handle_next(context) + } + } + } +} diff --git a/src/expressions/evaluation/evaluator.rs b/src/expressions/evaluation/evaluator.rs new file mode 100644 index 00000000..b1e6426e --- /dev/null +++ b/src/expressions/evaluation/evaluator.rs @@ -0,0 +1,434 @@ +#![allow(unused)] // TODO: Remove when places are properly late-bound +use super::*; + +pub(in super::super) struct ExpressionEvaluator<'a, K: Expressionable> { + nodes: &'a [ExpressionNode], + stack: EvaluationStack, +} + +impl<'a> ExpressionEvaluator<'a, Source> { + pub(in super::super) fn new(nodes: &'a [ExpressionNode]) -> Self { + Self { + nodes, + stack: EvaluationStack::new(), + } + } + + pub(in super::super) fn evaluate( + mut self, + root: ExpressionNodeId, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + let mut next_action = + NextActionInner::ReadNodeAsValue(root, RequestedValueOwnership::Owned); + + loop { + match self.step(next_action, interpreter)? { + StepResult::Continue(continue_action) => { + next_action = continue_action.0; + } + StepResult::Return(value) => { + return Ok(value); + } + } + } + } + + fn step( + &mut self, + action: NextActionInner, + interpreter: &mut Interpreter, + ) -> ExecutionResult { + Ok(StepResult::Continue(match action { + NextActionInner::ReadNodeAsValue(node, ownership) => self.nodes[node.0] + .handle_as_value( + interpreter, + Context { + request: ownership, + stack: &mut self.stack, + }, + )?, + NextActionInner::ReadNodeAsAssignee(node, value) => self.nodes[node.0] + .handle_as_assignee( + interpreter, + Context { + stack: &mut self.stack, + request: (), + }, + self.nodes, + node, + value, + )?, + NextActionInner::ReadNodeAsPlace(node, ownership) => self.nodes[node.0] + .handle_as_place( + interpreter, + Context { + stack: &mut self.stack, + request: ownership, + }, + )?, + NextActionInner::HandleReturnedItem(item) => { + let top_of_stack = match self.stack.handlers.pop() { + Some(top) => top, + None => { + // This aligns with the request for an owned value in evaluate + return Ok(StepResult::Return(item.expect_owned_value())); + } + }; + top_of_stack.handle_item(&mut self.stack, item)? + } + })) + } +} + +pub(super) struct EvaluationStack { + /// The stack of operations which are waiting to handle an item / value / assignment completion to continue execution + handlers: Vec, +} + +impl EvaluationStack { + pub(super) fn new() -> Self { + Self { + handlers: Vec::new(), + } + } +} + +pub(super) enum StepResult { + Continue(NextAction), + Return(ExpressionValue), +} + +pub(super) struct NextAction(NextActionInner); + +impl NextAction { + pub(super) fn return_owned(value: ExpressionValue) -> Self { + NextActionInner::HandleReturnedItem(EvaluationItem::OwnedValue(value)).into() + } + + pub(super) fn return_mutable(mut_ref: MutableValueReference) -> Self { + NextActionInner::HandleReturnedItem(EvaluationItem::MutableReference { mut_ref }).into() + } + + pub(super) fn return_shared( + shared_ref: SharedValueReference, + reason_not_mutable: Option, + ) -> Self { + NextActionInner::HandleReturnedItem(EvaluationItem::SharedReference { + shared_ref, + reason_not_mutable, + }) + .into() + } +} + +enum NextActionInner { + /// Enters an expression node to output a value + ReadNodeAsValue(ExpressionNodeId, RequestedValueOwnership), + // Enters an expression node for assignment purposes + ReadNodeAsAssignee(ExpressionNodeId, ExpressionValue), + // Enters an expression node to output a place (a location in memory) + // A place can be thought of as a mutable reference for e.g. a += operation + ReadNodeAsPlace(ExpressionNodeId, RequestedPlaceOwnership), + HandleReturnedItem(EvaluationItem), +} + +impl From for NextAction { + fn from(value: NextActionInner) -> Self { + Self(value) + } +} + +pub(super) enum EvaluationItem { + OwnedValue(ExpressionValue), + SharedReference { + shared_ref: SharedValueReference, + /// This is only populated if we request a "late bound" reference, and fail to resolve + /// a mutable reference. + reason_not_mutable: Option, + }, + MutableReference { + mut_ref: MutableValueReference, + }, + AssignmentCompletion(AssignmentCompletion), +} + +impl EvaluationItem { + pub(super) fn expect_owned_value(self) -> ExpressionValue { + match self { + EvaluationItem::OwnedValue(value) => value, + _ => panic!("expect_owned_value() called on a non-owned-value EvaluationItem"), + } + } + + pub(super) fn expect_any_place(self) -> Place { + match self { + EvaluationItem::MutableReference { mut_ref } => Place::MutableReference { mut_ref }, + EvaluationItem::SharedReference { + shared_ref, + reason_not_mutable, + } => Place::SharedReference { + shared_ref, + reason_not_mutable, + }, + _ => panic!("expect_any_place() called on a non-place EvaluationItem"), + } + } + + pub(super) fn expect_shared_ref(self) -> SharedValueReference { + match self { + EvaluationItem::SharedReference { shared_ref, .. } => shared_ref, + _ => { + panic!("expect_shared_reference() called on a non-shared-reference EvaluationItem") + } + } + } + + pub(super) fn expect_mutable_ref(self) -> MutableValueReference { + match self { + EvaluationItem::MutableReference { mut_ref } => mut_ref, + _ => panic!("expect_mutable_place() called on a non-mutable-place EvaluationItem"), + } + } + + pub(super) fn expect_assignment_complete(self) -> AssignmentCompletion { + match self { + EvaluationItem::AssignmentCompletion(completion) => completion, + _ => panic!( + "expect_assignment_complete() called on a non-assignment-completion EvaluationItem" + ), + } + } +} + +/// See the [rust reference] for a good description of assignee vs place. +/// +/// [rust reference]: https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions +pub(super) enum AnyEvaluationHandler { + Value(AnyValueFrame, RequestedValueOwnership), + Place(AnyPlaceFrame, RequestedPlaceOwnership), + Assignment(AnyAssignmentFrame), +} + +impl AnyEvaluationHandler { + fn handle_item( + self, + stack: &mut EvaluationStack, + item: EvaluationItem, + ) -> ExecutionResult { + match self { + AnyEvaluationHandler::Value(handler, ownership) => handler.handle_item( + Context { + stack, + request: ownership, + }, + item, + ), + AnyEvaluationHandler::Place(handler, ownership) => handler.handle_item( + Context { + stack, + request: ownership, + }, + item, + ), + AnyEvaluationHandler::Assignment(handler) => { + handler.handle_item(Context { stack, request: () }, item) + } + } + } +} + +pub(super) struct Context<'a, T: EvaluationItemType> { + stack: &'a mut EvaluationStack, + request: T::RequestConstraints, +} + +impl<'a, T: EvaluationItemType> Context<'a, T> { + pub(super) fn handle_node_as_value>( + self, + handler: H, + node: ExpressionNodeId, + requested_ownership: RequestedValueOwnership, + ) -> NextAction { + self.stack + .handlers + .push(T::into_unkinded_handler(handler.into_any(), self.request)); + NextActionInner::ReadNodeAsValue(node, requested_ownership).into() + } + + pub(super) fn handle_node_as_place>( + self, + handler: H, + node: ExpressionNodeId, + requested_ownership: RequestedPlaceOwnership, + ) -> NextAction { + self.stack + .handlers + .push(T::into_unkinded_handler(handler.into_any(), self.request)); + NextActionInner::ReadNodeAsPlace(node, requested_ownership).into() + } + + pub(super) fn handle_node_as_assignment>( + self, + handler: H, + node: ExpressionNodeId, + value: ExpressionValue, + ) -> NextAction { + self.stack + .handlers + .push(T::into_unkinded_handler(handler.into_any(), self.request)); + NextActionInner::ReadNodeAsAssignee(node, value).into() + } +} + +pub(super) trait EvaluationFrame: Sized { + type ReturnType: EvaluationItemType; + + fn into_any(self) -> ::AnyHandler; + + fn handle_item( + self, + context: Context, + item: EvaluationItem, + ) -> ExecutionResult; +} + +pub(super) trait EvaluationItemType { + type RequestConstraints; + type AnyHandler; + fn into_unkinded_handler( + handler: Self::AnyHandler, + request: Self::RequestConstraints, + ) -> AnyEvaluationHandler; +} + +pub(super) struct ValueType; +pub(super) type ValueContext<'a> = Context<'a, ValueType>; + +impl EvaluationItemType for ValueType { + type RequestConstraints = RequestedValueOwnership; + type AnyHandler = AnyValueFrame; + + fn into_unkinded_handler( + handler: Self::AnyHandler, + request: Self::RequestConstraints, + ) -> AnyEvaluationHandler { + AnyEvaluationHandler::Value(handler, request) + } +} + +impl<'a> Context<'a, ValueType> { + pub(super) fn requested_ownership(&self) -> RequestedValueOwnership { + self.request + } + + pub(super) fn return_owned_value(self, value: ExpressionValue) -> NextAction { + match self.request { + RequestedValueOwnership::LateBound | RequestedValueOwnership::Owned => { + NextAction::return_owned(value) + } + RequestedValueOwnership::SharedReference => { + NextAction::return_shared(SharedSubPlace::new_from_owned(value), None) + } + RequestedValueOwnership::MutableReference => { + NextAction::return_mutable(MutableSubPlace::new_from_owned(value)) + } + } + } + + pub(super) fn return_mut_ref(self, mut_ref: MutableValueReference) -> NextAction { + match self.request { + RequestedValueOwnership::LateBound + | RequestedValueOwnership::MutableReference => NextAction::return_mutable(mut_ref), + RequestedValueOwnership::Owned + | RequestedValueOwnership::SharedReference => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), + } + } + + pub(super) fn return_ref( + self, + shared_ref: SharedValueReference, + reason_not_mutable: Option, + ) -> NextAction { + match self.request { + RequestedValueOwnership::LateBound + | RequestedValueOwnership::SharedReference => NextAction::return_shared(shared_ref, reason_not_mutable), + RequestedValueOwnership::Owned + | RequestedValueOwnership::MutableReference => panic!("Returning a reference should only be used when the requested ownership is shared or late-bound"), + } + } +} + +pub(super) struct PlaceType; + +pub(super) type PlaceContext<'a> = Context<'a, PlaceType>; + +impl EvaluationItemType for PlaceType { + type RequestConstraints = RequestedPlaceOwnership; + type AnyHandler = AnyPlaceFrame; + + fn into_unkinded_handler( + handler: Self::AnyHandler, + request: Self::RequestConstraints, + ) -> AnyEvaluationHandler { + AnyEvaluationHandler::Place(handler, request) + } +} + +impl<'a> Context<'a, PlaceType> { + pub(super) fn requested_ownership(&self) -> RequestedPlaceOwnership { + self.request + } + + pub(super) fn return_any(self, place: Place) -> NextAction { + match place { + Place::MutableReference { mut_ref } => self.return_mutable(mut_ref), + Place::SharedReference { + shared_ref, + reason_not_mutable, + } => self.return_shared(shared_ref, reason_not_mutable), + } + } + + pub(super) fn return_mutable(self, mut_ref: MutableValueReference) -> NextAction { + match self.request { + RequestedPlaceOwnership::LateBound + | RequestedPlaceOwnership::MutableReference => NextAction::return_mutable(mut_ref), + RequestedPlaceOwnership::SharedReference => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), + } + } + + pub(super) fn return_shared( + self, + shared_ref: SharedValueReference, + reason_not_mutable: Option, + ) -> NextAction { + match self.request { + RequestedPlaceOwnership::LateBound + | RequestedPlaceOwnership::SharedReference => NextAction::return_shared(shared_ref, reason_not_mutable), + RequestedPlaceOwnership::MutableReference => panic!("Returning a shared reference should only be used when the requested ownership is shared or late-bound"), + } + } +} + +pub(super) struct AssignmentType; + +pub(super) type AssignmentContext<'a> = Context<'a, AssignmentType>; + +impl EvaluationItemType for AssignmentType { + type RequestConstraints = (); + type AnyHandler = AnyAssignmentFrame; + + fn into_unkinded_handler(handler: Self::AnyHandler, _request: ()) -> AnyEvaluationHandler { + AnyEvaluationHandler::Assignment(handler) + } +} + +impl<'a> Context<'a, AssignmentType> { + pub(super) fn return_assignment_completion(self, span_range: SpanRange) -> NextAction { + NextActionInner::HandleReturnedItem(EvaluationItem::AssignmentCompletion( + AssignmentCompletion { span_range }, + )) + .into() + } +} diff --git a/src/expressions/evaluation/mod.rs b/src/expressions/evaluation/mod.rs new file mode 100644 index 00000000..e8aba6f8 --- /dev/null +++ b/src/expressions/evaluation/mod.rs @@ -0,0 +1,13 @@ +mod assignment_frames; +mod evaluator; +mod node_conversion; +mod place_frames; +mod type_resolution; +mod value_frames; + +use super::*; +use assignment_frames::*; +pub(super) use evaluator::ExpressionEvaluator; +use evaluator::*; +use place_frames::*; +use value_frames::*; diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs new file mode 100644 index 00000000..103c113c --- /dev/null +++ b/src/expressions/evaluation/node_conversion.rs @@ -0,0 +1,144 @@ +use super::*; + +impl ExpressionNode { + pub(super) fn handle_as_value( + &self, + interpreter: &mut Interpreter, + context: Context, + ) -> ExecutionResult { + Ok(match self { + ExpressionNode::Leaf(leaf) => { + context.return_owned_value(Source::evaluate_leaf(leaf, interpreter)?) + } + ExpressionNode::Grouped { delim_span, inner } => { + GroupBuilder::start(context, delim_span, *inner) + } + ExpressionNode::Array { brackets, items } => { + ArrayBuilder::start(context, brackets, items) + } + ExpressionNode::Object { braces, entries } => { + ObjectBuilder::start(context, braces, entries)? + } + ExpressionNode::UnaryOperation { operation, input } => { + UnaryOperationBuilder::start(context, operation.clone(), *input) + } + ExpressionNode::BinaryOperation { + operation, + left_input, + right_input, + } => { + BinaryOperationBuilder::start(context, operation.clone(), *left_input, *right_input) + } + ExpressionNode::Property { node, access } => { + ValuePropertyAccessBuilder::start(context, access.clone(), *node) + } + ExpressionNode::Index { + node, + access, + index, + } => ValueIndexAccessBuilder::start(context, *node, *access, *index), + ExpressionNode::Range { + left, + range_limits, + right, + } => RangeBuilder::start(context, left, range_limits, right), + ExpressionNode::Assignment { + assignee, + equals_token, + value, + } => AssignmentBuilder::start(context, *assignee, *equals_token, *value), + ExpressionNode::CompoundAssignment { + place, + operation, + value, + } => CompoundAssignmentBuilder::start(context, *place, *operation, *value), + // TODO - Change this to follow outline in changelog + ExpressionNode::MethodCall { + node, + method, + parameters, + } => MethodCallBuilder::start(context, *node, method.clone(), parameters), + }) + } + + pub(super) fn handle_as_assignee( + &self, + _: &mut Interpreter, + context: AssignmentContext, + nodes: &[ExpressionNode], + self_node_id: ExpressionNodeId, + value: ExpressionValue, + ) -> ExecutionResult { + Ok(match self { + ExpressionNode::Leaf(SourceExpressionLeaf::Variable(_)) + | ExpressionNode::Index { .. } + | ExpressionNode::Property { .. } => PlaceAssigner::start(context, self_node_id, value), + ExpressionNode::Leaf(SourceExpressionLeaf::Discarded(underscore)) => { + context.return_assignment_completion(SpanRange::new_between(*underscore, value)) + } + ExpressionNode::Array { + brackets, + items: assignee_item_node_ids, + } => { + ArrayBasedAssigner::start(context, nodes, brackets, assignee_item_node_ids, value)? + } + ExpressionNode::Object { braces, entries } => { + ObjectBasedAssigner::start(context, braces, entries, value)? + } + ExpressionNode::Grouped { inner, .. } => GroupedAssigner::start(context, *inner, value), + other => { + return other + .operator_span_range() + .execution_err("This type of expression is not supported as an assignee. You may wish to use `_` to ignore the value."); + } + }) + } + + pub(super) fn handle_as_place( + &self, + interpreter: &mut Interpreter, + context: PlaceContext, + ) -> ExecutionResult { + Ok(match self { + ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => { + let variable_ref = variable.reference(interpreter)?; + match context.requested_ownership() { + RequestedPlaceOwnership::LateBound => { + match variable_ref.into_mut() { + Ok(place) => context.return_mutable(place), + Err(ExecutionInterrupt::Error(reason_not_mutable)) => { + // If we get an error with a mutable and shared reference, a mutable reference must already exist. + // We can just propogate the error from taking the shared reference, it should be good enough. + let shared_place = + variable.reference(interpreter)?.into_shared()?; + context.return_shared(shared_place, Some(reason_not_mutable)) + } + // Propogate any other errors, these shouldn't happen mind + Err(err) => Err(err)?, + } + } + RequestedPlaceOwnership::SharedReference => { + context.return_shared(variable_ref.into_shared()?, None) + } + RequestedPlaceOwnership::MutableReference => { + context.return_mutable(variable_ref.into_mut()?) + } + } + } + ExpressionNode::Index { + node, + access, + index, + } => PlaceIndexer::start(context, *node, *access, *index), + ExpressionNode::Property { node, access, .. } => { + PlacePropertyAccessor::start(context, *node, access.clone()) + } + ExpressionNode::Grouped { inner, .. } => PlaceGrouper::start(context, *inner), + other => { + return other + .operator_span_range() + .execution_err("This expression cannot be resolved into a memory location."); + } + }) + } +} diff --git a/src/expressions/evaluation/place_frames.rs b/src/expressions/evaluation/place_frames.rs new file mode 100644 index 00000000..126b262b --- /dev/null +++ b/src/expressions/evaluation/place_frames.rs @@ -0,0 +1,192 @@ +#![allow(unused)] // TODO: Remove when places are properly late-bound +use super::*; + +/// A rough equivalent of a Rust place (lvalue), as per: +/// https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions +/// +/// In preinterpret, references are (currently) only to variables, or sub-values of variables. +/// +/// # Late Binding +/// +/// Sometimes, a value which can be accessed, but we don't yet know *how* we need to access it. +/// In this case, we attempt to load it as a Mutable place, and failing that, as a Shared place. +/// +/// ## Example of requirement +/// For example, if we have `x[a].y(z)`, we first need to resolve the type of `x[a]` to know +/// whether `x[a]` takes a shared reference, mutable reference or an owned value. +/// +/// So instead, we take the most powerful access we can have for `x[a]`, and convert it later. +pub(super) enum Place { + MutableReference { + mut_ref: MutableValueReference, + }, + SharedReference { + shared_ref: SharedValueReference, + reason_not_mutable: Option, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RequestedPlaceOwnership { + LateBound, + SharedReference, + MutableReference, +} + +/// Handlers which return a Place +pub(super) enum AnyPlaceFrame { + Grouped(PlaceGrouper), + Indexed(PlaceIndexer), + PropertyAccessed(PlacePropertyAccessor), +} + +impl AnyPlaceFrame { + pub(super) fn handle_item( + self, + context: Context, + item: EvaluationItem, + ) -> ExecutionResult { + match self { + Self::Grouped(frame) => frame.handle_item(context, item), + Self::Indexed(frame) => frame.handle_item(context, item), + Self::PropertyAccessed(frame) => frame.handle_item(context, item), + } + } +} + +struct PrivateUnit; + +pub(super) struct PlaceGrouper(PrivateUnit); + +impl PlaceGrouper { + pub(super) fn start(context: PlaceContext, inner: ExpressionNodeId) -> NextAction { + let frame = Self(PrivateUnit); + let ownership = context.requested_ownership(); + context.handle_node_as_place(frame, inner, ownership) + } +} + +impl EvaluationFrame for PlaceGrouper { + type ReturnType = PlaceType; + + fn into_any(self) -> AnyPlaceFrame { + AnyPlaceFrame::Grouped(self) + } + + fn handle_item( + self, + context: PlaceContext, + item: EvaluationItem, + ) -> ExecutionResult { + Ok(context.return_any(item.expect_any_place())) + } +} + +pub(super) struct PlaceIndexer { + access: IndexAccess, + state: PlaceIndexerPath, +} + +enum PlaceIndexerPath { + PlacePath { index: ExpressionNodeId }, + IndexPath { place: Place }, +} + +impl PlaceIndexer { + pub(super) fn start( + context: PlaceContext, + source: ExpressionNodeId, + access: IndexAccess, + index: ExpressionNodeId, + ) -> NextAction { + let frame = Self { + access, + state: PlaceIndexerPath::PlacePath { index }, + }; + let ownership = context.requested_ownership(); + context.handle_node_as_place(frame, source, ownership) + } +} + +impl EvaluationFrame for PlaceIndexer { + type ReturnType = PlaceType; + + fn into_any(self) -> AnyPlaceFrame { + AnyPlaceFrame::Indexed(self) + } + + fn handle_item( + mut self, + context: PlaceContext, + item: EvaluationItem, + ) -> ExecutionResult { + Ok(match self.state { + PlaceIndexerPath::PlacePath { index } => { + let place = item.expect_any_place(); + self.state = PlaceIndexerPath::IndexPath { place }; + // If we do my_obj["my_key"] = 1 then the "my_key" place is created, + // so mutable reference indexing takes an owned index. + context.handle_node_as_value(self, index, RequestedValueOwnership::Owned) + } + PlaceIndexerPath::IndexPath { place } => { + let index = item.expect_owned_value(); + match place { + Place::MutableReference { mut_ref } => context.return_mutable( + mut_ref.resolve_indexed_with_autocreate(self.access, index)?, + ), + Place::SharedReference { + shared_ref, + reason_not_mutable, + } => context.return_shared( + shared_ref.resolve_indexed(self.access, &index)?, + reason_not_mutable, + ), + } + } + }) + } +} + +pub(super) struct PlacePropertyAccessor { + access: PropertyAccess, +} + +impl PlacePropertyAccessor { + pub(super) fn start( + context: PlaceContext, + source: ExpressionNodeId, + access: PropertyAccess, + ) -> NextAction { + let frame = Self { access }; + let requested_ownership = context.requested_ownership(); + context.handle_node_as_place(frame, source, requested_ownership) + } +} + +impl EvaluationFrame for PlacePropertyAccessor { + type ReturnType = PlaceType; + + fn into_any(self) -> AnyPlaceFrame { + AnyPlaceFrame::PropertyAccessed(self) + } + + fn handle_item( + self, + context: PlaceContext, + item: EvaluationItem, + ) -> ExecutionResult { + let place = item.expect_any_place(); + Ok(match place { + Place::MutableReference { mut_ref } => { + context.return_mutable(mut_ref.resolve_property(self.access)?) + } + Place::SharedReference { + shared_ref, + reason_not_mutable, + } => context.return_shared( + shared_ref.resolve_property(self.access)?, + reason_not_mutable, + ), + }) + } +} diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs new file mode 100644 index 00000000..0e8a3616 --- /dev/null +++ b/src/expressions/evaluation/type_resolution.rs @@ -0,0 +1,40 @@ +#![allow(unused)] // TODO: Remove when type resolution is implemented +use super::*; + +pub(super) trait ResolvedTypeDetails { + /// Whether the type can be transparently cloned. + fn supports_copy(&self) -> bool; + + /// Resolves a method for this resolved type with the given arguments. + fn resolve_method(&self, name: &str, num_arguments: usize) -> ExecutionResult; +} + +pub(super) struct ResolvedType { + inner: Box, +} + +impl ResolvedTypeDetails for ResolvedType { + fn supports_copy(&self) -> bool { + self.inner.supports_copy() + } + + fn resolve_method(&self, name: &str, num_arguments: usize) -> ExecutionResult { + self.inner.resolve_method(name, num_arguments) + } +} + +pub(super) struct ResolvedMethod { + // TODO: Add some reference to the code here + object_location_kind: RequestedValueOwnership, + parameter_location_kinds: Vec, +} + +impl ResolvedMethod { + fn execute( + &self, + object: ResolvedValue, + parameters: Vec, + ) -> ExecutionResult { + unimplemented!("Method execution is not implemented yet"); + } +} diff --git a/src/expressions/evaluation/value_frames.rs b/src/expressions/evaluation/value_frames.rs new file mode 100644 index 00000000..89eebcbd --- /dev/null +++ b/src/expressions/evaluation/value_frames.rs @@ -0,0 +1,797 @@ +#![allow(unused)] // TODO: Remove when values are properly late-bound +use super::*; + +/// # Late Binding +/// +/// Sometimes, a value which can be accessed, but we don't yet know *how* we need to access it. +/// In this case, we attempt to load it as a Mutable place, and failing that, as a Shared place. +/// +/// ## Example of requirement +/// For example, if we have `x.y(z)`, we first need to resolve the type of `x` to know +/// whether `y` takes a shared reference, mutable reference or an owned value. +/// +/// So instead, we take the most powerful access we can have, and convert it later. +pub(crate) enum ResolvedValue { + /// This has been requested as an owned value. + Owned(ExpressionValue), + /// This has been requested as a mutable reference. + Mutable(MutableValueReference), + /// This has been requested as a shared reference. + Shared { + shared_ref: SharedValueReference, + reason_not_mutable: Option, + }, +} + +impl ResolvedValue { + /// The requirement is just for error messages, and should be "A ", and will be filled like: + /// "{usage} must to be an owned value, but it is a reference to a non-copyable value kind" + pub(crate) fn into_owned_value(self, usage: &str) -> ExecutionResult { + let reference = match self { + ResolvedValue::Owned(value) => return Ok(value), + ResolvedValue::Shared { .. } | ResolvedValue::Mutable { .. } => self.as_ref(), + }; + // TODO: Add check that the value's type supports Auto-Clone (kinda like copy) + // reference.type_ref().assert_auto_clone(usage)?; + Ok(reference.clone()) + } + + pub(crate) fn as_value_ref(&self) -> &ExpressionValue { + match self { + ResolvedValue::Owned(value) => value, + ResolvedValue::Mutable(reference) => reference.as_ref(), + ResolvedValue::Shared { shared_ref, .. } => shared_ref.as_ref(), + } + } + + pub(crate) fn as_value_mut(&mut self) -> ExecutionResult<&mut ExpressionValue> { + match self { + ResolvedValue::Owned(value) => Ok(value), + ResolvedValue::Mutable(reference) => Ok(reference.as_mut()), + ResolvedValue::Shared { + shared_ref, + reason_not_mutable: Some(reason_not_mutable), + } => shared_ref.execution_err(format!( + "Cannot get a mutable reference: {}", + reason_not_mutable + )), + ResolvedValue::Shared { + shared_ref, + reason_not_mutable: None, + } => shared_ref.execution_err("Cannot get a mutable reference: Unknown reason"), + } + } +} + +impl WithSpanExt for ResolvedValue { + fn with_span(self, span: Span) -> Self { + match self { + ResolvedValue::Owned(value) => ResolvedValue::Owned(value.with_span(span)), + ResolvedValue::Mutable(reference) => ResolvedValue::Mutable(reference.with_span(span)), + ResolvedValue::Shared { + shared_ref, + reason_not_mutable, + } => ResolvedValue::Shared { + shared_ref: shared_ref.with_span(span), + reason_not_mutable, + }, + } + } +} + +impl AsRef for ResolvedValue { + fn as_ref(&self) -> &ExpressionValue { + self.as_value_ref() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RequestedValueOwnership { + LateBound, + Owned, + SharedReference, + MutableReference, +} + +/// Handlers which return a Value +pub(super) enum AnyValueFrame { + Group(GroupBuilder), + Array(ArrayBuilder), + Object(Box), + UnaryOperation(UnaryOperationBuilder), + BinaryOperation(BinaryOperationBuilder), + PropertyAccess(ValuePropertyAccessBuilder), + IndexAccess(ValueIndexAccessBuilder), + Range(RangeBuilder), + Assignment(AssignmentBuilder), + CompoundAssignment(CompoundAssignmentBuilder), + MethodCall(MethodCallBuilder), +} + +impl AnyValueFrame { + pub(super) fn handle_item( + self, + context: Context, + item: EvaluationItem, + ) -> ExecutionResult { + match self { + AnyValueFrame::Group(frame) => frame.handle_item(context, item), + AnyValueFrame::Array(frame) => frame.handle_item(context, item), + AnyValueFrame::Object(frame) => frame.handle_item(context, item), + AnyValueFrame::UnaryOperation(frame) => frame.handle_item(context, item), + AnyValueFrame::BinaryOperation(frame) => frame.handle_item(context, item), + AnyValueFrame::PropertyAccess(frame) => frame.handle_item(context, item), + AnyValueFrame::IndexAccess(frame) => frame.handle_item(context, item), + AnyValueFrame::Range(frame) => frame.handle_item(context, item), + AnyValueFrame::Assignment(frame) => frame.handle_item(context, item), + AnyValueFrame::CompoundAssignment(frame) => frame.handle_item(context, item), + AnyValueFrame::MethodCall(frame) => frame.handle_item(context, item), + } + } +} + +pub(super) struct GroupBuilder { + span: Span, +} + +impl GroupBuilder { + pub(super) fn start( + context: ValueContext, + delim_span: &DelimSpan, + inner: ExpressionNodeId, + ) -> NextAction { + let frame = Self { + span: delim_span.join(), + }; + context.handle_node_as_value(frame, inner, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for GroupBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::Group(self) + } + + fn handle_item( + self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + let inner = item.expect_owned_value(); + Ok(context.return_owned_value(inner.with_span(self.span))) + } +} + +pub(super) struct ArrayBuilder { + span: Span, + unevaluated_items: Vec, + evaluated_items: Vec, +} + +impl ArrayBuilder { + pub(super) fn start( + context: ValueContext, + brackets: &Brackets, + items: &[ExpressionNodeId], + ) -> NextAction { + Self { + span: brackets.join(), + unevaluated_items: items.to_vec(), + evaluated_items: Vec::with_capacity(items.len()), + } + .next(context) + } + + pub(super) fn next(self, context: ValueContext) -> NextAction { + match self + .unevaluated_items + .get(self.evaluated_items.len()) + .cloned() + { + Some(next) => context.handle_node_as_value(self, next, RequestedValueOwnership::Owned), + None => context.return_owned_value(ExpressionValue::Array(ExpressionArray { + items: self.evaluated_items, + span_range: self.span.span_range(), + })), + } + } +} + +impl EvaluationFrame for ArrayBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::Array(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + let value = item.expect_owned_value(); + self.evaluated_items.push(value); + Ok(self.next(context)) + } +} + +pub(super) struct ObjectBuilder { + span: Span, + pending: Option, + unevaluated_entries: Vec<(ObjectKey, ExpressionNodeId)>, + evaluated_entries: BTreeMap, +} + +enum PendingEntryPath { + OnIndexKeyBranch { + access: IndexAccess, + value_node: ExpressionNodeId, + }, + OnValueBranch { + key: String, + key_span: Span, + }, +} + +impl ObjectBuilder { + pub(super) fn start( + context: ValueContext, + braces: &Braces, + entries: &[(ObjectKey, ExpressionNodeId)], + ) -> ExecutionResult { + Box::new(Self { + span: braces.join(), + unevaluated_entries: entries.to_vec(), + evaluated_entries: BTreeMap::new(), + pending: None, + }) + .next(context) + } + + fn next(mut self: Box, context: ValueContext) -> ExecutionResult { + Ok( + match self + .unevaluated_entries + .get(self.evaluated_entries.len()) + .cloned() + { + Some((ObjectKey::Identifier(ident), value_node)) => { + let key = ident.to_string(); + if self.evaluated_entries.contains_key(&key) { + return ident + .execution_err(format!("The key {} has already been set", key)); + } + self.pending = Some(PendingEntryPath::OnValueBranch { + key, + key_span: ident.span(), + }); + context.handle_node_as_value(self, value_node, RequestedValueOwnership::Owned) + } + Some((ObjectKey::Indexed { access, index }, value_node)) => { + self.pending = Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }); + context.handle_node_as_value(self, index, RequestedValueOwnership::Owned) + } + None => context + .return_owned_value(self.evaluated_entries.to_value(self.span.span_range())), + }, + ) + } +} + +impl EvaluationFrame for Box { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::Object(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + let pending = self.pending.take(); + Ok(match pending { + Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }) => { + let value = item.expect_owned_value(); + let key = value.expect_string("An object key")?.value; + if self.evaluated_entries.contains_key(&key) { + return access.execution_err(format!("The key {} has already been set", key)); + } + self.pending = Some(PendingEntryPath::OnValueBranch { + key, + key_span: access.span(), + }); + context.handle_node_as_value(self, value_node, RequestedValueOwnership::Owned) + } + Some(PendingEntryPath::OnValueBranch { key, key_span }) => { + let value = item.expect_owned_value(); + let entry = ObjectEntry { key_span, value }; + self.evaluated_entries.insert(key, entry); + self.next(context)? + } + None => { + unreachable!("Should not receive a value without a pending handler set") + } + }) + } +} + +pub(super) struct UnaryOperationBuilder { + operation: UnaryOperation, +} + +impl UnaryOperationBuilder { + pub(super) fn start( + context: ValueContext, + operation: UnaryOperation, + input: ExpressionNodeId, + ) -> NextAction { + let frame = Self { operation }; + // TODO: Change to be LateBound to resolve what it actually should be + context.handle_node_as_value(frame, input, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for UnaryOperationBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::UnaryOperation(self) + } + + fn handle_item( + self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + let value = item.expect_owned_value(); + Ok(context.return_owned_value(self.operation.evaluate(value)?)) + } +} + +pub(super) struct BinaryOperationBuilder { + operation: BinaryOperation, + state: BinaryPath, +} + +enum BinaryPath { + OnLeftBranch { right: ExpressionNodeId }, + OnRightBranch { left: ExpressionValue }, +} + +impl BinaryOperationBuilder { + pub(super) fn start( + context: ValueContext, + operation: BinaryOperation, + left: ExpressionNodeId, + right: ExpressionNodeId, + ) -> NextAction { + let frame = Self { + operation, + state: BinaryPath::OnLeftBranch { right }, + }; + // TODO: Change to be LateBound to resolve what it actually should be + context.handle_node_as_value(frame, left, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for BinaryOperationBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::BinaryOperation(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + Ok(match self.state { + BinaryPath::OnLeftBranch { right } => { + let value = item.expect_owned_value(); + if let Some(result) = self.operation.lazy_evaluate(&value)? { + context.return_owned_value(result) + } else { + self.state = BinaryPath::OnRightBranch { left: value }; + context.handle_node_as_value( + self, + right, + // TODO: Change to late bound as the operation may dictate what ownership it needs for its right operand + RequestedValueOwnership::Owned, + ) + } + } + BinaryPath::OnRightBranch { left } => { + let value = item.expect_owned_value(); + context.return_owned_value(self.operation.evaluate(left, value)?) + } + }) + } +} + +pub(super) struct ValuePropertyAccessBuilder { + access: PropertyAccess, +} + +impl ValuePropertyAccessBuilder { + pub(super) fn start( + context: ValueContext, + access: PropertyAccess, + node: ExpressionNodeId, + ) -> NextAction { + let frame = Self { access }; + // TODO: Change to propagate context from value, and not always clone! + let ownership = RequestedValueOwnership::Owned; + context.handle_node_as_value(frame, node, ownership) + } +} + +impl EvaluationFrame for ValuePropertyAccessBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::PropertyAccess(self) + } + + fn handle_item( + self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + let value = item.expect_owned_value(); + Ok(context.return_owned_value(value.into_property(self.access)?)) + } +} + +pub(super) struct ValueIndexAccessBuilder { + access: IndexAccess, + state: IndexPath, +} + +enum IndexPath { + OnSourceBranch { index: ExpressionNodeId }, + OnIndexBranch { object: ExpressionValue }, +} + +impl ValueIndexAccessBuilder { + pub(super) fn start( + context: ValueContext, + source: ExpressionNodeId, + access: IndexAccess, + index: ExpressionNodeId, + ) -> NextAction { + let frame = Self { + access, + state: IndexPath::OnSourceBranch { index }, + }; + // TODO: Change to propagate context from value, and not always clone + context.handle_node_as_value(frame, source, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for ValueIndexAccessBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::IndexAccess(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + Ok(match self.state { + IndexPath::OnSourceBranch { index } => { + let value = item.expect_owned_value(); + self.state = IndexPath::OnIndexBranch { object: value }; + context.handle_node_as_value( + self, + index, + // We can use a &index for reading values from our array + RequestedValueOwnership::SharedReference, + ) + } + IndexPath::OnIndexBranch { object } => { + let value = item.expect_shared_ref(); + context.return_owned_value(object.into_indexed(self.access, value.as_ref())?) + } + }) + } +} + +pub(super) struct RangeBuilder { + range_limits: syn::RangeLimits, + state: RangePath, +} + +enum RangePath { + OnLeftBranch { right: Option }, + OnRightBranch { left: Option }, +} + +impl RangeBuilder { + pub(super) fn start( + context: ValueContext, + left: &Option, + range_limits: &syn::RangeLimits, + right: &Option, + ) -> NextAction { + match (left, right) { + (None, None) => match range_limits { + syn::RangeLimits::HalfOpen(token) => { + let inner = ExpressionRangeInner::RangeFull { token: *token }; + context.return_owned_value(inner.to_value(token.span_range())) + } + syn::RangeLimits::Closed(_) => { + unreachable!( + "A closed range should have been given a right in continue_range(..)" + ) + } + }, + (None, Some(right)) => context.handle_node_as_value( + Self { + range_limits: *range_limits, + state: RangePath::OnRightBranch { left: None }, + }, + *right, + RequestedValueOwnership::Owned, + ), + (Some(left), right) => context.handle_node_as_value( + Self { + range_limits: *range_limits, + state: RangePath::OnLeftBranch { right: *right }, + }, + *left, + RequestedValueOwnership::Owned, + ), + } + } +} + +impl EvaluationFrame for RangeBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::Range(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + // TODO: Change to not always clone the value + let value = item.expect_owned_value(); + Ok(match (self.state, self.range_limits) { + (RangePath::OnLeftBranch { right: Some(right) }, _) => { + self.state = RangePath::OnRightBranch { left: Some(value) }; + context.handle_node_as_value(self, right, RequestedValueOwnership::Owned) + } + (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::HalfOpen(token)) => { + let inner = ExpressionRangeInner::RangeFrom { + start_inclusive: value, + token, + }; + context.return_owned_value(inner.to_value(token.span_range())) + } + (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::Closed(_)) => { + unreachable!("A closed range should have been given a right in continue_range(..)") + } + (RangePath::OnRightBranch { left: Some(left) }, syn::RangeLimits::HalfOpen(token)) => { + let inner = ExpressionRangeInner::Range { + start_inclusive: left, + token, + end_exclusive: value, + }; + context.return_owned_value(inner.to_value(token.span_range())) + } + (RangePath::OnRightBranch { left: Some(left) }, syn::RangeLimits::Closed(token)) => { + let inner = ExpressionRangeInner::RangeInclusive { + start_inclusive: left, + token, + end_inclusive: value, + }; + context.return_owned_value(inner.to_value(token.span_range())) + } + (RangePath::OnRightBranch { left: None }, syn::RangeLimits::HalfOpen(token)) => { + let inner = ExpressionRangeInner::RangeTo { + token, + end_exclusive: value, + }; + context.return_owned_value(inner.to_value(token.span_range())) + } + (RangePath::OnRightBranch { left: None }, syn::RangeLimits::Closed(token)) => { + let inner = ExpressionRangeInner::RangeToInclusive { + token, + end_inclusive: value, + }; + context.return_owned_value(inner.to_value(token.span_range())) + } + }) + } +} + +pub(super) struct AssignmentBuilder { + #[allow(unused)] + equals_token: Token![=], + state: AssignmentPath, +} + +enum AssignmentPath { + OnValueBranch { assignee: ExpressionNodeId }, + OnAwaitingAssignment, +} + +impl AssignmentBuilder { + pub(super) fn start( + context: ValueContext, + assignee: ExpressionNodeId, + equals_token: Token![=], + value: ExpressionNodeId, + ) -> NextAction { + let frame = Self { + equals_token, + state: AssignmentPath::OnValueBranch { assignee }, + }; + context.handle_node_as_value(frame, value, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for AssignmentBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::Assignment(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + Ok(match self.state { + AssignmentPath::OnValueBranch { assignee } => { + let value = item.expect_owned_value(); + self.state = AssignmentPath::OnAwaitingAssignment; + context.handle_node_as_assignment(self, assignee, value) + } + AssignmentPath::OnAwaitingAssignment => { + let AssignmentCompletion { span_range } = item.expect_assignment_complete(); + context.return_owned_value(ExpressionValue::None(span_range)) + } + }) + } +} + +pub(super) struct CompoundAssignmentBuilder { + operation: CompoundAssignmentOperation, + state: CompoundAssignmentPath, +} + +enum CompoundAssignmentPath { + OnValueBranch { place: ExpressionNodeId }, + OnPlaceBranch { value: ExpressionValue }, +} + +impl CompoundAssignmentBuilder { + pub(super) fn start( + context: ValueContext, + place: ExpressionNodeId, + operation: CompoundAssignmentOperation, + value: ExpressionNodeId, + ) -> NextAction { + let frame = Self { + operation, + state: CompoundAssignmentPath::OnValueBranch { place }, + }; + context.handle_node_as_value(frame, value, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for CompoundAssignmentBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::CompoundAssignment(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + Ok(match self.state { + CompoundAssignmentPath::OnValueBranch { place } => { + let value = item.expect_owned_value(); + self.state = CompoundAssignmentPath::OnPlaceBranch { value }; + // TODO: Resolve as LateBound, and then convert to what is needed based on the operation + context.handle_node_as_place(self, place, RequestedPlaceOwnership::MutableReference) + } + CompoundAssignmentPath::OnPlaceBranch { value } => { + let mut mutable_reference = item.expect_mutable_ref(); + let span_range = + SpanRange::new_between(mutable_reference.span_range(), value.span_range()); + mutable_reference.as_mut().handle_compound_assignment( + &self.operation, + value, + span_range, + )?; + context.return_owned_value(ExpressionValue::None(span_range)) + } + }) + } +} + +pub(super) struct MethodCallBuilder { + method: MethodAccess, + unevaluated_parameters_stack: Vec, + evaluated_parameters: Vec, + state: MethodCallPath, +} + +enum MethodCallPath { + CallerPath, + ArgumentsPath { caller: ExpressionValue }, +} + +impl MethodCallBuilder { + pub(super) fn start( + context: ValueContext, + caller: ExpressionNodeId, + method: MethodAccess, + parameters: &[ExpressionNodeId], + ) -> NextAction { + let frame = Self { + method, + unevaluated_parameters_stack: parameters.iter().rev().cloned().collect(), + evaluated_parameters: Vec::with_capacity(parameters.len()), + state: MethodCallPath::CallerPath, + }; + // TODO: Change to resolve the caller as LateBound, in order to resolve the ownership for the + // caller and parameters based on the method signature + context.handle_node_as_value(frame, caller, RequestedValueOwnership::Owned) + } +} + +impl EvaluationFrame for MethodCallBuilder { + type ReturnType = ValueType; + + fn into_any(self) -> AnyValueFrame { + AnyValueFrame::MethodCall(self) + } + + fn handle_item( + mut self, + context: ValueContext, + item: EvaluationItem, + ) -> ExecutionResult { + match self.state { + MethodCallPath::CallerPath => { + let caller = item.expect_owned_value(); + self.state = MethodCallPath::ArgumentsPath { caller }; + } + MethodCallPath::ArgumentsPath { .. } => { + let argument = item.expect_owned_value(); + self.evaluated_parameters.push(argument); + } + }; + Ok(match self.unevaluated_parameters_stack.pop() { + Some(parameter) => { + context.handle_node_as_value(self, parameter, RequestedValueOwnership::Owned) + } + None => { + let caller = match self.state { + MethodCallPath::CallerPath => unreachable!("Already updated above"), + MethodCallPath::ArgumentsPath { caller } => caller, + }; + context + .return_owned_value(caller.call_method(self.method, self.evaluated_parameters)?) + } + }) + } +} diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index 0c2c5980..f81b9a95 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -86,10 +86,10 @@ impl ExpressionInteger { } } - pub(crate) fn expect_usize(self) -> ExecutionResult { - Ok(match self.value { + pub(crate) fn expect_usize(&self) -> ExecutionResult { + Ok(match &self.value { ExpressionIntegerValue::Untyped(input) => input.parse_as()?, - ExpressionIntegerValue::Usize(input) => input, + ExpressionIntegerValue::Usize(input) => *input, _ => { return self .span_range diff --git a/src/expressions/object.rs b/src/expressions/object.rs index d50bba65..60772e43 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -61,11 +61,11 @@ impl ExpressionObject { pub(super) fn into_indexed( mut self, access: IndexAccess, - index: ExpressionValue, + index: &ExpressionValue, ) -> ExecutionResult { let span_range = SpanRange::new_between(self.span_range, access.span_range()); - let key = index.expect_string("An object key")?.value; - Ok(self.remove_or_none(&key, span_range)) + let key = index.expect_str("An object key")?; + Ok(self.remove_or_none(key, span_range)) } pub(super) fn into_property( @@ -101,7 +101,7 @@ impl ExpressionObject { } } - pub(super) fn index_mut( + pub(super) fn index_mut_with_autocreate( &mut self, access: IndexAccess, index: ExpressionValue, @@ -110,6 +110,18 @@ impl ExpressionObject { Ok(self.mut_entry_or_create(key, access.span())) } + pub(super) fn index_ref( + &self, + access: IndexAccess, + index: &ExpressionValue, + ) -> ExecutionResult<&ExpressionValue> { + let key = index.expect_str("An object key")?; + let entry = self.entries.get(key).ok_or_else(|| { + access.execution_error(format!("The object does not have a field named `{}`", key)) + })?; + Ok(&entry.value) + } + pub(super) fn property_mut( &mut self, access: PropertyAccess, @@ -117,6 +129,14 @@ impl ExpressionObject { Ok(self.mut_entry_or_create(access.property.to_string(), access.property.span())) } + pub(super) fn property_ref(&self, access: PropertyAccess) -> ExecutionResult<&ExpressionValue> { + let key = access.property.to_string(); + let entry = self.entries.get(&key).ok_or_else(|| { + access.execution_error(format!("The object does not have a field named `{}`", key)) + })?; + Ok(&entry.value) + } + fn mut_entry_or_create(&mut self, key: String, key_span: Span) -> &mut ExpressionValue { use std::collections::btree_map::*; match self.entries.entry(key) { diff --git a/src/expressions/range.rs b/src/expressions/range.rs index b07291e6..968df510 100644 --- a/src/expressions/range.rs +++ b/src/expressions/range.rs @@ -56,12 +56,12 @@ impl ExpressionRange { } pub(crate) fn resolve_to_index_range( - self, + &self, array: &ExpressionArray, ) -> ExecutionResult> { let mut start = 0; let mut end = array.items.len(); - Ok(match *self.inner { + Ok(match &*self.inner { ExpressionRangeInner::Range { start_inclusive, end_exclusive, diff --git a/src/expressions/value.rs b/src/expressions/value.rs index b1a712e8..8e4f5fd5 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -284,6 +284,17 @@ impl ExpressionValue { } } + pub(crate) fn expect_str(&self, place_descriptor: &str) -> ExecutionResult<&str> { + match self { + ExpressionValue::String(value) => Ok(&value.value), + other => other.execution_err(format!( + "{} must be a string, but it is {}", + place_descriptor, + other.articled_value_type(), + )), + } + } + pub(crate) fn expect_string(self, place_descriptor: &str) -> ExecutionResult { match self { ExpressionValue::String(value) => Ok(value), @@ -295,6 +306,20 @@ impl ExpressionValue { } } + pub(crate) fn ref_expect_string( + &self, + place_descriptor: &str, + ) -> ExecutionResult<&ExpressionString> { + match self { + ExpressionValue::String(value) => Ok(value), + other => other.execution_err(format!( + "{} must be a string, but it is {}", + place_descriptor, + other.articled_value_type(), + )), + } + } + pub(crate) fn expect_array(self, place_descriptor: &str) -> ExecutionResult { match self { ExpressionValue::Array(value) => Ok(value), @@ -437,7 +462,7 @@ impl ExpressionValue { } } - pub(super) fn into_indexed(self, access: IndexAccess, index: Self) -> ExecutionResult { + pub(super) fn into_indexed(self, access: IndexAccess, index: &Self) -> ExecutionResult { match self { ExpressionValue::Array(array) => array.into_indexed(access, index), ExpressionValue::Object(object) => object.into_indexed(access, index), @@ -445,14 +470,22 @@ impl ExpressionValue { } } - pub(crate) fn index_mut( + pub(crate) fn index_mut_with_autocreate( &mut self, access: IndexAccess, index: Self, ) -> ExecutionResult<&mut Self> { match self { - ExpressionValue::Array(array) => array.index_mut(access, index), - ExpressionValue::Object(object) => object.index_mut(access, index), + ExpressionValue::Array(array) => array.index_mut(access, &index), + ExpressionValue::Object(object) => object.index_mut_with_autocreate(access, index), + other => access.execution_err(format!("Cannot index into a {}", other.value_type())), + } + } + + pub(crate) fn index_ref(&self, access: IndexAccess, index: &Self) -> ExecutionResult<&Self> { + match self { + ExpressionValue::Array(array) => array.index_ref(access, index), + ExpressionValue::Object(object) => object.index_ref(access, index), other => access.execution_err(format!("Cannot index into a {}", other.value_type())), } } @@ -517,6 +550,16 @@ impl ExpressionValue { } } + pub(crate) fn property_ref(&self, access: PropertyAccess) -> ExecutionResult<&Self> { + match self { + ExpressionValue::Object(object) => object.property_ref(access), + other => access.execution_err(format!( + "Cannot access properties on a {}", + other.value_type() + )), + } + } + fn span_range_mut(&mut self) -> &mut SpanRange { match self { Self::None(span_range) => span_range, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 5622fe3a..4bcdb61f 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -97,7 +97,7 @@ impl NoOutputCommandDefinition for SetCommand { let variable_data = variable.reference(interpreter)?; content.interpret_into( interpreter, - variable_data.into_mut()?.into_stream()?.value_mut(), + variable_data.into_mut()?.into_stream()?.as_mut(), )?; } SetArguments::SetVariablesEmpty { variables } => { diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index f1614bba..62bd10e0 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -161,8 +161,12 @@ impl VariableReference { .with_span_range(self.variable_span_range)) } - pub(crate) fn into_mut(self) -> ExecutionResult> { - MutableReference::new(self) + pub(crate) fn into_mut(self) -> ExecutionResult { + MutableValueReference::new_from_variable(self) + } + + pub(crate) fn into_shared(self) -> ExecutionResult { + SharedValueReference::new_from_variable(self) } } @@ -172,15 +176,30 @@ impl HasSpanRange for VariableReference { } } -pub(crate) struct MutableReference { - mut_cell: MutRcRefCell, +pub(crate) type MutableValueReference = MutableSubPlace; + +/// A mutable reference to a value inside a variable; along with a span of the whole access. +/// For example, for `x.y[4]`, this captures both: +/// * The mutable reference to the location under `x` +/// * The lexical span of the tokens `x.y[4]` +pub(crate) struct MutableSubPlace { + mut_cell: MutSubRcRefCell, span_range: SpanRange, } -impl MutableReference { - fn new(reference: VariableReference) -> ExecutionResult { +impl MutableSubPlace { + pub(crate) fn new_from_owned(value: ExpressionValue) -> Self { + let span_range = value.span_range(); + Self { + // Unwrap is safe because it's a new refcell + mut_cell: MutSubRcRefCell::new(Rc::new(RefCell::new(value))).unwrap(), + span_range, + } + } + + fn new_from_variable(reference: VariableReference) -> ExecutionResult { Ok(Self { - mut_cell: MutRcRefCell::new(reference.data).map_err(|_| { + mut_cell: MutSubRcRefCell::new(reference.data).map_err(|_| { reference.variable_span_range.execution_error( "The variable cannot be modified as it is already being modified", ) @@ -189,27 +208,27 @@ impl MutableReference { }) } - pub(crate) fn into_stream(self) -> ExecutionResult> { + pub(crate) fn into_stream(self) -> ExecutionResult> { let stream = self.mut_cell.try_map(|value| match value { ExpressionValue::Stream(stream) => Ok(&mut stream.value), _ => self .span_range .execution_err("The variable is not a stream"), })?; - Ok(MutableReference { + Ok(MutableSubPlace { mut_cell: stream, span_range: self.span_range, }) } - pub(crate) fn resolve_indexed( + pub(crate) fn resolve_indexed_with_autocreate( self, access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult { let indexed = self .mut_cell - .try_map(|value| value.index_mut(access, index))?; + .try_map(|value| value.index_mut_with_autocreate(access, index))?; Ok(Self { mut_cell: indexed, span_range: SpanRange::new_between(self.span_range.start(), access.span()), @@ -230,14 +249,108 @@ impl MutableReference { } } -impl MutableReference { - pub(crate) fn value_mut(&mut self) -> &mut T { +impl AsMut for MutableSubPlace { + fn as_mut(&mut self) -> &mut T { &mut self.mut_cell } } -impl HasSpanRange for MutableReference { +impl AsRef for MutableSubPlace { + fn as_ref(&self) -> &T { + &self.mut_cell + } +} + +impl HasSpanRange for MutableSubPlace { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl WithSpanExt for MutableSubPlace { + fn with_span(self, span: Span) -> Self { + Self { + mut_cell: self.mut_cell, + span_range: SpanRange::new_single(span), + } + } +} + +pub(crate) type SharedValueReference = SharedSubPlace; + +/// A mutable reference to a value inside a variable; along with a span of the whole access. +/// For example, for `x.y[4]`, this captures both: +/// * The mutable reference to the location under `x` +/// * The lexical span of the tokens `x.y[4]` +pub(crate) struct SharedSubPlace { + shared_cell: SharedSubRcRefCell, + span_range: SpanRange, +} + +impl SharedSubPlace { + pub(crate) fn new_from_owned(value: ExpressionValue) -> Self { + let span_range = value.span_range(); + Self { + // Unwrap is safe because it's a new refcell + shared_cell: SharedSubRcRefCell::new(Rc::new(RefCell::new(value))).unwrap(), + span_range, + } + } + + fn new_from_variable(reference: VariableReference) -> ExecutionResult { + Ok(Self { + shared_cell: SharedSubRcRefCell::new(reference.data).map_err(|_| { + reference + .variable_span_range + .execution_error("The variable cannot be read as it is already being modified") + })?, + span_range: reference.variable_span_range, + }) + } + + pub(crate) fn resolve_indexed( + self, + access: IndexAccess, + index: &ExpressionValue, + ) -> ExecutionResult { + let indexed = self + .shared_cell + .try_map(|value| value.index_ref(access, index))?; + Ok(Self { + shared_cell: indexed, + span_range: SpanRange::new_between(self.span_range.start(), access.span()), + }) + } + + pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { + let span_range = SpanRange::new_between(self.span_range.start(), access.span_range()); + let indexed = self + .shared_cell + .try_map(|value| value.property_ref(access))?; + Ok(Self { + shared_cell: indexed, + span_range, + }) + } +} + +impl AsRef for SharedSubPlace { + fn as_ref(&self) -> &T { + &self.shared_cell + } +} + +impl HasSpanRange for SharedSubPlace { fn span_range(&self) -> SpanRange { self.span_range } } + +impl WithSpanExt for SharedSubPlace { + fn with_span(self, span: Span) -> Self { + Self { + shared_cell: self.shared_cell, + span_range: SpanRange::new_single(span), + } + } +} diff --git a/src/misc/mut_rc_ref_cell.rs b/src/misc/mut_rc_ref_cell.rs index 8c7f1e4b..f3425079 100644 --- a/src/misc/mut_rc_ref_cell.rs +++ b/src/misc/mut_rc_ref_cell.rs @@ -2,7 +2,9 @@ use crate::internal_prelude::*; use std::cell::*; use std::rc::Rc; -pub(crate) struct MutRcRefCell { +/// A mutable reference to a sub-value `U` inside a [`Rc>`]. +/// Only one [`MutSubRcRefCell`] can exist at a time for a given [`Rc>`]. +pub(crate) struct MutSubRcRefCell { /// This is actually a reference to the contents of the RefCell /// but we store it using `unsafe` as `'static`, and use unsafe blocks /// to ensure it's dropped first. @@ -12,7 +14,7 @@ pub(crate) struct MutRcRefCell { pointed_at: Rc>, } -impl MutRcRefCell { +impl MutSubRcRefCell { pub(crate) fn new(pointed_at: Rc>) -> Result { let ref_mut = pointed_at.try_borrow_mut()?; Ok(Self { @@ -26,11 +28,11 @@ impl MutRcRefCell { } } -impl MutRcRefCell { +impl MutSubRcRefCell { pub(crate) fn try_map( self, f: impl FnOnce(&mut U) -> Result<&mut V, E>, - ) -> Result, E> { + ) -> Result, E> { let mut error = None; let outcome = RefMut::filter_map(self.ref_mut, |inner| match f(inner) { Ok(value) => Some(value), @@ -40,7 +42,7 @@ impl MutRcRefCell { } }); match outcome { - Ok(ref_mut) => Ok(MutRcRefCell { + Ok(ref_mut) => Ok(MutSubRcRefCell { ref_mut, pointed_at: self.pointed_at, }), @@ -49,15 +51,72 @@ impl MutRcRefCell { } } -impl DerefMut for MutRcRefCell { +impl DerefMut for MutSubRcRefCell { fn deref_mut(&mut self) -> &mut U { &mut self.ref_mut } } -impl Deref for MutRcRefCell { +impl Deref for MutSubRcRefCell { type Target = U; fn deref(&self) -> &U { &self.ref_mut } } + +/// A shared (immutable) reference to a sub-value `U` inside a [`Rc>`]. +/// Many [`SharedSubRcRefCell`] can exist at the same time for a given [`Rc>`], +/// but if any exist, then no [`MutSubRcRefCell`] can exist. +pub(crate) struct SharedSubRcRefCell { + /// This is actually a reference to the contents of the RefCell + /// but we store it using `unsafe` as `'static`, and use unsafe blocks + /// to ensure it's dropped first. + /// + /// SAFETY: This *must* appear before `pointed_at` so that it's dropped first. + shared_ref: Ref<'static, U>, + pointed_at: Rc>, +} + +impl SharedSubRcRefCell { + pub(crate) fn new(pointed_at: Rc>) -> Result { + let shared_ref = pointed_at.try_borrow()?; + Ok(Self { + // SAFETY: We must ensure that this lifetime lives as long as the + // reference to pointed_at (i.e. the RefCell). + // This is guaranteed by the fact that the only time we drop the RefCell + // is when we drop the MutRcRefCell, and we ensure that the RefMut is dropped first. + shared_ref: unsafe { std::mem::transmute::, Ref<'static, T>>(shared_ref) }, + pointed_at, + }) + } +} + +impl SharedSubRcRefCell { + pub(crate) fn try_map( + self, + f: impl FnOnce(&U) -> Result<&V, E>, + ) -> Result, E> { + let mut error = None; + let outcome = Ref::filter_map(self.shared_ref, |inner| match f(inner) { + Ok(value) => Some(value), + Err(e) => { + error = Some(e); + None + } + }); + match outcome { + Ok(shared_ref) => Ok(SharedSubRcRefCell { + shared_ref, + pointed_at: self.pointed_at, + }), + Err(_) => Err(error.unwrap()), + } + } +} + +impl Deref for SharedSubRcRefCell { + type Target = U; + fn deref(&self) -> &U { + &self.shared_ref + } +} diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index 6b2bb4fb..abb59c16 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -256,7 +256,7 @@ impl HandleTransformation for ExplicitTransformStream { content.handle_transform( input, interpreter, - reference.into_mut()?.into_stream()?.value_mut(), + reference.into_mut()?.into_stream()?.as_mut(), )?; } ExplicitTransformStreamArguments::Discard { content, .. } => { diff --git a/src/transformation/variable_binding.rs b/src/transformation/variable_binding.rs index 119798d6..580df19f 100644 --- a/src/transformation/variable_binding.rs +++ b/src/transformation/variable_binding.rs @@ -189,29 +189,25 @@ impl HandleTransformation for VariableBinding { let reference = self.reference(interpreter)?; input .parse::()? - .push_as_token_tree(reference.into_mut()?.into_stream()?.value_mut()); + .push_as_token_tree(reference.into_mut()?.into_stream()?.as_mut()); } VariableBinding::GroupedAppendFlattened { .. } => { let reference = self.reference(interpreter)?; input .parse::()? - .flatten_into(reference.into_mut()?.into_stream()?.value_mut()); + .flatten_into(reference.into_mut()?.into_stream()?.as_mut()); } VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { let reference = self.reference(interpreter)?; - reference - .into_mut()? - .into_stream()? - .value_mut() - .push_grouped( - |inner| until.handle_parse_into(input, inner), - Delimiter::None, - marker.span, - )?; + reference.into_mut()?.into_stream()?.as_mut().push_grouped( + |inner| until.handle_parse_into(input, inner), + Delimiter::None, + marker.span, + )?; } VariableBinding::FlattenedAppendFlattened { until, .. } => { let reference = self.reference(interpreter)?; - until.handle_parse_into(input, reference.into_mut()?.into_stream()?.value_mut())?; + until.handle_parse_into(input, reference.into_mut()?.into_stream()?.as_mut())?; } } Ok(()) diff --git a/tests/compilation_failures/expressions/index_into_discarded_place.stderr b/tests/compilation_failures/expressions/index_into_discarded_place.stderr index c6da3ce2..7403ee1e 100644 --- a/tests/compilation_failures/expressions/index_into_discarded_place.stderr +++ b/tests/compilation_failures/expressions/index_into_discarded_place.stderr @@ -1,5 +1,5 @@ -error: Cannot index into a discarded value - --> tests/compilation_failures/expressions/index_into_discarded_place.rs:5:12 +error: This expression cannot be resolved into a memory location. + --> tests/compilation_failures/expressions/index_into_discarded_place.rs:5:11 | 5 | #(_[0] = 10) - | ^^^ + | ^ From 7b5febd733377361863623fa792b736a312116a3 Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 18 Sep 2025 16:44:01 +0100 Subject: [PATCH 121/126] wip: Method resolutions GATs and Macros --- CHANGELOG.md | 9 +- src/expressions/evaluation/evaluator.rs | 46 +- src/expressions/evaluation/example.rs | 271 ++++++++++++ src/expressions/evaluation/mod.rs | 2 + src/expressions/evaluation/node_conversion.rs | 1 - src/expressions/evaluation/place_frames.rs | 4 +- src/expressions/evaluation/type_resolution.rs | 416 +++++++++++++++++- src/expressions/evaluation/value_frames.rs | 104 ++++- src/expressions/object.rs | 6 +- src/expressions/value.rs | 73 ++- src/extensions/errors_and_spans.rs | 6 + src/interpretation/interpreter.rs | 167 +++++-- src/misc/mut_rc_ref_cell.rs | 34 +- 13 files changed, 990 insertions(+), 149 deletions(-) create mode 100644 src/expressions/evaluation/example.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c03a74..dd2e1c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,7 +118,14 @@ Inside a transform stream, the following grammar is supported: * Then we can add `.push(x)` on array and stream * Scrap `#>>x` etc in favour of `#(a.push(@XXX))` * Also improve support to make it easier to add methods on built-in types or (in future) user-designed types -* Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write +* Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write i.e. https://crates.io/crates/crabtime - thoughts on crabtime: + => Looks great! + => Why don't they use a cheap hash of the code as a cache key? + - The cachability is a big win compared to preinterpret (although preinterpret is faster on first run) + => I can't imagine the span-chasing / error messages are great, because everything is translated to/from strings between the processes + ... I wonder if there's any way to improve this? Plausibly you could use the proc-macro bridge encoding scheme as per https://blog.jetbrains.com/rust/2022/07/07/procedural-macros-under-the-hood-part-ii/ to send handles onwards, or even delegate directly somehow? + - The spans are better in preinterpret + => Parsing isn't really a thing - they're not going after full macro stuff, probably wise * No clone required for testing equality of streams, objects and arrays * Some kind of reference support * Add new `.clone()` method diff --git a/src/expressions/evaluation/evaluator.rs b/src/expressions/evaluation/evaluator.rs index b1e6426e..53df9db7 100644 --- a/src/expressions/evaluation/evaluator.rs +++ b/src/expressions/evaluation/evaluator.rs @@ -106,12 +106,12 @@ impl NextAction { NextActionInner::HandleReturnedItem(EvaluationItem::OwnedValue(value)).into() } - pub(super) fn return_mutable(mut_ref: MutableValueReference) -> Self { + pub(super) fn return_mutable(mut_ref: MutableValue) -> Self { NextActionInner::HandleReturnedItem(EvaluationItem::MutableReference { mut_ref }).into() } pub(super) fn return_shared( - shared_ref: SharedValueReference, + shared_ref: SharedValue, reason_not_mutable: Option, ) -> Self { NextActionInner::HandleReturnedItem(EvaluationItem::SharedReference { @@ -142,13 +142,13 @@ impl From for NextAction { pub(super) enum EvaluationItem { OwnedValue(ExpressionValue), SharedReference { - shared_ref: SharedValueReference, + shared_ref: SharedValue, /// This is only populated if we request a "late bound" reference, and fail to resolve /// a mutable reference. reason_not_mutable: Option, }, MutableReference { - mut_ref: MutableValueReference, + mut_ref: MutableValue, }, AssignmentCompletion(AssignmentCompletion), } @@ -161,6 +161,21 @@ impl EvaluationItem { } } + pub(super) fn expect_any_value(self) -> ResolvedValue { + match self { + EvaluationItem::OwnedValue(value) => ResolvedValue::Owned(value), + EvaluationItem::MutableReference { mut_ref } => ResolvedValue::Mutable(mut_ref), + EvaluationItem::SharedReference { + shared_ref, + .. + } => ResolvedValue::Shared { + shared_ref: shared_ref, + reason_not_mutable: None, + }, + _ => panic!("expect_any_value() called on a non-value EvaluationItem"), + } + } + pub(super) fn expect_any_place(self) -> Place { match self { EvaluationItem::MutableReference { mut_ref } => Place::MutableReference { mut_ref }, @@ -175,7 +190,7 @@ impl EvaluationItem { } } - pub(super) fn expect_shared_ref(self) -> SharedValueReference { + pub(super) fn expect_shared_ref(self) -> SharedValue { match self { EvaluationItem::SharedReference { shared_ref, .. } => shared_ref, _ => { @@ -184,7 +199,7 @@ impl EvaluationItem { } } - pub(super) fn expect_mutable_ref(self) -> MutableValueReference { + pub(super) fn expect_mutable_ref(self) -> MutableValue { match self { EvaluationItem::MutableReference { mut_ref } => mut_ref, _ => panic!("expect_mutable_place() called on a non-mutable-place EvaluationItem"), @@ -322,6 +337,17 @@ impl<'a> Context<'a, ValueType> { self.request } + pub(super) fn return_any_value(self, value: ResolvedValue) -> NextAction { + match value { + ResolvedValue::Owned(value) => self.return_owned_value(value), + ResolvedValue::Mutable(mut_ref) => self.return_mut_ref(mut_ref), + ResolvedValue::Shared { + shared_ref, + reason_not_mutable, + } => self.return_ref(shared_ref, reason_not_mutable), + } + } + pub(super) fn return_owned_value(self, value: ExpressionValue) -> NextAction { match self.request { RequestedValueOwnership::LateBound | RequestedValueOwnership::Owned => { @@ -336,7 +362,7 @@ impl<'a> Context<'a, ValueType> { } } - pub(super) fn return_mut_ref(self, mut_ref: MutableValueReference) -> NextAction { + pub(super) fn return_mut_ref(self, mut_ref: MutableValue) -> NextAction { match self.request { RequestedValueOwnership::LateBound | RequestedValueOwnership::MutableReference => NextAction::return_mutable(mut_ref), @@ -347,7 +373,7 @@ impl<'a> Context<'a, ValueType> { pub(super) fn return_ref( self, - shared_ref: SharedValueReference, + shared_ref: SharedValue, reason_not_mutable: Option, ) -> NextAction { match self.request { @@ -390,7 +416,7 @@ impl<'a> Context<'a, PlaceType> { } } - pub(super) fn return_mutable(self, mut_ref: MutableValueReference) -> NextAction { + pub(super) fn return_mutable(self, mut_ref: MutableValue) -> NextAction { match self.request { RequestedPlaceOwnership::LateBound | RequestedPlaceOwnership::MutableReference => NextAction::return_mutable(mut_ref), @@ -400,7 +426,7 @@ impl<'a> Context<'a, PlaceType> { pub(super) fn return_shared( self, - shared_ref: SharedValueReference, + shared_ref: SharedValue, reason_not_mutable: Option, ) -> NextAction { match self.request { diff --git a/src/expressions/evaluation/example.rs b/src/expressions/evaluation/example.rs new file mode 100644 index 00000000..d82f86de --- /dev/null +++ b/src/expressions/evaluation/example.rs @@ -0,0 +1,271 @@ +use super::*; + +macro_rules! handle_arg_mapping { + // No more args + ([$(,)?] [$($bindings:tt)*]) => { + $($bindings)* + }; + // By shared reference + ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let $arg: &$ty = <$ty as ResolvableArgument>::resolve_ref($arg.as_ref())?; + ]) + }; + // By captured shared reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let tmp = $arg.into_shared_reference(); + let $arg: CapturedRef<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_ref(value))?; + ]) + }; + // SharedValue is an alias for CapturedRef + ([$arg:ident : SharedValue, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let $arg = $arg.into_shared_reference(); + ]) + }; + // By mutable reference + ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let mut tmp = $arg.into_mutable_reference("This argument")?; + let $arg: &mut $ty = <$ty as ResolvableArgument>::resolve_mut(tmp.as_mut()); + ]) + }; + // By captured mutable reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let mut tmp = $arg.into_mutable_reference("This argument")?; + let $arg: CapturedMut<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_mut(value))?; + ]) + }; + // MutableValue is an alias for CapturedMut + ([$arg:ident : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let $arg = $arg.into_mutable_reference("This argument")?; + ]) + }; + // By value + ([$arg:ident : $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let tmp = $arg.into_owned_value("This argument")?; + let $arg: $ty = <$ty as ResolvableArgument>::resolve_owned(tmp)?; + ]) + }; +} + +macro_rules! handle_arg_ownerships { + // No more args + ([$(,)?] [$($outputs:tt)*]) => { + vec![$($outputs)*] + }; + // By shared reference + ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + }; + // By captured shared reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + }; + // SharedValue is an alias for CapturedRef + ([$arg:ident : SharedValue, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + }; + // By mutable reference + ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + }; + // By captured mutable reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + }; + // MutableValue is an alias for CapturedMut + ([$arg:ident : MutableValue, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + }; + // By value + ([$arg:ident : $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Owned,]) + }; +} + +macro_rules! count { + () => { 0 }; + ($head:tt $($tail:tt)*) => { 1 + count!($($tail)*) }; +} + +trait ResolvableArgument: Sized { + fn resolve_owned(value: ExpressionValue) -> ExecutionResult; + fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self>; + fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self>; +} + +// Creating an inner method vastly improves IDE support when writing the method body +macro_rules! handle_define_inner_method { + ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?] $body:block $output_ty:ty) => { + fn $method_name($($arg: $ty),*) -> ExecutionResult<$output_ty> { + $body + } + }; +} + +macro_rules! handle_arg_separation { + ([$($arg:ident : $ty:ty),* $(,)?], $all_arguments:ident, $output_span_range:ident) => { + const LEN: usize = count!($($arg)*); + let Ok([ + $($arg,)* + ]) = <[ResolvedValue; LEN]>::try_from($all_arguments) else { + return $output_span_range.execution_err(format!("Expected {LEN} argument/s")); + }; + }; +} + +macro_rules! handle_call_inner_method { + ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?]) => { + $method_name($($arg),*) + }; +} + +macro_rules! wrap_method { + (($($args:tt)*) -> ExecutionResult<$output_ty:ty> $body:block) => { + MethodInterface { + method: Box::new(move | + all_arguments: Vec, + output_span_range: SpanRange, + | -> ExecutionResult { + handle_define_inner_method!(inner_method [$($args)*] $body $output_ty); + handle_arg_separation!([$($args)*], all_arguments, output_span_range); + handle_arg_mapping!([$($args)*,] []); + let output: ExecutionResult<$output_ty> = handle_call_inner_method!(inner_method [$($args)*]); + <$output_ty as ResolvableOutput>::to_resolved_value(output?, output_span_range) + }), + argument_ownerships: handle_arg_ownerships!([$($args)*,] []), + } + }; +} + +pub(super) struct MethodInterface { + method: Box<(dyn Fn(Vec, SpanRange) -> ExecutionResult)>, + argument_ownerships: Vec, +} + +impl MethodInterface { + pub fn execute( + &self, + parameters: Vec, + span_range: SpanRange + ) -> ExecutionResult { + (self.method)(parameters, span_range) + } +} + +impl ResolvableArgument for ExpressionValue { + fn resolve_owned(value: ExpressionValue) -> ExecutionResult { + Ok(value) + } + + fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self> { + Ok(value) + } + + fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self> { + Ok(value) + } +} + +impl ResolvableArgument for ExpressionArray { + fn resolve_owned(value: ExpressionValue) -> ExecutionResult { + match value { + ExpressionValue::Array(value) => Ok(value), + _ => value.execution_err("Expected array"), + } + } + + fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self> { + match value { + ExpressionValue::Array(value) => Ok(value), + _ => value.execution_err("Expected array"), + } + } + + fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self> { + match value { + ExpressionValue::Array(value) => Ok(value), + _ => value.execution_err("Expected array"), + } + } +} + +impl ResolvableArgument for u8 { + fn resolve_owned(value: ExpressionValue) -> ExecutionResult { + match value { + ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), + _ => value.execution_err("Expected u8"), + } + } + + fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self> { + match value { + ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), + _ => value.execution_err("Expected u8"), + } + } + + fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self> { + match value { + ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), + _ => value.execution_err("Expected u8"), + } + } +} + +#[diagnostic::on_unimplemented( + message = "`ResolvableOutput` is not implemented for `{Self}`", + note = "`ResolvableOutput` is not implemented for `CapturedRef` or `CapturedMut` unless `X` is `ExpressionValue`. If we wish to change this, we'd need to have some way to represent some kind of `ExpressionReference`, i.e. a `Typed>` rather than a `CapturedRef>`", +)] +trait ResolvableOutput { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult; +} + +impl ResolvableOutput for CapturedRef { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Shared { + shared_ref: self.update_span_range(|_| output_span_range), + reason_not_mutable: Some(syn::Error::new(output_span_range.join_into_span_else_start(), "It was output from a method a captured shared reference")), + }) + } +} + +impl ResolvableOutput for CapturedMut { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Mutable(self.update_span_range(|_| output_span_range))) + } +} + +impl ResolvableOutput for T { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Owned(self.to_value(output_span_range))) + } +} + +#[test] +fn test2() { + // Example usage: + let span_range = SpanRange::new_single(Span::call_site()); + let x = ResolvableOutput::to_resolved_value(vec![10u8.to_value(span_range)], span_range).unwrap(); + let y = ResolvableOutput::to_resolved_value(0usize, span_range).unwrap(); + + let method1 = wrap_method!((a: CapturedRef, index: &ExpressionValue) -> ExecutionResult { + let access = IndexAccess { brackets: Brackets { delim_span: Group::new(Delimiter::Brace, TokenStream::new()).delim_span() }}; + a.try_map(|a, _| a.index_ref(access, index)) + }); + let output = method1.execute(vec![x, y], span_range).unwrap(); + + assert_eq!(*u8::resolve_ref(output.as_value_ref()).unwrap(), 10u8); +} \ No newline at end of file diff --git a/src/expressions/evaluation/mod.rs b/src/expressions/evaluation/mod.rs index e8aba6f8..af89bb87 100644 --- a/src/expressions/evaluation/mod.rs +++ b/src/expressions/evaluation/mod.rs @@ -4,6 +4,7 @@ mod node_conversion; mod place_frames; mod type_resolution; mod value_frames; +mod example; use super::*; use assignment_frames::*; @@ -11,3 +12,4 @@ pub(super) use evaluator::ExpressionEvaluator; use evaluator::*; use place_frames::*; use value_frames::*; +pub(super) use type_resolution::*; diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs index 103c113c..ea8065dd 100644 --- a/src/expressions/evaluation/node_conversion.rs +++ b/src/expressions/evaluation/node_conversion.rs @@ -52,7 +52,6 @@ impl ExpressionNode { operation, value, } => CompoundAssignmentBuilder::start(context, *place, *operation, *value), - // TODO - Change this to follow outline in changelog ExpressionNode::MethodCall { node, method, diff --git a/src/expressions/evaluation/place_frames.rs b/src/expressions/evaluation/place_frames.rs index 126b262b..ddc6aad6 100644 --- a/src/expressions/evaluation/place_frames.rs +++ b/src/expressions/evaluation/place_frames.rs @@ -18,10 +18,10 @@ use super::*; /// So instead, we take the most powerful access we can have for `x[a]`, and convert it later. pub(super) enum Place { MutableReference { - mut_ref: MutableValueReference, + mut_ref: MutableValue, }, SharedReference { - shared_ref: SharedValueReference, + shared_ref: SharedValue, reason_not_mutable: Option, }, } diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs index 0e8a3616..1391b549 100644 --- a/src/expressions/evaluation/type_resolution.rs +++ b/src/expressions/evaluation/type_resolution.rs @@ -2,39 +2,425 @@ use super::*; pub(super) trait ResolvedTypeDetails { - /// Whether the type can be transparently cloned. - fn supports_copy(&self) -> bool; + /// This should be true for types which users expect to have value + /// semantics, but false for mutable types / types with reference + /// semantics. + /// + /// This indicates if an &x can be converted to an x via cloning + /// when doing method resolution. + fn supports_transparent_cloning(&self) -> bool; /// Resolves a method for this resolved type with the given arguments. - fn resolve_method(&self, name: &str, num_arguments: usize) -> ExecutionResult; + fn resolve_method(&self, method: &MethodAccess, num_arguments: usize) -> ExecutionResult; + + // TODO: Eventually we can migrate operations under this umbrella too + // fn resolve_unary_operation(&self, operation: UnaryOperation) -> ExecutionResult; + // fn resolve_binary_operation(&self, operation: BinaryOperation) -> ExecutionResult; } -pub(super) struct ResolvedType { - inner: Box, +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ValueKind { + None, + Integer, + Float, + Boolean, + String, + Char, + UnsupportedLiteral, + Array, + Object, + Stream, + Range, + Iterator, } -impl ResolvedTypeDetails for ResolvedType { - fn supports_copy(&self) -> bool { - self.inner.supports_copy() +impl ResolvedTypeDetails for ValueKind { + fn supports_transparent_cloning(&self) -> bool { + match self { + ValueKind::None => true, + ValueKind::Integer => true, + ValueKind::Float => true, + ValueKind::Boolean => true, + ValueKind::String => true, + ValueKind::Char => true, + ValueKind::UnsupportedLiteral => false, + ValueKind::Array => false, + ValueKind::Object => false, + ValueKind::Stream => false, + ValueKind::Range => true, + ValueKind::Iterator => false, + } } - fn resolve_method(&self, name: &str, num_arguments: usize) -> ExecutionResult { - self.inner.resolve_method(name, num_arguments) + fn resolve_method(&self, method: &MethodAccess, num_arguments: usize) -> ExecutionResult { + let method_name = method.method.to_string(); + match (self, method_name.as_str(), num_arguments) { + // (ValueKind::Array, "len", 0) => Ok(ResolvedMethod::new(array_len)), + // (ValueKind::Stream, "len", 0) => Ok(ResolvedMethod::new(stream_len)), + _ => method.execution_err(format!("{self:?} has no method `{method_name}` with {num_arguments} arguments")), + } } } + pub(super) struct ResolvedMethod { - // TODO: Add some reference to the code here - object_location_kind: RequestedValueOwnership, - parameter_location_kinds: Vec, + method: WrappedMethod, + argument_ownerships: Vec, } impl ResolvedMethod { - fn execute( + // fn new( + // method: fn(SelfType, Arguments) -> ExecutionResult, + // ) -> Self + // where + // SelfType: ResolvableArgument + 'static, + // Arguments: ResolvableArguments + 'static, + // Output: ResolvableOutput + 'static, + // { + // // TODO - Find some way of avoiding creating a new box for each method call (e.g. using a cache) + // // Could also consider using a GAT by upgrading to Rust 1.65 + // Self { + // method: Box::new(move | + // self_value: ResolvedValue, + // arguments: Vec, + // output_span_range: SpanRange, + // | -> ExecutionResult { + // SelfType::run_resolved(self_value, |self_value| { + // Arguments::run_with_arguments(arguments, |arguments| { + // method(self_value, arguments)?.to_value(output_span_range) + // }, output_span_range) + // }) + // }), + // argument_ownerships: Arguments::ownerships(), + // } + // } + + pub fn execute( &self, object: ResolvedValue, parameters: Vec, + span_range: SpanRange ) -> ExecutionResult { - unimplemented!("Method execution is not implemented yet"); + (self.method)(object, parameters, span_range) + } + + pub fn ownerships(&self) -> &[RequestedValueOwnership] { + &self.argument_ownerships + } +} + +fn array_len(self_value: &ExpressionArray, arguments: ()) -> ExecutionResult { + Ok(self_value.items.len()) +} + +fn stream_len(self_value: &ExpressionStream, arguments: ()) -> ExecutionResult { + Ok(self_value.value.len()) +} + +type WrappedMethod = Box<(dyn Fn(ResolvedValue, Vec, SpanRange) -> ExecutionResult)>; +/* +trait ResolvableOutput { + fn to_value(self, output_span_range: SpanRange) -> ExecutionResult; +} + +impl ResolvableOutput for T { + fn to_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Owned(self.to_value(output_span_range))) + } +} + +use arguments::*; + +mod arguments { + use super::*; + + // FRAMEWORK TO DO DISJOINT TRAIT IMPLEMENTATIONS + + pub(super) trait ResolvableArgument: Sized { + fn run_resolved(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult; + fn resolve(value: ResolvedValue) -> ExecutionResult; + fn ownership() -> RequestedValueOwnership; + } + + pub(super) trait ResolvableArguments: Sized { + fn run_with_arguments(value: Vec, inner: impl FnOnce(Self) -> ExecutionResult, arguments_span: SpanRange) -> ExecutionResult; + fn ownerships() -> Vec; + } + + impl ResolvableArguments for () { + fn run_with_arguments(value: Vec, inner: impl FnOnce(Self) -> ExecutionResult, arguments_span: SpanRange) -> ExecutionResult { + let Ok([ + // No arguments + ]) = <[ResolvedValue; 0]>::try_from(value) else { + return arguments_span.execution_err("Expected 0 arguments"); + }; + inner(( + // No arguments + )) + } + + fn ownerships() -> Vec { + vec![] + } + } + + impl ResolvableArguments for (T,) { + fn run_with_arguments(value: Vec, inner: impl FnOnce(Self) -> ExecutionResult, arguments_span: SpanRange) -> ExecutionResult { + let Ok([ + value0, + ]) = <[ResolvedValue; 1]>::try_from(value) else { + return arguments_span.execution_err("Expected 1 argument"); + }; + + T::run_resolved(value0, |value0| { + // Further nesting... + inner((value0,)) + }) + } + + fn ownerships() -> Vec { + vec![T::ownership()] + } + } + + // TODO: Add more tuples, maybe using a macro to generate them + + trait ArgumentKind {} + struct ArgumentKindOwned; + impl ArgumentKind for ArgumentKindOwned {} + struct ArgumentKindDisposedRef; + impl ArgumentKind for ArgumentKindDisposedRef {} + struct ArgumentKindCapturedRef; + impl ArgumentKind for ArgumentKindCapturedRef {} + struct ArgumentKindDisposedRefMut; + impl ArgumentKind for ArgumentKindDisposedRefMut {} + struct ArgumentKindCapturedRefMut; + impl ArgumentKind for ArgumentKindCapturedRefMut {} + + trait InferredArgumentType: Sized { + type ArgumentKind: ArgumentKind; + } + + trait ResolvableArgumentAs: Sized { + fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult; + fn ownership() -> RequestedValueOwnership; + } + + impl::ArgumentKind>> ResolvableArgument for T { + fn run_resolved(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + ::ArgumentKind>>::run_with_argument(value, inner) + } + + fn ownership() -> RequestedValueOwnership { + ::ArgumentKind>>::ownership() + } + } + + // TODO: + // The automatic conversion of &XXX in a method into this might just not be feasible; because stacked borrows go from + // out to in; and here we want to go from in (the arguments of an inner method) to out. + // Instead, we might need to use a macro-based approach rather than just leveraging the type system. + + // trait ResolvableRef + // where for<'a> &'a Self: InferredArgumentType + // { + // fn resolve_from_value<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self>; + // } + + // impl<'o, T: ResolvableRef> ResolvableArgumentAs for &'o T + // where for<'a> &'a T: InferredArgumentType + // { + // fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + // let output = { + // let value = value.as_value_ref(); + + // let output = inner( + // // This needs to be 'o to match Self and work with the callback to match up with the defined function... + // // But then this implies that 'o is smaller than this function call, which doesn't work. + // ::resolve_from_value(value)? + // )?; + // output + // }; + // Ok(output) + // // let mut value = value.as_value_ref(); + // // inner( + // // >::resolve_from_value(value)? + // // ) + // } + + // fn ownership() -> RequestedValueOwnership { + // RequestedValueOwnership::SharedReference + // } + // } + + trait ResolvableRef<'a>: InferredArgumentType + { + fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult; + } + + impl<'a, T: ResolvableRef<'a>> ResolvableArgumentAs for T { + fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + let mut value = value.as_value_ref(); + inner( + >::resolve_from_value(value)? + ) + } + + fn ownership() -> RequestedValueOwnership { + RequestedValueOwnership::SharedReference + } + } + + trait ResolvableCapturedRef: InferredArgumentType { + fn resolve_from_value(value: SharedValue) -> ExecutionResult; + } + + impl<'a, T: ResolvableCapturedRef> ResolvableArgumentAs for T { + fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + let mut value = value.into_shared_reference(); + inner( + ::resolve_from_value(value)? + ) + } + + fn ownership() -> RequestedValueOwnership { + RequestedValueOwnership::SharedReference + } + } + + trait ResolvableMutRef<'a>: InferredArgumentType { + fn resolve_from_value(value: &'a mut ExpressionValue) -> ExecutionResult; + } + + impl ResolvableMutRef<'a>> ResolvableArgumentAs for T { + fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + let mut value = value.into_mutable_reference("This argument")?; + inner( + >::resolve_from_value(value.as_mut())? + ) + } + + fn ownership() -> RequestedValueOwnership { + RequestedValueOwnership::MutableReference + } + } + + trait ResolvableCapturedMutRef: InferredArgumentType { + fn resolve_from_value(value: MutableValue) -> ExecutionResult; + } + + impl ResolvableArgumentAs for T { + fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + let mut value = value.into_mutable_reference("This argument")?; + inner( + ::resolve_from_value(value)? + ) + } + + fn ownership() -> RequestedValueOwnership { + RequestedValueOwnership::MutableReference + } + } + + trait ResolvableOwned: InferredArgumentType { + fn resolve_from_value(value: ExpressionValue) -> ExecutionResult; + } + + impl ResolvableArgumentAs for T { + fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { + let mut value = value.into_owned_value("This argument")?; + inner( + ::resolve_from_value(value)? + ) + } + + fn ownership() -> RequestedValueOwnership { + RequestedValueOwnership::Owned + } + } + + // IMPLEMENTATIONS + + impl InferredArgumentType for &'_ ExpressionValue { + type ArgumentKind = ArgumentKindDisposedRef; + } + + impl<'a> ResolvableRef<'a> for &'a ExpressionValue { + fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult { + Ok(value) + } + } + + impl InferredArgumentType for SharedSubPlace { + type ArgumentKind = ArgumentKindCapturedRef; + } + + impl ResolvableCapturedRef for SharedSubPlace + where + for<'a> &'a T: ResolvableRef<'a>, + { + fn resolve_from_value(value: SharedValue) -> ExecutionResult { + value.resolve_internally_mapped(|value| <&'_ T as ResolvableRef<'_>>::resolve_from_value(value)) + } + } + + impl InferredArgumentType for &'_ mut ExpressionValue { + type ArgumentKind = ArgumentKindDisposedRefMut; + } + + impl<'a> ResolvableMutRef<'a> for &'a mut ExpressionValue { + fn resolve_from_value(value: &'a mut ExpressionValue) -> ExecutionResult { + Ok(value) + } + } + + impl InferredArgumentType for MutableSubPlace { + type ArgumentKind = ArgumentKindCapturedRefMut; + } + + impl ResolvableCapturedMutRef for MutableSubPlace + where + for<'a> &'a mut T: ResolvableMutRef<'a>, + { + fn resolve_from_value(value: MutableValue) -> ExecutionResult { + value.resolve_internally_mapped(|value, _| <&'_ mut T as ResolvableMutRef<'_>>::resolve_from_value(value)) + } + } + + impl InferredArgumentType for ExpressionValue { + type ArgumentKind = ArgumentKindOwned; + } + + impl ResolvableOwned for ExpressionValue { + fn resolve_from_value(value: ExpressionValue) -> ExecutionResult { + Ok(value) + } + } + + impl InferredArgumentType for &'_ ExpressionArray { + type ArgumentKind = ArgumentKindDisposedRef; + } + + impl<'a> ResolvableRef<'a> for &'a ExpressionArray { + fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult { + match value { + ExpressionValue::Array(arr) => Ok(arr), + _ => value.execution_err("Expected array"), + } + } + } + + impl InferredArgumentType for &'_ ExpressionStream { + type ArgumentKind = ArgumentKindDisposedRef; + } + + impl<'a> ResolvableRef<'a> for &'a ExpressionStream { + fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult { + match value { + ExpressionValue::Stream(stream) => Ok(stream), + _ => value.execution_err("Expected stream"), + } + } } } + */ \ No newline at end of file diff --git a/src/expressions/evaluation/value_frames.rs b/src/expressions/evaluation/value_frames.rs index 89eebcbd..986ffaea 100644 --- a/src/expressions/evaluation/value_frames.rs +++ b/src/expressions/evaluation/value_frames.rs @@ -1,4 +1,6 @@ -#![allow(unused)] // TODO: Remove when values are properly late-bound +#![allow(unused)] use crate::expressions::evaluation::type_resolution::{ResolvedMethod, ResolvedTypeDetails}; + +// TODO: Remove when values are properly late-bound use super::*; /// # Late Binding @@ -15,27 +17,64 @@ pub(crate) enum ResolvedValue { /// This has been requested as an owned value. Owned(ExpressionValue), /// This has been requested as a mutable reference. - Mutable(MutableValueReference), + Mutable(MutableValue), /// This has been requested as a shared reference. Shared { - shared_ref: SharedValueReference, + shared_ref: SharedValue, reason_not_mutable: Option, }, } impl ResolvedValue { /// The requirement is just for error messages, and should be "A ", and will be filled like: - /// "{usage} must to be an owned value, but it is a reference to a non-copyable value kind" + /// "{usage} expects an owned value, but receives a reference..." pub(crate) fn into_owned_value(self, usage: &str) -> ExecutionResult { let reference = match self { ResolvedValue::Owned(value) => return Ok(value), ResolvedValue::Shared { .. } | ResolvedValue::Mutable { .. } => self.as_ref(), }; - // TODO: Add check that the value's type supports Auto-Clone (kinda like copy) - // reference.type_ref().assert_auto_clone(usage)?; + if !reference.kind().supports_transparent_cloning() { + return reference.execution_err(format!( + "{} expects an owned value, but receives a reference and {} does not support transparent cloning. You may wish to use .clone() explicitly.", + usage, + reference.articled_value_type(), + )); + } Ok(reference.clone()) } + /// The requirement is just for error messages, and should be "A ", and will be filled like: + /// "{usage} expects an owned value, but receives a reference..." + pub(crate) fn into_mutable_reference(self, usage: &str) -> ExecutionResult { + Ok(match self { + // It might be a bug if a user calls a mutable method on a floating owned value, + // but it might be annoying to force them to do `.as_mut()` everywhere. + // Let's leave this as-is for now. + ResolvedValue::Owned(value) => MutableValue::new_from_owned(value), + ResolvedValue::Mutable(reference) => reference, + ResolvedValue::Shared { shared_ref, reason_not_mutable: Some(reason_not_mutable), } => return shared_ref.execution_err(format!( + "{} expects a mutable reference, but only receives a shared reference because {reason_not_mutable}.", + usage + )), + ResolvedValue::Shared { shared_ref, reason_not_mutable: None, } => return shared_ref.execution_err(format!( + "{} expects a mutable reference, but only receives a shared reference.", + usage + )), + }) + } + + pub(crate) fn into_shared_reference(self) -> SharedValue { + match self { + ResolvedValue::Owned(value) => SharedValue::new_from_owned(value), + ResolvedValue::Mutable(reference) => reference.to_shared(), + ResolvedValue::Shared { shared_ref, .. } => shared_ref, + } + } + + pub(crate) fn kind(&self) -> ValueKind { + self.as_value_ref().kind() + } + pub(crate) fn as_value_ref(&self) -> &ExpressionValue { match self { ResolvedValue::Owned(value) => value, @@ -63,6 +102,12 @@ impl ResolvedValue { } } +impl HasSpanRange for ResolvedValue { + fn span_range(&self) -> SpanRange { + self.as_value_ref().span_range() + } +} + impl WithSpanExt for ResolvedValue { fn with_span(self, span: Span) -> Self { match self { @@ -443,7 +488,7 @@ impl EvaluationFrame for ValuePropertyAccessBuilder { item: EvaluationItem, ) -> ExecutionResult { let value = item.expect_owned_value(); - Ok(context.return_owned_value(value.into_property(self.access)?)) + Ok(context.return_owned_value(value.into_property(&self.access)?)) } } @@ -729,14 +774,17 @@ impl EvaluationFrame for CompoundAssignmentBuilder { pub(super) struct MethodCallBuilder { method: MethodAccess, - unevaluated_parameters_stack: Vec, - evaluated_parameters: Vec, + unevaluated_parameters_stack: Vec<(ExpressionNodeId, RequestedValueOwnership)>, + evaluated_parameters: Vec, state: MethodCallPath, } enum MethodCallPath { CallerPath, - ArgumentsPath { caller: ExpressionValue }, + ArgumentsPath { + caller: ResolvedValue, + method: ResolvedMethod, + }, } impl MethodCallBuilder { @@ -748,13 +796,15 @@ impl MethodCallBuilder { ) -> NextAction { let frame = Self { method, - unevaluated_parameters_stack: parameters.iter().rev().cloned().collect(), + unevaluated_parameters_stack: parameters.iter().rev() + .map(|x| { + // This is just a placeholder - we'll fix it up shortly + (*x, RequestedValueOwnership::LateBound) + }).collect(), evaluated_parameters: Vec::with_capacity(parameters.len()), state: MethodCallPath::CallerPath, }; - // TODO: Change to resolve the caller as LateBound, in order to resolve the ownership for the - // caller and parameters based on the method signature - context.handle_node_as_value(frame, caller, RequestedValueOwnership::Owned) + context.handle_node_as_value(frame, caller, RequestedValueOwnership::LateBound) } } @@ -770,27 +820,35 @@ impl EvaluationFrame for MethodCallBuilder { context: ValueContext, item: EvaluationItem, ) -> ExecutionResult { + // Handle expected item based on current state match self.state { MethodCallPath::CallerPath => { - let caller = item.expect_owned_value(); - self.state = MethodCallPath::ArgumentsPath { caller }; + let caller = item.expect_any_value(); + let method = caller.kind().resolve_method(&self.method, self.unevaluated_parameters_stack.len())?; + assert!(method.ownerships().len() == self.unevaluated_parameters_stack.len(), "The method resolution should ensure the argument count is correct"); + let argument_ownerships_stack = method.ownerships().iter().rev(); + for ((_, requested_ownership), ownership) in self.unevaluated_parameters_stack.iter_mut().zip(argument_ownerships_stack) { + *requested_ownership = *ownership; + } + self.state = MethodCallPath::ArgumentsPath { caller, method, }; } MethodCallPath::ArgumentsPath { .. } => { - let argument = item.expect_owned_value(); + let argument = item.expect_any_value(); self.evaluated_parameters.push(argument); } }; + // Now plan the next action Ok(match self.unevaluated_parameters_stack.pop() { - Some(parameter) => { - context.handle_node_as_value(self, parameter, RequestedValueOwnership::Owned) + Some((parameter, ownership)) => { + context.handle_node_as_value(self, parameter, ownership) } None => { - let caller = match self.state { + let (caller, method) = match self.state { MethodCallPath::CallerPath => unreachable!("Already updated above"), - MethodCallPath::ArgumentsPath { caller } => caller, + MethodCallPath::ArgumentsPath { caller, method } => (caller, method), }; - context - .return_owned_value(caller.call_method(self.method, self.evaluated_parameters)?) + let output = method.execute(caller, self.evaluated_parameters, self.method.span_range())?; + context.return_any_value(output) } }) } diff --git a/src/expressions/object.rs b/src/expressions/object.rs index 60772e43..e91b1d35 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -70,7 +70,7 @@ impl ExpressionObject { pub(super) fn into_property( mut self, - access: PropertyAccess, + access: &PropertyAccess, ) -> ExecutionResult { let span_range = SpanRange::new_between(self.span_range, access.span_range()); let key = access.property.to_string(); @@ -124,12 +124,12 @@ impl ExpressionObject { pub(super) fn property_mut( &mut self, - access: PropertyAccess, + access: &PropertyAccess, ) -> ExecutionResult<&mut ExpressionValue> { Ok(self.mut_entry_or_create(access.property.to_string(), access.property.span())) } - pub(super) fn property_ref(&self, access: PropertyAccess) -> ExecutionResult<&ExpressionValue> { + pub(super) fn property_ref(&self, access: &PropertyAccess) -> ExecutionResult<&ExpressionValue> { let key = access.property.to_string(); let entry = self.entries.get(&key).ok_or_else(|| { access.execution_error(format!("The object does not have a field named `{}`", key)) diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 8e4f5fd5..2ca6f67d 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -248,6 +248,23 @@ impl ExpressionValue { }) } + pub(crate) fn kind(&self) -> ValueKind { + match self { + ExpressionValue::None(_) => ValueKind::None, + ExpressionValue::Integer(_) => ValueKind::Integer, + ExpressionValue::Float(_) => ValueKind::Float, + ExpressionValue::Boolean(_) => ValueKind::Boolean, + ExpressionValue::String(_) => ValueKind::String, + ExpressionValue::Char(_) => ValueKind::Char, + ExpressionValue::Array(_) => ValueKind::Array, + ExpressionValue::Object(_) => ValueKind::Object, + ExpressionValue::Stream(_) => ValueKind::Stream, + ExpressionValue::Range(_) => ValueKind::Range, + ExpressionValue::Iterator(_) => ValueKind::Iterator, + ExpressionValue::UnsupportedLiteral(_) => ValueKind::UnsupportedLiteral, + } + } + pub(crate) fn is_none(&self) -> bool { matches!(self, ExpressionValue::None(_)) } @@ -490,47 +507,7 @@ impl ExpressionValue { } } - pub(crate) fn call_method( - self, - method: MethodAccess, - parameters: Vec, - ) -> ExecutionResult { - // TODO: Make this more extensible / usable - match self { - ExpressionValue::Array(array) => { - if method.method.to_string().as_str() == "len" { - match parameters.as_slice() { - [] => {} - _ => { - return method.execution_err( - "The `len` method does not take parameters".to_string(), - ) - } - } - let len = array.items.len(); - return Ok(len.to_value(method.span_range())); - } - } - ExpressionValue::Stream(stream) => { - if method.method.to_string().as_str() == "len" { - match parameters.as_slice() { - [] => {} - _ => { - return method.execution_err( - "The `len` method does not take parameters".to_string(), - ) - } - } - let len = stream.value.len(); - return Ok(len.to_value(method.span_range())); - } - } - _ => {} - } - method.execution_err("Methods are not currently supported") - } - - pub(crate) fn into_property(self, access: PropertyAccess) -> ExecutionResult { + pub(crate) fn into_property(self, access: &PropertyAccess) -> ExecutionResult { match self { ExpressionValue::Object(object) => object.into_property(access), other => access.execution_err(format!( @@ -540,7 +517,7 @@ impl ExpressionValue { } } - pub(crate) fn property_mut(&mut self, access: PropertyAccess) -> ExecutionResult<&mut Self> { + pub(crate) fn property_mut(&mut self, access: &PropertyAccess) -> ExecutionResult<&mut Self> { match self { ExpressionValue::Object(object) => object.property_mut(access), other => access.execution_err(format!( @@ -550,7 +527,7 @@ impl ExpressionValue { } } - pub(crate) fn property_ref(&self, access: PropertyAccess) -> ExecutionResult<&Self> { + pub(crate) fn property_ref(&self, access: &PropertyAccess) -> ExecutionResult<&Self> { match self { ExpressionValue::Object(object) => object.property_ref(access), other => access.execution_err(format!( @@ -672,11 +649,13 @@ impl ExpressionValue { Ok(()) } + pub(crate) fn into_debug_string(self) -> ExecutionResult { + self.concat_recursive(&ConcatBehaviour::debug()) + } + pub(crate) fn into_debug_string_value(self) -> ExecutionResult { let span_range = self.span_range(); - let value = self - .concat_recursive(&ConcatBehaviour::debug())? - .to_value(span_range); + let value = self.into_debug_string()?.to_value(span_range); Ok(value) } @@ -811,7 +790,7 @@ pub(super) trait HasValueType { let first_char = value_type.chars().next().unwrap(); match first_char { 'a' | 'e' | 'i' | 'o' | 'u' => format!("an {}", value_type), - _ => value_type.to_string(), + _ => format!("a {}", value_type), } } } diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index 0187f65c..a4767089 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -65,6 +65,12 @@ impl HasSpanRange for T { } } +impl HasSpanRange for &SpanRange { + fn span_range(&self) -> SpanRange { + **self + } +} + /// [`syn::spanned`] is potentially unexpectedly expensive, and has the /// limitation that it uses [`proc_macro::Span::join`] and falls back to the /// span of the first token when not available. diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 62bd10e0..2d4afbca 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -161,12 +161,12 @@ impl VariableReference { .with_span_range(self.variable_span_range)) } - pub(crate) fn into_mut(self) -> ExecutionResult { - MutableValueReference::new_from_variable(self) + pub(crate) fn into_mut(self) -> ExecutionResult { + MutableValue::new_from_variable(self) } - pub(crate) fn into_shared(self) -> ExecutionResult { - SharedValueReference::new_from_variable(self) + pub(crate) fn into_shared(self) -> ExecutionResult { + SharedValue::new_from_variable(self) } } @@ -176,7 +176,8 @@ impl HasSpanRange for VariableReference { } } -pub(crate) type MutableValueReference = MutableSubPlace; +pub(crate) type CapturedMut = MutableSubPlace; +pub(crate) type MutableValue = MutableSubPlace; /// A mutable reference to a value inside a variable; along with a span of the whole access. /// For example, for `x.y[4]`, this captures both: @@ -187,6 +188,45 @@ pub(crate) struct MutableSubPlace { span_range: SpanRange, } +impl MutableSubPlace { + pub(crate) fn to_shared(self) -> SharedSubPlace { + SharedSubPlace { + shared_cell: self.mut_cell.to_shared(), + span_range: self.span_range, + } + } + + pub(crate) fn map( + self, + value_map: impl for<'a> FnOnce(&'a mut T) -> &'a mut V, + ) -> MutableSubPlace { + MutableSubPlace { + mut_cell: self.mut_cell.map(value_map), + span_range: self.span_range, + } + } + + pub(crate) fn try_map( + self, + value_map: impl for<'a, 'b> FnOnce(&'a mut T, &'b SpanRange) -> ExecutionResult<&'a mut V>, + ) -> ExecutionResult> { + Ok(MutableSubPlace { + mut_cell: self.mut_cell.try_map(|value| value_map(value, &self.span_range))?, + span_range: self.span_range, + }) + } + + pub(crate) fn update_span_range( + self, + span_range_map: impl FnOnce(SpanRange) -> SpanRange, + ) -> Self { + Self { + mut_cell: self.mut_cell, + span_range: span_range_map(self.span_range), + } + } +} + impl MutableSubPlace { pub(crate) fn new_from_owned(value: ExpressionValue) -> Self { let span_range = value.span_range(); @@ -209,16 +249,14 @@ impl MutableSubPlace { } pub(crate) fn into_stream(self) -> ExecutionResult> { - let stream = self.mut_cell.try_map(|value| match value { - ExpressionValue::Stream(stream) => Ok(&mut stream.value), - _ => self - .span_range - .execution_err("The variable is not a stream"), - })?; - Ok(MutableSubPlace { - mut_cell: stream, - span_range: self.span_range, - }) + self.try_map( + |value, span_range| { + match value { + ExpressionValue::Stream(stream) => Ok(&mut stream.value), + _ => span_range.execution_err("The variable is not a stream"), + } + }, + ) } pub(crate) fn resolve_indexed_with_autocreate( @@ -226,22 +264,15 @@ impl MutableSubPlace { access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult { - let indexed = self - .mut_cell - .try_map(|value| value.index_mut_with_autocreate(access, index))?; - Ok(Self { - mut_cell: indexed, - span_range: SpanRange::new_between(self.span_range.start(), access.span()), - }) + self + .update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + .try_map(|value, _| value.index_mut_with_autocreate(access, index)) } pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { - let span_range = SpanRange::new_between(self.span_range.start(), access.span_range()); - let indexed = self.mut_cell.try_map(|value| value.property_mut(access))?; - Ok(Self { - mut_cell: indexed, - span_range, - }) + self + .update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + .try_map(|value, _| value.property_mut(&access)) } pub(crate) fn set(&mut self, content: impl ToExpressionValue) { @@ -276,7 +307,22 @@ impl WithSpanExt for MutableSubPlace { } } -pub(crate) type SharedValueReference = SharedSubPlace; +impl Deref for MutableSubPlace { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.mut_cell + } +} + +impl DerefMut for MutableSubPlace { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.mut_cell + } +} + +pub(crate) type CapturedRef = SharedSubPlace; +pub(crate) type SharedValue = SharedSubPlace; /// A mutable reference to a value inside a variable; along with a span of the whole access. /// For example, for `x.y[4]`, this captures both: @@ -287,6 +333,38 @@ pub(crate) struct SharedSubPlace { span_range: SpanRange, } +impl SharedSubPlace { + pub(crate) fn try_map( + self, + value_map: impl for<'a, 'b> FnOnce(&'a T, &'b SpanRange) -> ExecutionResult<&'a V>, + ) -> ExecutionResult> { + Ok(SharedSubPlace { + shared_cell: self.shared_cell.try_map(|value| value_map(value, &self.span_range))?, + span_range: self.span_range, + }) + } + + pub(crate) fn map( + self, + value_map: impl FnOnce(&T) -> &V, + ) -> ExecutionResult> { + Ok(SharedSubPlace { + shared_cell: self.shared_cell.map(value_map), + span_range: self.span_range, + }) + } + + pub(crate) fn update_span_range( + self, + span_range_map: impl FnOnce(SpanRange) -> SpanRange, + ) -> Self { + Self { + shared_cell: self.shared_cell, + span_range: span_range_map(self.span_range), + } + } +} + impl SharedSubPlace { pub(crate) fn new_from_owned(value: ExpressionValue) -> Self { let span_range = value.span_range(); @@ -313,24 +391,15 @@ impl SharedSubPlace { access: IndexAccess, index: &ExpressionValue, ) -> ExecutionResult { - let indexed = self - .shared_cell - .try_map(|value| value.index_ref(access, index))?; - Ok(Self { - shared_cell: indexed, - span_range: SpanRange::new_between(self.span_range.start(), access.span()), - }) + self + .update_span_range(|old_span| SpanRange::new_between(old_span, access)) + .try_map(|value, _| value.index_ref(access, index)) } pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { - let span_range = SpanRange::new_between(self.span_range.start(), access.span_range()); - let indexed = self - .shared_cell - .try_map(|value| value.property_ref(access))?; - Ok(Self { - shared_cell: indexed, - span_range, - }) + self + .update_span_range(|old_span| SpanRange::new_between(old_span, access.span_range())) + .try_map(|value, _| value.property_ref(&access)) } } @@ -340,13 +409,21 @@ impl AsRef for SharedSubPlace { } } +impl Deref for SharedSubPlace { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.shared_cell + } +} + impl HasSpanRange for SharedSubPlace { fn span_range(&self) -> SpanRange { self.span_range } } -impl WithSpanExt for SharedSubPlace { +impl WithSpanExt for SharedSubPlace { fn with_span(self, span: Span) -> Self { Self { shared_cell: self.shared_cell, diff --git a/src/misc/mut_rc_ref_cell.rs b/src/misc/mut_rc_ref_cell.rs index f3425079..fab84d12 100644 --- a/src/misc/mut_rc_ref_cell.rs +++ b/src/misc/mut_rc_ref_cell.rs @@ -29,6 +29,29 @@ impl MutSubRcRefCell { } impl MutSubRcRefCell { + pub(crate) fn to_shared(self) -> SharedSubRcRefCell { + let ptr = self.ref_mut.deref() as *const U; + drop(self.ref_mut); + // SAFETY: + // - The pointer was previously a reference, so it is safe to deference it here + // (the pointer is pointing into the Rc> which hasn't moved) + // - All our invariants for SharedSubRcRefCell / MutSubRcRefCell are maintained + unsafe { + // The unwrap cannot panic because we just held a mutable borrow, we're not in Sync land, so no-one else can have a borrow. + SharedSubRcRefCell::new(self.pointed_at).unwrap().map(|_| &*ptr) + } + } + + pub(crate) fn map( + self, + f: impl FnOnce(&mut U) -> &mut V, + ) -> MutSubRcRefCell { + MutSubRcRefCell { + ref_mut: RefMut::map(self.ref_mut, f), + pointed_at: self.pointed_at, + } + } + pub(crate) fn try_map( self, f: impl FnOnce(&mut U) -> Result<&mut V, E>, @@ -84,14 +107,21 @@ impl SharedSubRcRefCell { // SAFETY: We must ensure that this lifetime lives as long as the // reference to pointed_at (i.e. the RefCell). // This is guaranteed by the fact that the only time we drop the RefCell - // is when we drop the MutRcRefCell, and we ensure that the RefMut is dropped first. + // is when we drop the SharedSubRcRefCell, and we ensure that the Ref is dropped first. shared_ref: unsafe { std::mem::transmute::, Ref<'static, T>>(shared_ref) }, - pointed_at, + pointed_at: pointed_at, }) } } impl SharedSubRcRefCell { + pub(crate) fn map(self, f: impl FnOnce(&U) -> &V) -> SharedSubRcRefCell { + SharedSubRcRefCell { + shared_ref: Ref::map(self.shared_ref, f), + pointed_at: self.pointed_at, + } + } + pub(crate) fn try_map( self, f: impl FnOnce(&U) -> Result<&V, E>, From f6e927c8885601d24c8a749252bbcd99430c71fa Mon Sep 17 00:00:00 2001 From: David Edey Date: Thu, 18 Sep 2025 22:48:37 +0100 Subject: [PATCH 122/126] feature: Added support for methods taking mutable references --- src/expressions/evaluation/evaluator.rs | 15 +- src/expressions/evaluation/example.rs | 271 -------- src/expressions/evaluation/mod.rs | 3 +- src/expressions/evaluation/node_conversion.rs | 48 +- src/expressions/evaluation/place_frames.rs | 25 - src/expressions/evaluation/type_resolution.rs | 643 +++++++++--------- src/expressions/evaluation/value_frames.rs | 95 +-- src/expressions/expression.rs | 26 - src/expressions/object.rs | 5 +- src/expressions/value.rs | 16 + src/interpretation/interpreter.rs | 91 ++- src/misc/mut_rc_ref_cell.rs | 14 +- tests/expressions.rs | 20 + 13 files changed, 555 insertions(+), 717 deletions(-) delete mode 100644 src/expressions/evaluation/example.rs diff --git a/src/expressions/evaluation/evaluator.rs b/src/expressions/evaluation/evaluator.rs index 53df9db7..565ec2ea 100644 --- a/src/expressions/evaluation/evaluator.rs +++ b/src/expressions/evaluation/evaluator.rs @@ -165,11 +165,8 @@ impl EvaluationItem { match self { EvaluationItem::OwnedValue(value) => ResolvedValue::Owned(value), EvaluationItem::MutableReference { mut_ref } => ResolvedValue::Mutable(mut_ref), - EvaluationItem::SharedReference { + EvaluationItem::SharedReference { shared_ref, .. } => ResolvedValue::Shared { shared_ref, - .. - } => ResolvedValue::Shared { - shared_ref: shared_ref, reason_not_mutable: None, }, _ => panic!("expect_any_value() called on a non-value EvaluationItem"), @@ -348,6 +345,16 @@ impl<'a> Context<'a, ValueType> { } } + pub(super) fn return_any_place(self, value: Place) -> NextAction { + match value { + Place::MutableReference { mut_ref } => self.return_mut_ref(mut_ref), + Place::SharedReference { + shared_ref, + reason_not_mutable, + } => self.return_ref(shared_ref, reason_not_mutable), + } + } + pub(super) fn return_owned_value(self, value: ExpressionValue) -> NextAction { match self.request { RequestedValueOwnership::LateBound | RequestedValueOwnership::Owned => { diff --git a/src/expressions/evaluation/example.rs b/src/expressions/evaluation/example.rs deleted file mode 100644 index d82f86de..00000000 --- a/src/expressions/evaluation/example.rs +++ /dev/null @@ -1,271 +0,0 @@ -use super::*; - -macro_rules! handle_arg_mapping { - // No more args - ([$(,)?] [$($bindings:tt)*]) => { - $($bindings)* - }; - // By shared reference - ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let $arg: &$ty = <$ty as ResolvableArgument>::resolve_ref($arg.as_ref())?; - ]) - }; - // By captured shared reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let tmp = $arg.into_shared_reference(); - let $arg: CapturedRef<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_ref(value))?; - ]) - }; - // SharedValue is an alias for CapturedRef - ([$arg:ident : SharedValue, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let $arg = $arg.into_shared_reference(); - ]) - }; - // By mutable reference - ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let mut tmp = $arg.into_mutable_reference("This argument")?; - let $arg: &mut $ty = <$ty as ResolvableArgument>::resolve_mut(tmp.as_mut()); - ]) - }; - // By captured mutable reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let mut tmp = $arg.into_mutable_reference("This argument")?; - let $arg: CapturedMut<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_mut(value))?; - ]) - }; - // MutableValue is an alias for CapturedMut - ([$arg:ident : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let $arg = $arg.into_mutable_reference("This argument")?; - ]) - }; - // By value - ([$arg:ident : $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let tmp = $arg.into_owned_value("This argument")?; - let $arg: $ty = <$ty as ResolvableArgument>::resolve_owned(tmp)?; - ]) - }; -} - -macro_rules! handle_arg_ownerships { - // No more args - ([$(,)?] [$($outputs:tt)*]) => { - vec![$($outputs)*] - }; - // By shared reference - ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) - }; - // By captured shared reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) - }; - // SharedValue is an alias for CapturedRef - ([$arg:ident : SharedValue, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) - }; - // By mutable reference - ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) - }; - // By captured mutable reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) - }; - // MutableValue is an alias for CapturedMut - ([$arg:ident : MutableValue, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) - }; - // By value - ([$arg:ident : $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Owned,]) - }; -} - -macro_rules! count { - () => { 0 }; - ($head:tt $($tail:tt)*) => { 1 + count!($($tail)*) }; -} - -trait ResolvableArgument: Sized { - fn resolve_owned(value: ExpressionValue) -> ExecutionResult; - fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self>; - fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self>; -} - -// Creating an inner method vastly improves IDE support when writing the method body -macro_rules! handle_define_inner_method { - ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?] $body:block $output_ty:ty) => { - fn $method_name($($arg: $ty),*) -> ExecutionResult<$output_ty> { - $body - } - }; -} - -macro_rules! handle_arg_separation { - ([$($arg:ident : $ty:ty),* $(,)?], $all_arguments:ident, $output_span_range:ident) => { - const LEN: usize = count!($($arg)*); - let Ok([ - $($arg,)* - ]) = <[ResolvedValue; LEN]>::try_from($all_arguments) else { - return $output_span_range.execution_err(format!("Expected {LEN} argument/s")); - }; - }; -} - -macro_rules! handle_call_inner_method { - ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?]) => { - $method_name($($arg),*) - }; -} - -macro_rules! wrap_method { - (($($args:tt)*) -> ExecutionResult<$output_ty:ty> $body:block) => { - MethodInterface { - method: Box::new(move | - all_arguments: Vec, - output_span_range: SpanRange, - | -> ExecutionResult { - handle_define_inner_method!(inner_method [$($args)*] $body $output_ty); - handle_arg_separation!([$($args)*], all_arguments, output_span_range); - handle_arg_mapping!([$($args)*,] []); - let output: ExecutionResult<$output_ty> = handle_call_inner_method!(inner_method [$($args)*]); - <$output_ty as ResolvableOutput>::to_resolved_value(output?, output_span_range) - }), - argument_ownerships: handle_arg_ownerships!([$($args)*,] []), - } - }; -} - -pub(super) struct MethodInterface { - method: Box<(dyn Fn(Vec, SpanRange) -> ExecutionResult)>, - argument_ownerships: Vec, -} - -impl MethodInterface { - pub fn execute( - &self, - parameters: Vec, - span_range: SpanRange - ) -> ExecutionResult { - (self.method)(parameters, span_range) - } -} - -impl ResolvableArgument for ExpressionValue { - fn resolve_owned(value: ExpressionValue) -> ExecutionResult { - Ok(value) - } - - fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self> { - Ok(value) - } - - fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self> { - Ok(value) - } -} - -impl ResolvableArgument for ExpressionArray { - fn resolve_owned(value: ExpressionValue) -> ExecutionResult { - match value { - ExpressionValue::Array(value) => Ok(value), - _ => value.execution_err("Expected array"), - } - } - - fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self> { - match value { - ExpressionValue::Array(value) => Ok(value), - _ => value.execution_err("Expected array"), - } - } - - fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self> { - match value { - ExpressionValue::Array(value) => Ok(value), - _ => value.execution_err("Expected array"), - } - } -} - -impl ResolvableArgument for u8 { - fn resolve_owned(value: ExpressionValue) -> ExecutionResult { - match value { - ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), - _ => value.execution_err("Expected u8"), - } - } - - fn resolve_ref<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self> { - match value { - ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), - _ => value.execution_err("Expected u8"), - } - } - - fn resolve_mut<'a>(value: &'a mut ExpressionValue) -> ExecutionResult<&'a mut Self> { - match value { - ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), - _ => value.execution_err("Expected u8"), - } - } -} - -#[diagnostic::on_unimplemented( - message = "`ResolvableOutput` is not implemented for `{Self}`", - note = "`ResolvableOutput` is not implemented for `CapturedRef` or `CapturedMut` unless `X` is `ExpressionValue`. If we wish to change this, we'd need to have some way to represent some kind of `ExpressionReference`, i.e. a `Typed>` rather than a `CapturedRef>`", -)] -trait ResolvableOutput { - fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult; -} - -impl ResolvableOutput for CapturedRef { - fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { - Ok(ResolvedValue::Shared { - shared_ref: self.update_span_range(|_| output_span_range), - reason_not_mutable: Some(syn::Error::new(output_span_range.join_into_span_else_start(), "It was output from a method a captured shared reference")), - }) - } -} - -impl ResolvableOutput for CapturedMut { - fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { - Ok(ResolvedValue::Mutable(self.update_span_range(|_| output_span_range))) - } -} - -impl ResolvableOutput for T { - fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { - Ok(ResolvedValue::Owned(self.to_value(output_span_range))) - } -} - -#[test] -fn test2() { - // Example usage: - let span_range = SpanRange::new_single(Span::call_site()); - let x = ResolvableOutput::to_resolved_value(vec![10u8.to_value(span_range)], span_range).unwrap(); - let y = ResolvableOutput::to_resolved_value(0usize, span_range).unwrap(); - - let method1 = wrap_method!((a: CapturedRef, index: &ExpressionValue) -> ExecutionResult { - let access = IndexAccess { brackets: Brackets { delim_span: Group::new(Delimiter::Brace, TokenStream::new()).delim_span() }}; - a.try_map(|a, _| a.index_ref(access, index)) - }); - let output = method1.execute(vec![x, y], span_range).unwrap(); - - assert_eq!(*u8::resolve_ref(output.as_value_ref()).unwrap(), 10u8); -} \ No newline at end of file diff --git a/src/expressions/evaluation/mod.rs b/src/expressions/evaluation/mod.rs index af89bb87..66b6d29f 100644 --- a/src/expressions/evaluation/mod.rs +++ b/src/expressions/evaluation/mod.rs @@ -4,12 +4,11 @@ mod node_conversion; mod place_frames; mod type_resolution; mod value_frames; -mod example; use super::*; use assignment_frames::*; pub(super) use evaluator::ExpressionEvaluator; use evaluator::*; use place_frames::*; -use value_frames::*; pub(super) use type_resolution::*; +use value_frames::*; diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs index ea8065dd..758b6fca 100644 --- a/src/expressions/evaluation/node_conversion.rs +++ b/src/expressions/evaluation/node_conversion.rs @@ -8,7 +8,40 @@ impl ExpressionNode { ) -> ExecutionResult { Ok(match self { ExpressionNode::Leaf(leaf) => { - context.return_owned_value(Source::evaluate_leaf(leaf, interpreter)?) + match leaf { + SourceExpressionLeaf::Command(command) => { + // TODO: Allow returning reference + context.return_owned_value(command.clone().interpret_to_value(interpreter)?) + } + SourceExpressionLeaf::Discarded(token) => { + return token.execution_err("This cannot be used in a value expression"); + } + SourceExpressionLeaf::Variable(variable_path) => { + // TODO: Allow block to return reference + let variable_ref = variable_path.reference(interpreter)?; + match context.requested_ownership() { + RequestedValueOwnership::LateBound => { + context.return_any_place(variable_ref.into_late_bound()?) + } + RequestedValueOwnership::SharedReference => { + context.return_ref(variable_ref.into_shared()?, None) + } + RequestedValueOwnership::MutableReference => { + context.return_mut_ref(variable_ref.into_mut()?) + } + RequestedValueOwnership::Owned => { + // TODO: Change this but fix tests which are breaking + // context.return_owned_value(variable_ref.get_value_transparently_cloned()?) + context.return_owned_value(variable_ref.get_value_cloned()?) + } + } + } + SourceExpressionLeaf::ExpressionBlock(block) => { + // TODO: Allow block to return reference + context.return_owned_value(block.interpret_to_value(interpreter)?) + } + SourceExpressionLeaf::Value(value) => context.return_owned_value(value.clone()), + } } ExpressionNode::Grouped { delim_span, inner } => { GroupBuilder::start(context, delim_span, *inner) @@ -103,18 +136,7 @@ impl ExpressionNode { let variable_ref = variable.reference(interpreter)?; match context.requested_ownership() { RequestedPlaceOwnership::LateBound => { - match variable_ref.into_mut() { - Ok(place) => context.return_mutable(place), - Err(ExecutionInterrupt::Error(reason_not_mutable)) => { - // If we get an error with a mutable and shared reference, a mutable reference must already exist. - // We can just propogate the error from taking the shared reference, it should be good enough. - let shared_place = - variable.reference(interpreter)?.into_shared()?; - context.return_shared(shared_place, Some(reason_not_mutable)) - } - // Propogate any other errors, these shouldn't happen mind - Err(err) => Err(err)?, - } + context.return_any(variable_ref.into_late_bound()?) } RequestedPlaceOwnership::SharedReference => { context.return_shared(variable_ref.into_shared()?, None) diff --git a/src/expressions/evaluation/place_frames.rs b/src/expressions/evaluation/place_frames.rs index ddc6aad6..5f63ebe9 100644 --- a/src/expressions/evaluation/place_frames.rs +++ b/src/expressions/evaluation/place_frames.rs @@ -1,31 +1,6 @@ #![allow(unused)] // TODO: Remove when places are properly late-bound use super::*; -/// A rough equivalent of a Rust place (lvalue), as per: -/// https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions -/// -/// In preinterpret, references are (currently) only to variables, or sub-values of variables. -/// -/// # Late Binding -/// -/// Sometimes, a value which can be accessed, but we don't yet know *how* we need to access it. -/// In this case, we attempt to load it as a Mutable place, and failing that, as a Shared place. -/// -/// ## Example of requirement -/// For example, if we have `x[a].y(z)`, we first need to resolve the type of `x[a]` to know -/// whether `x[a]` takes a shared reference, mutable reference or an owned value. -/// -/// So instead, we take the most powerful access we can have for `x[a]`, and convert it later. -pub(super) enum Place { - MutableReference { - mut_ref: MutableValue, - }, - SharedReference { - shared_ref: SharedValue, - reason_not_mutable: Option, - }, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum RequestedPlaceOwnership { LateBound, diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs index 1391b549..9585df04 100644 --- a/src/expressions/evaluation/type_resolution.rs +++ b/src/expressions/evaluation/type_resolution.rs @@ -1,17 +1,168 @@ -#![allow(unused)] // TODO: Remove when type resolution is implemented +#![allow(unused)] // TODO: Remove when places are properly late-bound use super::*; -pub(super) trait ResolvedTypeDetails { +#[macro_use] +mod macros { + macro_rules! handle_arg_mapping { + // No more args + ([$(,)?] [$($bindings:tt)*]) => { + $($bindings)* + }; + // By shared reference + ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let $arg: &$ty = <$ty as ResolvableArgument>::resolve_from_ref($arg.as_ref())?; + ]) + }; + // By captured shared reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let tmp = $arg.into_shared_reference(); + let $arg: CapturedRef<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_ref(value))?; + ]) + }; + // SharedValue is an alias for CapturedRef + ([$arg:ident : SharedValue, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let $arg = $arg.into_shared_reference(); + ]) + }; + // By mutable reference + ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let mut tmp = $arg.into_mutable_reference()?; + let $arg: &mut $ty = <$ty as ResolvableArgument>::resolve_from_mut(tmp.as_mut())?; + ]) + }; + // By captured mutable reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let mut tmp = $arg.into_mutable_reference()?; + let $arg: CapturedMut<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_mut(value))?; + ]) + }; + // MutableValue is an alias for CapturedMut + ([$arg:ident : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let $arg = $arg.into_mutable_reference()?; + ]) + }; + // By value + ([$arg:ident : $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + handle_arg_mapping!([$($rest)*] [ + $($bindings)* + let tmp = $arg.into_owned_value()?; + let $arg: $ty = <$ty as ResolvableArgument>::resolve_from_owned(tmp)?; + ]) + }; + } + + macro_rules! handle_arg_ownerships { + // No more args + ([$(,)?] [$($outputs:tt)*]) => { + vec![$($outputs)*] + }; + // By shared reference + ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + }; + // By captured shared reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + }; + // SharedValue is an alias for CapturedRef + ([$arg:ident : SharedValue, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + }; + // By mutable reference + ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + }; + // By captured mutable reference (i.e. can return a sub-reference from it) + ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + }; + // MutableValue is an alias for CapturedMut + ([$arg:ident : MutableValue, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + }; + // By value + ([$arg:ident : $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Owned,]) + }; + } + + macro_rules! count { + () => { 0 }; + ($head:tt $($tail:tt)*) => { 1 + count!($($tail)*) }; + } + + // Creating an inner method vastly improves IDE support when writing the method body + macro_rules! handle_define_inner_method { + ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?] $body:block $output_ty:ty) => { + fn $method_name($($arg: $ty),*) -> ExecutionResult<$output_ty> { + $body + } + }; + } + + macro_rules! handle_arg_separation { + ([$($arg:ident : $ty:ty),* $(,)?], $all_arguments:ident, $output_span_range:ident) => { + const LEN: usize = count!($($arg)*); + let Ok([ + $($arg,)* + ]) = <[ResolvedValue; LEN]>::try_from($all_arguments) else { + return $output_span_range.execution_err(format!("Expected {LEN} argument/s")); + }; + }; + } + + macro_rules! handle_call_inner_method { + ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?]) => { + $method_name($($arg),*) + }; + } + + macro_rules! wrap_method { + (($($args:tt)*) -> ExecutionResult<$output_ty:ty> $body:block) => { + MethodInterface { + method: Box::new(move | + all_arguments: Vec, + output_span_range: SpanRange, + | -> ExecutionResult { + handle_define_inner_method!(inner_method [$($args)*] $body $output_ty); + handle_arg_separation!([$($args)*], all_arguments, output_span_range); + handle_arg_mapping!([$($args)*,] []); + let output: ExecutionResult<$output_ty> = handle_call_inner_method!(inner_method [$($args)*]); + <$output_ty as ResolvableOutput>::to_resolved_value(output?, output_span_range) + }), + argument_ownerships: handle_arg_ownerships!([$($args)*,] []), + } + }; + } +} + +pub(crate) trait ResolvedTypeDetails { /// This should be true for types which users expect to have value /// semantics, but false for mutable types / types with reference /// semantics. - /// + /// /// This indicates if an &x can be converted to an x via cloning /// when doing method resolution. fn supports_transparent_cloning(&self) -> bool; /// Resolves a method for this resolved type with the given arguments. - fn resolve_method(&self, method: &MethodAccess, num_arguments: usize) -> ExecutionResult; + fn resolve_method( + &self, + method: &MethodAccess, + num_arguments: usize, + ) -> ExecutionResult; // TODO: Eventually we can migrate operations under this umbrella too // fn resolve_unary_operation(&self, operation: UnaryOperation) -> ExecutionResult; @@ -52,80 +203,108 @@ impl ResolvedTypeDetails for ValueKind { } } - fn resolve_method(&self, method: &MethodAccess, num_arguments: usize) -> ExecutionResult { + fn resolve_method( + &self, + method: &MethodAccess, + num_arguments: usize, + ) -> ExecutionResult { let method_name = method.method.to_string(); - match (self, method_name.as_str(), num_arguments) { - // (ValueKind::Array, "len", 0) => Ok(ResolvedMethod::new(array_len)), - // (ValueKind::Stream, "len", 0) => Ok(ResolvedMethod::new(stream_len)), - _ => method.execution_err(format!("{self:?} has no method `{method_name}` with {num_arguments} arguments")), - } + let method = match (self, method_name.as_str(), num_arguments) { + (_, "clone", 0) => { + wrap_method! {(this: &ExpressionValue) -> ExecutionResult { + Ok(this.clone()) + }} + } + (_, "as_mut", 0) => { + wrap_method! {(this: ExpressionValue) -> ExecutionResult { + Ok(MutableValue::new_from_owned(this)) + }} + } + (_, "debug", 0) => wrap_method! {(this: &ExpressionValue) -> ExecutionResult { + this.clone().into_debug_string() + }}, + (ValueKind::Array, "len", 0) => { + wrap_method! {(this: &ExpressionArray) -> ExecutionResult { + Ok(this.items.len()) + }} + } + (ValueKind::Array, "push", 1) => { + wrap_method! {(this: &mut ExpressionArray, item: ExpressionValue) -> ExecutionResult<()> { + this.items.push(item); + Ok(()) + }} + } + (ValueKind::Stream, "len", 0) => { + wrap_method! {(this: &ExpressionStream) -> ExecutionResult { + Ok(this.value.len()) + }} + } + _ => { + return method.execution_err(format!( + "{self:?} has no method `{method_name}` with {num_arguments} arguments" + )) + } + }; + Ok(method) } } - -pub(super) struct ResolvedMethod { - method: WrappedMethod, +pub(crate) struct MethodInterface { + method: Box<(dyn Fn(Vec, SpanRange) -> ExecutionResult)>, argument_ownerships: Vec, } -impl ResolvedMethod { - // fn new( - // method: fn(SelfType, Arguments) -> ExecutionResult, - // ) -> Self - // where - // SelfType: ResolvableArgument + 'static, - // Arguments: ResolvableArguments + 'static, - // Output: ResolvableOutput + 'static, - // { - // // TODO - Find some way of avoiding creating a new box for each method call (e.g. using a cache) - // // Could also consider using a GAT by upgrading to Rust 1.65 - // Self { - // method: Box::new(move | - // self_value: ResolvedValue, - // arguments: Vec, - // output_span_range: SpanRange, - // | -> ExecutionResult { - // SelfType::run_resolved(self_value, |self_value| { - // Arguments::run_with_arguments(arguments, |arguments| { - // method(self_value, arguments)?.to_value(output_span_range) - // }, output_span_range) - // }) - // }), - // argument_ownerships: Arguments::ownerships(), - // } - // } - +impl MethodInterface { pub fn execute( &self, - object: ResolvedValue, - parameters: Vec, - span_range: SpanRange + arguments: Vec, + span_range: SpanRange, ) -> ExecutionResult { - (self.method)(object, parameters, span_range) + (self.method)(arguments, span_range) } - pub fn ownerships(&self) -> &[RequestedValueOwnership] { + pub(super) fn ownerships(&self) -> &[RequestedValueOwnership] { &self.argument_ownerships } } -fn array_len(self_value: &ExpressionArray, arguments: ()) -> ExecutionResult { - Ok(self_value.items.len()) -} +use outputs::*; -fn stream_len(self_value: &ExpressionStream, arguments: ()) -> ExecutionResult { - Ok(self_value.value.len()) -} +mod outputs { + use super::*; -type WrappedMethod = Box<(dyn Fn(ResolvedValue, Vec, SpanRange) -> ExecutionResult)>; -/* -trait ResolvableOutput { - fn to_value(self, output_span_range: SpanRange) -> ExecutionResult; -} + #[diagnostic::on_unimplemented( + message = "`ResolvableOutput` is not implemented for `{Self}`", + note = "`ResolvableOutput` is not implemented for `CapturedRef` or `CapturedMut` unless `X` is `ExpressionValue`. If we wish to change this, we'd need to have some way to represent some kind of `ExpressionReference`, i.e. a `Typed>` rather than a `CapturedRef>`" + )] + pub(crate) trait ResolvableOutput { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult; + } + + impl ResolvableOutput for CapturedRef { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Shared { + shared_ref: self.update_span_range(|_| output_span_range), + reason_not_mutable: Some(syn::Error::new( + output_span_range.join_into_span_else_start(), + "It was output from a method a captured shared reference", + )), + }) + } + } + + impl ResolvableOutput for CapturedMut { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Mutable( + self.update_span_range(|_| output_span_range), + )) + } + } -impl ResolvableOutput for T { - fn to_value(self, output_span_range: SpanRange) -> ExecutionResult { - Ok(ResolvedValue::Owned(self.to_value(output_span_range))) + impl ResolvableOutput for T { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Owned(self.to_value(output_span_range))) + } } } @@ -134,293 +313,149 @@ use arguments::*; mod arguments { use super::*; - // FRAMEWORK TO DO DISJOINT TRAIT IMPLEMENTATIONS - - pub(super) trait ResolvableArgument: Sized { - fn run_resolved(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult; - fn resolve(value: ResolvedValue) -> ExecutionResult; - fn ownership() -> RequestedValueOwnership; + pub(crate) trait ResolveAs { + fn resolve_as(self) -> ExecutionResult; } - pub(super) trait ResolvableArguments: Sized { - fn run_with_arguments(value: Vec, inner: impl FnOnce(Self) -> ExecutionResult, arguments_span: SpanRange) -> ExecutionResult; - fn ownerships() -> Vec; - } - - impl ResolvableArguments for () { - fn run_with_arguments(value: Vec, inner: impl FnOnce(Self) -> ExecutionResult, arguments_span: SpanRange) -> ExecutionResult { - let Ok([ - // No arguments - ]) = <[ResolvedValue; 0]>::try_from(value) else { - return arguments_span.execution_err("Expected 0 arguments"); - }; - inner(( - // No arguments - )) - } - - fn ownerships() -> Vec { - vec![] + impl ResolveAs for ExpressionValue { + fn resolve_as(self) -> ExecutionResult { + T::resolve_from_owned(self) } } - impl ResolvableArguments for (T,) { - fn run_with_arguments(value: Vec, inner: impl FnOnce(Self) -> ExecutionResult, arguments_span: SpanRange) -> ExecutionResult { - let Ok([ - value0, - ]) = <[ResolvedValue; 1]>::try_from(value) else { - return arguments_span.execution_err("Expected 1 argument"); - }; - - T::run_resolved(value0, |value0| { - // Further nesting... - inner((value0,)) - }) - } - - fn ownerships() -> Vec { - vec![T::ownership()] + impl<'a, T: ResolvableArgument> ResolveAs<&'a T> for &'a ExpressionValue { + fn resolve_as(self) -> ExecutionResult<&'a T> { + T::resolve_from_ref(self) } } - // TODO: Add more tuples, maybe using a macro to generate them - - trait ArgumentKind {} - struct ArgumentKindOwned; - impl ArgumentKind for ArgumentKindOwned {} - struct ArgumentKindDisposedRef; - impl ArgumentKind for ArgumentKindDisposedRef {} - struct ArgumentKindCapturedRef; - impl ArgumentKind for ArgumentKindCapturedRef {} - struct ArgumentKindDisposedRefMut; - impl ArgumentKind for ArgumentKindDisposedRefMut {} - struct ArgumentKindCapturedRefMut; - impl ArgumentKind for ArgumentKindCapturedRefMut {} - - trait InferredArgumentType: Sized { - type ArgumentKind: ArgumentKind; - } - - trait ResolvableArgumentAs: Sized { - fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult; - fn ownership() -> RequestedValueOwnership; - } - - impl::ArgumentKind>> ResolvableArgument for T { - fn run_resolved(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - ::ArgumentKind>>::run_with_argument(value, inner) - } - - fn ownership() -> RequestedValueOwnership { - ::ArgumentKind>>::ownership() + impl<'a, T: ResolvableArgument> ResolveAs<&'a mut T> for &'a mut ExpressionValue { + fn resolve_as(self) -> ExecutionResult<&'a mut T> { + T::resolve_from_mut(self) } } - // TODO: - // The automatic conversion of &XXX in a method into this might just not be feasible; because stacked borrows go from - // out to in; and here we want to go from in (the arguments of an inner method) to out. - // Instead, we might need to use a macro-based approach rather than just leveraging the type system. - - // trait ResolvableRef - // where for<'a> &'a Self: InferredArgumentType - // { - // fn resolve_from_value<'a>(value: &'a ExpressionValue) -> ExecutionResult<&'a Self>; - // } - - // impl<'o, T: ResolvableRef> ResolvableArgumentAs for &'o T - // where for<'a> &'a T: InferredArgumentType - // { - // fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - // let output = { - // let value = value.as_value_ref(); - - // let output = inner( - // // This needs to be 'o to match Self and work with the callback to match up with the defined function... - // // But then this implies that 'o is smaller than this function call, which doesn't work. - // ::resolve_from_value(value)? - // )?; - // output - // }; - // Ok(output) - // // let mut value = value.as_value_ref(); - // // inner( - // // >::resolve_from_value(value)? - // // ) - // } - - // fn ownership() -> RequestedValueOwnership { - // RequestedValueOwnership::SharedReference - // } - // } - - trait ResolvableRef<'a>: InferredArgumentType - { - fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult; + pub(crate) trait ResolvableArgument: Sized { + fn resolve_from_owned(value: ExpressionValue) -> ExecutionResult; + fn resolve_from_ref(value: &ExpressionValue) -> ExecutionResult<&Self>; + fn resolve_from_mut(value: &mut ExpressionValue) -> ExecutionResult<&mut Self>; } - impl<'a, T: ResolvableRef<'a>> ResolvableArgumentAs for T { - fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - let mut value = value.as_value_ref(); - inner( - >::resolve_from_value(value)? - ) + impl ResolvableArgument for ExpressionValue { + fn resolve_from_owned(value: ExpressionValue) -> ExecutionResult { + Ok(value) } - fn ownership() -> RequestedValueOwnership { - RequestedValueOwnership::SharedReference + fn resolve_from_ref(value: &ExpressionValue) -> ExecutionResult<&Self> { + Ok(value) } - } - - trait ResolvableCapturedRef: InferredArgumentType { - fn resolve_from_value(value: SharedValue) -> ExecutionResult; - } - impl<'a, T: ResolvableCapturedRef> ResolvableArgumentAs for T { - fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - let mut value = value.into_shared_reference(); - inner( - ::resolve_from_value(value)? - ) - } - - fn ownership() -> RequestedValueOwnership { - RequestedValueOwnership::SharedReference + fn resolve_from_mut(value: &mut ExpressionValue) -> ExecutionResult<&mut Self> { + Ok(value) } } - trait ResolvableMutRef<'a>: InferredArgumentType { - fn resolve_from_value(value: &'a mut ExpressionValue) -> ExecutionResult; - } - - impl ResolvableMutRef<'a>> ResolvableArgumentAs for T { - fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - let mut value = value.into_mutable_reference("This argument")?; - inner( - >::resolve_from_value(value.as_mut())? - ) - } - - fn ownership() -> RequestedValueOwnership { - RequestedValueOwnership::MutableReference - } - } + macro_rules! impl_resolvable_argument_for { + (($value:ident) -> $type:ty $body:block) => { + impl ResolvableArgument for $type { + fn resolve_from_owned($value: ExpressionValue) -> ExecutionResult { + $body + } - trait ResolvableCapturedMutRef: InferredArgumentType { - fn resolve_from_value(value: MutableValue) -> ExecutionResult; - } + fn resolve_from_ref($value: &ExpressionValue) -> ExecutionResult<&Self> { + $body + } - impl ResolvableArgumentAs for T { - fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - let mut value = value.into_mutable_reference("This argument")?; - inner( - ::resolve_from_value(value)? - ) - } - - fn ownership() -> RequestedValueOwnership { - RequestedValueOwnership::MutableReference - } + fn resolve_from_mut($value: &mut ExpressionValue) -> ExecutionResult<&mut Self> { + $body + } + } + }; } - trait ResolvableOwned: InferredArgumentType { - fn resolve_from_value(value: ExpressionValue) -> ExecutionResult; - } + pub(crate) use impl_resolvable_argument_for; - impl ResolvableArgumentAs for T { - fn run_with_argument(value: ResolvedValue, inner: impl FnOnce(Self) -> ExecutionResult) -> ExecutionResult { - let mut value = value.into_owned_value("This argument")?; - inner( - ::resolve_from_value(value)? - ) + impl_resolvable_argument_for! {(value) -> ExpressionInteger { + match value { + ExpressionValue::Integer(value) => Ok(value), + _ => value.execution_err("Expected integer"), } - - fn ownership() -> RequestedValueOwnership { - RequestedValueOwnership::Owned - } - } - - // IMPLEMENTATIONS + }} - impl InferredArgumentType for &'_ ExpressionValue { - type ArgumentKind = ArgumentKindDisposedRef; - } - - impl<'a> ResolvableRef<'a> for &'a ExpressionValue { - fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult { - Ok(value) + impl_resolvable_argument_for! {(value) -> u8 { + match value { + ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::U8(x), ..}) => Ok(x), + _ => value.execution_err("Expected u8"), } - } - - impl InferredArgumentType for SharedSubPlace { - type ArgumentKind = ArgumentKindCapturedRef; - } + }} - impl ResolvableCapturedRef for SharedSubPlace - where - for<'a> &'a T: ResolvableRef<'a>, - { - fn resolve_from_value(value: SharedValue) -> ExecutionResult { - value.resolve_internally_mapped(|value| <&'_ T as ResolvableRef<'_>>::resolve_from_value(value)) + impl_resolvable_argument_for! {(value) -> usize { + match value { + ExpressionValue::Integer(ExpressionInteger { value: ExpressionIntegerValue::Usize(x), ..}) => Ok(x), + _ => value.execution_err("Expected usize"), } - } - - impl InferredArgumentType for &'_ mut ExpressionValue { - type ArgumentKind = ArgumentKindDisposedRefMut; - } + }} - impl<'a> ResolvableMutRef<'a> for &'a mut ExpressionValue { - fn resolve_from_value(value: &'a mut ExpressionValue) -> ExecutionResult { - Ok(value) + impl_resolvable_argument_for! {(value) -> ExpressionFloat { + match value { + ExpressionValue::Float(value) => Ok(value), + _ => value.execution_err("Expected float"), } - } + }} - impl InferredArgumentType for MutableSubPlace { - type ArgumentKind = ArgumentKindCapturedRefMut; - } + impl_resolvable_argument_for! {(value) -> ExpressionBoolean { + match value { + ExpressionValue::Boolean(value) => Ok(value), + _ => value.execution_err("Expected boolean"), + } + }} - impl ResolvableCapturedMutRef for MutableSubPlace - where - for<'a> &'a mut T: ResolvableMutRef<'a>, - { - fn resolve_from_value(value: MutableValue) -> ExecutionResult { - value.resolve_internally_mapped(|value, _| <&'_ mut T as ResolvableMutRef<'_>>::resolve_from_value(value)) + impl_resolvable_argument_for! {(value) -> ExpressionString { + match value { + ExpressionValue::String(value) => Ok(value), + _ => value.execution_err("Expected string"), } - } + }} - impl InferredArgumentType for ExpressionValue { - type ArgumentKind = ArgumentKindOwned; - } + impl_resolvable_argument_for! {(value) -> ExpressionChar { + match value { + ExpressionValue::Char(value) => Ok(value), + _ => value.execution_err("Expected char"), + } + }} - impl ResolvableOwned for ExpressionValue { - fn resolve_from_value(value: ExpressionValue) -> ExecutionResult { - Ok(value) + impl_resolvable_argument_for! {(value) -> ExpressionArray { + match value { + ExpressionValue::Array(value) => Ok(value), + _ => value.execution_err("Expected array"), } - } + }} - impl InferredArgumentType for &'_ ExpressionArray { - type ArgumentKind = ArgumentKindDisposedRef; - } + impl_resolvable_argument_for! {(value) -> ExpressionObject { + match value { + ExpressionValue::Object(value) => Ok(value), + _ => value.execution_err("Expected object"), + } + }} - impl<'a> ResolvableRef<'a> for &'a ExpressionArray { - fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult { - match value { - ExpressionValue::Array(arr) => Ok(arr), - _ => value.execution_err("Expected array"), - } + impl_resolvable_argument_for! {(value) -> ExpressionStream { + match value { + ExpressionValue::Stream(value) => Ok(value), + _ => value.execution_err("Expected stream"), } - } + }} - impl InferredArgumentType for &'_ ExpressionStream { - type ArgumentKind = ArgumentKindDisposedRef; - } + impl_resolvable_argument_for! {(value) -> ExpressionRange { + match value { + ExpressionValue::Range(value) => Ok(value), + _ => value.execution_err("Expected range"), + } + }} - impl<'a> ResolvableRef<'a> for &'a ExpressionStream { - fn resolve_from_value(value: &'a ExpressionValue) -> ExecutionResult { - match value { - ExpressionValue::Stream(stream) => Ok(stream), - _ => value.execution_err("Expected stream"), - } + impl_resolvable_argument_for! {(value) -> ExpressionIterator { + match value { + ExpressionValue::Iterator(value) => Ok(value), + _ => value.execution_err("Expected iterator"), } - } + }} } - */ \ No newline at end of file diff --git a/src/expressions/evaluation/value_frames.rs b/src/expressions/evaluation/value_frames.rs index 986ffaea..cbec5be4 100644 --- a/src/expressions/evaluation/value_frames.rs +++ b/src/expressions/evaluation/value_frames.rs @@ -1,6 +1,4 @@ -#![allow(unused)] use crate::expressions::evaluation::type_resolution::{ResolvedMethod, ResolvedTypeDetails}; - -// TODO: Remove when values are properly late-bound +#![allow(unused)] // TODO: Remove when places are properly late-bound use super::*; /// # Late Binding @@ -26,47 +24,33 @@ pub(crate) enum ResolvedValue { } impl ResolvedValue { - /// The requirement is just for error messages, and should be "A ", and will be filled like: - /// "{usage} expects an owned value, but receives a reference..." - pub(crate) fn into_owned_value(self, usage: &str) -> ExecutionResult { + pub(crate) fn into_owned_value(self) -> ExecutionResult { let reference = match self { ResolvedValue::Owned(value) => return Ok(value), ResolvedValue::Shared { .. } | ResolvedValue::Mutable { .. } => self.as_ref(), }; - if !reference.kind().supports_transparent_cloning() { - return reference.execution_err(format!( - "{} expects an owned value, but receives a reference and {} does not support transparent cloning. You may wish to use .clone() explicitly.", - usage, - reference.articled_value_type(), - )); - } - Ok(reference.clone()) + reference.try_transparent_clone() } /// The requirement is just for error messages, and should be "A ", and will be filled like: /// "{usage} expects an owned value, but receives a reference..." - pub(crate) fn into_mutable_reference(self, usage: &str) -> ExecutionResult { + pub(crate) fn into_mutable_reference(self) -> ExecutionResult { Ok(match self { - // It might be a bug if a user calls a mutable method on a floating owned value, - // but it might be annoying to force them to do `.as_mut()` everywhere. - // Let's leave this as-is for now. - ResolvedValue::Owned(value) => MutableValue::new_from_owned(value), + ResolvedValue::Owned(value) => { + return value.execution_err("A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference.".to_string()) + }, ResolvedValue::Mutable(reference) => reference, ResolvedValue::Shared { shared_ref, reason_not_mutable: Some(reason_not_mutable), } => return shared_ref.execution_err(format!( - "{} expects a mutable reference, but only receives a shared reference because {reason_not_mutable}.", - usage - )), - ResolvedValue::Shared { shared_ref, reason_not_mutable: None, } => return shared_ref.execution_err(format!( - "{} expects a mutable reference, but only receives a shared reference.", - usage + "A mutable reference is required, but only a shared reference was received because {reason_not_mutable}." )), + ResolvedValue::Shared { shared_ref, reason_not_mutable: None, } => return shared_ref.execution_err("A mutable reference is required, but only a shared reference was received.".to_string()), }) } pub(crate) fn into_shared_reference(self) -> SharedValue { match self { ResolvedValue::Owned(value) => SharedValue::new_from_owned(value), - ResolvedValue::Mutable(reference) => reference.to_shared(), + ResolvedValue::Mutable(reference) => reference.into_shared(), ResolvedValue::Shared { shared_ref, .. } => shared_ref, } } @@ -775,15 +759,14 @@ impl EvaluationFrame for CompoundAssignmentBuilder { pub(super) struct MethodCallBuilder { method: MethodAccess, unevaluated_parameters_stack: Vec<(ExpressionNodeId, RequestedValueOwnership)>, - evaluated_parameters: Vec, state: MethodCallPath, } enum MethodCallPath { CallerPath, ArgumentsPath { - caller: ResolvedValue, - method: ResolvedMethod, + method: MethodInterface, + evaluated_arguments_including_caller: Vec, }, } @@ -796,12 +779,14 @@ impl MethodCallBuilder { ) -> NextAction { let frame = Self { method, - unevaluated_parameters_stack: parameters.iter().rev() + unevaluated_parameters_stack: parameters + .iter() + .rev() .map(|x| { // This is just a placeholder - we'll fix it up shortly (*x, RequestedValueOwnership::LateBound) - }).collect(), - evaluated_parameters: Vec::with_capacity(parameters.len()), + }) + .collect(), state: MethodCallPath::CallerPath, }; context.handle_node_as_value(frame, caller, RequestedValueOwnership::LateBound) @@ -824,17 +809,40 @@ impl EvaluationFrame for MethodCallBuilder { match self.state { MethodCallPath::CallerPath => { let caller = item.expect_any_value(); - let method = caller.kind().resolve_method(&self.method, self.unevaluated_parameters_stack.len())?; - assert!(method.ownerships().len() == self.unevaluated_parameters_stack.len(), "The method resolution should ensure the argument count is correct"); - let argument_ownerships_stack = method.ownerships().iter().rev(); - for ((_, requested_ownership), ownership) in self.unevaluated_parameters_stack.iter_mut().zip(argument_ownerships_stack) { + let method = caller + .kind() + .resolve_method(&self.method, self.unevaluated_parameters_stack.len())?; + assert!( + method.ownerships().len() == 1 + self.unevaluated_parameters_stack.len(), + "The method resolution should ensure the argument count is correct" + ); + + // We skip 1 to ignore the caller + let argument_ownerships_stack = method.ownerships().iter().skip(1).rev(); + for ((_, requested_ownership), ownership) in self + .unevaluated_parameters_stack + .iter_mut() + .zip(argument_ownerships_stack) + { *requested_ownership = *ownership; } - self.state = MethodCallPath::ArgumentsPath { caller, method, }; + + self.state = MethodCallPath::ArgumentsPath { + evaluated_arguments_including_caller: { + let mut params = + Vec::with_capacity(1 + self.unevaluated_parameters_stack.len()); + params.push(caller); + params + }, + method, + }; } - MethodCallPath::ArgumentsPath { .. } => { + MethodCallPath::ArgumentsPath { + evaluated_arguments_including_caller: ref mut evaluated_parameters_including_caller, + .. + } => { let argument = item.expect_any_value(); - self.evaluated_parameters.push(argument); + evaluated_parameters_including_caller.push(argument); } }; // Now plan the next action @@ -843,11 +851,14 @@ impl EvaluationFrame for MethodCallBuilder { context.handle_node_as_value(self, parameter, ownership) } None => { - let (caller, method) = match self.state { + let (arguments, method) = match self.state { MethodCallPath::CallerPath => unreachable!("Already updated above"), - MethodCallPath::ArgumentsPath { caller, method } => (caller, method), + MethodCallPath::ArgumentsPath { + evaluated_arguments_including_caller, + method, + } => (evaluated_arguments_including_caller, method), }; - let output = method.execute(caller, self.evaluated_parameters, self.method.span_range())?; + let output = method.execute(arguments, self.method.span_range())?; context.return_any_value(output) } }) diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index 91d0bebe..da3b6e76 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -221,27 +221,6 @@ impl Expressionable for Source { } } } - - fn evaluate_leaf( - leaf: &Self::Leaf, - interpreter: &mut Self::EvaluationContext, - ) -> ExecutionResult { - Ok(match leaf { - SourceExpressionLeaf::Command(command) => { - command.clone().interpret_to_value(interpreter)? - } - SourceExpressionLeaf::Discarded(token) => { - return token.execution_err("This cannot be used in a value expression"); - } - SourceExpressionLeaf::Variable(variable_path) => { - variable_path.interpret_to_value(interpreter)? - } - SourceExpressionLeaf::ExpressionBlock(block) => { - block.interpret_to_value(interpreter)? - } - SourceExpressionLeaf::Value(value) => value.clone(), - }) - } } impl Parse for Expression { @@ -352,9 +331,4 @@ pub(super) trait Expressionable: Sized { input: &mut ParseStreamStack, parent_stack_frame: &ExpressionStackFrame, ) -> ParseResult; - - fn evaluate_leaf( - leaf: &Self::Leaf, - context: &mut Self::EvaluationContext, - ) -> ExecutionResult; } diff --git a/src/expressions/object.rs b/src/expressions/object.rs index e91b1d35..aec81822 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -129,7 +129,10 @@ impl ExpressionObject { Ok(self.mut_entry_or_create(access.property.to_string(), access.property.span())) } - pub(super) fn property_ref(&self, access: &PropertyAccess) -> ExecutionResult<&ExpressionValue> { + pub(super) fn property_ref( + &self, + access: &PropertyAccess, + ) -> ExecutionResult<&ExpressionValue> { let key = access.property.to_string(); let entry = self.entries.get(&key).ok_or_else(|| { access.execution_error(format!("The object does not have a field named `{}`", key)) diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 2ca6f67d..6c4ab267 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -22,6 +22,12 @@ pub(crate) trait ToExpressionValue: Sized { fn to_value(self, span_range: SpanRange) -> ExpressionValue; } +impl ToExpressionValue for () { + fn to_value(self, span_range: SpanRange) -> ExpressionValue { + ExpressionValue::None(span_range) + } +} + impl ExpressionValue { pub(crate) fn for_literal(literal: Literal) -> Self { // The unwrap should be safe because all Literal should be parsable @@ -56,6 +62,16 @@ impl ExpressionValue { } } + pub(crate) fn try_transparent_clone(&self) -> ExecutionResult { + if !self.kind().supports_transparent_cloning() { + return self.execution_err(format!( + "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .clone() explicitly.", + self.articled_value_type() + )); + } + Ok(self.clone()) + } + pub(super) fn expect_value_pair( self, operation: &impl Operation, diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 2d4afbca..1b37bc2e 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -1,3 +1,4 @@ +#![allow(unused)] // TODO: Remove when places are properly late-bound use crate::internal_prelude::*; use super::IsVariable; @@ -153,6 +154,14 @@ impl VariableReference { }) } + // Gets the cloned expression value, setting the span range appropriately + pub(crate) fn get_value_transparently_cloned(&self) -> ExecutionResult { + Ok(self + .get_value_ref()? + .try_transparent_clone()? + .with_span_range(self.variable_span_range)) + } + // Gets the cloned expression value, setting the span range appropriately pub(crate) fn get_value_cloned(&self) -> ExecutionResult { Ok(self @@ -168,6 +177,48 @@ impl VariableReference { pub(crate) fn into_shared(self) -> ExecutionResult { SharedValue::new_from_variable(self) } + + pub(crate) fn into_late_bound(self) -> ExecutionResult { + match self.clone().into_mut() { + Ok(value) => Ok(Place::MutableReference { mut_ref: value }), + Err(ExecutionInterrupt::Error(reason_not_mutable)) => { + // If we get an error with a mutable and shared reference, a mutable reference must already exist. + // We can just propogate the error from taking the shared reference, it should be good enough. + let value = self.into_shared()?; + Ok(Place::SharedReference { + shared_ref: value, + reason_not_mutable: Some(reason_not_mutable), + }) + } + // Propogate any other errors, these shouldn't happen mind + Err(err) => Err(err), + } + } +} + +/// A rough equivalent of a Rust place (lvalue), as per: +/// https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions +/// +/// In preinterpret, references are (currently) only to variables, or sub-values of variables. +/// +/// # Late Binding +/// +/// Sometimes, a value which can be accessed, but we don't yet know *how* we need to access it. +/// In this case, we attempt to load it as a Mutable place, and failing that, as a Shared place. +/// +/// ## Example of requirement +/// For example, if we have `x[a].y(z)`, we first need to resolve the type of `x[a]` to know +/// whether `x[a]` takes a shared reference, mutable reference or an owned value. +/// +/// So instead, we take the most powerful access we can have for `x[a]`, and convert it later. +pub(crate) enum Place { + MutableReference { + mut_ref: MutableValue, + }, + SharedReference { + shared_ref: SharedValue, + reason_not_mutable: Option, + }, } impl HasSpanRange for VariableReference { @@ -189,9 +240,9 @@ pub(crate) struct MutableSubPlace { } impl MutableSubPlace { - pub(crate) fn to_shared(self) -> SharedSubPlace { + pub(crate) fn into_shared(self) -> SharedSubPlace { SharedSubPlace { - shared_cell: self.mut_cell.to_shared(), + shared_cell: self.mut_cell.into_shared(), span_range: self.span_range, } } @@ -211,13 +262,15 @@ impl MutableSubPlace { value_map: impl for<'a, 'b> FnOnce(&'a mut T, &'b SpanRange) -> ExecutionResult<&'a mut V>, ) -> ExecutionResult> { Ok(MutableSubPlace { - mut_cell: self.mut_cell.try_map(|value| value_map(value, &self.span_range))?, + mut_cell: self + .mut_cell + .try_map(|value| value_map(value, &self.span_range))?, span_range: self.span_range, }) } pub(crate) fn update_span_range( - self, + self, span_range_map: impl FnOnce(SpanRange) -> SpanRange, ) -> Self { Self { @@ -249,14 +302,10 @@ impl MutableSubPlace { } pub(crate) fn into_stream(self) -> ExecutionResult> { - self.try_map( - |value, span_range| { - match value { - ExpressionValue::Stream(stream) => Ok(&mut stream.value), - _ => span_range.execution_err("The variable is not a stream"), - } - }, - ) + self.try_map(|value, span_range| match value { + ExpressionValue::Stream(stream) => Ok(&mut stream.value), + _ => span_range.execution_err("The variable is not a stream"), + }) } pub(crate) fn resolve_indexed_with_autocreate( @@ -264,14 +313,12 @@ impl MutableSubPlace { access: IndexAccess, index: ExpressionValue, ) -> ExecutionResult { - self - .update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) .try_map(|value, _| value.index_mut_with_autocreate(access, index)) } pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { - self - .update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) .try_map(|value, _| value.property_mut(&access)) } @@ -339,7 +386,9 @@ impl SharedSubPlace { value_map: impl for<'a, 'b> FnOnce(&'a T, &'b SpanRange) -> ExecutionResult<&'a V>, ) -> ExecutionResult> { Ok(SharedSubPlace { - shared_cell: self.shared_cell.try_map(|value| value_map(value, &self.span_range))?, + shared_cell: self + .shared_cell + .try_map(|value| value_map(value, &self.span_range))?, span_range: self.span_range, }) } @@ -355,7 +404,7 @@ impl SharedSubPlace { } pub(crate) fn update_span_range( - self, + self, span_range_map: impl FnOnce(SpanRange) -> SpanRange, ) -> Self { Self { @@ -391,14 +440,12 @@ impl SharedSubPlace { access: IndexAccess, index: &ExpressionValue, ) -> ExecutionResult { - self - .update_span_range(|old_span| SpanRange::new_between(old_span, access)) + self.update_span_range(|old_span| SpanRange::new_between(old_span, access)) .try_map(|value, _| value.index_ref(access, index)) } pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { - self - .update_span_range(|old_span| SpanRange::new_between(old_span, access.span_range())) + self.update_span_range(|old_span| SpanRange::new_between(old_span, access.span_range())) .try_map(|value, _| value.property_ref(&access)) } } diff --git a/src/misc/mut_rc_ref_cell.rs b/src/misc/mut_rc_ref_cell.rs index fab84d12..d32af9a0 100644 --- a/src/misc/mut_rc_ref_cell.rs +++ b/src/misc/mut_rc_ref_cell.rs @@ -1,3 +1,4 @@ +#![allow(unused)] // TODO: Remove when places are properly late-bound use crate::internal_prelude::*; use std::cell::*; use std::rc::Rc; @@ -29,7 +30,7 @@ impl MutSubRcRefCell { } impl MutSubRcRefCell { - pub(crate) fn to_shared(self) -> SharedSubRcRefCell { + pub(crate) fn into_shared(self) -> SharedSubRcRefCell { let ptr = self.ref_mut.deref() as *const U; drop(self.ref_mut); // SAFETY: @@ -38,14 +39,13 @@ impl MutSubRcRefCell { // - All our invariants for SharedSubRcRefCell / MutSubRcRefCell are maintained unsafe { // The unwrap cannot panic because we just held a mutable borrow, we're not in Sync land, so no-one else can have a borrow. - SharedSubRcRefCell::new(self.pointed_at).unwrap().map(|_| &*ptr) + SharedSubRcRefCell::new(self.pointed_at) + .unwrap() + .map(|_| &*ptr) } } - pub(crate) fn map( - self, - f: impl FnOnce(&mut U) -> &mut V, - ) -> MutSubRcRefCell { + pub(crate) fn map(self, f: impl FnOnce(&mut U) -> &mut V) -> MutSubRcRefCell { MutSubRcRefCell { ref_mut: RefMut::map(self.ref_mut, f), pointed_at: self.pointed_at, @@ -109,7 +109,7 @@ impl SharedSubRcRefCell { // This is guaranteed by the fact that the only time we drop the RefCell // is when we drop the SharedSubRcRefCell, and we ensure that the Ref is dropped first. shared_ref: unsafe { std::mem::transmute::, Ref<'static, T>>(shared_ref) }, - pointed_at: pointed_at, + pointed_at, }) } } diff --git a/tests/expressions.rs b/tests/expressions.rs index f18b6446..5a85e8cb 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -476,4 +476,24 @@ fn test_method_calls() { ), 2 + 3 ); + + preinterpret_assert_eq!( + #( + let x = [1, 2, 3]; + x.push(5); + x.push(2); + x.debug() + ), + "[1, 2, 3, 5, 2]" + ); + // Push returns None + preinterpret_assert_eq!( + #([1, 2, 3].as_mut().push(4).debug()), + "None" + ); + // Converting to mut and then to shared works + preinterpret_assert_eq!( + #([].as_mut().len().debug()), + "0usize" + ); } From 327319d1fbb3971677d82bbfa8ca9b6ad1229a46 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 19 Sep 2025 12:45:55 +0100 Subject: [PATCH 123/126] feature: Add take() --- CHANGELOG.md | 129 ++++++++---------- src/expressions/evaluation/evaluator.rs | 2 +- src/expressions/evaluation/node_conversion.rs | 9 +- src/expressions/evaluation/place_frames.rs | 2 +- src/expressions/evaluation/type_resolution.rs | 86 +++++++----- src/expressions/evaluation/value_frames.rs | 18 +-- src/expressions/object.rs | 4 +- src/expressions/value.rs | 8 +- src/interpretation/interpreter.rs | 8 +- src/misc/mut_rc_ref_cell.rs | 1 - tests/expressions.rs | 9 ++ 11 files changed, 144 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd2e1c91..e093e784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,41 +97,25 @@ Inside a transform stream, the following grammar is supported: ### To come * Method calls continued - * Add support for &mut methods like `push(..)`... - * WIP: Finish the late binding / TODOs in the evaluation module - * WIP: Add in basic method resolution, even if just with hardcoded strings for now! - * CHALLENGE: Given a.b(c,d,e) we need to resolve: - * What method / code to use - * Whether a, c, d and e should be resolved as &, &mut or owned - * SOLUTION: - * We change `evaluation.rs` to start by: - * Resolving a value's reference path as a `MutRef | SharedRef` at the same time (basically taking a `MutRef` if we can)... We probably want value resolution to take a `ValueOwnership::Any|Owned|MutRef|SharedRef` to guide this process. - * We'll need to back-propogate through places, so will also need to support `SharedRef` - in `Place` and have a `PlaceOwnership::Any|MutRef|SharedRef` - => e.g. if we're resolving `x.add(arr[3])` and want to read `arr[3]` as a shared reference. - => or if we're resolve `arr[3].to_string()` we want to read `arr[3]` as `LateBound` and then resolve to shared. - * Existing resolutions likely convert to an `Owned`, possibly via a clone... Although some of these can be fixed too. - * When resolving a method call, we start by requesting a `PermittedValueKind::Any` for `a` and then getting a `&a` from the`Owned | MutRef | SharedRef`; and alongside the name `b`, and possibly # of arguments, we resolve a method definition (or error) - * The method definition tells us whether we need `a` to be `Owned | MutRef | SharedRef`, and similarly for each argument. And the type of `&a` should tell us whether - it is copy (i.e. can be transparently cloned from a ref to an owned if needed) - * We can then resolve the correct value for each argument, and execute the method. - * Then we can add `.push(x)` on array and stream - * Scrap `#>>x` etc in favour of `#(a.push(@XXX))` - * Also improve support to make it easier to add methods on built-in types or (in future) user-designed types -* Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write i.e. https://crates.io/crates/crabtime - thoughts on crabtime: - => Looks great! - => Why don't they use a cheap hash of the code as a cache key? - - The cachability is a big win compared to preinterpret (although preinterpret is faster on first run) - => I can't imagine the span-chasing / error messages are great, because everything is translated to/from strings between the processes - ... I wonder if there's any way to improve this? Plausibly you could use the proc-macro bridge encoding scheme as per https://blog.jetbrains.com/rust/2022/07/07/procedural-macros-under-the-hood-part-ii/ to send handles onwards, or even delegate directly somehow? - - The spans are better in preinterpret - => Parsing isn't really a thing - they're not going after full macro stuff, probably wise -* No clone required for testing equality of streams, objects and arrays - * Some kind of reference support - * Add new `.clone()` method + * Replace `as debug` and `[!debug! ..]` with `.debug()` + * Add support for `RequestedValueOwnership::SharedOrOwned` (CoW behaviour) + and equivalent `SharedOrOwned`. This can be used for utilities like `debug()` + * Scrap `#>>x` etc in favour of `#(x.push(@TOKEN))` + * TODO[access-refactor] + * TODO[range-refactor] & some kind of more thought through typed reference support - e.g. slices, mutable arrays? + * TODO[operation-refactor] + * No clone required for testing equality of streams, objects and arrays + * Add better way of defining methods once / lazily, and binding them to + an object type. +* Consider: + * Changing evaluation to allow `ResolvedValue` (i.e. allow references) + * Removing span range from value: + * Moving it to an `ResolvedValue::OwnedValue` and `OwnedValue(Value, SpanRange)` + * Using `EvaluationError` (without a span!) inside the calculation, and adding the span in the evaluator (nb. it may still need to be able to propogate an `ExecutionInterrupt` internally) + * Using ResolvedValue in place of ExpressionValue e.g. inside arrays +* Introduce `~(...)` and `r~(...)` streams instead of `[!stream! ...]` and `[!raw! ...]` * Introduce interpreter stack frames - * Read the `REVERSION` comment in the `Parsers Revisited` section below to consider approaches, which will work with possibly needing - to revert state if a parser fails. + * Read the `REVERSION` comment in the `Parsers Revisited` section below to consider approaches, which will work with possibly needing to revert state if a parser fails. * Design the data model => is it some kind of linked list of frames? * If we introduce functions in future, then a reference is a closure, in _lexical_ scope, which is different from stack frames. * We probably don't want to allow closures to start with. @@ -145,7 +129,6 @@ Inside a transform stream, the following grammar is supported: * To avoid confusion (such as below) and teach the user to only include #var where necessary, only expression _blocks_ are allowed in an expression. * Confusion example: `let x; x = #(let x = 123; 5)`. This isn't allowed in normal rust because the inside is a `{ .. }` which defines a new scope. * The `#()` syntax can be used in certain places (such as parse streams) -* Introduce `~(...)` and `r~(...)` streams instead of `[!stream! ...]` and `[!raw! ...]` * Support for/while/loop/break/continue inside expressions * And allow them to start with an expression block with `{}` or `#{}` or a stream block with `~{}` QUESTION: Do we require either `#{}` or `~{}`? Maybe! That way we can give a helpful error message. @@ -189,9 +172,17 @@ Inside a transform stream, the following grammar is supported: }] ``` * Support `#(x[..])` syntax for indexing streams, like with arrays - * `#(x[0])` returns the item at that position of the array / OR the value at that position of the stream (using `INFER_TOKEN_TREE`) + * `#(x[0])` returns the value at that position of the stream (using `INFER_TOKEN_TREE`) * `#(x[0..3])` returns a TokenStream * `#(x[0..=3])` returns a TokenStream +* Compare to https://www.reddit.com/r/rust/comments/1j42fgi/media_introducing_eval_macro_a_new_way_to_write i.e. https://crates.io/crates/crabtime - thoughts on crabtime: + => Looks great! + => Why don't they use a cheap hash of the code as a cache key? + - The cachability is a big win compared to preinterpret (although preinterpret is faster on first run) + => I can't imagine the span-chasing / error messages are great, because everything is translated to/from strings between the processes + ... I wonder if there's any way to improve this? Plausibly you could use the proc-macro bridge encoding scheme as per https://blog.jetbrains.com/rust/2022/07/07/procedural-macros-under-the-hood-part-ii/ to send handles onwards, or even delegate directly somehow? + - The spans are better in preinterpret + => Parsing isn't really a thing - they're not going after full macro stuff, probably wise * Add `LiteralPattern` (wrapping a `Literal`) * Add `Eq` support on composite types and streams * Consider: @@ -326,7 +317,7 @@ Inside a transform stream, the following grammar is supported: // * Instead, we suggest people to use a match statement or something // * There is still an issue of REVERSION - "what happens to external state mutated this iteration repetition when a repetition is not possible?" // * This appears in lots of places: -// * In ? or * blocks where an error isn't fatal, but should continue +// * In ? or * blocks where a greedy parse attempt isn't fatal, but should continue // * In match arms, considering various stream parsers, some of which might fail after mutating state // * In nested * blocks, with different levels of reversion // * But it's only a problem with state lexically captured from a parent block @@ -339,6 +330,8 @@ Inside a transform stream, the following grammar is supported: // to parse further until that scope is closed. (this needs to handle nested repetitions). // (D) Error on revert - at reversion time, we check there have been no changes to variables below that depth in the stack tree // (say by recording a "lowest_stack_touched: usize"), and panic if so; and tell people to use `return` instead; or move state changes to the end. +// We could even prevent parsing in a conditional block after the reversion. +// [!Brilliant!] We could introduce a @[REQUIRE ] which takes the parent conditional block out of conditional mode and makes it a hard error instead. This could dramatically improve error messages, and allow parsing after mutation :). (Ideally they'd be some way of applying it to a specific conditional block, but I think parent is good enough) // // ==> D might be easiest, most flexible AND most performant... But it might not cover enough use cases. // ... but I think advising people to only mutate lexical-closure'd state at the end, when a parse is confirmed, is reasonably... @@ -406,38 +399,36 @@ for { the_trait, the_type } in [!parse! { input, parser: @IMPL_ITEM,* }] ~{ } ``` -* Pushed to 0.4: - * Performance: - * Use a small-vec optimization in some places - * Get rid of needless cloning of commands/variables etc - * Support `+=` inside expressions to allow appending of token streams - * Variable reference would need to be a sub-type of stream - * Then `#x += [] + []` could resolve to two variable reference appends and then a return null - * Iterators: - * Iterator value type (with an inbuilt iteration count / limit check) - * Allow `for` lazily reading from iterators - * Support unbounded iterators `[!range! xx..]` - * Fork of syn to: - * Fix issues in Rust Analyzer - * Add support for a more general `TokenBuffer`, and ensure that Cursor can work in a backwards-compatible way with that buffer. Support: - * Storing a length - * Embedding tokens directly without putting them into a `Group` - * Possibling embedding a reference to a slice buffer inside a group - * Ability to parse to a TokenBuffer or TokenBufferSlice - * Possibly allowing some kind of embedding of Tokens whichcan be converted into a TokenStream. - * Currently, `ParseBuffer` stores `unexpected` and has drop glue which is a hacky abstraction. We'll need to think of an alternative. Perhaps we change `ParseBuffer` to operate on top of a `TokenBuffer` ?? - * Allow variables to use CoW semantics. Variables can be Owned(ParseBuffer) or `Slice(ParseBufferSlice), where a ParseBufferSlice is some form of reference counting to a ParseBufferCore, and a FromLocation and ToLocation which are assumed to be at the same level. - * Permit `[!parse_while! (!stream! ...) from #x { ... }]` - * Fix `any_punct()` to ignore none groups - * Groups can either be: - * Raw Groups - * Or created groups, where we store `DelimSpan` for re-parsing and accessing the open/close delimiters (this will let us improve `invalid_content_wrong_group`) - * In future - improve performance of some other parts of syn - * Better error messages - * See e.g. invalid_content_too_short where ideally the error message would be on the last token in the stream. Perhaps End gets a span from the previous error? - * See e.g. invalid_content_too_long where `unexpected token` is quite vague. - Maybe we can't sensibly do better though... - * Further syn parsings (e.g. item, fields, etc) +### Pushed to 0.4: +* Performance: + * Use a small-vec optimization in some places + * Get rid of needless cloning of commands/variables etc + * Avoid needless token stream clones: Have `x += y` take `y` as OwnedOrRef, and either handles it as owned or shared reference (by first cloning) +* Iterators: + * Iterator value type (with an inbuilt iteration count / limit check) + * Allow `for` lazily reading from iterators + * Support unbounded iterators `[!range! xx..]` +* Fork of syn to: + * Fix issues in Rust Analyzer + * Add support for a more general `TokenBuffer`, and ensure that Cursor can work in a backwards-compatible way with that buffer. Support: + * Storing a length + * Embedding tokens directly without putting them into a `Group` + * Possibling embedding a reference to a slice buffer inside a group + * Ability to parse to a TokenBuffer or TokenBufferSlice + * Possibly allowing some kind of embedding of Tokens whichcan be converted into a TokenStream. + * Currently, `ParseBuffer` stores `unexpected` and has drop glue which is a hacky abstraction. We'll need to think of an alternative. Perhaps we change `ParseBuffer` to operate on top of a `TokenBuffer` ?? + * Allow variables to use CoW semantics. Variables can be Owned(ParseBuffer) or `Slice(ParseBufferSlice), where a ParseBufferSlice is some form of reference counting to a ParseBufferCore, and a FromLocation and ToLocation which are assumed to be at the same level. + * Permit `[!parse_while! (!stream! ...) from #x { ... }]` + * Fix `any_punct()` to ignore none groups + * Groups can either be: + * Raw Groups + * Or created groups, where we store `DelimSpan` for re-parsing and accessing the open/close delimiters (this will let us improve `invalid_content_wrong_group`) + * In future - improve performance of some other parts of syn + * Better error messages + * See e.g. invalid_content_too_short where ideally the error message would be on the last token in the stream. Perhaps End gets a span from the previous error? + * See e.g. invalid_content_too_long where `unexpected token` is quite vague. + Maybe we can't sensibly do better though... +* Further syn parsings (e.g. item, fields, etc) # Major Version 0.2 diff --git a/src/expressions/evaluation/evaluator.rs b/src/expressions/evaluation/evaluator.rs index 565ec2ea..d8dc032a 100644 --- a/src/expressions/evaluation/evaluator.rs +++ b/src/expressions/evaluation/evaluator.rs @@ -1,4 +1,4 @@ -#![allow(unused)] // TODO: Remove when places are properly late-bound +#![allow(unused)] // TODO[unused-clearup] use super::*; pub(in super::super) struct ExpressionEvaluator<'a, K: Expressionable> { diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs index 758b6fca..0e930827 100644 --- a/src/expressions/evaluation/node_conversion.rs +++ b/src/expressions/evaluation/node_conversion.rs @@ -10,14 +10,13 @@ impl ExpressionNode { ExpressionNode::Leaf(leaf) => { match leaf { SourceExpressionLeaf::Command(command) => { - // TODO: Allow returning reference + // TODO[interpret_to_value]: Allow command to return a reference context.return_owned_value(command.clone().interpret_to_value(interpreter)?) } SourceExpressionLeaf::Discarded(token) => { return token.execution_err("This cannot be used in a value expression"); } SourceExpressionLeaf::Variable(variable_path) => { - // TODO: Allow block to return reference let variable_ref = variable_path.reference(interpreter)?; match context.requested_ownership() { RequestedValueOwnership::LateBound => { @@ -30,14 +29,12 @@ impl ExpressionNode { context.return_mut_ref(variable_ref.into_mut()?) } RequestedValueOwnership::Owned => { - // TODO: Change this but fix tests which are breaking - // context.return_owned_value(variable_ref.get_value_transparently_cloned()?) - context.return_owned_value(variable_ref.get_value_cloned()?) + context.return_owned_value(variable_ref.get_value_transparently_cloned()?) } } } SourceExpressionLeaf::ExpressionBlock(block) => { - // TODO: Allow block to return reference + // TODO[interpret_to_value]: Allow block to return reference context.return_owned_value(block.interpret_to_value(interpreter)?) } SourceExpressionLeaf::Value(value) => context.return_owned_value(value.clone()), diff --git a/src/expressions/evaluation/place_frames.rs b/src/expressions/evaluation/place_frames.rs index 5f63ebe9..2de9566d 100644 --- a/src/expressions/evaluation/place_frames.rs +++ b/src/expressions/evaluation/place_frames.rs @@ -1,4 +1,4 @@ -#![allow(unused)] // TODO: Remove when places are properly late-bound +#![allow(unused)] // TODO[unused-clearup] use super::*; #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs index 9585df04..748046ea 100644 --- a/src/expressions/evaluation/type_resolution.rs +++ b/src/expressions/evaluation/type_resolution.rs @@ -1,4 +1,6 @@ -#![allow(unused)] // TODO: Remove when places are properly late-bound +#![allow(unused)] use std::mem; + +// TODO[unused-clearup] use super::*; #[macro_use] @@ -9,56 +11,56 @@ mod macros { $($bindings)* }; // By shared reference - ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : &$ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let $arg: &$ty = <$ty as ResolvableArgument>::resolve_from_ref($arg.as_ref())?; + let handle_arg_name!($($arg_part)+): &$ty = <$ty as ResolvableArgument>::resolve_from_ref(handle_arg_name!($($arg_part)+).as_ref())?; ]) }; // By captured shared reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : CapturedRef<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let tmp = $arg.into_shared_reference(); - let $arg: CapturedRef<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_ref(value))?; + let tmp = handle_arg_name!($($arg_part)+).into_shared_reference(); + let handle_arg_name!($($arg_part)+): CapturedRef<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_ref(value))?; ]) }; // SharedValue is an alias for CapturedRef - ([$arg:ident : SharedValue, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : SharedValue, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let $arg = $arg.into_shared_reference(); + let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_shared_reference(); ]) }; // By mutable reference - ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : &mut $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let mut tmp = $arg.into_mutable_reference()?; - let $arg: &mut $ty = <$ty as ResolvableArgument>::resolve_from_mut(tmp.as_mut())?; + let mut tmp = handle_arg_name!($($arg_part)+).into_mutable_reference()?; + let handle_arg_name!($($arg_part)+): &mut $ty = <$ty as ResolvableArgument>::resolve_from_mut(tmp.as_mut())?; ]) }; // By captured mutable reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : CapturedMut<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let mut tmp = $arg.into_mutable_reference()?; - let $arg: CapturedMut<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_mut(value))?; + let mut tmp = handle_arg_name!($($arg_part)+).into_mutable_reference()?; + let handle_arg_name!($($arg_part)+): CapturedMut<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_mut(value))?; ]) }; // MutableValue is an alias for CapturedMut - ([$arg:ident : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let $arg = $arg.into_mutable_reference()?; + let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_mutable_reference()?; ]) }; // By value - ([$arg:ident : $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let tmp = $arg.into_owned_value()?; - let $arg: $ty = <$ty as ResolvableArgument>::resolve_from_owned(tmp)?; + let tmp = handle_arg_name!($($arg_part)+).into_owned_value()?; + let handle_arg_name!($($arg_part)+): $ty = <$ty as ResolvableArgument>::resolve_from_owned(tmp)?; ]) }; } @@ -69,31 +71,31 @@ mod macros { vec![$($outputs)*] }; // By shared reference - ([$arg:ident : &$ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : &$ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) }; // By captured shared reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedRef<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : CapturedRef<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) }; // SharedValue is an alias for CapturedRef - ([$arg:ident : SharedValue, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : SharedValue, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) }; // By mutable reference - ([$arg:ident : &mut $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : &mut $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) }; // By captured mutable reference (i.e. can return a sub-reference from it) - ([$arg:ident : CapturedMut<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : CapturedMut<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) }; // MutableValue is an alias for CapturedMut - ([$arg:ident : MutableValue, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : MutableValue, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) }; // By value - ([$arg:ident : $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Owned,]) }; } @@ -105,18 +107,28 @@ mod macros { // Creating an inner method vastly improves IDE support when writing the method body macro_rules! handle_define_inner_method { - ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?] $body:block $output_ty:ty) => { - fn $method_name($($arg: $ty),*) -> ExecutionResult<$output_ty> { + // The $arg_part+ allows for mut x in the argument list + ($method_name:ident [$($($arg_part:ident)+ : $ty:ty),* $(,)?] $body:block $output_ty:ty) => { + fn $method_name($($($arg_part)+: $ty),*) -> ExecutionResult<$output_ty> { $body } }; } + macro_rules! handle_arg_name { + (mut $name:ident) => { + $name + }; + ($name:ident) => { + $name + }; + } + macro_rules! handle_arg_separation { - ([$($arg:ident : $ty:ty),* $(,)?], $all_arguments:ident, $output_span_range:ident) => { - const LEN: usize = count!($($arg)*); + ([$($($arg_part:ident)+ : $ty:ty),* $(,)?], $all_arguments:ident, $output_span_range:ident) => { + const LEN: usize = count!($($ty)*); let Ok([ - $($arg,)* + $(handle_arg_name!($($arg_part)+),)* ]) = <[ResolvedValue; LEN]>::try_from($all_arguments) else { return $output_span_range.execution_err(format!("Expected {LEN} argument/s")); }; @@ -124,8 +136,8 @@ mod macros { } macro_rules! handle_call_inner_method { - ($method_name:ident [$($arg:ident : $ty:ty),* $(,)?]) => { - $method_name($($arg),*) + ($method_name:ident [$($($arg_part:ident)+ : $ty:ty),* $(,)?]) => { + $method_name($(handle_arg_name!($($arg_part)+)),*) }; } @@ -164,7 +176,7 @@ pub(crate) trait ResolvedTypeDetails { num_arguments: usize, ) -> ExecutionResult; - // TODO: Eventually we can migrate operations under this umbrella too + // TODO[operation-refactor]: Eventually we can migrate operations under this umbrella too // fn resolve_unary_operation(&self, operation: UnaryOperation) -> ExecutionResult; // fn resolve_binary_operation(&self, operation: BinaryOperation) -> ExecutionResult; } @@ -215,6 +227,12 @@ impl ResolvedTypeDetails for ValueKind { Ok(this.clone()) }} } + (_, "take", 0) => { + wrap_method! {(mut this: MutableValue) -> ExecutionResult { + let span_range = this.span_range(); + Ok(mem::replace(this.deref_mut(), ExpressionValue::None(span_range))) + }} + } (_, "as_mut", 0) => { wrap_method! {(this: ExpressionValue) -> ExecutionResult { Ok(MutableValue::new_from_owned(this)) diff --git a/src/expressions/evaluation/value_frames.rs b/src/expressions/evaluation/value_frames.rs index cbec5be4..69255809 100644 --- a/src/expressions/evaluation/value_frames.rs +++ b/src/expressions/evaluation/value_frames.rs @@ -1,4 +1,4 @@ -#![allow(unused)] // TODO: Remove when places are properly late-bound +#![allow(unused)] // TODO[unused-clearup] use super::*; /// # Late Binding @@ -29,7 +29,7 @@ impl ResolvedValue { ResolvedValue::Owned(value) => return Ok(value), ResolvedValue::Shared { .. } | ResolvedValue::Mutable { .. } => self.as_ref(), }; - reference.try_transparent_clone() + reference.try_transparent_clone(self.span_range()) } /// The requirement is just for error messages, and should be "A ", and will be filled like: @@ -359,7 +359,7 @@ impl UnaryOperationBuilder { input: ExpressionNodeId, ) -> NextAction { let frame = Self { operation }; - // TODO: Change to be LateBound to resolve what it actually should be + // TODO[operation-refactor]: Change to be LateBound to resolve what it actually should be context.handle_node_as_value(frame, input, RequestedValueOwnership::Owned) } } @@ -402,7 +402,7 @@ impl BinaryOperationBuilder { operation, state: BinaryPath::OnLeftBranch { right }, }; - // TODO: Change to be LateBound to resolve what it actually should be + // TODO[operation-refactor]: Change to be LateBound to resolve what it actually should be context.handle_node_as_value(frame, left, RequestedValueOwnership::Owned) } } @@ -429,7 +429,7 @@ impl EvaluationFrame for BinaryOperationBuilder { context.handle_node_as_value( self, right, - // TODO: Change to late bound as the operation may dictate what ownership it needs for its right operand + // TODO[operation-refactor]: Change to late bound as the operation may dictate what ownership it needs for its right operand RequestedValueOwnership::Owned, ) } @@ -453,7 +453,7 @@ impl ValuePropertyAccessBuilder { node: ExpressionNodeId, ) -> NextAction { let frame = Self { access }; - // TODO: Change to propagate context from value, and not always clone! + // TODO[access-refactor]: Change to propagate context from value, and not always clone! let ownership = RequestedValueOwnership::Owned; context.handle_node_as_value(frame, node, ownership) } @@ -497,7 +497,7 @@ impl ValueIndexAccessBuilder { access, state: IndexPath::OnSourceBranch { index }, }; - // TODO: Change to propagate context from value, and not always clone + // TODO[access-refactor]: Change to propagate context from value, and not always clone context.handle_node_as_value(frame, source, RequestedValueOwnership::Owned) } } @@ -594,7 +594,7 @@ impl EvaluationFrame for RangeBuilder { context: ValueContext, item: EvaluationItem, ) -> ExecutionResult { - // TODO: Change to not always clone the value + // TODO[range-refactor]: Change to not always clone the value let value = item.expect_owned_value(); Ok(match (self.state, self.range_limits) { (RangePath::OnLeftBranch { right: Some(right) }, _) => { @@ -738,7 +738,7 @@ impl EvaluationFrame for CompoundAssignmentBuilder { CompoundAssignmentPath::OnValueBranch { place } => { let value = item.expect_owned_value(); self.state = CompoundAssignmentPath::OnPlaceBranch { value }; - // TODO: Resolve as LateBound, and then convert to what is needed based on the operation + // TODO[assignment-refactor]: Resolve as LateBound, and then convert to what is needed based on the operation context.handle_node_as_place(self, place, RequestedPlaceOwnership::MutableReference) } CompoundAssignmentPath::OnPlaceBranch { value } => { diff --git a/src/expressions/object.rs b/src/expressions/object.rs index aec81822..c53c593d 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -201,8 +201,6 @@ impl ExpressionObject { Ok(()) } - // TODO: Make ObjectValidation a trait, and have it implemented by the static - // define_field_inputs! macro pub(crate) fn validate(&self, validation: &impl ObjectValidate) -> ExecutionResult<()> { let mut missing_fields = Vec::new(); for (field_name, _) in validation.required_fields() { @@ -276,7 +274,7 @@ impl ToExpressionValue for BTreeMap { } } -#[allow(unused)] +#[allow(unused)] // TODO[unused-clearup] pub(crate) struct ObjectValidation { // Should ideally be an indexmap fields: Vec<(String, FieldDefinition)>, diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 6c4ab267..34ab3042 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -62,14 +62,14 @@ impl ExpressionValue { } } - pub(crate) fn try_transparent_clone(&self) -> ExecutionResult { + pub(crate) fn try_transparent_clone(&self, new_span_range: SpanRange) -> ExecutionResult { if !self.kind().supports_transparent_cloning() { - return self.execution_err(format!( - "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .clone() explicitly.", + return new_span_range.execution_err(format!( + "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .take() or .clone() explicitly.", self.articled_value_type() )); } - Ok(self.clone()) + Ok(self.clone().with_span_range(new_span_range)) } pub(super) fn expect_value_pair( diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 1b37bc2e..81abf746 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -1,4 +1,3 @@ -#![allow(unused)] // TODO: Remove when places are properly late-bound use crate::internal_prelude::*; use super::IsVariable; @@ -156,10 +155,9 @@ impl VariableReference { // Gets the cloned expression value, setting the span range appropriately pub(crate) fn get_value_transparently_cloned(&self) -> ExecutionResult { - Ok(self + self .get_value_ref()? - .try_transparent_clone()? - .with_span_range(self.variable_span_range)) + .try_transparent_clone(self.variable_span_range) } // Gets the cloned expression value, setting the span range appropriately @@ -247,6 +245,7 @@ impl MutableSubPlace { } } + #[allow(unused)] pub(crate) fn map( self, value_map: impl for<'a> FnOnce(&'a mut T) -> &'a mut V, @@ -393,6 +392,7 @@ impl SharedSubPlace { }) } + #[allow(unused)] pub(crate) fn map( self, value_map: impl FnOnce(&T) -> &V, diff --git a/src/misc/mut_rc_ref_cell.rs b/src/misc/mut_rc_ref_cell.rs index d32af9a0..b602b96f 100644 --- a/src/misc/mut_rc_ref_cell.rs +++ b/src/misc/mut_rc_ref_cell.rs @@ -1,4 +1,3 @@ -#![allow(unused)] // TODO: Remove when places are properly late-bound use crate::internal_prelude::*; use std::cell::*; use std::rc::Rc; diff --git a/tests/expressions.rs b/tests/expressions.rs index 5a85e8cb..244f5e0c 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -496,4 +496,13 @@ fn test_method_calls() { #([].as_mut().len().debug()), "0usize" ); + preinterpret_assert_eq!( + #( + let x = [1, 2, 3]; + let y = x.take(); + // x is now None + x.debug() + " - " + y.debug() + ), + "None - [1, 2, 3]" + ); } From 758a6e8d5b0b7849e18cb08810e34b47b08770be Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 19 Sep 2025 15:13:21 +0100 Subject: [PATCH 124/126] feature: Added `debug()` method which throws an error --- CHANGELOG.md | 18 +- README.md | 2 +- src/expressions/array.rs | 3 - src/expressions/boolean.rs | 1 - src/expressions/character.rs | 1 - src/expressions/evaluation/node_conversion.rs | 5 +- src/expressions/evaluation/type_resolution.rs | 13 +- src/expressions/float.rs | 2 - src/expressions/integer.rs | 4 - src/expressions/iterator.rs | 3 - src/expressions/object.rs | 15 +- src/expressions/operations.rs | 3 - src/expressions/stream.rs | 3 - src/expressions/string.rs | 1 - src/expressions/value.rs | 32 ++-- src/interpretation/command.rs | 1 - src/interpretation/commands/core_commands.rs | 35 ---- src/interpretation/interpreter.rs | 3 +- .../expressions/debug_method.rs | 7 + .../expressions/debug_method.stderr | 5 + tests/core.rs | 20 ++- tests/expressions.rs | 89 +++++----- tests/tokens.rs | 158 +++++++++--------- tests/transforming.rs | 47 +++--- 24 files changed, 218 insertions(+), 253 deletions(-) create mode 100644 tests/compilation_failures/expressions/debug_method.rs create mode 100644 tests/compilation_failures/expressions/debug_method.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index e093e784..68113b1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ * `[!error! ...]` to output a compile error. * `[!set! #x += ...]` to performantly add extra characters to a variable's stream. * `[!set! _ = ...]` interprets its arguments but then ignores any outputs. - * `[!debug! ...]` to output its interpreted contents including none-delimited groups. Useful for debugging the content of variables. * `[!stream! ...]` can be used to just output its interpreted contents. It's useful to create a stream value inside an expression. * `[!reinterpret! ...]` is like an `eval` command in scripting languages. It takes a stream, and parses/interprets it. * `[!settings! { ... }]` can be used to adjust the iteration limit. @@ -65,6 +64,16 @@ The following operators are supported: * Casting with `as` including to untyped integers/floats with `as int` and `as float`, to a grouped stream with `as group` and to a flattened stream with `as stream`. * () and none-delimited groups for precedence +The following methods are supported: +* On all values: + * `.clone()` - converts a reference to a mutable value. You will be told in an error if this is needed. + * `.as_mut()` - converts an owned value to a mutable value. You will be told in an error if this is needed. + * `.take()` - takes the value from a mutable reference, and replaces it with `None`. Useful instead of cloning. + * `.debug()` - a debugging aid whilst writing code. Causes a compile error with the content of the value. Equivalent to `[!error! #(x.debug_string())]` + * `.debug_string()` - returns the value's contents as a string for debugging purposes +* On arrays: `len()` and `push()` +* On streams: `len()` + An expression also supports embedding commands `[!xxx! ...]`, other expression blocks, variables and flattened variables. The value type outputted by a command depends on the command. ### Transforming @@ -97,12 +106,13 @@ Inside a transform stream, the following grammar is supported: ### To come * Method calls continued - * Replace `as debug` and `[!debug! ..]` with `.debug()` + * Add `debug_error()` * Add support for `RequestedValueOwnership::SharedOrOwned` (CoW behaviour) - and equivalent `SharedOrOwned`. This can be used for utilities like `debug()` + and equivalent `SharedOrOwned`. This can be used to create a `OwnedOrCloned` parameter for utilities like `debug()` which is more performant * Scrap `#>>x` etc in favour of `#(x.push(@TOKEN))` * TODO[access-refactor] * TODO[range-refactor] & some kind of more thought through typed reference support - e.g. slices, mutable arrays? + * Re-enable TODO[auto-cloning] * TODO[operation-refactor] * No clone required for testing equality of streams, objects and arrays * Add better way of defining methods once / lazily, and binding them to @@ -148,7 +158,7 @@ Inside a transform stream, the following grammar is supported: * Implement the following named parsers * `@TOKEN_TREE` * `@TOKEN_OR_GROUP_CONTENT` - Literal, Ident, Punct or None-group content. - * `@INFER_TOKEN_TREE` - Infers values, falls back to Stream + * `@INFER_TOKEN_TREE` - Infers parsing as a value, falls back to Stream * `@[ANY_GROUP ...]` * `@REST` * `@[FORK @{ ...parser... }]` the parser block creates a `commit=false` variable, if this is set to `commit=true` then it commits the fork. diff --git a/README.md b/README.md index 6f2172e0..9a77cf9f 100644 --- a/README.md +++ b/README.md @@ -441,7 +441,7 @@ Other boolean commands could be possible, similar to numeric commands: * `[!tokens_eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example: * `[!tokens_eq! (3 4) (3 4)]` outputs `true` because the token stream ignores spacing. * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals. - * This can be effectively done already with `#([!debug! #x] == [!debug! #y])` + * This can be effectively done already with `#(x.debug_string() == y.debug_string())` * `[!str_split! { input: Value, separator: Value, }]` * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string. diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 0e03eaf0..22f5eddc 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -37,9 +37,6 @@ impl ExpressionArray { self.concat_recursive_into(&mut output, &ConcatBehaviour::standard())?; output }), - CastTarget::DebugString => { - operation.output(self.items).into_debug_string_value()? - } CastTarget::Boolean | CastTarget::Char | CastTarget::Integer(_) diff --git a/src/expressions/boolean.rs b/src/expressions/boolean.rs index ded7da28..362fe584 100644 --- a/src/expressions/boolean.rs +++ b/src/expressions/boolean.rs @@ -46,7 +46,6 @@ impl ExpressionBoolean { } CastTarget::Boolean => operation.output(input), CastTarget::String => operation.output(input.to_string()), - CastTarget::DebugString => operation.output(input.to_string()), CastTarget::Stream => { operation.output(operation.output(input).into_new_output_stream( Grouping::Flattened, diff --git a/src/expressions/character.rs b/src/expressions/character.rs index 6d8a6d0c..a3720d28 100644 --- a/src/expressions/character.rs +++ b/src/expressions/character.rs @@ -45,7 +45,6 @@ impl ExpressionChar { CastTarget::Char => operation.output(char), CastTarget::Boolean | CastTarget::Float(_) => return operation.unsupported(self), CastTarget::String => operation.output(char.to_string()), - CastTarget::DebugString => operation.output(format!("{:?}", char)), CastTarget::Stream => { operation.output(operation.output(char).into_new_output_stream( Grouping::Flattened, diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs index 0e930827..6bc8b38e 100644 --- a/src/expressions/evaluation/node_conversion.rs +++ b/src/expressions/evaluation/node_conversion.rs @@ -28,9 +28,8 @@ impl ExpressionNode { RequestedValueOwnership::MutableReference => { context.return_mut_ref(variable_ref.into_mut()?) } - RequestedValueOwnership::Owned => { - context.return_owned_value(variable_ref.get_value_transparently_cloned()?) - } + RequestedValueOwnership::Owned => context + .return_owned_value(variable_ref.get_value_transparently_cloned()?), } } SourceExpressionLeaf::ExpressionBlock(block) => { diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs index 748046ea..c9a92153 100644 --- a/src/expressions/evaluation/type_resolution.rs +++ b/src/expressions/evaluation/type_resolution.rs @@ -1,4 +1,5 @@ -#![allow(unused)] use std::mem; +#![allow(unused)] +use std::mem; // TODO[unused-clearup] use super::*; @@ -238,8 +239,14 @@ impl ResolvedTypeDetails for ValueKind { Ok(MutableValue::new_from_owned(this)) }} } - (_, "debug", 0) => wrap_method! {(this: &ExpressionValue) -> ExecutionResult { - this.clone().into_debug_string() + (_, "debug_string", 0) => { + wrap_method! {(this: &ExpressionValue) -> ExecutionResult { + this.clone().into_debug_string() + }} + } + (_, "debug", 0) => wrap_method! {(this: SharedValue) -> ExecutionResult { + let message = this.clone().into_debug_string()?; + this.execution_err(message) }}, (ValueKind::Array, "len", 0) => { wrap_method! {(this: &ExpressionArray) -> ExecutionResult { diff --git a/src/expressions/float.rs b/src/expressions/float.rs index 81a5ebc8..3c185aa4 100644 --- a/src/expressions/float.rs +++ b/src/expressions/float.rs @@ -175,7 +175,6 @@ impl UntypedFloat { CastTarget::Float(FloatKind::F32) => operation.output(input as f32), CastTarget::Float(FloatKind::F64) => operation.output(input), CastTarget::String => operation.output(input.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Boolean | CastTarget::Char => { return operation.execution_err("This cast is not supported") } @@ -335,7 +334,6 @@ macro_rules! impl_float_operations { CastTarget::Float(FloatKind::F32) => operation.output(self as f32), CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, StreamOutputBehaviour::Standard)?), CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, StreamOutputBehaviour::Standard)?), diff --git a/src/expressions/integer.rs b/src/expressions/integer.rs index f81b9a95..03df69a7 100644 --- a/src/expressions/integer.rs +++ b/src/expressions/integer.rs @@ -304,7 +304,6 @@ impl UntypedInteger { return operation.execution_err("This cast is not supported") } CastTarget::String => operation.output(input.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Stream => { operation.output(operation.output(self).into_new_output_stream( Grouping::Flattened, @@ -592,7 +591,6 @@ macro_rules! impl_unsigned_unary_operations { CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, StreamOutputBehaviour::Standard)?), CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, StreamOutputBehaviour::Standard)?), } @@ -630,7 +628,6 @@ macro_rules! impl_signed_unary_operations { CastTarget::Float(FloatKind::F64) => operation.output(self as f64), CastTarget::Boolean | CastTarget::Char => return operation.execution_err("This cast is not supported"), CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Stream => operation.output(operation.output(self).into_new_output_stream(Grouping::Flattened, StreamOutputBehaviour::Standard)?), CastTarget::Group => operation.output(operation.output(self).into_new_output_stream(Grouping::Grouped, StreamOutputBehaviour::Standard)?), } @@ -675,7 +672,6 @@ impl HandleUnaryOperation for u8 { return operation.execution_err("This cast is not supported") } CastTarget::String => operation.output(self.to_string()), - CastTarget::DebugString => operation.output(self).into_debug_string_value()?, CastTarget::Stream => { operation.output(operation.output(self).into_new_output_stream( Grouping::Flattened, diff --git a/src/expressions/iterator.rs b/src/expressions/iterator.rs index c0d59a2e..0dec18db 100644 --- a/src/expressions/iterator.rs +++ b/src/expressions/iterator.rs @@ -67,9 +67,6 @@ impl ExpressionIterator { self.concat_recursive_into(&mut output, &ConcatBehaviour::standard())?; output }), - CastTarget::DebugString => { - operation.output(self.iterator).into_debug_string_value()? - } CastTarget::Boolean | CastTarget::Char | CastTarget::Integer(_) diff --git a/src/expressions/object.rs b/src/expressions/object.rs index c53c593d..59f1ac1d 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -21,25 +21,18 @@ impl ExpressionObject { self, operation: OutputSpanned, ) -> ExecutionResult { - Ok(match operation.operation { - UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => { - return operation.unsupported(self) - } + match operation.operation { + UnaryOperation::Neg { .. } | UnaryOperation::Not { .. } => operation.unsupported(self), UnaryOperation::Cast { target, .. } => match target { - CastTarget::DebugString => { - operation.output(self.entries).into_debug_string_value()? - } CastTarget::String | CastTarget::Stream | CastTarget::Group | CastTarget::Boolean | CastTarget::Char | CastTarget::Integer(_) - | CastTarget::Float(_) => { - return operation.unsupported(self); - } + | CastTarget::Float(_) => operation.unsupported(self), }, - }) + } } pub(super) fn handle_integer_binary_operation( diff --git a/src/expressions/operations.rs b/src/expressions/operations.rs index d584dc25..947d0ba9 100644 --- a/src/expressions/operations.rs +++ b/src/expressions/operations.rs @@ -136,7 +136,6 @@ pub(super) enum CastTarget { Float(FloatKind), Boolean, String, - DebugString, Char, Stream, Group, @@ -168,7 +167,6 @@ impl FromStr for CastTarget { "stream" => CastTarget::Stream, "group" => CastTarget::Group, "string" => CastTarget::String, - "debug" => CastTarget::DebugString, _ => return Err(()), }) } @@ -195,7 +193,6 @@ impl CastTarget { CastTarget::Float(FloatKind::F64) => "as f64", CastTarget::Boolean => "as bool", CastTarget::String => "as string", - CastTarget::DebugString => "as debug", CastTarget::Char => "as char", CastTarget::Stream => "as stream", CastTarget::Group => "as group", diff --git a/src/expressions/stream.rs b/src/expressions/stream.rs index 40bbeeb2..1fa44a85 100644 --- a/src/expressions/stream.rs +++ b/src/expressions/stream.rs @@ -24,9 +24,6 @@ impl ExpressionStream { self.concat_recursive_into(&mut output, &ConcatBehaviour::standard()); output }), - CastTarget::DebugString => { - operation.output(self.value).into_debug_string_value()? - } CastTarget::Stream => operation.output(self.value), CastTarget::Group => { operation.output(operation.output(self.value).into_new_output_stream( diff --git a/src/expressions/string.rs b/src/expressions/string.rs index 3ac122c4..fd56f748 100644 --- a/src/expressions/string.rs +++ b/src/expressions/string.rs @@ -39,7 +39,6 @@ impl ExpressionString { )?) } CastTarget::String => operation.output(self.value), - CastTarget::DebugString => operation.output(format!("{:?}", self.value)), CastTarget::Boolean | CastTarget::Char | CastTarget::Integer(_) diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 34ab3042..3df33da8 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -62,13 +62,17 @@ impl ExpressionValue { } } - pub(crate) fn try_transparent_clone(&self, new_span_range: SpanRange) -> ExecutionResult { - if !self.kind().supports_transparent_cloning() { - return new_span_range.execution_err(format!( - "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .take() or .clone() explicitly.", - self.articled_value_type() - )); - } + pub(crate) fn try_transparent_clone( + &self, + new_span_range: SpanRange, + ) -> ExecutionResult { + // TODO[auto-cloning]: + // if !self.kind().supports_transparent_cloning() { + // return new_span_range.execution_err(format!( + // "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .take() or .clone() explicitly.", + // self.articled_value_type() + // )); + // } Ok(self.clone().with_span_range(new_span_range)) } @@ -408,13 +412,7 @@ impl ExpressionValue { operation: OutputSpanned, ) -> ExecutionResult { match self { - ExpressionValue::None(_) => match operation.operation { - UnaryOperation::Cast { - target: CastTarget::DebugString, - .. - } => ExpressionValue::None(operation.output_span_range).into_debug_string_value(), - _ => operation.unsupported(self), - }, + ExpressionValue::None(_) => operation.unsupported(self), ExpressionValue::Integer(value) => value.handle_unary_operation(operation), ExpressionValue::Float(value) => value.handle_unary_operation(operation), ExpressionValue::Boolean(value) => value.handle_unary_operation(operation), @@ -669,12 +667,6 @@ impl ExpressionValue { self.concat_recursive(&ConcatBehaviour::debug()) } - pub(crate) fn into_debug_string_value(self) -> ExecutionResult { - let span_range = self.span_range(); - let value = self.into_debug_string()?.to_value(span_range); - Ok(value) - } - pub(crate) fn concat_recursive(self, behaviour: &ConcatBehaviour) -> ExecutionResult { let mut output = String::new(); self.concat_recursive_into(&mut output, behaviour)?; diff --git a/src/interpretation/command.rs b/src/interpretation/command.rs index 128e3a75..685b8257 100644 --- a/src/interpretation/command.rs +++ b/src/interpretation/command.rs @@ -348,7 +348,6 @@ define_command_enums! { ReinterpretCommand, SettingsCommand, ErrorCommand, - DebugCommand, // Concat & Type Convert Commands StringCommand, diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index 4bcdb61f..acad2415 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -374,38 +374,3 @@ impl NoOutputCommandDefinition for ErrorCommand { error_span.execution_err(error_message) } } - -#[derive(Clone)] -pub(crate) struct DebugCommand { - span: Span, - inner: SourceExpression, -} - -impl CommandType for DebugCommand { - type OutputKind = OutputKindValue; -} - -impl ValueCommandDefinition for DebugCommand { - const COMMAND_NAME: &'static str = "debug"; - - fn parse(arguments: CommandArguments) -> ParseResult { - arguments.fully_parse_or_error( - |input| { - Ok(Self { - span: arguments.command_span(), - inner: input.parse()?, - }) - }, - "Expected [!debug! ]. To provide a stream, use [!debug! [!stream! ...]]", - ) - } - - fn execute(self, interpreter: &mut Interpreter) -> ExecutionResult { - let value = self - .inner - .interpret_to_value(interpreter)? - .with_span(self.span) - .into_debug_string_value()?; - Ok(value) - } -} diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 81abf746..8afc41a6 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -155,8 +155,7 @@ impl VariableReference { // Gets the cloned expression value, setting the span range appropriately pub(crate) fn get_value_transparently_cloned(&self) -> ExecutionResult { - self - .get_value_ref()? + self.get_value_ref()? .try_transparent_clone(self.variable_span_range) } diff --git a/tests/compilation_failures/expressions/debug_method.rs b/tests/compilation_failures/expressions/debug_method.rs new file mode 100644 index 00000000..4d6162cf --- /dev/null +++ b/tests/compilation_failures/expressions/debug_method.rs @@ -0,0 +1,7 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #(let x = [1, 2]; x.debug()) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/debug_method.stderr b/tests/compilation_failures/expressions/debug_method.stderr new file mode 100644 index 00000000..fc932ac4 --- /dev/null +++ b/tests/compilation_failures/expressions/debug_method.stderr @@ -0,0 +1,5 @@ +error: [1, 2] + --> tests/compilation_failures/expressions/debug_method.rs:5:27 + | +5 | #(let x = [1, 2]; x.debug()) + | ^ diff --git a/tests/core.rs b/tests/core.rs index 467804de..bd08d8bd 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -109,11 +109,13 @@ fn test_debug() { // It keeps the semantic punctuation spacing intact // (e.g. it keeps 'a and >> together) preinterpret_assert_eq!( - [!debug! [!stream! impl<'a, T> MyStruct<'a, T> { - pub fn new() -> Self { - !($crate::Test::CONSTANT >> 5 > 1) - } - }]], + #( + [!stream! impl<'a, T> MyStruct<'a, T> { + pub fn new() -> Self { + !($crate::Test::CONSTANT >> 5 > 1) + } + }].debug_string() + ), "[!stream! impl < 'a , T > MyStruct < 'a , T > { pub fn new () -> Self { ! ($ crate :: Test :: CONSTANT >> 5 > 1) } }]" ); // It shows transparent groups @@ -121,10 +123,10 @@ fn test_debug() { // because it doesn't stick [!raw! ...] around things which could be confused // for the preinterpret grammar. Perhaps it could/should in future. preinterpret_assert_eq!( - { - [!set! #x = Hello (World)] - [!debug! [!stream! #x [!raw! #test] "and" [!raw! ##] #..x]] - }, + #( + let x = [!stream! Hello (World)]; + [!stream! #x [!raw! #test] "and" [!raw! ##] #..x].debug_string() + ), r###"[!stream! [!group! Hello (World)] # test "and" ## Hello (World)]"### ); } diff --git a/tests/expressions.rs b/tests/expressions.rs index 244f5e0c..db071002 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -44,7 +44,7 @@ fn test_basic_evaluate_works() { preinterpret_assert_eq!(#(let six_as_sum = 3 + 3; six_as_sum * six_as_sum), 36); preinterpret_assert_eq!(#( let partial_sum = [!stream! + 2]; - [!debug! [!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]] + ([!stream! #([!stream! 5] + partial_sum) =] + [!reinterpret! [!raw! #](5 #..partial_sum)]).debug_string() ), "[!stream! [!group! 5 + 2] = [!group! 7]]"); preinterpret_assert_eq!(#(1 + (1..2) as int), 2); preinterpret_assert_eq!(#("hello" == "world"), false); @@ -56,19 +56,19 @@ fn test_basic_evaluate_works() { preinterpret_assert_eq!(#('A' < 'B'), true); preinterpret_assert_eq!(#("Zoo" > "Aardvark"), true); preinterpret_assert_eq!( - #([!debug! "Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group]), + #(("Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group).debug_string()), r#"[!stream! "Hello" "World" 2 [!group! 2]]"# ); preinterpret_assert_eq!( - [!debug! #([1, 2, 1 + 2, 4])], + #([1, 2, 1 + 2, 4].debug_string()), "[1, 2, 3, 4]" ); preinterpret_assert_eq!( - [!debug! #([1 + (3 + 4), [5,] + [], [[6, 7],]] + [123])], + #(([1 + (3 + 4), [5,] + [], [[6, 7],]] + [123]).debug_string()), "[8, [5], [[6, 7]], 123]" ); preinterpret_assert_eq!( - #([!debug! "Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group]), + #(("Hello" as stream + "World" as stream + (1 + 1) as stream + (1 + 1) as group).debug_string()), r#"[!stream! "Hello" "World" 2 [!group! 2]]"# ); } @@ -139,7 +139,7 @@ fn assign_works() { preinterpret_assert_eq!( #( let x = 5 + 5; - [!debug! x] + x.debug_string() ), "10" ); @@ -201,18 +201,18 @@ fn test_range() { ); preinterpret_assert_eq!({ [!string! #(('a'..='f') as stream)] }, "abcdef"); preinterpret_assert_eq!( - { [!debug! -1i8..3i8] }, + #((-1i8..3i8).debug_string()), "-1i8..3i8" ); // Large ranges are allowed, but are subject to limits at iteration time - preinterpret_assert_eq!({ [!debug! 0..10000] }, "0..10000"); - preinterpret_assert_eq!({ [!debug! ..5 + 5] }, "..10"); - preinterpret_assert_eq!({ [!debug! ..=9] }, "..=9"); - preinterpret_assert_eq!({ [!debug! ..] }, ".."); - preinterpret_assert_eq!({ [!debug![.., ..]] }, "[.., ..]"); - preinterpret_assert_eq!({ [!debug![..[1, 2..], ..]] }, "[..[1, 2..], ..]"); - preinterpret_assert_eq!({ [!debug! 4 + 7..=10] }, "11..=10"); + preinterpret_assert_eq!(#((0..10000).debug_string()), "0..10000"); + preinterpret_assert_eq!(#((..5 + 5).debug_string()), "..10"); + preinterpret_assert_eq!(#((..=9).debug_string()), "..=9"); + preinterpret_assert_eq!(#((..).debug_string()), ".."); + preinterpret_assert_eq!(#([.., ..].debug_string()), "[.., ..]"); + preinterpret_assert_eq!(#([..[1, 2..], ..].debug_string()), "[..[1, 2..], ..]"); + preinterpret_assert_eq!(#((4 + 7..=10).debug_string()), "11..=10"); preinterpret_assert_eq!( [!for! i in 0..10000000 { [!if! i == 5 { @@ -222,7 +222,7 @@ fn test_range() { }], "5" ); - // preinterpret_assert_eq!({ [!debug! 0..10000 as iterator] }, "[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ..<9980 further items>]"); + // preinterpret_assert_eq!(#((0..10000 as iterator).debug_string()), "[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ..<9980 further items>]"); } #[test] @@ -239,7 +239,7 @@ fn test_array_indexing() { let x = [0, 0, 0]; x[0] = 2; (x[1 + 1]) = x[0]; - x as debug + x.debug_string() ), "[2, 0, 2]" ); @@ -247,42 +247,42 @@ fn test_array_indexing() { preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[..] as debug + x[..].debug_string() ), "[1, 2, 3, 4, 5]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[0..0] as debug + x[0..0].debug_string() ), "[]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[2..=2] as debug + x[2..=2].debug_string() ), "[3]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[..=2] as debug + x[..=2].debug_string() ), "[1, 2, 3]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[..4] as debug + x[..4].debug_string() ), "[1, 2, 3, 4]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[2..] as debug + x[2..].debug_string() ), "[3, 4, 5]" ); @@ -296,7 +296,7 @@ fn test_array_place_destructurings() { let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; [a, b, _, _, c] = x; - [a, b, c] as debug + [a, b, c].debug_string() ), "[1, 2, 5]" ); @@ -305,7 +305,7 @@ fn test_array_place_destructurings() { let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; [a, b, c, ..] = x; - [a, b, c] as debug + [a, b, c].debug_string() ), "[1, 2, 3]" ); @@ -314,7 +314,7 @@ fn test_array_place_destructurings() { let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; [.., a, b] = x; - [a, b, c] as debug + [a, b, c].debug_string() ), "[4, 5, 0]" ); @@ -323,7 +323,7 @@ fn test_array_place_destructurings() { let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; [a, .., b, c] = x; - [a, b, c] as debug + [a, b, c].debug_string() ), "[1, 4, 5]" ); @@ -333,7 +333,7 @@ fn test_array_place_destructurings() { let out = [[0, 0], 0]; let a = 0; let b = 0; let c = 0; [out[1], .., out[0][0], out[0][1]] = [1, 2, 3, 4, 5]; - out as debug + out.debug_string() ), "[[4, 5], 1]" ); @@ -343,10 +343,11 @@ fn test_array_place_destructurings() { let a = [0, 0, 0, 0, 0]; let b = 3; let c = 0; + // Demonstrates right-associativity of = let _ = c = [a[2], _] = [4, 5]; let _ = a[1] += 2; let _ = b = 2; - [a, b, c] as debug + [a, b, c].debug_string() ), "[[0, 2, 4, 0, 0], 2, None]" ); @@ -357,7 +358,7 @@ fn test_array_place_destructurings() { let a = [0, 0]; let b = 0; a[b] += #(b += 1; 5); - a as debug + a.debug_string() ), "[0, 5]" ); @@ -382,35 +383,35 @@ fn test_array_pattern_destructurings() { preinterpret_assert_eq!( #( let [a, b, _, _, c] = [1, 2, 3, 4, 5]; - [a, b, c] as debug + [a, b, c].debug_string() ), "[1, 2, 5]" ); preinterpret_assert_eq!( #( let [a, b, c, ..] = [1, 2, 3, 4, 5]; - [a, b, c] as debug + [a, b, c].debug_string() ), "[1, 2, 3]" ); preinterpret_assert_eq!( #( let [.., a, b] = [1, 2, 3, 4, 5]; - [a, b] as debug + [a, b].debug_string() ), "[4, 5]" ); preinterpret_assert_eq!( #( let [a, .., b, c] = [1, 2, 3, 4, 5]; - [a, b, c] as debug + [a, b, c].debug_string() ), "[1, 4, 5]" ); preinterpret_assert_eq!( #( let [a, .., b, c] = [[1, "a"], 2, 3, 4, 5]; - [a, b, c] as debug + [a, b, c].debug_string() ), r#"[[1, "a"], 4, 5]"# ); @@ -426,25 +427,25 @@ fn test_objects() { x["x y z"] = 4; x["z\" test"] = {}; x.y = 5; - x as debug + x.debug_string() ), r#"{ a: {}, b: "Hello", hello: 1, world: 2, ["x y z"]: 4, y: 5, ["z\" test"]: {} }"# ); preinterpret_assert_eq!( #( - { prop1: 1 }["prop1"] as debug + { prop1: 1 }["prop1"].debug_string() ), r#"1"# ); preinterpret_assert_eq!( #( - { prop1: 1 }["prop2"] as debug + { prop1: 1 }["prop2"].debug_string() ), r#"None"# ); preinterpret_assert_eq!( #( - { prop1: 1 }.prop1 as debug + { prop1: 1 }.prop1.debug_string() ), r#"1"# ); @@ -454,14 +455,14 @@ fn test_objects() { let b; let z; { a, y: [_, b], z } = { a: 1, y: [5, 7] }; - { a, b, z } as debug + { a, b, z }.debug_string() ), r#"{ a: 1, b: 7, z: None }"# ); preinterpret_assert_eq!( #( let { a, y: [_, b], ["c"]: c, [r#"two "words"#]: x, z } = { a: 1, y: [5, 7], ["two \"words"]: {}, }; - { a, b, c, x, z } as debug + { a, b, c, x, z }.debug_string() ), r#"{ a: 1, b: 7, c: None, x: {}, z: None }"# ); @@ -482,18 +483,18 @@ fn test_method_calls() { let x = [1, 2, 3]; x.push(5); x.push(2); - x.debug() + x.debug_string() ), "[1, 2, 3, 5, 2]" ); // Push returns None preinterpret_assert_eq!( - #([1, 2, 3].as_mut().push(4).debug()), + #([1, 2, 3].as_mut().push(4).debug_string()), "None" ); // Converting to mut and then to shared works preinterpret_assert_eq!( - #([].as_mut().len().debug()), + #([].as_mut().len().debug_string()), "0usize" ); preinterpret_assert_eq!( @@ -501,7 +502,7 @@ fn test_method_calls() { let x = [1, 2, 3]; let y = x.take(); // x is now None - x.debug() + " - " + y.debug() + x.debug_string() + " - " + y.debug_string() ), "None - [1, 2, 3]" ); diff --git a/tests/tokens.rs b/tests/tokens.rs index 82e152ae..46e95faa 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -235,77 +235,83 @@ fn test_split() { // Empty separators are allowed, and split on every token // In this case, drop_empty_start / drop_empty_end are ignored preinterpret_assert_eq!( - { - [!debug![!split! { + #( + [!split! { stream: [!stream! A::B], separator: [!stream!], - }]] - }, + }].debug_string() + ), "[[!stream! A], [!stream! :], [!stream! :], [!stream! B]]" ); // Double separators are allowed preinterpret_assert_eq!( - { - [!debug![!split! { + #( + [!split! { stream: [!stream! A::B::C], separator: [!stream! ::], - }]] - }, + }].debug_string() + ), "[[!stream! A], [!stream! B], [!stream! C]]" ); // Trailing separator is ignored by default preinterpret_assert_eq!( - { - [!debug![!split! { + #( + [!split! { stream: [!stream! Pizza, Mac and Cheese, Hamburger,], separator: [!stream! ,], - }]] - }, + }].debug_string() + ), "[[!stream! Pizza], [!stream! Mac and Cheese], [!stream! Hamburger]]" ); // By default, empty groups are included except at the end preinterpret_assert_eq!( - { - [!debug![!split! { + #( + ([!split! { stream: [!stream! ::A::B::::C::], separator: [!stream! ::], - }] as stream] - }, + }] as stream).debug_string() + ), "[!stream! [!group!] [!group! A] [!group! B] [!group!] [!group! C]]" ); // Stream and separator are both interpreted - preinterpret_assert_eq!({ - [!set! #x = ;] - [!debug! [!split! { - stream: [!stream! ;A;;B;C;D #..x E;], - separator: x, - drop_empty_start: true, - drop_empty_middle: true, - drop_empty_end: true, - }] as stream] - }, "[!stream! [!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); + preinterpret_assert_eq!( + #( + let x = [!stream! ;]; + ([!split! { + stream: [!stream! ;A;;B;C;D #..x E;], + separator: x, + drop_empty_start: true, + drop_empty_middle: true, + drop_empty_end: true, + }] as stream).debug_string() + ), + "[!stream! [!group! A] [!group! B] [!group! C] [!group! D] [!group! E]]"); // Drop empty false works - preinterpret_assert_eq!({ - [!set! #x = ;] - [!debug! [!split! { - stream: [!stream! ;A;;B;C;D #..x E;], - separator: x, - drop_empty_start: false, - drop_empty_middle: false, - drop_empty_end: false, - }] as stream] - }, "[!stream! [!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]"); + preinterpret_assert_eq!( + #( + let x = [!stream! ;]; + let output = [!split! { + stream: [!stream! ;A;;B;C;D #..x E;], + separator: x, + drop_empty_start: false, + drop_empty_middle: false, + drop_empty_end: false, + }] as stream; + output.debug_string() + ), + "[!stream! [!group!] [!group! A] [!group!] [!group! B] [!group! C] [!group! D] [!group! E] [!group!]]" + ); // Drop empty middle works preinterpret_assert_eq!( - { - [!debug![!split! { + #( + [!split! { stream: [!stream! ;A;;B;;;;E;], separator: [!stream! ;], drop_empty_start: false, drop_empty_middle: true, drop_empty_end: false, - }]] - }, + }].debug_string() + ), "[[!stream!], [!stream! A], [!stream! B], [!stream! E], [!stream!]]" ); } @@ -313,7 +319,7 @@ fn test_split() { #[test] fn test_comma_split() { preinterpret_assert_eq!( - { [!debug! [!comma_split! Pizza, Mac and Cheese, Hamburger,]] }, + #([!comma_split! Pizza, Mac and Cheese, Hamburger,].debug_string()), "[[!stream! Pizza], [!stream! Mac and Cheese], [!stream! Hamburger]]" ); } @@ -321,56 +327,53 @@ fn test_comma_split() { #[test] fn test_zip() { preinterpret_assert_eq!( - [!debug![ - !zip! [[!stream! Hello "Goodbye"], ["World", "Friend"]] - ]], + #([!zip! [[!stream! Hello "Goodbye"], ["World", "Friend"]]].debug_string()), r#"[[[!stream! Hello], "World"], ["Goodbye", "Friend"]]"#, ); preinterpret_assert_eq!( - { - [!set! #countries = "France" "Germany" "Italy"] - [!set! #flags = "🇫🇷" "🇩🇪" "🇮🇹"] - [!set! #capitals = "Paris" "Berlin" "Rome"] - [!debug! [!zip! [countries, flags, capitals]]] - }, + #( + let countries = [!stream! "France" "Germany" "Italy"]; + let flags = [!stream! "🇫🇷" "🇩🇪" "🇮🇹"]; + let capitals = [!stream! "Paris" "Berlin" "Rome"]; + [!zip! [countries, flags, capitals]].debug_string() + ), r#"[["France", "🇫🇷", "Paris"], ["Germany", "🇩🇪", "Berlin"], ["Italy", "🇮🇹", "Rome"]]"#, ); preinterpret_assert_eq!( - { - [!set! #longer = A B C D] - [!set! #shorter = 1 2 3] - [!debug! [!zip_truncated! [longer, shorter]]] - }, + #( + let longer = [!stream! A B C D]; + let shorter = [1, 2, 3]; + [!zip_truncated! [longer, shorter]].debug_string() + ), r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); preinterpret_assert_eq!( - { - [!set! #letters = A B C] - #(let numbers = [1, 2, 3]) - [!debug! [!zip! [letters, numbers]]] - }, + #( + let letters = [!stream! A B C]; + let numbers = [1, 2, 3]; + [!zip! [letters, numbers]].debug_string() + ), r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); preinterpret_assert_eq!( - { - [!set! #letters = A B C] - #(let numbers = [1, 2, 3]) - #(let combined = [letters, numbers]) - [!debug! [!zip! combined]] - }, + #( + let letters = [!stream! A B C]; + let numbers = [1, 2, 3]; + let combined = [letters, numbers]; + [!zip! combined].debug_string() + ), r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); preinterpret_assert_eq!( - { - [!set! #letters = A B C] - #(let numbers = [1, 2, 3]) - #(let letter = [letters, numbers]) - [!debug! [!zip! { number: numbers, letter: letters }]] - }, + #( + [!set! #letters = A B C]; + let numbers = [1, 2, 3]; + [!zip! { number: numbers, letter: letters }].debug_string() + ), r#"[{ letter: [!stream! A], number: 1 }, { letter: [!stream! B], number: 2 }, { letter: [!stream! C], number: 3 }]"#, ); - preinterpret_assert_eq!([!debug![!zip![]]], r#"[]"#,); - preinterpret_assert_eq!([!debug![!zip! {}]], r#"[]"#,); + preinterpret_assert_eq!(#([!zip![]].debug_string()), r#"[]"#); + preinterpret_assert_eq!(#([!zip! {}].debug_string()), r#"[]"#); } #[test] @@ -380,9 +383,10 @@ fn test_zip_with_for() { [!set! #countries = France Germany Italy] #(let flags = ["🇫🇷", "🇩🇪", "🇮🇹"]) [!set! #capitals = "Paris" "Berlin" "Rome"] - [!set! #facts = [!for! [country, flag, capital] in [!zip! [countries, flags, capitals]] { - [!string! "=> The capital of " #country " is " #capital " and its flag is " #flag] - }]] + #(let facts = []) + [!for! [country, flag, capital] in [!zip! [countries, flags, capitals]] { + #(facts.push([!string! "=> The capital of " #country " is " #capital " and its flag is " #flag])) + }] #("The facts are:\n" + [!intersperse! { items: facts, diff --git a/tests/transforming.rs b/tests/transforming.rs index 1b267db4..3fec1a34 100644 --- a/tests/transforming.rs +++ b/tests/transforming.rs @@ -57,7 +57,7 @@ fn test_variable_parsing() { #..>>..x // Matches stream and appends it flattened: do you agree ? ) = Why Hello Everyone ([!group! This is an exciting adventure] do you agree?)] - [!debug! x] + #(x.debug_string()) }, "[!stream! Why [!group! Hello Everyone] This is an exciting adventure do you agree ?]"); } @@ -78,7 +78,7 @@ fn test_ident_transformer() { preinterpret_assert_eq!({ [!set! #x =] [!let! The quick @(#x += @IDENT) fox jumps @(#x += @IDENT) the lazy dog = The quick brown fox jumps over the lazy dog] - [!debug! x] + #(x.debug_string()) }, "[!stream! brown over]"); } @@ -92,7 +92,7 @@ fn test_literal_transformer() { preinterpret_assert_eq!({ [!set! #x] [!let! @LITERAL @LITERAL @LITERAL @(#x += @LITERAL) @LITERAL @(#x += @LITERAL @LITERAL) = "Hello" 9 3.4 'c' 41u16 0b1010 r#"123"#] - [!debug! x] + #(x.debug_string()) }, "[!stream! 'c' 0b1010 r#\"123\"#]"); } @@ -100,18 +100,18 @@ fn test_literal_transformer() { fn test_punct_transformer() { preinterpret_assert_eq!({ [!let! The "quick" brown fox "jumps" @(#x = @PUNCT) = The "quick" brown fox "jumps"!] - [!debug! x] + #(x.debug_string()) }, "[!stream! !]"); // Test for ' which is treated weirdly by syn / rustc preinterpret_assert_eq!({ [!let! The "quick" fox isn 't brown and doesn @(#x = @PUNCT) t "jump" = The "quick" fox isn 't brown and doesn 't "jump"] - #(x as debug) + #(x.debug_string()) }, "[!stream! ']"); // Lots of punctuation, most of it ignored preinterpret_assert_eq!({ [!set! #x =] [!let! @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT @PUNCT @PUNCT @(#x += @PUNCT) @PUNCT @PUNCT @PUNCT = # ! $$ % ^ & * + = | @ : ;] - [!debug! x] + #(x.debug_string()) }, "[!stream! % |]"); } @@ -119,18 +119,18 @@ fn test_punct_transformer() { fn test_group_transformer() { preinterpret_assert_eq!({ [!let! The "quick" @[GROUP brown #x] "jumps" = The "quick" [!group! brown fox] "jumps"] - [!debug! x] + #(x.debug_string()) }, "[!stream! fox]"); preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said @[GROUP #..y]! = I said #x!] - [!debug! y] + #(y.debug_string()) }, "[!stream! \"hello\" \"world\"]"); // ... which is equivalent to this: preinterpret_assert_eq!({ [!set! #x = "hello" "world"] [!let! I said #y! = I said #x!] - [!debug! y] + #(y.debug_string()) }, "[!stream! \"hello\" \"world\"]"); } @@ -138,7 +138,7 @@ fn test_group_transformer() { fn test_none_output_commands_mid_parse() { preinterpret_assert_eq!({ [!let! The "quick" @(#x = @LITERAL) fox [!let! #y = #x] @(#x = @IDENT) = The "quick" "brown" fox jumps] - [!string! "#x = " [!debug! x] "; #y = "[!debug! y]] + [!string! "#x = " #(x.debug_string()) "; #y = "#(y.debug_string())] }, "#x = [!stream! jumps]; #y = \"brown\""); } @@ -171,27 +171,30 @@ fn test_exact_transformer() { fn test_parse_command_and_exact_transformer() { // The output stream is additive preinterpret_assert_eq!( - { [!debug! [!parse! [!stream! Hello World] with @(@IDENT @IDENT)]] }, + #([!parse! [!stream! Hello World] with @(@IDENT @IDENT)].debug_string()), "[!stream! Hello World]" ); // Substreams redirected to a variable are not included in the output preinterpret_assert_eq!( - { - [!debug! [!parse! [!stream! The quick brown fox] with @( + #( + [!parse! [!stream! The quick brown fox] with @( @[EXACT The] quick @IDENT @(#x = @IDENT) - )]] - }, + )].debug_string() + ), "[!stream! The brown]" ); // This tests that: // * Can nest EXACT and transform streams // * Can discard output with @(_ = ...) // * That EXACT ignores none-delimited groups, to make it more intuitive - preinterpret_assert_eq!({ - [!set! #x = [!group! fox]] - [!debug! [!parse! [!stream! The quick brown fox is a fox - right?!] with @( - // The outputs are only from the EXACT transformer - The quick @(_ = @IDENT) @[EXACT #x @(_ = @IDENT a) #..x - right?!] - )]] - }, "[!stream! fox fox - right ?!]"); + preinterpret_assert_eq!( + #( + [!set! #x = [!group! fox]]; + [!parse! [!stream! The quick brown fox is a fox - right?!] with @( + // The outputs are only from the EXACT transformer + The quick @(_ = @IDENT) @[EXACT #x @(_ = @IDENT a) #..x - right?!] + )].debug_string() + ), + "[!stream! fox fox - right ?!]" + ); } From 8264a03ea2e34ad68ae304f869f7a0270be95b82 Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 20 Sep 2025 02:56:05 +0100 Subject: [PATCH 125/126] refactor: Various renames --- CHANGELOG.md | 19 +- src/expressions/array.rs | 30 +- .../evaluation/assignment_frames.rs | 8 +- src/expressions/evaluation/evaluator.rs | 145 +++--- src/expressions/evaluation/node_conversion.rs | 30 +- src/expressions/evaluation/place_frames.rs | 35 +- src/expressions/evaluation/type_resolution.rs | 129 ++--- src/expressions/evaluation/value_frames.rs | 236 ++++++--- src/expressions/expression.rs | 2 +- src/expressions/object.rs | 46 +- src/expressions/value.rs | 37 +- src/extensions/errors_and_spans.rs | 6 + src/interpretation/bindings.rs | 482 ++++++++++++++++++ src/interpretation/commands/core_commands.rs | 2 +- src/interpretation/interpreter.rs | 378 +------------- src/interpretation/mod.rs | 4 + src/interpretation/source_stream.rs | 2 +- src/interpretation/variable.rs | 11 +- src/misc/parse_traits.rs | 6 +- src/transformation/exact_stream.rs | 2 +- src/transformation/mod.rs | 4 +- src/transformation/parse_utilities.rs | 2 +- src/transformation/transform_stream.rs | 8 +- ...variable_binding.rs => variable_parser.rs} | 64 +-- .../extend_variable_and_then_read_it.stderr | 2 +- ...ray_place_destructure_multiple_muts.stderr | 2 +- tests/expressions.rs | 29 +- tests/tokens.rs | 18 +- 28 files changed, 1009 insertions(+), 730 deletions(-) create mode 100644 src/interpretation/bindings.rs rename src/transformation/{variable_binding.rs => variable_parser.rs} (79%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68113b1c..218c059b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,21 +106,24 @@ Inside a transform stream, the following grammar is supported: ### To come * Method calls continued - * Add `debug_error()` - * Add support for `RequestedValueOwnership::SharedOrOwned` (CoW behaviour) - and equivalent `SharedOrOwned`. This can be used to create a `OwnedOrCloned` parameter for utilities like `debug()` which is more performant + * Consider how to align: + * `ResolvedValue::into_mutable` (used by LateBound Owned => Mutable Arg; such as a method object) + * `context::return_owned` (used by Owned returned when Mutable requested, + such as a method argument) + * Create `CopyOnWrite<..>` in `bindings.rs`, add `CopyOnWriteValue` support to `wrap_method!` and change it so that `debug_string` takes CowValue + * Add tests for TODO[access-refactor] (i.e. changing places to resolve correctly) + * Check if we actually need `RequestedPlaceOwnership::SharedReference` or `RequestedPlaceOwnership::LateBound` and potentially remove them if not + * Add more impl_resolvable_argument_for * Scrap `#>>x` etc in favour of `#(x.push(@TOKEN))` - * TODO[access-refactor] - * TODO[range-refactor] & some kind of more thought through typed reference support - e.g. slices, mutable arrays? - * Re-enable TODO[auto-cloning] + * TODO[range-refactor] & some kind of more thought through typed reference support - e.g. slices, mutable slices? * TODO[operation-refactor] - * No clone required for testing equality of streams, objects and arrays + * Including no clone required for testing equality of streams, objects and arrays * Add better way of defining methods once / lazily, and binding them to an object type. * Consider: * Changing evaluation to allow `ResolvedValue` (i.e. allow references) * Removing span range from value: - * Moving it to an `ResolvedValue::OwnedValue` and `OwnedValue(Value, SpanRange)` + * Moving it to an `OwnedValue` or `Owned` * Using `EvaluationError` (without a span!) inside the calculation, and adding the span in the evaluator (nb. it may still need to be able to propogate an `ExecutionInterrupt` internally) * Using ResolvedValue in place of ExpressionValue e.g. inside arrays * Introduce `~(...)` and `r~(...)` streams instead of `[!stream! ...]` and `[!raw! ...]` diff --git a/src/expressions/array.rs b/src/expressions/array.rs index 22f5eddc..abc2ee98 100644 --- a/src/expressions/array.rs +++ b/src/expressions/array.rs @@ -136,20 +136,38 @@ impl ExpressionArray { pub(super) fn index_mut( &mut self, - _access: IndexAccess, + access: IndexAccess, index: &ExpressionValue, ) -> ExecutionResult<&mut ExpressionValue> { - let index = self.resolve_valid_index(index, false)?; - Ok(&mut self.items[index]) + Ok(match index { + ExpressionValue::Integer(integer) => { + let index = self.resolve_valid_index_from_integer(integer, false)?; + &mut self.items[index] + } + ExpressionValue::Range(..) => { + // Temporary until we add slice types - we error here + return access.execution_err("Currently, a range-indexed array must be owned. Use `.take()` or `.clone()` before indexing [..]"); + } + _ => return index.execution_err("The index must be an integer or a range"), + }) } pub(super) fn index_ref( &self, - _access: IndexAccess, + access: IndexAccess, index: &ExpressionValue, ) -> ExecutionResult<&ExpressionValue> { - let index = self.resolve_valid_index(index, false)?; - Ok(&self.items[index]) + Ok(match index { + ExpressionValue::Integer(integer) => { + let index = self.resolve_valid_index_from_integer(integer, false)?; + &self.items[index] + } + ExpressionValue::Range(..) => { + // Temporary until we add slice types - we error here + return access.execution_err("Currently, a range-indexed array must be owned. Use `.take()` or `.clone()` before indexing [..]"); + } + _ => return index.execution_err("The index must be an integer or a range"), + }) } pub(super) fn resolve_valid_index( diff --git a/src/expressions/evaluation/assignment_frames.rs b/src/expressions/evaluation/assignment_frames.rs index eab6d06b..c126904d 100644 --- a/src/expressions/evaluation/assignment_frames.rs +++ b/src/expressions/evaluation/assignment_frames.rs @@ -40,7 +40,7 @@ impl PlaceAssigner { value: ExpressionValue, ) -> NextAction { let frame = Self { value }; - context.handle_node_as_place(frame, place, RequestedPlaceOwnership::MutableReference) + context.handle_node_as_place(frame, place, RequestedPlaceOwnership::Mutable) } } @@ -56,7 +56,7 @@ impl EvaluationFrame for PlaceAssigner { context: AssignmentContext, item: EvaluationItem, ) -> ExecutionResult { - let mut mutable_place = item.expect_mutable_ref(); + let mut mutable_place = item.expect_mutable(); let value = self.value; let span_range = SpanRange::new_between(mutable_place.span_range(), value.span_range()); mutable_place.set(value); @@ -285,7 +285,7 @@ impl ObjectBasedAssigner { self, index, // This only needs to be read-only, as we are just using it to work out which field/s to assign - RequestedValueOwnership::SharedReference, + RequestedValueOwnership::Shared, ) } None => context.return_assignment_completion(self.span_range), @@ -323,7 +323,7 @@ impl EvaluationFrame for Box { assignee_node, access, } => { - let index_place = item.expect_shared_ref(); + let index_place = item.expect_shared(); self.handle_index_value(context, access, index_place.as_ref(), assignee_node) } ObjectAssignmentState::WaitingForSubassignment => { diff --git a/src/expressions/evaluation/evaluator.rs b/src/expressions/evaluation/evaluator.rs index d8dc032a..920efd3b 100644 --- a/src/expressions/evaluation/evaluator.rs +++ b/src/expressions/evaluation/evaluator.rs @@ -72,7 +72,7 @@ impl<'a> ExpressionEvaluator<'a, Source> { Some(top) => top, None => { // This aligns with the request for an owned value in evaluate - return Ok(StepResult::Return(item.expect_owned_value())); + return Ok(StepResult::Return(item.expect_owned_value().into_inner())); } }; top_of_stack.handle_item(&mut self.stack, item)? @@ -102,20 +102,21 @@ pub(super) enum StepResult { pub(super) struct NextAction(NextActionInner); impl NextAction { - pub(super) fn return_owned(value: ExpressionValue) -> Self { + pub(super) fn return_owned(value: OwnedValue) -> Self { NextActionInner::HandleReturnedItem(EvaluationItem::OwnedValue(value)).into() } pub(super) fn return_mutable(mut_ref: MutableValue) -> Self { - NextActionInner::HandleReturnedItem(EvaluationItem::MutableReference { mut_ref }).into() + NextActionInner::HandleReturnedItem(EvaluationItem::MutableReference { mutable: mut_ref }) + .into() } pub(super) fn return_shared( - shared_ref: SharedValue, + shared: SharedValue, reason_not_mutable: Option, ) -> Self { NextActionInner::HandleReturnedItem(EvaluationItem::SharedReference { - shared_ref, + shared, reason_not_mutable, }) .into() @@ -140,33 +141,41 @@ impl From for NextAction { } pub(super) enum EvaluationItem { - OwnedValue(ExpressionValue), + OwnedValue(OwnedValue), SharedReference { - shared_ref: SharedValue, + shared: SharedValue, /// This is only populated if we request a "late bound" reference, and fail to resolve /// a mutable reference. reason_not_mutable: Option, }, MutableReference { - mut_ref: MutableValue, + mutable: MutableValue, }, AssignmentCompletion(AssignmentCompletion), } impl EvaluationItem { - pub(super) fn expect_owned_value(self) -> ExpressionValue { + pub(super) fn expect_owned_value(self) -> OwnedValue { match self { EvaluationItem::OwnedValue(value) => value, _ => panic!("expect_owned_value() called on a non-owned-value EvaluationItem"), } } + pub(super) fn expect_cow_value(self) -> CowValue { + match self { + EvaluationItem::OwnedValue(value) => CowValue::Owned(value), + EvaluationItem::SharedReference { shared, .. } => CowValue::Shared(shared), + _ => panic!("expect_cow_value() called on a non-cow-value EvaluationItem"), + } + } + pub(super) fn expect_any_value(self) -> ResolvedValue { match self { EvaluationItem::OwnedValue(value) => ResolvedValue::Owned(value), - EvaluationItem::MutableReference { mut_ref } => ResolvedValue::Mutable(mut_ref), - EvaluationItem::SharedReference { shared_ref, .. } => ResolvedValue::Shared { - shared_ref, + EvaluationItem::MutableReference { mutable } => ResolvedValue::Mutable(mutable), + EvaluationItem::SharedReference { shared, .. } => ResolvedValue::Shared { + shared, reason_not_mutable: None, }, _ => panic!("expect_any_value() called on a non-value EvaluationItem"), @@ -175,31 +184,31 @@ impl EvaluationItem { pub(super) fn expect_any_place(self) -> Place { match self { - EvaluationItem::MutableReference { mut_ref } => Place::MutableReference { mut_ref }, + EvaluationItem::MutableReference { mutable } => Place::Mutable { mutable }, EvaluationItem::SharedReference { - shared_ref, + shared, reason_not_mutable, - } => Place::SharedReference { - shared_ref, + } => Place::Shared { + shared, reason_not_mutable, }, _ => panic!("expect_any_place() called on a non-place EvaluationItem"), } } - pub(super) fn expect_shared_ref(self) -> SharedValue { + pub(super) fn expect_shared(self) -> SharedValue { match self { - EvaluationItem::SharedReference { shared_ref, .. } => shared_ref, + EvaluationItem::SharedReference { shared, .. } => shared, _ => { - panic!("expect_shared_reference() called on a non-shared-reference EvaluationItem") + panic!("expect_shared() called on a non-shared-reference EvaluationItem") } } } - pub(super) fn expect_mutable_ref(self) -> MutableValue { + pub(super) fn expect_mutable(self) -> MutableValue { match self { - EvaluationItem::MutableReference { mut_ref } => mut_ref, - _ => panic!("expect_mutable_place() called on a non-mutable-place EvaluationItem"), + EvaluationItem::MutableReference { mutable } => mutable, + _ => panic!("expect_mutable() called on a non-mutable-place EvaluationItem"), } } @@ -334,61 +343,67 @@ impl<'a> Context<'a, ValueType> { self.request } - pub(super) fn return_any_value(self, value: ResolvedValue) -> NextAction { - match value { - ResolvedValue::Owned(value) => self.return_owned_value(value), - ResolvedValue::Mutable(mut_ref) => self.return_mut_ref(mut_ref), + pub(super) fn return_any_value(self, value: ResolvedValue) -> ExecutionResult { + Ok(match value { + ResolvedValue::Owned(owned) => self.return_owned(owned), + ResolvedValue::Mutable(mutable) => self.return_mutable(mutable), ResolvedValue::Shared { - shared_ref, + shared, reason_not_mutable, - } => self.return_ref(shared_ref, reason_not_mutable), - } + } => self.return_shared(shared, reason_not_mutable)?, + }) } - pub(super) fn return_any_place(self, value: Place) -> NextAction { - match value { - Place::MutableReference { mut_ref } => self.return_mut_ref(mut_ref), - Place::SharedReference { - shared_ref, + pub(super) fn return_any_place(self, value: Place) -> ExecutionResult { + Ok(match value { + Place::Mutable { mutable } => self.return_mutable(mutable), + Place::Shared { + shared, reason_not_mutable, - } => self.return_ref(shared_ref, reason_not_mutable), - } + } => self.return_shared(shared, reason_not_mutable)?, + }) } - pub(super) fn return_owned_value(self, value: ExpressionValue) -> NextAction { + pub(super) fn return_owned(self, value: impl Into) -> NextAction { + let value = value.into(); match self.request { - RequestedValueOwnership::LateBound | RequestedValueOwnership::Owned => { - NextAction::return_owned(value) - } - RequestedValueOwnership::SharedReference => { - NextAction::return_shared(SharedSubPlace::new_from_owned(value), None) + RequestedValueOwnership::LateBound + | RequestedValueOwnership::CopyOnWrite + | RequestedValueOwnership::Owned => NextAction::return_owned(value), + RequestedValueOwnership::Shared => { + NextAction::return_shared(Shared::new_from_owned(value), None) } - RequestedValueOwnership::MutableReference => { - NextAction::return_mutable(MutableSubPlace::new_from_owned(value)) + RequestedValueOwnership::Mutable => { + NextAction::return_mutable(Mutable::new_from_owned(value)) } } } - pub(super) fn return_mut_ref(self, mut_ref: MutableValue) -> NextAction { + pub(super) fn return_mutable(self, mut_ref: MutableValue) -> NextAction { match self.request { RequestedValueOwnership::LateBound - | RequestedValueOwnership::MutableReference => NextAction::return_mutable(mut_ref), + | RequestedValueOwnership::Mutable => NextAction::return_mutable(mut_ref), RequestedValueOwnership::Owned - | RequestedValueOwnership::SharedReference => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), + | RequestedValueOwnership::CopyOnWrite + | RequestedValueOwnership::Shared => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), } } - pub(super) fn return_ref( + pub(super) fn return_shared( self, - shared_ref: SharedValue, + shared: SharedValue, reason_not_mutable: Option, - ) -> NextAction { - match self.request { + ) -> ExecutionResult { + Ok(match self.request { RequestedValueOwnership::LateBound - | RequestedValueOwnership::SharedReference => NextAction::return_shared(shared_ref, reason_not_mutable), - RequestedValueOwnership::Owned - | RequestedValueOwnership::MutableReference => panic!("Returning a reference should only be used when the requested ownership is shared or late-bound"), - } + | RequestedValueOwnership::CopyOnWrite + | RequestedValueOwnership::Shared => NextAction::return_shared(shared, reason_not_mutable), + RequestedValueOwnership::Owned => { + // This can happen if a requested of "Owned" was mapped to "CopyOnWrite" + NextAction::return_owned(shared.transparent_clone()?) + } + RequestedValueOwnership::Mutable => panic!("Returning a reference should only be used when the requested ownership is shared or late-bound"), + }) } } @@ -415,31 +430,31 @@ impl<'a> Context<'a, PlaceType> { pub(super) fn return_any(self, place: Place) -> NextAction { match place { - Place::MutableReference { mut_ref } => self.return_mutable(mut_ref), - Place::SharedReference { - shared_ref, + Place::Mutable { mutable } => self.return_mutable(mutable), + Place::Shared { + shared, reason_not_mutable, - } => self.return_shared(shared_ref, reason_not_mutable), + } => self.return_shared(shared, reason_not_mutable), } } - pub(super) fn return_mutable(self, mut_ref: MutableValue) -> NextAction { + pub(super) fn return_mutable(self, mutable: MutableValue) -> NextAction { match self.request { RequestedPlaceOwnership::LateBound - | RequestedPlaceOwnership::MutableReference => NextAction::return_mutable(mut_ref), - RequestedPlaceOwnership::SharedReference => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), + | RequestedPlaceOwnership::Mutable => NextAction::return_mutable(mutable), + RequestedPlaceOwnership::Shared => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), } } pub(super) fn return_shared( self, - shared_ref: SharedValue, + shared: SharedValue, reason_not_mutable: Option, ) -> NextAction { match self.request { RequestedPlaceOwnership::LateBound - | RequestedPlaceOwnership::SharedReference => NextAction::return_shared(shared_ref, reason_not_mutable), - RequestedPlaceOwnership::MutableReference => panic!("Returning a shared reference should only be used when the requested ownership is shared or late-bound"), + | RequestedPlaceOwnership::Shared => NextAction::return_shared(shared, reason_not_mutable), + RequestedPlaceOwnership::Mutable => panic!("Returning a shared reference should only be used when the requested ownership is shared or late-bound"), } } } diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs index 6bc8b38e..bf2da4b2 100644 --- a/src/expressions/evaluation/node_conversion.rs +++ b/src/expressions/evaluation/node_conversion.rs @@ -11,32 +11,34 @@ impl ExpressionNode { match leaf { SourceExpressionLeaf::Command(command) => { // TODO[interpret_to_value]: Allow command to return a reference - context.return_owned_value(command.clone().interpret_to_value(interpreter)?) + context.return_owned(command.clone().interpret_to_value(interpreter)?) } SourceExpressionLeaf::Discarded(token) => { return token.execution_err("This cannot be used in a value expression"); } SourceExpressionLeaf::Variable(variable_path) => { - let variable_ref = variable_path.reference(interpreter)?; + let variable_ref = variable_path.binding(interpreter)?; match context.requested_ownership() { RequestedValueOwnership::LateBound => { - context.return_any_place(variable_ref.into_late_bound()?) + context.return_any_place(variable_ref.into_late_bound()?)? } - RequestedValueOwnership::SharedReference => { - context.return_ref(variable_ref.into_shared()?, None) + RequestedValueOwnership::Shared + | RequestedValueOwnership::CopyOnWrite => { + context.return_shared(variable_ref.into_shared()?, None)? } - RequestedValueOwnership::MutableReference => { - context.return_mut_ref(variable_ref.into_mut()?) + RequestedValueOwnership::Mutable => { + context.return_mutable(variable_ref.into_mut()?) + } + RequestedValueOwnership::Owned => { + context.return_owned(variable_ref.into_transparently_cloned()?) } - RequestedValueOwnership::Owned => context - .return_owned_value(variable_ref.get_value_transparently_cloned()?), } } SourceExpressionLeaf::ExpressionBlock(block) => { // TODO[interpret_to_value]: Allow block to return reference - context.return_owned_value(block.interpret_to_value(interpreter)?) + context.return_owned(block.interpret_to_value(interpreter)?) } - SourceExpressionLeaf::Value(value) => context.return_owned_value(value.clone()), + SourceExpressionLeaf::Value(value) => context.return_owned(value.clone()), } } ExpressionNode::Grouped { delim_span, inner } => { @@ -129,15 +131,15 @@ impl ExpressionNode { ) -> ExecutionResult { Ok(match self { ExpressionNode::Leaf(SourceExpressionLeaf::Variable(variable)) => { - let variable_ref = variable.reference(interpreter)?; + let variable_ref = variable.binding(interpreter)?; match context.requested_ownership() { RequestedPlaceOwnership::LateBound => { context.return_any(variable_ref.into_late_bound()?) } - RequestedPlaceOwnership::SharedReference => { + RequestedPlaceOwnership::Shared => { context.return_shared(variable_ref.into_shared()?, None) } - RequestedPlaceOwnership::MutableReference => { + RequestedPlaceOwnership::Mutable => { context.return_mutable(variable_ref.into_mut()?) } } diff --git a/src/expressions/evaluation/place_frames.rs b/src/expressions/evaluation/place_frames.rs index 2de9566d..6dc0956b 100644 --- a/src/expressions/evaluation/place_frames.rs +++ b/src/expressions/evaluation/place_frames.rs @@ -4,8 +4,8 @@ use super::*; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum RequestedPlaceOwnership { LateBound, - SharedReference, - MutableReference, + Shared, + Mutable, } /// Handlers which return a Place @@ -100,20 +100,20 @@ impl EvaluationFrame for PlaceIndexer { let place = item.expect_any_place(); self.state = PlaceIndexerPath::IndexPath { place }; // If we do my_obj["my_key"] = 1 then the "my_key" place is created, - // so mutable reference indexing takes an owned index. - context.handle_node_as_value(self, index, RequestedValueOwnership::Owned) + // so mutable reference indexing takes an owned index... + // But we auto-clone the key in that case, so we can still pass a shared ref + context.handle_node_as_value(self, index, RequestedValueOwnership::Shared) } PlaceIndexerPath::IndexPath { place } => { - let index = item.expect_owned_value(); + let index = item.expect_shared(); match place { - Place::MutableReference { mut_ref } => context.return_mutable( - mut_ref.resolve_indexed_with_autocreate(self.access, index)?, - ), - Place::SharedReference { - shared_ref, + Place::Mutable { mutable: mut_ref } => context + .return_mutable(mut_ref.resolve_indexed(self.access, &index, true)?), + Place::Shared { + shared, reason_not_mutable, } => context.return_shared( - shared_ref.resolve_indexed(self.access, &index)?, + shared.resolve_indexed(self.access, &index)?, reason_not_mutable, ), } @@ -152,16 +152,13 @@ impl EvaluationFrame for PlacePropertyAccessor { ) -> ExecutionResult { let place = item.expect_any_place(); Ok(match place { - Place::MutableReference { mut_ref } => { - context.return_mutable(mut_ref.resolve_property(self.access)?) + Place::Mutable { mutable } => { + context.return_mutable(mutable.resolve_property(&self.access, true)?) } - Place::SharedReference { - shared_ref, - reason_not_mutable, - } => context.return_shared( - shared_ref.resolve_property(self.access)?, + Place::Shared { + shared, reason_not_mutable, - ), + } => context.return_shared(shared.resolve_property(&self.access)?, reason_not_mutable), }) } } diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs index c9a92153..79764ff6 100644 --- a/src/expressions/evaluation/type_resolution.rs +++ b/src/expressions/evaluation/type_resolution.rs @@ -11,57 +11,49 @@ mod macros { ([$(,)?] [$($bindings:tt)*]) => { $($bindings)* }; - // By shared reference - ([$($arg_part:ident)+ : &$ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { - handle_arg_mapping!([$($rest)*] [ - $($bindings)* - let handle_arg_name!($($arg_part)+): &$ty = <$ty as ResolvableArgument>::resolve_from_ref(handle_arg_name!($($arg_part)+).as_ref())?; - ]) - }; // By captured shared reference (i.e. can return a sub-reference from it) - ([$($arg_part:ident)+ : CapturedRef<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : Shared<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let tmp = handle_arg_name!($($arg_part)+).into_shared_reference(); - let handle_arg_name!($($arg_part)+): CapturedRef<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_ref(value))?; + let tmp = handle_arg_name!($($arg_part)+).into_shared(); + let handle_arg_name!($($arg_part)+): Shared<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_ref(value))?; ]) }; - // SharedValue is an alias for CapturedRef + // SharedValue is an alias for Shared ([$($arg_part:ident)+ : SharedValue, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_shared_reference(); + let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_shared(); ]) }; - // By mutable reference - ([$($arg_part:ident)+ : &mut $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + // By captured mutable reference (i.e. can return a sub-reference from it) + ([$($arg_part:ident)+ : Mutable<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let mut tmp = handle_arg_name!($($arg_part)+).into_mutable_reference()?; - let handle_arg_name!($($arg_part)+): &mut $ty = <$ty as ResolvableArgument>::resolve_from_mut(tmp.as_mut())?; + let mut tmp = handle_arg_name!($($arg_part)+).into_mutable()?; + let handle_arg_name!($($arg_part)+): Mutable<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_mut(value))?; ]) }; - // By captured mutable reference (i.e. can return a sub-reference from it) - ([$($arg_part:ident)+ : CapturedMut<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { + // MutableValue is an alias for Mutable + ([$($arg_part:ident)+ : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let mut tmp = handle_arg_name!($($arg_part)+).into_mutable_reference()?; - let handle_arg_name!($($arg_part)+): CapturedMut<$ty> = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_mut(value))?; + let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_mutable()?; ]) }; - // MutableValue is an alias for CapturedMut - ([$($arg_part:ident)+ : MutableValue, $($rest:tt)*] [$($bindings:tt)*]) => { + // By value + ([$($arg_part:ident)+ : Owned<$ty:ty>, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_mutable_reference()?; + let tmp = handle_arg_name!($($arg_part)+).into_owned()?; + let handle_arg_name!($($arg_part)+): $ty = tmp.try_map(|value, _| <$ty as ResolvableArgument>::resolve_from_owned(value))?; ]) }; // By value - ([$($arg_part:ident)+ : $ty:ty, $($rest:tt)*] [$($bindings:tt)*]) => { + ([$($arg_part:ident)+ : OwnedValue, $($rest:tt)*] [$($bindings:tt)*]) => { handle_arg_mapping!([$($rest)*] [ $($bindings)* - let tmp = handle_arg_name!($($arg_part)+).into_owned_value()?; - let handle_arg_name!($($arg_part)+): $ty = <$ty as ResolvableArgument>::resolve_from_owned(tmp)?; + let handle_arg_name!($($arg_part)+) = handle_arg_name!($($arg_part)+).into_owned()?; ]) }; } @@ -71,32 +63,28 @@ mod macros { ([$(,)?] [$($outputs:tt)*]) => { vec![$($outputs)*] }; - // By shared reference - ([$($arg_part:ident)+ : &$ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) - }; // By captured shared reference (i.e. can return a sub-reference from it) - ([$($arg_part:ident)+ : CapturedRef<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) + ([$($arg_part:ident)+ : Shared<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Shared,]) }; - // SharedValue is an alias for CapturedRef + // SharedValue is an alias for Shared ([$($arg_part:ident)+ : SharedValue, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::SharedReference,]) - }; - // By mutable reference - ([$($arg_part:ident)+ : &mut $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Shared,]) }; // By captured mutable reference (i.e. can return a sub-reference from it) - ([$($arg_part:ident)+ : CapturedMut<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + ([$($arg_part:ident)+ : Mutable<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Mutable,]) }; - // MutableValue is an alias for CapturedMut + // MutableValue is an alias for Mutable ([$($arg_part:ident)+ : MutableValue, $($rest:tt)*] [$($outputs:tt)*]) => { - handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::MutableReference,]) + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Mutable,]) }; // By value - ([$($arg_part:ident)+ : $ty:ty, $($rest:tt)*] [$($outputs:tt)*]) => { + ([$($arg_part:ident)+ : Owned<$ty:ty>, $($rest:tt)*] [$($outputs:tt)*]) => { + handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Owned,]) + }; + // By value + ([$($arg_part:ident)+ : OwnedValue, $($rest:tt)*] [$($outputs:tt)*]) => { handle_arg_ownerships!([$($rest)*] [$($outputs)* RequestedValueOwnership::Owned,]) }; } @@ -163,8 +151,8 @@ mod macros { pub(crate) trait ResolvedTypeDetails { /// This should be true for types which users expect to have value - /// semantics, but false for mutable types / types with reference - /// semantics. + /// semantics, but false for types which are expensive to clone or + /// are expected to have reference semantics. /// /// This indicates if an &x can be converted to an x via cloning /// when doing method resolution. @@ -205,12 +193,16 @@ impl ResolvedTypeDetails for ValueKind { ValueKind::Integer => true, ValueKind::Float => true, ValueKind::Boolean => true, + // Strings are value-like, so it makes sense to transparently clone them ValueKind::String => true, ValueKind::Char => true, ValueKind::UnsupportedLiteral => false, ValueKind::Array => false, ValueKind::Object => false, - ValueKind::Stream => false, + // It's super common to want to embed a stream in another stream + // Having to embed it as #(type_name.clone()) instead of + // #type_name would be awkward + ValueKind::Stream => true, ValueKind::Range => true, ValueKind::Iterator => false, } @@ -224,7 +216,7 @@ impl ResolvedTypeDetails for ValueKind { let method_name = method.method.to_string(); let method = match (self, method_name.as_str(), num_arguments) { (_, "clone", 0) => { - wrap_method! {(this: &ExpressionValue) -> ExecutionResult { + wrap_method! {(this: SharedValue) -> ExecutionResult { Ok(this.clone()) }} } @@ -235,12 +227,12 @@ impl ResolvedTypeDetails for ValueKind { }} } (_, "as_mut", 0) => { - wrap_method! {(this: ExpressionValue) -> ExecutionResult { + wrap_method! {(this: OwnedValue) -> ExecutionResult { Ok(MutableValue::new_from_owned(this)) }} } (_, "debug_string", 0) => { - wrap_method! {(this: &ExpressionValue) -> ExecutionResult { + wrap_method! {(this: SharedValue) -> ExecutionResult { this.clone().into_debug_string() }} } @@ -249,18 +241,18 @@ impl ResolvedTypeDetails for ValueKind { this.execution_err(message) }}, (ValueKind::Array, "len", 0) => { - wrap_method! {(this: &ExpressionArray) -> ExecutionResult { + wrap_method! {(this: Shared) -> ExecutionResult { Ok(this.items.len()) }} } (ValueKind::Array, "push", 1) => { - wrap_method! {(this: &mut ExpressionArray, item: ExpressionValue) -> ExecutionResult<()> { - this.items.push(item); + wrap_method! {(mut this: Mutable, item: OwnedValue) -> ExecutionResult<()> { + this.items.push(item.into()); Ok(()) }} } (ValueKind::Stream, "len", 0) => { - wrap_method! {(this: &ExpressionStream) -> ExecutionResult { + wrap_method! {(this: Shared) -> ExecutionResult { Ok(this.value.len()) }} } @@ -300,16 +292,16 @@ mod outputs { #[diagnostic::on_unimplemented( message = "`ResolvableOutput` is not implemented for `{Self}`", - note = "`ResolvableOutput` is not implemented for `CapturedRef` or `CapturedMut` unless `X` is `ExpressionValue`. If we wish to change this, we'd need to have some way to represent some kind of `ExpressionReference`, i.e. a `Typed>` rather than a `CapturedRef>`" + note = "`ResolvableOutput` is not implemented for `Shared` or `Mutable` unless `X` is `ExpressionValue`. If we wish to change this, we'd need to have some way to represent some kind of `ExpressionReference`, i.e. a `Typed>` rather than a `Shared>`" )] pub(crate) trait ResolvableOutput { fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult; } - impl ResolvableOutput for CapturedRef { + impl ResolvableOutput for Shared { fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { Ok(ResolvedValue::Shared { - shared_ref: self.update_span_range(|_| output_span_range), + shared: self.update_span_range(|_| output_span_range), reason_not_mutable: Some(syn::Error::new( output_span_range.join_into_span_else_start(), "It was output from a method a captured shared reference", @@ -318,7 +310,7 @@ mod outputs { } } - impl ResolvableOutput for CapturedMut { + impl ResolvableOutput for Mutable { fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { Ok(ResolvedValue::Mutable( self.update_span_range(|_| output_span_range), @@ -326,14 +318,24 @@ mod outputs { } } + impl ResolvableOutput for Owned { + fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { + Ok(ResolvedValue::Owned( + self.update_span_range(|_| output_span_range), + )) + } + } + impl ResolvableOutput for T { fn to_resolved_value(self, output_span_range: SpanRange) -> ExecutionResult { - Ok(ResolvedValue::Owned(self.to_value(output_span_range))) + Ok(ResolvedValue::Owned( + self.to_value(output_span_range).into(), + )) } } } -use arguments::*; +pub(crate) use arguments::*; mod arguments { use super::*; @@ -442,6 +444,15 @@ mod arguments { } }} + impl<'a> ResolveAs<&'a str> for &'a ExpressionValue { + fn resolve_as(self) -> ExecutionResult<&'a str> { + match self { + ExpressionValue::String(s) => Ok(&s.value), + _ => self.execution_err("Expected string"), + } + } + } + impl_resolvable_argument_for! {(value) -> ExpressionChar { match value { ExpressionValue::Char(value) => Ok(value), diff --git a/src/expressions/evaluation/value_frames.rs b/src/expressions/evaluation/value_frames.rs index 69255809..8ae4f2eb 100644 --- a/src/expressions/evaluation/value_frames.rs +++ b/src/expressions/evaluation/value_frames.rs @@ -13,45 +13,43 @@ use super::*; /// So instead, we take the most powerful access we can have, and convert it later. pub(crate) enum ResolvedValue { /// This has been requested as an owned value. - Owned(ExpressionValue), + Owned(OwnedValue), /// This has been requested as a mutable reference. Mutable(MutableValue), /// This has been requested as a shared reference. Shared { - shared_ref: SharedValue, + shared: SharedValue, reason_not_mutable: Option, }, } impl ResolvedValue { - pub(crate) fn into_owned_value(self) -> ExecutionResult { - let reference = match self { - ResolvedValue::Owned(value) => return Ok(value), - ResolvedValue::Shared { .. } | ResolvedValue::Mutable { .. } => self.as_ref(), - }; - reference.try_transparent_clone(self.span_range()) + pub(crate) fn into_owned(self) -> ExecutionResult { + match self { + ResolvedValue::Owned(value) => Ok(value), + ResolvedValue::Shared { shared, .. } => shared.transparent_clone(), + ResolvedValue::Mutable(mutable) => mutable.transparent_clone(), + } } - /// The requirement is just for error messages, and should be "A ", and will be filled like: - /// "{usage} expects an owned value, but receives a reference..." - pub(crate) fn into_mutable_reference(self) -> ExecutionResult { + pub(crate) fn into_mutable(self) -> ExecutionResult { Ok(match self { ResolvedValue::Owned(value) => { return value.execution_err("A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference.".to_string()) }, ResolvedValue::Mutable(reference) => reference, - ResolvedValue::Shared { shared_ref, reason_not_mutable: Some(reason_not_mutable), } => return shared_ref.execution_err(format!( + ResolvedValue::Shared { shared, reason_not_mutable: Some(reason_not_mutable), } => return shared.execution_err(format!( "A mutable reference is required, but only a shared reference was received because {reason_not_mutable}." )), - ResolvedValue::Shared { shared_ref, reason_not_mutable: None, } => return shared_ref.execution_err("A mutable reference is required, but only a shared reference was received.".to_string()), + ResolvedValue::Shared { shared, reason_not_mutable: None, } => return shared.execution_err("A mutable reference is required, but only a shared reference was received.".to_string()), }) } - pub(crate) fn into_shared_reference(self) -> SharedValue { + pub(crate) fn into_shared(self) -> SharedValue { match self { ResolvedValue::Owned(value) => SharedValue::new_from_owned(value), ResolvedValue::Mutable(reference) => reference.into_shared(), - ResolvedValue::Shared { shared_ref, .. } => shared_ref, + ResolvedValue::Shared { shared, .. } => shared, } } @@ -61,27 +59,27 @@ impl ResolvedValue { pub(crate) fn as_value_ref(&self) -> &ExpressionValue { match self { - ResolvedValue::Owned(value) => value, - ResolvedValue::Mutable(reference) => reference.as_ref(), - ResolvedValue::Shared { shared_ref, .. } => shared_ref.as_ref(), + ResolvedValue::Owned(owned) => owned.as_ref(), + ResolvedValue::Mutable(mutable) => mutable.as_ref(), + ResolvedValue::Shared { shared, .. } => shared.as_ref(), } } pub(crate) fn as_value_mut(&mut self) -> ExecutionResult<&mut ExpressionValue> { match self { - ResolvedValue::Owned(value) => Ok(value), + ResolvedValue::Owned(value) => Ok(value.as_mut()), ResolvedValue::Mutable(reference) => Ok(reference.as_mut()), ResolvedValue::Shared { - shared_ref, + shared, reason_not_mutable: Some(reason_not_mutable), - } => shared_ref.execution_err(format!( + } => shared.execution_err(format!( "Cannot get a mutable reference: {}", reason_not_mutable )), ResolvedValue::Shared { - shared_ref, + shared, reason_not_mutable: None, - } => shared_ref.execution_err("Cannot get a mutable reference: Unknown reason"), + } => shared.execution_err("Cannot get a mutable reference: Unknown reason"), } } } @@ -98,10 +96,10 @@ impl WithSpanExt for ResolvedValue { ResolvedValue::Owned(value) => ResolvedValue::Owned(value.with_span(span)), ResolvedValue::Mutable(reference) => ResolvedValue::Mutable(reference.with_span(span)), ResolvedValue::Shared { - shared_ref, + shared, reason_not_mutable, } => ResolvedValue::Shared { - shared_ref: shared_ref.with_span(span), + shared: shared.with_span(span), reason_not_mutable, }, } @@ -114,12 +112,54 @@ impl AsRef for ResolvedValue { } } +pub(crate) enum CowValue { + /// This has been requested as an owned value. + Owned(OwnedValue), + /// This has been requested as a mutable reference. + Shared(SharedValue), +} + +impl CowValue { + pub(crate) fn into_owned(self) -> ExecutionResult { + match self { + CowValue::Owned(owned) => Ok(owned), + // A CoW value is used in place of a mutable reference. + CowValue::Shared(shared) => shared.transparent_clone(), + } + } + + pub(crate) fn kind(&self) -> ValueKind { + self.as_value_ref().kind() + } + + pub(crate) fn as_value_ref(&self) -> &ExpressionValue { + match self { + CowValue::Owned(owned) => owned.as_ref(), + CowValue::Shared(shared) => shared.as_ref(), + } + } +} + +impl HasSpanRange for CowValue { + fn span_range(&self) -> SpanRange { + self.as_value_ref().span_range() + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum RequestedValueOwnership { - LateBound, + /// Receives an Owned value (or an error!) Owned, - SharedReference, - MutableReference, + /// Receives a Shared Reference + Shared, + /// Receives a Mutable Reference (or an error!) + Mutable, + /// Receives any of Owned, SharedReference or MutableReference, depending on what + /// is available. + /// This can then be used to resolve the value kind, and use the correct one. + LateBound, + /// Receives either a SharedReference or Owned. + CopyOnWrite, } /// Handlers which return a Value @@ -189,7 +229,7 @@ impl EvaluationFrame for GroupBuilder { item: EvaluationItem, ) -> ExecutionResult { let inner = item.expect_owned_value(); - Ok(context.return_owned_value(inner.with_span(self.span))) + Ok(context.return_owned(inner.update_span_range(|_| self.span.into()))) } } @@ -220,7 +260,7 @@ impl ArrayBuilder { .cloned() { Some(next) => context.handle_node_as_value(self, next, RequestedValueOwnership::Owned), - None => context.return_owned_value(ExpressionValue::Array(ExpressionArray { + None => context.return_owned(ExpressionValue::Array(ExpressionArray { items: self.evaluated_items, span_range: self.span.span_range(), })), @@ -241,7 +281,7 @@ impl EvaluationFrame for ArrayBuilder { item: EvaluationItem, ) -> ExecutionResult { let value = item.expect_owned_value(); - self.evaluated_items.push(value); + self.evaluated_items.push(value.into_inner()); Ok(self.next(context)) } } @@ -302,8 +342,9 @@ impl ObjectBuilder { self.pending = Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }); context.handle_node_as_value(self, index, RequestedValueOwnership::Owned) } - None => context - .return_owned_value(self.evaluated_entries.to_value(self.span.span_range())), + None => { + context.return_owned(self.evaluated_entries.to_value(self.span.span_range())) + } }, ) } @@ -325,7 +366,7 @@ impl EvaluationFrame for Box { Ok(match pending { Some(PendingEntryPath::OnIndexKeyBranch { access, value_node }) => { let value = item.expect_owned_value(); - let key = value.expect_string("An object key")?.value; + let key = value.into_inner().expect_string("An object key")?.value; if self.evaluated_entries.contains_key(&key) { return access.execution_err(format!("The key {} has already been set", key)); } @@ -336,7 +377,7 @@ impl EvaluationFrame for Box { context.handle_node_as_value(self, value_node, RequestedValueOwnership::Owned) } Some(PendingEntryPath::OnValueBranch { key, key_span }) => { - let value = item.expect_owned_value(); + let value = item.expect_owned_value().into_inner(); let entry = ObjectEntry { key_span, value }; self.evaluated_entries.insert(key, entry); self.next(context)? @@ -376,8 +417,8 @@ impl EvaluationFrame for UnaryOperationBuilder { context: ValueContext, item: EvaluationItem, ) -> ExecutionResult { - let value = item.expect_owned_value(); - Ok(context.return_owned_value(self.operation.evaluate(value)?)) + let value = item.expect_owned_value().into_inner(); + Ok(context.return_owned(self.operation.evaluate(value)?)) } } @@ -421,9 +462,9 @@ impl EvaluationFrame for BinaryOperationBuilder { ) -> ExecutionResult { Ok(match self.state { BinaryPath::OnLeftBranch { right } => { - let value = item.expect_owned_value(); + let value = item.expect_owned_value().into_inner(); if let Some(result) = self.operation.lazy_evaluate(&value)? { - context.return_owned_value(result) + context.return_owned(result) } else { self.state = BinaryPath::OnRightBranch { left: value }; context.handle_node_as_value( @@ -435,8 +476,8 @@ impl EvaluationFrame for BinaryOperationBuilder { } } BinaryPath::OnRightBranch { left } => { - let value = item.expect_owned_value(); - context.return_owned_value(self.operation.evaluate(left, value)?) + let value = item.expect_owned_value().into_inner(); + context.return_owned(self.operation.evaluate(left, value)?) } }) } @@ -453,8 +494,13 @@ impl ValuePropertyAccessBuilder { node: ExpressionNodeId, ) -> NextAction { let frame = Self { access }; - // TODO[access-refactor]: Change to propagate context from value, and not always clone! - let ownership = RequestedValueOwnership::Owned; + let ownership = match context.requested_ownership() { + // If we need an owned value, we can try resolving a reference and + // clone the outputted value if needed - which can be much cheaper. + // e.g. `let x = arr[0]` only copies `arr[0]` instead of the whole array. + RequestedValueOwnership::Owned => RequestedValueOwnership::CopyOnWrite, + other => other, + }; context.handle_node_as_value(frame, node, ownership) } } @@ -471,8 +517,24 @@ impl EvaluationFrame for ValuePropertyAccessBuilder { context: ValueContext, item: EvaluationItem, ) -> ExecutionResult { - let value = item.expect_owned_value(); - Ok(context.return_owned_value(value.into_property(&self.access)?)) + let value = item.expect_any_value(); + Ok(match value { + ResolvedValue::Owned(value) => { + let output = value.resolve_property(&self.access)?; + context.return_owned(output) + } + ResolvedValue::Mutable(mutable) => { + let output = mutable.resolve_property(&self.access, false)?; + context.return_mutable(output) + } + ResolvedValue::Shared { + shared, + reason_not_mutable, + } => { + let output = shared.resolve_property(&self.access)?; + context.return_shared(output, reason_not_mutable)? + } + }) } } @@ -483,7 +545,7 @@ pub(super) struct ValueIndexAccessBuilder { enum IndexPath { OnSourceBranch { index: ExpressionNodeId }, - OnIndexBranch { object: ExpressionValue }, + OnIndexBranch { source: ResolvedValue }, } impl ValueIndexAccessBuilder { @@ -497,8 +559,14 @@ impl ValueIndexAccessBuilder { access, state: IndexPath::OnSourceBranch { index }, }; - // TODO[access-refactor]: Change to propagate context from value, and not always clone - context.handle_node_as_value(frame, source, RequestedValueOwnership::Owned) + let ownership = match context.requested_ownership() { + // If we need an owned value, we can try resolving a reference and + // clone the outputted value if needed - which can be much cheaper. + // e.g. `let x = obj.xyz` only copies `obj.xyz` instead of the whole object. + RequestedValueOwnership::Owned => RequestedValueOwnership::CopyOnWrite, + other => other, + }; + context.handle_node_as_value(frame, source, ownership) } } @@ -516,18 +584,37 @@ impl EvaluationFrame for ValueIndexAccessBuilder { ) -> ExecutionResult { Ok(match self.state { IndexPath::OnSourceBranch { index } => { - let value = item.expect_owned_value(); - self.state = IndexPath::OnIndexBranch { object: value }; + let value = item.expect_any_value(); + self.state = IndexPath::OnIndexBranch { source: value }; context.handle_node_as_value( self, index, - // We can use a &index for reading values from our array - RequestedValueOwnership::SharedReference, + // This is a value, so we are _accessing it_ and can't create values + // (that's only possible in a place!) - therefore we don't need an owned key, + // and can use &index for reading values from our array + RequestedValueOwnership::Shared, ) } - IndexPath::OnIndexBranch { object } => { - let value = item.expect_shared_ref(); - context.return_owned_value(object.into_indexed(self.access, value.as_ref())?) + IndexPath::OnIndexBranch { source } => { + let index = item.expect_shared(); + let is_range = matches!(index.kind(), ValueKind::Range); + match source { + ResolvedValue::Owned(value) => { + let output = value.resolve_indexed(self.access, index.as_ref())?; + context.return_owned(output) + } + ResolvedValue::Mutable(mutable) => { + let output = mutable.resolve_indexed(self.access, index.as_ref(), false)?; + context.return_mutable(output) + } + ResolvedValue::Shared { + shared, + reason_not_mutable, + } => { + let output = shared.resolve_indexed(self.access, index.as_ref())?; + context.return_shared(output, reason_not_mutable)? + } + } } }) } @@ -554,7 +641,7 @@ impl RangeBuilder { (None, None) => match range_limits { syn::RangeLimits::HalfOpen(token) => { let inner = ExpressionRangeInner::RangeFull { token: *token }; - context.return_owned_value(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range())) } syn::RangeLimits::Closed(_) => { unreachable!( @@ -595,7 +682,7 @@ impl EvaluationFrame for RangeBuilder { item: EvaluationItem, ) -> ExecutionResult { // TODO[range-refactor]: Change to not always clone the value - let value = item.expect_owned_value(); + let value = item.expect_owned_value().into_inner(); Ok(match (self.state, self.range_limits) { (RangePath::OnLeftBranch { right: Some(right) }, _) => { self.state = RangePath::OnRightBranch { left: Some(value) }; @@ -606,7 +693,7 @@ impl EvaluationFrame for RangeBuilder { start_inclusive: value, token, }; - context.return_owned_value(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range())) } (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::Closed(_)) => { unreachable!("A closed range should have been given a right in continue_range(..)") @@ -617,7 +704,7 @@ impl EvaluationFrame for RangeBuilder { token, end_exclusive: value, }; - context.return_owned_value(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range())) } (RangePath::OnRightBranch { left: Some(left) }, syn::RangeLimits::Closed(token)) => { let inner = ExpressionRangeInner::RangeInclusive { @@ -625,21 +712,21 @@ impl EvaluationFrame for RangeBuilder { token, end_inclusive: value, }; - context.return_owned_value(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range())) } (RangePath::OnRightBranch { left: None }, syn::RangeLimits::HalfOpen(token)) => { let inner = ExpressionRangeInner::RangeTo { token, end_exclusive: value, }; - context.return_owned_value(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range())) } (RangePath::OnRightBranch { left: None }, syn::RangeLimits::Closed(token)) => { let inner = ExpressionRangeInner::RangeToInclusive { token, end_inclusive: value, }; - context.return_owned_value(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range())) } }) } @@ -685,13 +772,13 @@ impl EvaluationFrame for AssignmentBuilder { ) -> ExecutionResult { Ok(match self.state { AssignmentPath::OnValueBranch { assignee } => { - let value = item.expect_owned_value(); + let value = item.expect_owned_value().into_inner(); self.state = AssignmentPath::OnAwaitingAssignment; context.handle_node_as_assignment(self, assignee, value) } AssignmentPath::OnAwaitingAssignment => { let AssignmentCompletion { span_range } = item.expect_assignment_complete(); - context.return_owned_value(ExpressionValue::None(span_range)) + context.return_owned(ExpressionValue::None(span_range)) } }) } @@ -736,21 +823,18 @@ impl EvaluationFrame for CompoundAssignmentBuilder { ) -> ExecutionResult { Ok(match self.state { CompoundAssignmentPath::OnValueBranch { place } => { - let value = item.expect_owned_value(); + let value = item.expect_owned_value().into_inner(); self.state = CompoundAssignmentPath::OnPlaceBranch { value }; // TODO[assignment-refactor]: Resolve as LateBound, and then convert to what is needed based on the operation - context.handle_node_as_place(self, place, RequestedPlaceOwnership::MutableReference) + context.handle_node_as_place(self, place, RequestedPlaceOwnership::Mutable) } CompoundAssignmentPath::OnPlaceBranch { value } => { - let mut mutable_reference = item.expect_mutable_ref(); - let span_range = - SpanRange::new_between(mutable_reference.span_range(), value.span_range()); - mutable_reference.as_mut().handle_compound_assignment( - &self.operation, - value, - span_range, - )?; - context.return_owned_value(ExpressionValue::None(span_range)) + let mut mutable = item.expect_mutable(); + let span_range = SpanRange::new_between(mutable.span_range(), value.span_range()); + mutable + .as_mut() + .handle_compound_assignment(&self.operation, value, span_range)?; + context.return_owned(ExpressionValue::None(span_range)) } }) } @@ -859,7 +943,7 @@ impl EvaluationFrame for MethodCallBuilder { } => (evaluated_arguments_including_caller, method), }; let output = method.execute(arguments, self.method.span_range())?; - context.return_any_value(output) + context.return_any_value(output)? } }) } diff --git a/src/expressions/expression.rs b/src/expressions/expression.rs index da3b6e76..a8abdde6 100644 --- a/src/expressions/expression.rs +++ b/src/expressions/expression.rs @@ -69,7 +69,7 @@ impl Expressionable for Source { SourcePeekMatch::ExpressionBlock(_) => { UnaryAtom::Leaf(Self::Leaf::ExpressionBlock(input.parse()?)) } - SourcePeekMatch::AppendVariableBinding => { + SourcePeekMatch::AppendVariableParser => { return input .parse_err("Append variable operations are not supported in an expression") } diff --git a/src/expressions/object.rs b/src/expressions/object.rs index 59f1ac1d..b53a5c44 100644 --- a/src/expressions/object.rs +++ b/src/expressions/object.rs @@ -94,13 +94,14 @@ impl ExpressionObject { } } - pub(super) fn index_mut_with_autocreate( + pub(super) fn index_mut( &mut self, access: IndexAccess, - index: ExpressionValue, + index: &ExpressionValue, + auto_create: bool, ) -> ExecutionResult<&mut ExpressionValue> { - let key = index.expect_string("An object key")?.value; - Ok(self.mut_entry_or_create(key, access.span())) + let index: &str = index.resolve_as()?; + self.mut_entry(index.to_string(), access.span(), auto_create) } pub(super) fn index_ref( @@ -118,8 +119,13 @@ impl ExpressionObject { pub(super) fn property_mut( &mut self, access: &PropertyAccess, + auto_create: bool, ) -> ExecutionResult<&mut ExpressionValue> { - Ok(self.mut_entry_or_create(access.property.to_string(), access.property.span())) + self.mut_entry( + access.property.to_string(), + access.property.span(), + auto_create, + ) } pub(super) fn property_ref( @@ -133,19 +139,31 @@ impl ExpressionObject { Ok(&entry.value) } - fn mut_entry_or_create(&mut self, key: String, key_span: Span) -> &mut ExpressionValue { + fn mut_entry( + &mut self, + key: String, + key_span: Span, + auto_create: bool, + ) -> ExecutionResult<&mut ExpressionValue> { use std::collections::btree_map::*; - match self.entries.entry(key) { + Ok(match self.entries.entry(key) { Entry::Occupied(entry) => &mut entry.into_mut().value, Entry::Vacant(entry) => { - &mut entry - .insert(ObjectEntry { - key_span, - value: ExpressionValue::None(key_span.span_range()), - }) - .value + if auto_create { + &mut entry + .insert(ObjectEntry { + key_span, + value: ExpressionValue::None(key_span.span_range()), + }) + .value + } else { + return key_span.execution_err(format!( + "No property found for key `{}`", + entry.into_key() + )); + } } - } + }) } pub(crate) fn concat_recursive_into( diff --git a/src/expressions/value.rs b/src/expressions/value.rs index 3df33da8..d7d7f814 100644 --- a/src/expressions/value.rs +++ b/src/expressions/value.rs @@ -66,13 +66,12 @@ impl ExpressionValue { &self, new_span_range: SpanRange, ) -> ExecutionResult { - // TODO[auto-cloning]: - // if !self.kind().supports_transparent_cloning() { - // return new_span_range.execution_err(format!( - // "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .take() or .clone() explicitly.", - // self.articled_value_type() - // )); - // } + if !self.kind().supports_transparent_cloning() { + return new_span_range.execution_err(format!( + "An owned value is required, but a reference was received, and {} does not support transparent cloning. You may wish to use .take() or .clone() explicitly.", + self.articled_value_type() + )); + } Ok(self.clone().with_span_range(new_span_range)) } @@ -493,7 +492,7 @@ impl ExpressionValue { } } - pub(super) fn into_indexed(self, access: IndexAccess, index: &Self) -> ExecutionResult { + pub(crate) fn into_indexed(self, access: IndexAccess, index: &Self) -> ExecutionResult { match self { ExpressionValue::Array(array) => array.into_indexed(access, index), ExpressionValue::Object(object) => object.into_indexed(access, index), @@ -501,14 +500,15 @@ impl ExpressionValue { } } - pub(crate) fn index_mut_with_autocreate( + pub(crate) fn index_mut( &mut self, access: IndexAccess, - index: Self, + index: &Self, + auto_create: bool, ) -> ExecutionResult<&mut Self> { match self { - ExpressionValue::Array(array) => array.index_mut(access, &index), - ExpressionValue::Object(object) => object.index_mut_with_autocreate(access, index), + ExpressionValue::Array(array) => array.index_mut(access, index), + ExpressionValue::Object(object) => object.index_mut(access, index, auto_create), other => access.execution_err(format!("Cannot index into a {}", other.value_type())), } } @@ -531,9 +531,13 @@ impl ExpressionValue { } } - pub(crate) fn property_mut(&mut self, access: &PropertyAccess) -> ExecutionResult<&mut Self> { + pub(crate) fn property_mut( + &mut self, + access: &PropertyAccess, + auto_create: bool, + ) -> ExecutionResult<&mut Self> { match self { - ExpressionValue::Object(object) => object.property_mut(access), + ExpressionValue::Object(object) => object.property_mut(access, auto_create), other => access.execution_err(format!( "Cannot access properties on a {}", other.value_type() @@ -568,11 +572,6 @@ impl ExpressionValue { } } - pub(crate) fn with_span(mut self, source_span: Span) -> ExpressionValue { - *self.span_range_mut() = source_span.span_range(); - self - } - pub(crate) fn with_span_range(mut self, source_span_range: SpanRange) -> ExpressionValue { *self.span_range_mut() = source_span_range; self diff --git a/src/extensions/errors_and_spans.rs b/src/extensions/errors_and_spans.rs index a4767089..a96e5d8f 100644 --- a/src/extensions/errors_and_spans.rs +++ b/src/extensions/errors_and_spans.rs @@ -140,6 +140,12 @@ impl ToTokens for SpanRange { } } +impl From for SpanRange { + fn from(span: Span) -> Self { + Self::new_single(span) + } +} + impl HasSpanRange for SpanRange { fn span_range(&self) -> SpanRange { *self diff --git a/src/interpretation/bindings.rs b/src/interpretation/bindings.rs new file mode 100644 index 00000000..ab3fce77 --- /dev/null +++ b/src/interpretation/bindings.rs @@ -0,0 +1,482 @@ +use super::*; + +use std::cell::*; +use std::rc::Rc; + +pub(super) struct VariableContent { + value: Rc>, + #[allow(unused)] + definition_span_range: SpanRange, +} + +impl VariableContent { + pub(super) fn new(tokens: ExpressionValue, definition_span_range: SpanRange) -> Self { + Self { + value: Rc::new(RefCell::new(tokens)), + definition_span_range, + } + } + + pub(super) fn binding(&self, variable: &(impl IsVariable + ?Sized)) -> VariableBinding { + VariableBinding { + data: self.value.clone(), + variable_span_range: variable.span_range(), + } + } +} + +#[derive(Clone)] +pub(crate) struct VariableBinding { + data: Rc>, + variable_span_range: SpanRange, +} + +impl VariableBinding { + /// Gets the cloned expression value, setting the span range appropriately + /// This only works if the value can be transparently cloned + pub(crate) fn into_transparently_cloned(self) -> ExecutionResult { + self.into_shared()?.transparent_clone() + } + + /// Gets the cloned expression value, setting the span range appropriately + /// This works for any value, but may be more expensive + pub(crate) fn into_expensively_cloned(self) -> ExecutionResult { + Ok(self.into_shared()?.expensive_clone()) + } + + pub(crate) fn into_mut(self) -> ExecutionResult { + MutableValue::new_from_variable(self) + } + + pub(crate) fn into_shared(self) -> ExecutionResult { + SharedValue::new_from_variable(self) + } + + pub(crate) fn into_late_bound(self) -> ExecutionResult { + match self.clone().into_mut() { + Ok(value) => Ok(Place::Mutable { mutable: value }), + Err(ExecutionInterrupt::Error(reason_not_mutable)) => { + // If we get an error with a mutable and shared reference, a mutable reference must already exist. + // We can just propogate the error from taking the shared reference, it should be good enough. + let value = self.into_shared()?; + Ok(Place::Shared { + shared: value, + reason_not_mutable: Some(reason_not_mutable), + }) + } + // Propogate any other errors, these shouldn't happen mind + Err(err) => Err(err), + } + } +} + +/// A rough equivalent of a Rust place (lvalue), as per: +/// https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions +/// +/// In preinterpret, references are (currently) only to variables, or sub-values of variables. +/// +/// # Late Binding +/// +/// Sometimes, a value which can be accessed, but we don't yet know *how* we need to access it. +/// In this case, we attempt to load it as a Mutable place, and failing that, as a Shared place. +/// +/// ## Example of requirement +/// For example, if we have `x[a].y(z)`, we first need to resolve the type of `x[a]` to know +/// whether `x[a]` takes a shared reference, mutable reference or an owned value. +/// +/// So instead, we take the most powerful access we can have for `x[a]`, and convert it later. +pub(crate) enum Place { + Mutable { + mutable: MutableValue, + }, + Shared { + shared: SharedValue, + reason_not_mutable: Option, + }, +} + +impl HasSpanRange for VariableBinding { + fn span_range(&self) -> SpanRange { + self.variable_span_range + } +} + +pub(crate) type OwnedValue = Owned; + +/// A binding of an owned value along with a span of the whole access. +/// +/// For example, for `x.y[4]`, this would capture both: +/// * The owned value +/// * The lexical span of the tokens `x.y[4]` +pub(crate) struct Owned { + inner: T, + /// The span-range of the current binding to the value. + span_range: SpanRange, +} + +impl Owned { + pub(crate) fn new(inner: T, span_range: SpanRange) -> Self { + Self { inner, span_range } + } + + pub(crate) fn into_inner(self) -> T { + self.inner + } + + pub(crate) fn as_ref(&self) -> &T { + &self.inner + } + + pub(crate) fn as_mut(&mut self) -> &mut T { + &mut self.inner + } + + #[allow(unused)] + pub(crate) fn map(self, value_map: impl FnOnce(T, &SpanRange) -> V) -> Owned { + Owned { + inner: value_map(self.inner, &self.span_range), + span_range: self.span_range, + } + } + + pub(crate) fn try_map( + self, + value_map: impl FnOnce(T, &SpanRange) -> ExecutionResult, + ) -> ExecutionResult> { + Ok(Owned { + inner: value_map(self.inner, &self.span_range)?, + span_range: self.span_range, + }) + } + + pub(crate) fn update_span_range( + self, + span_range_map: impl FnOnce(SpanRange) -> SpanRange, + ) -> Self { + Self { + inner: self.inner, + span_range: span_range_map(self.span_range), + } + } +} + +impl OwnedValue { + pub(crate) fn resolve_indexed( + self, + access: IndexAccess, + index: &ExpressionValue, + ) -> ExecutionResult { + self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + .try_map(|value, _| value.into_indexed(access, index)) + } + + pub(crate) fn resolve_property(self, access: &PropertyAccess) -> ExecutionResult { + self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + .try_map(|value, _| value.into_property(access)) + } +} + +impl HasSpanRange for Owned { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl From for ExpressionValue { + fn from(value: OwnedValue) -> Self { + value.inner + } +} + +/// NOTE: This should be deleted when we remove the span range from ExpressionValue +impl From for OwnedValue { + fn from(value: ExpressionValue) -> Self { + let span_range = value.span_range(); + Self { + inner: value, + span_range, + } + } +} + +impl WithSpanExt for Owned { + fn with_span(self, span: Span) -> Self { + Self { + inner: self.inner, + span_range: SpanRange::new_single(span), + } + } +} + +pub(crate) type MutableValue = Mutable; + +/// A binding of a unique (mutable) reference to a value +/// (e.g. inside a variable) along with a span of the whole access. +/// +/// For example, for `x.y[4]`, this captures both: +/// * The mutable reference to the location under `x` +/// * The lexical span of the tokens `x.y[4]` +pub(crate) struct Mutable { + mut_cell: MutSubRcRefCell, + span_range: SpanRange, +} + +impl Mutable { + pub(crate) fn into_shared(self) -> Shared { + Shared { + shared_cell: self.mut_cell.into_shared(), + span_range: self.span_range, + } + } + + #[allow(unused)] + pub(crate) fn map( + self, + value_map: impl for<'a> FnOnce(&'a mut T) -> &'a mut V, + ) -> Mutable { + Mutable { + mut_cell: self.mut_cell.map(value_map), + span_range: self.span_range, + } + } + + pub(crate) fn try_map( + self, + value_map: impl for<'a, 'b> FnOnce(&'a mut T, &'b SpanRange) -> ExecutionResult<&'a mut V>, + ) -> ExecutionResult> { + Ok(Mutable { + mut_cell: self + .mut_cell + .try_map(|value| value_map(value, &self.span_range))?, + span_range: self.span_range, + }) + } + + pub(crate) fn update_span_range( + self, + span_range_map: impl FnOnce(SpanRange) -> SpanRange, + ) -> Self { + Self { + mut_cell: self.mut_cell, + span_range: span_range_map(self.span_range), + } + } +} + +impl Mutable { + pub(crate) fn new_from_owned(value: OwnedValue) -> Self { + let span_range = value.span_range; + Self { + // Unwrap is safe because it's a new refcell + mut_cell: MutSubRcRefCell::new(Rc::new(RefCell::new(value.inner))).unwrap(), + span_range, + } + } + + pub(crate) fn transparent_clone(&self) -> ExecutionResult { + let value = self.as_ref().try_transparent_clone(self.span_range)?; + Ok(OwnedValue::new(value, self.span_range)) + } + + fn new_from_variable(reference: VariableBinding) -> ExecutionResult { + Ok(Self { + mut_cell: MutSubRcRefCell::new(reference.data).map_err(|_| { + reference.variable_span_range.execution_error( + "The variable cannot be modified as it is already being modified", + ) + })?, + span_range: reference.variable_span_range, + }) + } + + pub(crate) fn into_stream(self) -> ExecutionResult> { + self.try_map(|value, span_range| match value { + ExpressionValue::Stream(stream) => Ok(&mut stream.value), + _ => span_range.execution_err("The variable is not a stream"), + }) + } + + pub(crate) fn resolve_indexed( + self, + access: IndexAccess, + index: &ExpressionValue, + auto_create: bool, + ) -> ExecutionResult { + self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + .try_map(|value, _| value.index_mut(access, index, auto_create)) + } + + pub(crate) fn resolve_property( + self, + access: &PropertyAccess, + auto_create: bool, + ) -> ExecutionResult { + self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) + .try_map(|value, _| value.property_mut(access, auto_create)) + } + + pub(crate) fn set(&mut self, content: impl ToExpressionValue) { + *self.mut_cell = content.to_value(self.span_range); + } +} + +impl AsMut for Mutable { + fn as_mut(&mut self) -> &mut T { + &mut self.mut_cell + } +} + +impl AsRef for Mutable { + fn as_ref(&self) -> &T { + &self.mut_cell + } +} + +impl HasSpanRange for Mutable { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl WithSpanExt for Mutable { + fn with_span(self, span: Span) -> Self { + Self { + mut_cell: self.mut_cell, + span_range: SpanRange::new_single(span), + } + } +} + +impl Deref for Mutable { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.mut_cell + } +} + +impl DerefMut for Mutable { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.mut_cell + } +} + +pub(crate) type SharedValue = Shared; + +/// A binding of a shared (immutable) reference to a value +/// (e.g. inside a variable) along with a span of the whole access. +/// +/// For example, for `x.y[4]`, this captures both: +/// * The mutable reference to the location under `x` +/// * The lexical span of the tokens `x.y[4]` +pub(crate) struct Shared { + shared_cell: SharedSubRcRefCell, + span_range: SpanRange, +} + +impl Shared { + pub(crate) fn try_map( + self, + value_map: impl for<'a, 'b> FnOnce(&'a T, &'b SpanRange) -> ExecutionResult<&'a V>, + ) -> ExecutionResult> { + Ok(Shared { + shared_cell: self + .shared_cell + .try_map(|value| value_map(value, &self.span_range))?, + span_range: self.span_range, + }) + } + + #[allow(unused)] + pub(crate) fn map(self, value_map: impl FnOnce(&T) -> &V) -> ExecutionResult> { + Ok(Shared { + shared_cell: self.shared_cell.map(value_map), + span_range: self.span_range, + }) + } + + pub(crate) fn update_span_range( + self, + span_range_map: impl FnOnce(SpanRange) -> SpanRange, + ) -> Self { + Self { + shared_cell: self.shared_cell, + span_range: span_range_map(self.span_range), + } + } +} + +impl Shared { + pub(crate) fn new_from_owned(value: OwnedValue) -> Self { + let span_range = value.span_range; + Self { + // Unwrap is safe because it's a new refcell + shared_cell: SharedSubRcRefCell::new(Rc::new(RefCell::new(value.inner))).unwrap(), + span_range, + } + } + + pub(crate) fn transparent_clone(&self) -> ExecutionResult { + let value = self.as_ref().try_transparent_clone(self.span_range)?; + Ok(OwnedValue::new(value, self.span_range)) + } + + pub(crate) fn expensive_clone(&self) -> OwnedValue { + let value = self.as_ref().clone().with_span_range(self.span_range); + OwnedValue::new(value, self.span_range) + } + + fn new_from_variable(reference: VariableBinding) -> ExecutionResult { + Ok(Self { + shared_cell: SharedSubRcRefCell::new(reference.data).map_err(|_| { + reference + .variable_span_range + .execution_error("The variable cannot be read as it is already being modified") + })?, + span_range: reference.variable_span_range, + }) + } + + pub(crate) fn resolve_indexed( + self, + access: IndexAccess, + index: &ExpressionValue, + ) -> ExecutionResult { + self.update_span_range(|old_span| SpanRange::new_between(old_span, access)) + .try_map(|value, _| value.index_ref(access, index)) + } + + pub(crate) fn resolve_property(self, access: &PropertyAccess) -> ExecutionResult { + self.update_span_range(|old_span| SpanRange::new_between(old_span, access.span_range())) + .try_map(|value, _| value.property_ref(access)) + } +} + +impl AsRef for Shared { + fn as_ref(&self) -> &T { + &self.shared_cell + } +} + +impl Deref for Shared { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.shared_cell + } +} + +impl HasSpanRange for Shared { + fn span_range(&self) -> SpanRange { + self.span_range + } +} + +impl WithSpanExt for Shared { + fn with_span(self, span: Span) -> Self { + Self { + shared_cell: self.shared_cell, + span_range: SpanRange::new_single(span), + } + } +} diff --git a/src/interpretation/commands/core_commands.rs b/src/interpretation/commands/core_commands.rs index acad2415..6075626a 100644 --- a/src/interpretation/commands/core_commands.rs +++ b/src/interpretation/commands/core_commands.rs @@ -94,7 +94,7 @@ impl NoOutputCommandDefinition for SetCommand { SetArguments::ExtendVariable { variable, content, .. } => { - let variable_data = variable.reference(interpreter)?; + let variable_data = variable.binding(interpreter)?; content.interpret_into( interpreter, variable_data.into_mut()?.into_stream()?.as_mut(), diff --git a/src/interpretation/interpreter.rs b/src/interpretation/interpreter.rs index 8afc41a6..e79e5dbc 100644 --- a/src/interpretation/interpreter.rs +++ b/src/interpretation/interpreter.rs @@ -1,8 +1,4 @@ -use crate::internal_prelude::*; - -use super::IsVariable; -use std::cell::*; -use std::rc::Rc; +use super::*; pub(crate) struct Interpreter { config: InterpreterConfig, @@ -25,12 +21,12 @@ impl Interpreter { self.variable_data.define_variable(variable, value) } - pub(crate) fn get_variable_reference( + pub(crate) fn resolve_variable_binding( &self, variable: &(impl IsVariable + ?Sized), make_error: impl FnOnce() -> SynError, - ) -> ExecutionResult { - self.variable_data.get_reference(variable, make_error) + ) -> ExecutionResult { + self.variable_data.resolve_binding(variable, make_error) } pub(crate) fn start_iteration_counter<'s, S: HasSpanRange>( @@ -67,42 +63,20 @@ impl VariableData { ); } - fn get_reference( + fn resolve_binding( &self, variable: &(impl IsVariable + ?Sized), make_error: impl FnOnce() -> SynError, - ) -> ExecutionResult { + ) -> ExecutionResult { let reference = self .variable_data .get(&variable.get_name()) .ok_or_else(make_error)? - .create_reference(variable); + .binding(variable); Ok(reference) } } -struct VariableContent { - value: Rc>, - #[allow(unused)] - definition_span_range: SpanRange, -} - -impl VariableContent { - fn new(tokens: ExpressionValue, definition_span_range: SpanRange) -> Self { - Self { - value: Rc::new(RefCell::new(tokens)), - definition_span_range, - } - } - - fn create_reference(&self, variable: &(impl IsVariable + ?Sized)) -> VariableReference { - VariableReference { - data: self.value.clone(), - variable_span_range: variable.span_range(), - } - } -} - pub(crate) struct IterationCounter<'a, S: HasSpanRange> { span_source: &'a S, count: usize, @@ -139,341 +113,3 @@ impl Default for InterpreterConfig { } } } - -#[derive(Clone)] -pub(crate) struct VariableReference { - data: Rc>, - variable_span_range: SpanRange, -} - -impl VariableReference { - pub(crate) fn get_value_ref(&self) -> ExecutionResult> { - self.data.try_borrow().map_err(|_| { - self.execution_error("The variable cannot be read if it is currently being modified") - }) - } - - // Gets the cloned expression value, setting the span range appropriately - pub(crate) fn get_value_transparently_cloned(&self) -> ExecutionResult { - self.get_value_ref()? - .try_transparent_clone(self.variable_span_range) - } - - // Gets the cloned expression value, setting the span range appropriately - pub(crate) fn get_value_cloned(&self) -> ExecutionResult { - Ok(self - .get_value_ref()? - .clone() - .with_span_range(self.variable_span_range)) - } - - pub(crate) fn into_mut(self) -> ExecutionResult { - MutableValue::new_from_variable(self) - } - - pub(crate) fn into_shared(self) -> ExecutionResult { - SharedValue::new_from_variable(self) - } - - pub(crate) fn into_late_bound(self) -> ExecutionResult { - match self.clone().into_mut() { - Ok(value) => Ok(Place::MutableReference { mut_ref: value }), - Err(ExecutionInterrupt::Error(reason_not_mutable)) => { - // If we get an error with a mutable and shared reference, a mutable reference must already exist. - // We can just propogate the error from taking the shared reference, it should be good enough. - let value = self.into_shared()?; - Ok(Place::SharedReference { - shared_ref: value, - reason_not_mutable: Some(reason_not_mutable), - }) - } - // Propogate any other errors, these shouldn't happen mind - Err(err) => Err(err), - } - } -} - -/// A rough equivalent of a Rust place (lvalue), as per: -/// https://doc.rust-lang.org/reference/expressions.html#place-expressions-and-value-expressions -/// -/// In preinterpret, references are (currently) only to variables, or sub-values of variables. -/// -/// # Late Binding -/// -/// Sometimes, a value which can be accessed, but we don't yet know *how* we need to access it. -/// In this case, we attempt to load it as a Mutable place, and failing that, as a Shared place. -/// -/// ## Example of requirement -/// For example, if we have `x[a].y(z)`, we first need to resolve the type of `x[a]` to know -/// whether `x[a]` takes a shared reference, mutable reference or an owned value. -/// -/// So instead, we take the most powerful access we can have for `x[a]`, and convert it later. -pub(crate) enum Place { - MutableReference { - mut_ref: MutableValue, - }, - SharedReference { - shared_ref: SharedValue, - reason_not_mutable: Option, - }, -} - -impl HasSpanRange for VariableReference { - fn span_range(&self) -> SpanRange { - self.variable_span_range - } -} - -pub(crate) type CapturedMut = MutableSubPlace; -pub(crate) type MutableValue = MutableSubPlace; - -/// A mutable reference to a value inside a variable; along with a span of the whole access. -/// For example, for `x.y[4]`, this captures both: -/// * The mutable reference to the location under `x` -/// * The lexical span of the tokens `x.y[4]` -pub(crate) struct MutableSubPlace { - mut_cell: MutSubRcRefCell, - span_range: SpanRange, -} - -impl MutableSubPlace { - pub(crate) fn into_shared(self) -> SharedSubPlace { - SharedSubPlace { - shared_cell: self.mut_cell.into_shared(), - span_range: self.span_range, - } - } - - #[allow(unused)] - pub(crate) fn map( - self, - value_map: impl for<'a> FnOnce(&'a mut T) -> &'a mut V, - ) -> MutableSubPlace { - MutableSubPlace { - mut_cell: self.mut_cell.map(value_map), - span_range: self.span_range, - } - } - - pub(crate) fn try_map( - self, - value_map: impl for<'a, 'b> FnOnce(&'a mut T, &'b SpanRange) -> ExecutionResult<&'a mut V>, - ) -> ExecutionResult> { - Ok(MutableSubPlace { - mut_cell: self - .mut_cell - .try_map(|value| value_map(value, &self.span_range))?, - span_range: self.span_range, - }) - } - - pub(crate) fn update_span_range( - self, - span_range_map: impl FnOnce(SpanRange) -> SpanRange, - ) -> Self { - Self { - mut_cell: self.mut_cell, - span_range: span_range_map(self.span_range), - } - } -} - -impl MutableSubPlace { - pub(crate) fn new_from_owned(value: ExpressionValue) -> Self { - let span_range = value.span_range(); - Self { - // Unwrap is safe because it's a new refcell - mut_cell: MutSubRcRefCell::new(Rc::new(RefCell::new(value))).unwrap(), - span_range, - } - } - - fn new_from_variable(reference: VariableReference) -> ExecutionResult { - Ok(Self { - mut_cell: MutSubRcRefCell::new(reference.data).map_err(|_| { - reference.variable_span_range.execution_error( - "The variable cannot be modified as it is already being modified", - ) - })?, - span_range: reference.variable_span_range, - }) - } - - pub(crate) fn into_stream(self) -> ExecutionResult> { - self.try_map(|value, span_range| match value { - ExpressionValue::Stream(stream) => Ok(&mut stream.value), - _ => span_range.execution_err("The variable is not a stream"), - }) - } - - pub(crate) fn resolve_indexed_with_autocreate( - self, - access: IndexAccess, - index: ExpressionValue, - ) -> ExecutionResult { - self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) - .try_map(|value, _| value.index_mut_with_autocreate(access, index)) - } - - pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { - self.update_span_range(|span_range| SpanRange::new_between(span_range, access.span_range())) - .try_map(|value, _| value.property_mut(&access)) - } - - pub(crate) fn set(&mut self, content: impl ToExpressionValue) { - *self.mut_cell = content.to_value(self.span_range); - } -} - -impl AsMut for MutableSubPlace { - fn as_mut(&mut self) -> &mut T { - &mut self.mut_cell - } -} - -impl AsRef for MutableSubPlace { - fn as_ref(&self) -> &T { - &self.mut_cell - } -} - -impl HasSpanRange for MutableSubPlace { - fn span_range(&self) -> SpanRange { - self.span_range - } -} - -impl WithSpanExt for MutableSubPlace { - fn with_span(self, span: Span) -> Self { - Self { - mut_cell: self.mut_cell, - span_range: SpanRange::new_single(span), - } - } -} - -impl Deref for MutableSubPlace { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.mut_cell - } -} - -impl DerefMut for MutableSubPlace { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.mut_cell - } -} - -pub(crate) type CapturedRef = SharedSubPlace; -pub(crate) type SharedValue = SharedSubPlace; - -/// A mutable reference to a value inside a variable; along with a span of the whole access. -/// For example, for `x.y[4]`, this captures both: -/// * The mutable reference to the location under `x` -/// * The lexical span of the tokens `x.y[4]` -pub(crate) struct SharedSubPlace { - shared_cell: SharedSubRcRefCell, - span_range: SpanRange, -} - -impl SharedSubPlace { - pub(crate) fn try_map( - self, - value_map: impl for<'a, 'b> FnOnce(&'a T, &'b SpanRange) -> ExecutionResult<&'a V>, - ) -> ExecutionResult> { - Ok(SharedSubPlace { - shared_cell: self - .shared_cell - .try_map(|value| value_map(value, &self.span_range))?, - span_range: self.span_range, - }) - } - - #[allow(unused)] - pub(crate) fn map( - self, - value_map: impl FnOnce(&T) -> &V, - ) -> ExecutionResult> { - Ok(SharedSubPlace { - shared_cell: self.shared_cell.map(value_map), - span_range: self.span_range, - }) - } - - pub(crate) fn update_span_range( - self, - span_range_map: impl FnOnce(SpanRange) -> SpanRange, - ) -> Self { - Self { - shared_cell: self.shared_cell, - span_range: span_range_map(self.span_range), - } - } -} - -impl SharedSubPlace { - pub(crate) fn new_from_owned(value: ExpressionValue) -> Self { - let span_range = value.span_range(); - Self { - // Unwrap is safe because it's a new refcell - shared_cell: SharedSubRcRefCell::new(Rc::new(RefCell::new(value))).unwrap(), - span_range, - } - } - - fn new_from_variable(reference: VariableReference) -> ExecutionResult { - Ok(Self { - shared_cell: SharedSubRcRefCell::new(reference.data).map_err(|_| { - reference - .variable_span_range - .execution_error("The variable cannot be read as it is already being modified") - })?, - span_range: reference.variable_span_range, - }) - } - - pub(crate) fn resolve_indexed( - self, - access: IndexAccess, - index: &ExpressionValue, - ) -> ExecutionResult { - self.update_span_range(|old_span| SpanRange::new_between(old_span, access)) - .try_map(|value, _| value.index_ref(access, index)) - } - - pub(crate) fn resolve_property(self, access: PropertyAccess) -> ExecutionResult { - self.update_span_range(|old_span| SpanRange::new_between(old_span, access.span_range())) - .try_map(|value, _| value.property_ref(&access)) - } -} - -impl AsRef for SharedSubPlace { - fn as_ref(&self) -> &T { - &self.shared_cell - } -} - -impl Deref for SharedSubPlace { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.shared_cell - } -} - -impl HasSpanRange for SharedSubPlace { - fn span_range(&self) -> SpanRange { - self.span_range - } -} - -impl WithSpanExt for SharedSubPlace { - fn with_span(self, span: Span) -> Self { - Self { - shared_cell: self.shared_cell, - span_range: SpanRange::new_single(span), - } - } -} diff --git a/src/interpretation/mod.rs b/src/interpretation/mod.rs index 1300bb4a..3019756c 100644 --- a/src/interpretation/mod.rs +++ b/src/interpretation/mod.rs @@ -1,3 +1,4 @@ +mod bindings; mod command; mod command_arguments; mod commands; @@ -8,6 +9,9 @@ mod source_code_block; mod source_stream; mod variable; +// Marked as use for expression sub-modules to use with a `use super::*` statement +use crate::internal_prelude::*; +pub(crate) use bindings::*; pub(crate) use command::*; pub(crate) use command_arguments::*; pub(crate) use interpret_traits::*; diff --git a/src/interpretation/source_stream.rs b/src/interpretation/source_stream.rs index b5bb006b..06099c2a 100644 --- a/src/interpretation/source_stream.rs +++ b/src/interpretation/source_stream.rs @@ -59,7 +59,7 @@ impl Parse for SourceItem { SourcePeekMatch::ExplicitTransformStream | SourcePeekMatch::Transformer(_) => { return input.parse_err("Destructurings are not supported here. If this wasn't intended to be a destructuring, replace @ with [!raw! @]"); } - SourcePeekMatch::AppendVariableBinding => { + SourcePeekMatch::AppendVariableParser => { return input.parse_err("Destructurings are not supported here."); } SourcePeekMatch::Punct(_) => SourceItem::Punct(input.parse_any_punct()?), diff --git a/src/interpretation/variable.rs b/src/interpretation/variable.rs index 79a1b47d..d46efdf5 100644 --- a/src/interpretation/variable.rs +++ b/src/interpretation/variable.rs @@ -12,7 +12,10 @@ pub(crate) trait IsVariable: HasSpanRange { } fn get_cloned_value(&self, interpreter: &Interpreter) -> ExecutionResult { - self.reference(interpreter)?.get_value_cloned() + Ok(self + .binding(interpreter)? + .into_expensively_cloned()? + .into()) } fn substitute_into( @@ -21,15 +24,15 @@ pub(crate) trait IsVariable: HasSpanRange { grouping: Grouping, output: &mut OutputStream, ) -> ExecutionResult<()> { - self.reference(interpreter)?.get_value_ref()?.output_to( + self.binding(interpreter)?.into_shared()?.output_to( grouping, output, StreamOutputBehaviour::Standard, ) } - fn reference(&self, interpreter: &Interpreter) -> ExecutionResult { - interpreter.get_variable_reference(self, || { + fn binding(&self, interpreter: &Interpreter) -> ExecutionResult { + interpreter.resolve_variable_binding(self, || { self.error("The variable does not already exist in the current scope") }) } diff --git a/src/misc/parse_traits.rs b/src/misc/parse_traits.rs index d5117345..986cdd0f 100644 --- a/src/misc/parse_traits.rs +++ b/src/misc/parse_traits.rs @@ -16,7 +16,7 @@ pub(crate) enum SourcePeekMatch { Command(Option), ExpressionBlock(Grouping), Variable(Grouping), - AppendVariableBinding, + AppendVariableParser, ExplicitTransformStream, Transformer(Option), Group(Delimiter), @@ -69,14 +69,14 @@ fn detect_preinterpret_grammar(cursor: syn::buffer::Cursor) -> SourcePeekMatch { } if let Some((_, next)) = next.punct_matching('>') { if next.punct_matching('>').is_some() { - return SourcePeekMatch::AppendVariableBinding; + return SourcePeekMatch::AppendVariableParser; } } } } if let Some((_, next)) = next.punct_matching('>') { if next.punct_matching('>').is_some() { - return SourcePeekMatch::AppendVariableBinding; + return SourcePeekMatch::AppendVariableParser; } } if next.group_matching(Delimiter::Parenthesis).is_some() { diff --git a/src/transformation/exact_stream.rs b/src/transformation/exact_stream.rs index e575e454..432ebb8a 100644 --- a/src/transformation/exact_stream.rs +++ b/src/transformation/exact_stream.rs @@ -71,7 +71,7 @@ impl Parse for ExactItem { SourcePeekMatch::Command(_) => Self::ExactCommandOutput(input.parse()?), SourcePeekMatch::Variable(_) => Self::ExactVariableOutput(input.parse()?), SourcePeekMatch::ExpressionBlock(_) => Self::ExactExpressionBlock(input.parse()?), - SourcePeekMatch::AppendVariableBinding => { + SourcePeekMatch::AppendVariableParser => { return input .parse_err("Append variable bindings are not supported in an EXACT stream") } diff --git a/src/transformation/mod.rs b/src/transformation/mod.rs index 6ad5d561..1c02bdf7 100644 --- a/src/transformation/mod.rs +++ b/src/transformation/mod.rs @@ -6,7 +6,7 @@ mod transform_stream; mod transformation_traits; mod transformer; mod transformers; -mod variable_binding; +mod variable_parser; pub(crate) use exact_stream::*; #[allow(unused)] @@ -17,4 +17,4 @@ pub(crate) use transform_stream::*; pub(crate) use transformation_traits::*; pub(crate) use transformer::*; pub(crate) use transformers::*; -pub(crate) use variable_binding::*; +pub(crate) use variable_parser::*; diff --git a/src/transformation/parse_utilities.rs b/src/transformation/parse_utilities.rs index 86fe984d..8d1290c0 100644 --- a/src/transformation/parse_utilities.rs +++ b/src/transformation/parse_utilities.rs @@ -72,7 +72,7 @@ impl ParseUntil { | SourcePeekMatch::Variable(_) | SourcePeekMatch::Transformer(_) | SourcePeekMatch::ExplicitTransformStream - | SourcePeekMatch::AppendVariableBinding + | SourcePeekMatch::AppendVariableParser | SourcePeekMatch::ExpressionBlock(_) => { return input .span() diff --git a/src/transformation/transform_stream.rs b/src/transformation/transform_stream.rs index abb59c16..f262c523 100644 --- a/src/transformation/transform_stream.rs +++ b/src/transformation/transform_stream.rs @@ -39,7 +39,7 @@ impl HandleTransformation for TransformSegment { #[derive(Clone)] pub(crate) enum TransformItem { Command(Command), - Variable(VariableBinding), + Variable(VariableParserKind), ExpressionBlock(ExpressionBlock), Transformer(Transformer), TransformStreamInput(ExplicitTransformStream), @@ -59,8 +59,8 @@ impl TransformItem { ) -> ParseResult { Ok(match input.peek_grammar() { SourcePeekMatch::Command(_) => Self::Command(input.parse()?), - SourcePeekMatch::Variable(_) | SourcePeekMatch::AppendVariableBinding => { - Self::Variable(VariableBinding::parse_until::(input)?) + SourcePeekMatch::Variable(_) | SourcePeekMatch::AppendVariableParser => { + Self::Variable(VariableParserKind::parse_until::(input)?) } SourcePeekMatch::ExpressionBlock(_) => Self::ExpressionBlock(input.parse()?), SourcePeekMatch::Group(_) => Self::ExactGroup(input.parse()?), @@ -252,7 +252,7 @@ impl HandleTransformation for ExplicitTransformStream { ExplicitTransformStreamArguments::ExtendToVariable { variable, content, .. } => { - let reference = variable.reference(interpreter)?; + let reference = variable.binding(interpreter)?; content.handle_transform( input, interpreter, diff --git a/src/transformation/variable_binding.rs b/src/transformation/variable_parser.rs similarity index 79% rename from src/transformation/variable_binding.rs rename to src/transformation/variable_parser.rs index 580df19f..b65eea6c 100644 --- a/src/transformation/variable_binding.rs +++ b/src/transformation/variable_parser.rs @@ -11,7 +11,7 @@ use crate::internal_prelude::*; /// * `#..>>..x` - Reads a stream, appends a stream #[derive(Clone)] #[allow(unused)] -pub(crate) enum VariableBinding { +pub(crate) enum VariableParserKind { /// #x - Reads a token tree, writes a stream (opposite of #x) Grouped { marker: Token![#], name: Ident }, /// #..x - Reads a stream, writes a stream (opposite of #..x) @@ -53,10 +53,10 @@ pub(crate) enum VariableBinding { }, } -impl VariableBinding { +impl VariableParserKind { #[allow(unused)] pub(crate) fn parse_only_unflattened_input(input: ParseStream) -> ParseResult { - let variable: VariableBinding = Self::parse_until::(input)?; + let variable: VariableParserKind = Self::parse_until::(input)?; if variable.is_flattened_input() { return variable .span_range() @@ -129,46 +129,46 @@ impl VariableBinding { } } -impl IsVariable for VariableBinding { +impl IsVariable for VariableParserKind { fn get_name(&self) -> String { let name_ident = match self { - VariableBinding::Grouped { name, .. } => name, - VariableBinding::Flattened { name, .. } => name, - VariableBinding::GroupedAppendGrouped { name, .. } => name, - VariableBinding::GroupedAppendFlattened { name, .. } => name, - VariableBinding::FlattenedAppendGrouped { name, .. } => name, - VariableBinding::FlattenedAppendFlattened { name, .. } => name, + VariableParserKind::Grouped { name, .. } => name, + VariableParserKind::Flattened { name, .. } => name, + VariableParserKind::GroupedAppendGrouped { name, .. } => name, + VariableParserKind::GroupedAppendFlattened { name, .. } => name, + VariableParserKind::FlattenedAppendGrouped { name, .. } => name, + VariableParserKind::FlattenedAppendFlattened { name, .. } => name, }; name_ident.to_string() } } -impl HasSpanRange for VariableBinding { +impl HasSpanRange for VariableParserKind { fn span_range(&self) -> SpanRange { let (marker, name) = match self { - VariableBinding::Grouped { marker, name, .. } => (marker, name), - VariableBinding::Flattened { marker, name, .. } => (marker, name), - VariableBinding::GroupedAppendGrouped { marker, name, .. } => (marker, name), - VariableBinding::GroupedAppendFlattened { marker, name, .. } => (marker, name), - VariableBinding::FlattenedAppendGrouped { marker, name, .. } => (marker, name), - VariableBinding::FlattenedAppendFlattened { marker, name, .. } => (marker, name), + VariableParserKind::Grouped { marker, name, .. } => (marker, name), + VariableParserKind::Flattened { marker, name, .. } => (marker, name), + VariableParserKind::GroupedAppendGrouped { marker, name, .. } => (marker, name), + VariableParserKind::GroupedAppendFlattened { marker, name, .. } => (marker, name), + VariableParserKind::FlattenedAppendGrouped { marker, name, .. } => (marker, name), + VariableParserKind::FlattenedAppendFlattened { marker, name, .. } => (marker, name), }; SpanRange::new_between(marker.span, name.span()) } } -impl VariableBinding { +impl VariableParserKind { pub(crate) fn is_flattened_input(&self) -> bool { matches!( self, - VariableBinding::Flattened { .. } - | VariableBinding::FlattenedAppendGrouped { .. } - | VariableBinding::FlattenedAppendFlattened { .. } + VariableParserKind::Flattened { .. } + | VariableParserKind::FlattenedAppendGrouped { .. } + | VariableParserKind::FlattenedAppendFlattened { .. } ) } } -impl HandleTransformation for VariableBinding { +impl HandleTransformation for VariableParserKind { fn handle_transform( &self, input: ParseStream, @@ -176,37 +176,37 @@ impl HandleTransformation for VariableBinding { _: &mut OutputStream, ) -> ExecutionResult<()> { match self { - VariableBinding::Grouped { .. } => { + VariableParserKind::Grouped { .. } => { let content = input.parse::()?.into_interpreted(); self.define_coerced(interpreter, content); } - VariableBinding::Flattened { until, .. } => { + VariableParserKind::Flattened { until, .. } => { let mut content = OutputStream::new(); until.handle_parse_into(input, &mut content)?; self.define_coerced(interpreter, content); } - VariableBinding::GroupedAppendGrouped { .. } => { - let reference = self.reference(interpreter)?; + VariableParserKind::GroupedAppendGrouped { .. } => { + let reference = self.binding(interpreter)?; input .parse::()? .push_as_token_tree(reference.into_mut()?.into_stream()?.as_mut()); } - VariableBinding::GroupedAppendFlattened { .. } => { - let reference = self.reference(interpreter)?; + VariableParserKind::GroupedAppendFlattened { .. } => { + let reference = self.binding(interpreter)?; input .parse::()? .flatten_into(reference.into_mut()?.into_stream()?.as_mut()); } - VariableBinding::FlattenedAppendGrouped { marker, until, .. } => { - let reference = self.reference(interpreter)?; + VariableParserKind::FlattenedAppendGrouped { marker, until, .. } => { + let reference = self.binding(interpreter)?; reference.into_mut()?.into_stream()?.as_mut().push_grouped( |inner| until.handle_parse_into(input, inner), Delimiter::None, marker.span, )?; } - VariableBinding::FlattenedAppendFlattened { until, .. } => { - let reference = self.reference(interpreter)?; + VariableParserKind::FlattenedAppendFlattened { until, .. } => { + let reference = self.binding(interpreter)?; until.handle_parse_into(input, reference.into_mut()?.into_stream()?.as_mut())?; } } diff --git a/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr b/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr index 8dd508e7..38bfe725 100644 --- a/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr +++ b/tests/compilation_failures/core/extend_variable_and_then_read_it.stderr @@ -1,4 +1,4 @@ -error: The variable cannot be read if it is currently being modified +error: The variable cannot be read as it is already being modified --> tests/compilation_failures/core/extend_variable_and_then_read_it.rs:6:35 | 6 | [!set! #variable += World #variable] diff --git a/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr b/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr index 467936ed..fe0f4c69 100644 --- a/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr +++ b/tests/compilation_failures/expressions/array_place_destructure_multiple_muts.stderr @@ -1,4 +1,4 @@ -error: The variable cannot be read if it is currently being modified +error: The variable cannot be read as it is already being modified --> tests/compilation_failures/expressions/array_place_destructure_multiple_muts.rs:5:33 | 5 | #(let arr = [0, 1]; arr[arr[1]] = 3) diff --git a/tests/expressions.rs b/tests/expressions.rs index db071002..ebbaf3bc 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -90,6 +90,7 @@ fn test_expression_precedence() { } #[test] +#[allow(clippy::zero_prefixed_literal)] fn test_very_long_expression_works() { preinterpret_assert_eq!( { @@ -247,42 +248,42 @@ fn test_array_indexing() { preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[..].debug_string() + x.take()[..].debug_string() ), "[1, 2, 3, 4, 5]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[0..0].debug_string() + x.take()[0..0].debug_string() ), "[]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[2..=2].debug_string() + x.take()[2..=2].debug_string() ), "[3]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[..=2].debug_string() + x.take()[..=2].debug_string() ), "[1, 2, 3]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[..4].debug_string() + x.take()[..4].debug_string() ), "[1, 2, 3, 4]" ); preinterpret_assert_eq!( #( let x = [1, 2, 3, 4, 5]; - x[2..].debug_string() + x.take()[2..].debug_string() ), "[3, 4, 5]" ); @@ -295,7 +296,7 @@ fn test_array_place_destructurings() { #( let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; - [a, b, _, _, c] = x; + [a, b, _, _, c] = x.take(); [a, b, c].debug_string() ), "[1, 2, 5]" @@ -304,7 +305,7 @@ fn test_array_place_destructurings() { #( let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; - [a, b, c, ..] = x; + [a, b, c, ..] = x.take(); [a, b, c].debug_string() ), "[1, 2, 3]" @@ -313,7 +314,7 @@ fn test_array_place_destructurings() { #( let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; - [.., a, b] = x; + [.., a, b] = x.take(); [a, b, c].debug_string() ), "[4, 5, 0]" @@ -322,7 +323,7 @@ fn test_array_place_destructurings() { #( let a = 0; let b = 0; let c = 0; let x = [1, 2, 3, 4, 5]; - [a, .., b, c] = x; + [a, .., b, c] = x.take(); [a, b, c].debug_string() ), "[1, 4, 5]" @@ -347,7 +348,7 @@ fn test_array_place_destructurings() { let _ = c = [a[2], _] = [4, 5]; let _ = a[1] += 2; let _ = b = 2; - [a, b, c].debug_string() + [a.take(), b, c].debug_string() ), "[[0, 2, 4, 0, 0], 2, None]" ); @@ -411,7 +412,7 @@ fn test_array_pattern_destructurings() { preinterpret_assert_eq!( #( let [a, .., b, c] = [[1, "a"], 2, 3, 4, 5]; - [a, b, c].debug_string() + [a.take(), b, c].debug_string() ), r#"[[1, "a"], 4, 5]"# ); @@ -423,7 +424,7 @@ fn test_objects() { #( let a = {}; let b = "Hello"; - let x = { a, hello: 1, ["world"]: 2, b }; + let x = { a: a.clone(), hello: 1, ["world"]: 2, b }; x["x y z"] = 4; x["z\" test"] = {}; x.y = 5; @@ -462,7 +463,7 @@ fn test_objects() { preinterpret_assert_eq!( #( let { a, y: [_, b], ["c"]: c, [r#"two "words"#]: x, z } = { a: 1, y: [5, 7], ["two \"words"]: {}, }; - { a, b, c, x, z }.debug_string() + { a, b, c, x: x.take(), z }.debug_string() ), r#"{ a: 1, b: 7, c: None, x: {}, z: None }"# ); diff --git a/tests/tokens.rs b/tests/tokens.rs index 46e95faa..69b02644 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -204,8 +204,8 @@ fn complex_cases_for_intersperse_and_input_types() { let final_separator = [" and "]; let add_trailing = false; [!intersperse! { - separator: separator, - final_separator: final_separator, + separator: separator.take(), + final_separator: final_separator.take(), add_trailing: add_trailing, items: people, }] as string @@ -343,7 +343,7 @@ fn test_zip() { #( let longer = [!stream! A B C D]; let shorter = [1, 2, 3]; - [!zip_truncated! [longer, shorter]].debug_string() + [!zip_truncated! [longer, shorter.take()]].debug_string() ), r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); @@ -351,7 +351,7 @@ fn test_zip() { #( let letters = [!stream! A B C]; let numbers = [1, 2, 3]; - [!zip! [letters, numbers]].debug_string() + [!zip! [letters, numbers.take()]].debug_string() ), r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); @@ -359,8 +359,8 @@ fn test_zip() { #( let letters = [!stream! A B C]; let numbers = [1, 2, 3]; - let combined = [letters, numbers]; - [!zip! combined].debug_string() + let combined = [letters, numbers.take()]; + [!zip! combined.take()].debug_string() ), r#"[[[!stream! A], 1], [[!stream! B], 2], [[!stream! C], 3]]"#, ); @@ -368,7 +368,7 @@ fn test_zip() { #( [!set! #letters = A B C]; let numbers = [1, 2, 3]; - [!zip! { number: numbers, letter: letters }].debug_string() + [!zip! { number: numbers.take(), letter: letters }].debug_string() ), r#"[{ letter: [!stream! A], number: 1 }, { letter: [!stream! B], number: 2 }, { letter: [!stream! C], number: 3 }]"#, ); @@ -384,12 +384,12 @@ fn test_zip_with_for() { #(let flags = ["🇫🇷", "🇩🇪", "🇮🇹"]) [!set! #capitals = "Paris" "Berlin" "Rome"] #(let facts = []) - [!for! [country, flag, capital] in [!zip! [countries, flags, capitals]] { + [!for! [country, flag, capital] in [!zip! [countries, flags.take(), capitals]] { #(facts.push([!string! "=> The capital of " #country " is " #capital " and its flag is " #flag])) }] #("The facts are:\n" + [!intersperse! { - items: facts, + items: facts.take(), separator: ["\n"], }] as string + "\n") }, From eb7d23d3b1e02d20e6853f2b92cb2381a636b61c Mon Sep 17 00:00:00 2001 From: David Edey Date: Sat, 20 Sep 2025 19:12:22 +0100 Subject: [PATCH 126/126] feature: Added `swap` to test mutable references --- CHANGELOG.md | 11 ++- src/expressions/evaluation/evaluator.rs | 49 ++++++------- src/expressions/evaluation/node_conversion.rs | 12 ++-- src/expressions/evaluation/place_frames.rs | 9 ++- src/expressions/evaluation/type_resolution.rs | 5 ++ src/expressions/evaluation/value_frames.rs | 68 +++++++++---------- .../late_bound_owned_to_mutable copy.rs | 11 +++ .../late_bound_owned_to_mutable copy.stderr | 5 ++ .../expressions/owned_to_mutable.rs | 11 +++ .../expressions/owned_to_mutable.stderr | 5 ++ tests/expressions.rs | 10 ++- 11 files changed, 127 insertions(+), 69 deletions(-) create mode 100644 tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.rs create mode 100644 tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.stderr create mode 100644 tests/compilation_failures/expressions/owned_to_mutable.rs create mode 100644 tests/compilation_failures/expressions/owned_to_mutable.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index 218c059b..8e199194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,13 +106,20 @@ Inside a transform stream, the following grammar is supported: ### To come * Method calls continued + * Create `CopyOnWrite<..>` in `bindings.rs`, add `CopyOnWriteValue` support to `wrap_method!` and change it so that `debug_string` takes CowValue + * ... maybe we just use `CopyOnWriteValue` everywhere instead of `OwnedValue`?? + * Check if we actually need `RequestedPlaceOwnership::SharedReference` or `RequestedPlaceOwnership::LateBound` and potentially remove them if not * Consider how to align: * `ResolvedValue::into_mutable` (used by LateBound Owned => Mutable Arg; such as a method object) * `context::return_owned` (used by Owned returned when Mutable requested, such as a method argument) - * Create `CopyOnWrite<..>` in `bindings.rs`, add `CopyOnWriteValue` support to `wrap_method!` and change it so that `debug_string` takes CowValue + * CURRENTLY WE HAVE: + * Mapping "context.return_shared(X)" => requested kind + * Late bound method arguments => needed argument + * PERHAPS WE HAVE: + * Late bound => Resolve similar to context.return_X + * In method call, we do expect/assert it's the right type * Add tests for TODO[access-refactor] (i.e. changing places to resolve correctly) - * Check if we actually need `RequestedPlaceOwnership::SharedReference` or `RequestedPlaceOwnership::LateBound` and potentially remove them if not * Add more impl_resolvable_argument_for * Scrap `#>>x` etc in favour of `#(x.push(@TOKEN))` * TODO[range-refactor] & some kind of more thought through typed reference support - e.g. slices, mutable slices? diff --git a/src/expressions/evaluation/evaluator.rs b/src/expressions/evaluation/evaluator.rs index 920efd3b..9b49c106 100644 --- a/src/expressions/evaluation/evaluator.rs +++ b/src/expressions/evaluation/evaluator.rs @@ -103,11 +103,11 @@ pub(super) struct NextAction(NextActionInner); impl NextAction { pub(super) fn return_owned(value: OwnedValue) -> Self { - NextActionInner::HandleReturnedItem(EvaluationItem::OwnedValue(value)).into() + NextActionInner::HandleReturnedItem(EvaluationItem::Owned(value)).into() } - pub(super) fn return_mutable(mut_ref: MutableValue) -> Self { - NextActionInner::HandleReturnedItem(EvaluationItem::MutableReference { mutable: mut_ref }) + pub(super) fn return_mutable(mutable: MutableValue) -> Self { + NextActionInner::HandleReturnedItem(EvaluationItem::Mutable { mutable }) .into() } @@ -115,7 +115,7 @@ impl NextAction { shared: SharedValue, reason_not_mutable: Option, ) -> Self { - NextActionInner::HandleReturnedItem(EvaluationItem::SharedReference { + NextActionInner::HandleReturnedItem(EvaluationItem::Shared { shared, reason_not_mutable, }) @@ -141,14 +141,14 @@ impl From for NextAction { } pub(super) enum EvaluationItem { - OwnedValue(OwnedValue), - SharedReference { + Owned(OwnedValue), + Shared { shared: SharedValue, /// This is only populated if we request a "late bound" reference, and fail to resolve /// a mutable reference. reason_not_mutable: Option, }, - MutableReference { + Mutable { mutable: MutableValue, }, AssignmentCompletion(AssignmentCompletion), @@ -157,24 +157,24 @@ pub(super) enum EvaluationItem { impl EvaluationItem { pub(super) fn expect_owned_value(self) -> OwnedValue { match self { - EvaluationItem::OwnedValue(value) => value, + EvaluationItem::Owned(value) => value, _ => panic!("expect_owned_value() called on a non-owned-value EvaluationItem"), } } pub(super) fn expect_cow_value(self) -> CowValue { match self { - EvaluationItem::OwnedValue(value) => CowValue::Owned(value), - EvaluationItem::SharedReference { shared, .. } => CowValue::Shared(shared), + EvaluationItem::Owned(value) => CowValue::Owned(value), + EvaluationItem::Shared { shared, .. } => CowValue::Shared(shared), _ => panic!("expect_cow_value() called on a non-cow-value EvaluationItem"), } } pub(super) fn expect_any_value(self) -> ResolvedValue { match self { - EvaluationItem::OwnedValue(value) => ResolvedValue::Owned(value), - EvaluationItem::MutableReference { mutable } => ResolvedValue::Mutable(mutable), - EvaluationItem::SharedReference { shared, .. } => ResolvedValue::Shared { + EvaluationItem::Owned(value) => ResolvedValue::Owned(value), + EvaluationItem::Mutable { mutable } => ResolvedValue::Mutable(mutable), + EvaluationItem::Shared { shared, .. } => ResolvedValue::Shared { shared, reason_not_mutable: None, }, @@ -184,8 +184,8 @@ impl EvaluationItem { pub(super) fn expect_any_place(self) -> Place { match self { - EvaluationItem::MutableReference { mutable } => Place::Mutable { mutable }, - EvaluationItem::SharedReference { + EvaluationItem::Mutable { mutable } => Place::Mutable { mutable }, + EvaluationItem::Shared { shared, reason_not_mutable, } => Place::Shared { @@ -198,7 +198,7 @@ impl EvaluationItem { pub(super) fn expect_shared(self) -> SharedValue { match self { - EvaluationItem::SharedReference { shared, .. } => shared, + EvaluationItem::Shared { shared, .. } => shared, _ => { panic!("expect_shared() called on a non-shared-reference EvaluationItem") } @@ -207,7 +207,7 @@ impl EvaluationItem { pub(super) fn expect_mutable(self) -> MutableValue { match self { - EvaluationItem::MutableReference { mutable } => mutable, + EvaluationItem::Mutable { mutable } => mutable, _ => panic!("expect_mutable() called on a non-mutable-place EvaluationItem"), } } @@ -345,7 +345,7 @@ impl<'a> Context<'a, ValueType> { pub(super) fn return_any_value(self, value: ResolvedValue) -> ExecutionResult { Ok(match value { - ResolvedValue::Owned(owned) => self.return_owned(owned), + ResolvedValue::Owned(owned) => self.return_owned(owned)?, ResolvedValue::Mutable(mutable) => self.return_mutable(mutable), ResolvedValue::Shared { shared, @@ -364,9 +364,9 @@ impl<'a> Context<'a, ValueType> { }) } - pub(super) fn return_owned(self, value: impl Into) -> NextAction { + pub(super) fn return_owned(self, value: impl Into) -> ExecutionResult { let value = value.into(); - match self.request { + Ok(match self.request { RequestedValueOwnership::LateBound | RequestedValueOwnership::CopyOnWrite | RequestedValueOwnership::Owned => NextAction::return_owned(value), @@ -374,15 +374,16 @@ impl<'a> Context<'a, ValueType> { NextAction::return_shared(Shared::new_from_owned(value), None) } RequestedValueOwnership::Mutable => { - NextAction::return_mutable(Mutable::new_from_owned(value)) + // This aligns with ResolveValue::into_mutable + return value.execution_err("A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference.") } - } + }) } - pub(super) fn return_mutable(self, mut_ref: MutableValue) -> NextAction { + pub(super) fn return_mutable(self, mutable: MutableValue) -> NextAction { match self.request { RequestedValueOwnership::LateBound - | RequestedValueOwnership::Mutable => NextAction::return_mutable(mut_ref), + | RequestedValueOwnership::Mutable => NextAction::return_mutable(mutable), RequestedValueOwnership::Owned | RequestedValueOwnership::CopyOnWrite | RequestedValueOwnership::Shared => panic!("Returning a mutable reference should only be used when the requested ownership is mutable or late-bound"), diff --git a/src/expressions/evaluation/node_conversion.rs b/src/expressions/evaluation/node_conversion.rs index bf2da4b2..aa280fdb 100644 --- a/src/expressions/evaluation/node_conversion.rs +++ b/src/expressions/evaluation/node_conversion.rs @@ -11,7 +11,7 @@ impl ExpressionNode { match leaf { SourceExpressionLeaf::Command(command) => { // TODO[interpret_to_value]: Allow command to return a reference - context.return_owned(command.clone().interpret_to_value(interpreter)?) + context.return_owned(command.clone().interpret_to_value(interpreter)?)? } SourceExpressionLeaf::Discarded(token) => { return token.execution_err("This cannot be used in a value expression"); @@ -30,22 +30,22 @@ impl ExpressionNode { context.return_mutable(variable_ref.into_mut()?) } RequestedValueOwnership::Owned => { - context.return_owned(variable_ref.into_transparently_cloned()?) + context.return_owned(variable_ref.into_transparently_cloned()?)? } } } SourceExpressionLeaf::ExpressionBlock(block) => { // TODO[interpret_to_value]: Allow block to return reference - context.return_owned(block.interpret_to_value(interpreter)?) + context.return_owned(block.interpret_to_value(interpreter)?)? } - SourceExpressionLeaf::Value(value) => context.return_owned(value.clone()), + SourceExpressionLeaf::Value(value) => context.return_owned(value.clone())?, } } ExpressionNode::Grouped { delim_span, inner } => { GroupBuilder::start(context, delim_span, *inner) } ExpressionNode::Array { brackets, items } => { - ArrayBuilder::start(context, brackets, items) + ArrayBuilder::start(context, brackets, items)? } ExpressionNode::Object { braces, entries } => { ObjectBuilder::start(context, braces, entries)? @@ -72,7 +72,7 @@ impl ExpressionNode { left, range_limits, right, - } => RangeBuilder::start(context, left, range_limits, right), + } => RangeBuilder::start(context, left, range_limits, right)?, ExpressionNode::Assignment { assignee, equals_token, diff --git a/src/expressions/evaluation/place_frames.rs b/src/expressions/evaluation/place_frames.rs index 6dc0956b..fe8cfb57 100644 --- a/src/expressions/evaluation/place_frames.rs +++ b/src/expressions/evaluation/place_frames.rs @@ -1,3 +1,8 @@ +//! A preinterpret place frame is just used for the target of an assignment. +//! The name is inspired by Rust places, but it is a subtly different concept. +//! +//! They're similar to mutable references, but behave slightly differently: +//! * They can create entries in objects, e.g. `x["new_key"] = value`` #![allow(unused)] // TODO[unused-clearup] use super::*; @@ -107,8 +112,8 @@ impl EvaluationFrame for PlaceIndexer { PlaceIndexerPath::IndexPath { place } => { let index = item.expect_shared(); match place { - Place::Mutable { mutable: mut_ref } => context - .return_mutable(mut_ref.resolve_indexed(self.access, &index, true)?), + Place::Mutable { mutable } => context + .return_mutable(mutable.resolve_indexed(self.access, &index, true)?), Place::Shared { shared, reason_not_mutable, diff --git a/src/expressions/evaluation/type_resolution.rs b/src/expressions/evaluation/type_resolution.rs index 79764ff6..831ebb3e 100644 --- a/src/expressions/evaluation/type_resolution.rs +++ b/src/expressions/evaluation/type_resolution.rs @@ -240,6 +240,11 @@ impl ResolvedTypeDetails for ValueKind { let message = this.clone().into_debug_string()?; this.execution_err(message) }}, + // Mostly just a test of mutable values + (_, "swap", 1) => wrap_method! {(mut a: MutableValue, mut b: MutableValue) -> ExecutionResult<()> { + mem::swap(a.deref_mut(), b.deref_mut()); + Ok(()) + }}, (ValueKind::Array, "len", 0) => { wrap_method! {(this: Shared) -> ExecutionResult { Ok(this.items.len()) diff --git a/src/expressions/evaluation/value_frames.rs b/src/expressions/evaluation/value_frames.rs index 8ae4f2eb..d7115a47 100644 --- a/src/expressions/evaluation/value_frames.rs +++ b/src/expressions/evaluation/value_frames.rs @@ -35,7 +35,7 @@ impl ResolvedValue { pub(crate) fn into_mutable(self) -> ExecutionResult { Ok(match self { ResolvedValue::Owned(value) => { - return value.execution_err("A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference.".to_string()) + return value.execution_err("A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference.") }, ResolvedValue::Mutable(reference) => reference, ResolvedValue::Shared { shared, reason_not_mutable: Some(reason_not_mutable), } => return shared.execution_err(format!( @@ -229,7 +229,7 @@ impl EvaluationFrame for GroupBuilder { item: EvaluationItem, ) -> ExecutionResult { let inner = item.expect_owned_value(); - Ok(context.return_owned(inner.update_span_range(|_| self.span.into()))) + context.return_owned(inner.update_span_range(|_| self.span.into())) } } @@ -244,7 +244,7 @@ impl ArrayBuilder { context: ValueContext, brackets: &Brackets, items: &[ExpressionNodeId], - ) -> NextAction { + ) -> ExecutionResult { Self { span: brackets.join(), unevaluated_items: items.to_vec(), @@ -253,8 +253,8 @@ impl ArrayBuilder { .next(context) } - pub(super) fn next(self, context: ValueContext) -> NextAction { - match self + pub(super) fn next(self, context: ValueContext) -> ExecutionResult { + Ok(match self .unevaluated_items .get(self.evaluated_items.len()) .cloned() @@ -263,8 +263,8 @@ impl ArrayBuilder { None => context.return_owned(ExpressionValue::Array(ExpressionArray { items: self.evaluated_items, span_range: self.span.span_range(), - })), - } + }))?, + }) } } @@ -282,7 +282,7 @@ impl EvaluationFrame for ArrayBuilder { ) -> ExecutionResult { let value = item.expect_owned_value(); self.evaluated_items.push(value.into_inner()); - Ok(self.next(context)) + self.next(context) } } @@ -343,7 +343,7 @@ impl ObjectBuilder { context.handle_node_as_value(self, index, RequestedValueOwnership::Owned) } None => { - context.return_owned(self.evaluated_entries.to_value(self.span.span_range())) + context.return_owned(self.evaluated_entries.to_value(self.span.span_range()))? } }, ) @@ -418,7 +418,7 @@ impl EvaluationFrame for UnaryOperationBuilder { item: EvaluationItem, ) -> ExecutionResult { let value = item.expect_owned_value().into_inner(); - Ok(context.return_owned(self.operation.evaluate(value)?)) + context.return_owned(self.operation.evaluate(value)?) } } @@ -464,7 +464,7 @@ impl EvaluationFrame for BinaryOperationBuilder { BinaryPath::OnLeftBranch { right } => { let value = item.expect_owned_value().into_inner(); if let Some(result) = self.operation.lazy_evaluate(&value)? { - context.return_owned(result) + context.return_owned(result)? } else { self.state = BinaryPath::OnRightBranch { left: value }; context.handle_node_as_value( @@ -477,7 +477,7 @@ impl EvaluationFrame for BinaryOperationBuilder { } BinaryPath::OnRightBranch { left } => { let value = item.expect_owned_value().into_inner(); - context.return_owned(self.operation.evaluate(left, value)?) + context.return_owned(self.operation.evaluate(left, value)?)? } }) } @@ -521,7 +521,7 @@ impl EvaluationFrame for ValuePropertyAccessBuilder { Ok(match value { ResolvedValue::Owned(value) => { let output = value.resolve_property(&self.access)?; - context.return_owned(output) + context.return_owned(output)? } ResolvedValue::Mutable(mutable) => { let output = mutable.resolve_property(&self.access, false)?; @@ -601,7 +601,7 @@ impl EvaluationFrame for ValueIndexAccessBuilder { match source { ResolvedValue::Owned(value) => { let output = value.resolve_indexed(self.access, index.as_ref())?; - context.return_owned(output) + context.return_owned(output)? } ResolvedValue::Mutable(mutable) => { let output = mutable.resolve_indexed(self.access, index.as_ref(), false)?; @@ -636,12 +636,12 @@ impl RangeBuilder { left: &Option, range_limits: &syn::RangeLimits, right: &Option, - ) -> NextAction { - match (left, right) { + ) -> ExecutionResult { + Ok(match (left, right) { (None, None) => match range_limits { syn::RangeLimits::HalfOpen(token) => { let inner = ExpressionRangeInner::RangeFull { token: *token }; - context.return_owned(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range()))? } syn::RangeLimits::Closed(_) => { unreachable!( @@ -665,7 +665,7 @@ impl RangeBuilder { *left, RequestedValueOwnership::Owned, ), - } + }) } } @@ -693,7 +693,7 @@ impl EvaluationFrame for RangeBuilder { start_inclusive: value, token, }; - context.return_owned(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range()))? } (RangePath::OnLeftBranch { right: None }, syn::RangeLimits::Closed(_)) => { unreachable!("A closed range should have been given a right in continue_range(..)") @@ -704,7 +704,7 @@ impl EvaluationFrame for RangeBuilder { token, end_exclusive: value, }; - context.return_owned(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range()))? } (RangePath::OnRightBranch { left: Some(left) }, syn::RangeLimits::Closed(token)) => { let inner = ExpressionRangeInner::RangeInclusive { @@ -712,21 +712,21 @@ impl EvaluationFrame for RangeBuilder { token, end_inclusive: value, }; - context.return_owned(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range()))? } (RangePath::OnRightBranch { left: None }, syn::RangeLimits::HalfOpen(token)) => { let inner = ExpressionRangeInner::RangeTo { token, end_exclusive: value, }; - context.return_owned(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range()))? } (RangePath::OnRightBranch { left: None }, syn::RangeLimits::Closed(token)) => { let inner = ExpressionRangeInner::RangeToInclusive { token, end_inclusive: value, }; - context.return_owned(inner.to_value(token.span_range())) + context.return_owned(inner.to_value(token.span_range()))? } }) } @@ -778,7 +778,7 @@ impl EvaluationFrame for AssignmentBuilder { } AssignmentPath::OnAwaitingAssignment => { let AssignmentCompletion { span_range } = item.expect_assignment_complete(); - context.return_owned(ExpressionValue::None(span_range)) + context.return_owned(ExpressionValue::None(span_range))? } }) } @@ -790,20 +790,20 @@ pub(super) struct CompoundAssignmentBuilder { } enum CompoundAssignmentPath { - OnValueBranch { place: ExpressionNodeId }, - OnPlaceBranch { value: ExpressionValue }, + OnValueBranch { target: ExpressionNodeId }, + OnTargetBranch { value: ExpressionValue }, } impl CompoundAssignmentBuilder { pub(super) fn start( context: ValueContext, - place: ExpressionNodeId, + target: ExpressionNodeId, operation: CompoundAssignmentOperation, value: ExpressionNodeId, ) -> NextAction { let frame = Self { operation, - state: CompoundAssignmentPath::OnValueBranch { place }, + state: CompoundAssignmentPath::OnValueBranch { target }, }; context.handle_node_as_value(frame, value, RequestedValueOwnership::Owned) } @@ -822,19 +822,19 @@ impl EvaluationFrame for CompoundAssignmentBuilder { item: EvaluationItem, ) -> ExecutionResult { Ok(match self.state { - CompoundAssignmentPath::OnValueBranch { place } => { + CompoundAssignmentPath::OnValueBranch { target } => { let value = item.expect_owned_value().into_inner(); - self.state = CompoundAssignmentPath::OnPlaceBranch { value }; - // TODO[assignment-refactor]: Resolve as LateBound, and then convert to what is needed based on the operation - context.handle_node_as_place(self, place, RequestedPlaceOwnership::Mutable) + self.state = CompoundAssignmentPath::OnTargetBranch { value }; + // TODO[compound-assignment-refactor]: Resolve as LateBound, and then convert to what is needed based on the operation + context.handle_node_as_value(self, target, RequestedValueOwnership::Mutable) } - CompoundAssignmentPath::OnPlaceBranch { value } => { + CompoundAssignmentPath::OnTargetBranch { value } => { let mut mutable = item.expect_mutable(); let span_range = SpanRange::new_between(mutable.span_range(), value.span_range()); mutable .as_mut() .handle_compound_assignment(&self.operation, value, span_range)?; - context.return_owned(ExpressionValue::None(span_range)) + context.return_owned(ExpressionValue::None(span_range))? } }) } diff --git a/tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.rs b/tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.rs new file mode 100644 index 00000000..3c3b3251 --- /dev/null +++ b/tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #( + let a = "a"; + "b".swap(a); + a + ) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.stderr b/tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.stderr new file mode 100644 index 00000000..03395987 --- /dev/null +++ b/tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.stderr @@ -0,0 +1,5 @@ +error: A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference. + --> tests/compilation_failures/expressions/late_bound_owned_to_mutable copy.rs:7:13 + | +7 | "b".swap(a); + | ^^^ diff --git a/tests/compilation_failures/expressions/owned_to_mutable.rs b/tests/compilation_failures/expressions/owned_to_mutable.rs new file mode 100644 index 00000000..8681de36 --- /dev/null +++ b/tests/compilation_failures/expressions/owned_to_mutable.rs @@ -0,0 +1,11 @@ +use preinterpret::*; + +fn main() { + let _ = preinterpret!{ + #( + let a = "a"; + a.swap("b"); + a + ) + }; +} \ No newline at end of file diff --git a/tests/compilation_failures/expressions/owned_to_mutable.stderr b/tests/compilation_failures/expressions/owned_to_mutable.stderr new file mode 100644 index 00000000..23e92cb5 --- /dev/null +++ b/tests/compilation_failures/expressions/owned_to_mutable.stderr @@ -0,0 +1,5 @@ +error: A mutable reference is required, but an owned value was received, this indicates a possible bug as the updated value won't be accessible. To proceed regardless, use `.as_mut()` to get a mutable reference. + --> tests/compilation_failures/expressions/owned_to_mutable.rs:7:20 + | +7 | a.swap("b"); + | ^^^ diff --git a/tests/expressions.rs b/tests/expressions.rs index ebbaf3bc..cc38c942 100644 --- a/tests/expressions.rs +++ b/tests/expressions.rs @@ -478,7 +478,6 @@ fn test_method_calls() { ), 2 + 3 ); - preinterpret_assert_eq!( #( let x = [1, 2, 3]; @@ -507,4 +506,13 @@ fn test_method_calls() { ), "None - [1, 2, 3]" ); + preinterpret_assert_eq!( + #( + let a = "a"; + let b = "b"; + a.swap(b); + [!string! #a " - " #b] + ), + "b - a" + ); }