v1.0.0 - Expressions, Parsing etc (WIP)

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ permissions:
   contents: read
 
 env:
-  RUSTFLAGS: -Dwarnings
+  RUSTFLAGS: -Dwarnings # Deny warnings - i.e. warnings cause build errors
 
 jobs:
   test:
@@ -31,6 +31,8 @@ jobs:
       - name: Enable type layout randomization
         run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV
         if: matrix.rust == 'nightly'
+      - name: Add TEST_RUST_MODE env variable
+        run: echo TEST_RUST_MODE=${{matrix.rust}} >> $GITHUB_ENV
       - run: cargo test
       - uses: actions/upload-artifact@v4
         if: matrix.rust == 'nightly' && always()
@@ -39,14 +41,14 @@ jobs:
           path: Cargo.lock
 
   msrv:
-    name: MSRV (1.61) Compiles
+    name: MSRV (1.63) Compiles
     runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: 1.61
+          toolchain: 1.63
       - run: cargo check
 
   style-check:
@@ -61,6 +63,17 @@ jobs:
           toolchain: stable
       - run: ./style-check.sh
 
+  book:
+    name: Rust Book Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - run: ./book/test.sh
+
   miri:
     name: Miri
     runs-on: ubuntu-latest

.github/workflows/publish_book.yml

@@ -0,0 +1,44 @@
+name: Publish book to Github Pages
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read  # to read the docs and build job
+  pages: write    # to deploy to pages
+  id-token: write # to verify the deployment originates from an appropriate source
+
+jobs:
+  # Build job - https://github.com/actions/upload-pages-artifact
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # Sets up the repository configuration for github pages to be deployed from an action
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      # Builds the book and outputs to book/book
+      - name: Build book
+        run: ./book/build.sh
+      # Creates a correctly compressed archive from the build HTML file, and outputs it as
+      # a `github-pages` artifact
+      - name: Upload static files as artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: book/book
+
+  # Deployment job - https://github.com/actions/deploy-pages
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      # Deploys the artifact `github-pages` created by the `upload-pages-artifact` job
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

Cargo.toml

@@ -12,12 +12,17 @@ keywords = ["macros", "declarative-macros", "toolkit", "interpreter", "preproces
 categories = ["development-tools", "compilers"]
 # MSRV 1.56.0 is the start of Edition 2021
 # MSRV 1.61.0 is the current MSRV of syn
+# MRSV 1.63.0 is needed to support RefMut::filter_map
 # If changing this, also update the local-check-msrv.sh script and ci.yml
-rust-version = "1.61"
+rust-version = "1.63"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-proc-macro2 = { version = "1.0" }
-syn = { version = "2.0", default-features = false, features = ["parsing"] }
+proc-macro2 = { version = "1.0.93" }
+syn = { version = "2.0.98", default-features = false, features = ["parsing", "derive", "printing", "clone-impls", "full"] }
+quote = { version = "1.0.38", default-features = false }
+
+[dev-dependencies]
+trybuild = { version = "1.0.110", features = ["diff"] }

KNOWN_ISSUES.md

@@ -0,0 +1,8 @@
+# Since Major Version 0.3.0
+
+* `[MAX LITERAL]` - In an expression, the maximum negative integer literal is not valid, for example `-128i8`.
+  * By comparison, in `rustc`, there is special handling for such literals, so that e.g. `-128i8` and `--(-128i8)` are accepted.
+  * In rust analyzer, before the full pass, it seems to store literals as u128 and perform wrapping arithmetic on them. On a save / full pass, it does appear to reject literals which are out of bounds (e.g. 150i8).
+  * We could fix this by special casing maximal signed integer literals (e.g. 128 for an i8) which have yet to be involved in a non-unary expression.
+* `&&` and `||` are not lazy in expressions
+  * This needs a custom expression parser, rather than just leveraging syn

README.md

@@ -334,10 +334,6 @@ macro_rules! impl_new_type {
 
 ## Future Extension Possibilities
 
-### Add github docs page / rust book
-
-Add a github docs page / rust book at this repository, to allow us to build out a suite of examples, like `serde` or the little book of macros.
-
 ### Destructuring / Parsing Syntax, and Declarative Macros 2.0
 
 I have a vision for having preinterpret effectively replace the use of declarative macros in the Rust ecosystem, by:
@@ -359,17 +355,19 @@ The idea is that we create two new tools:
 
 In more detail:
 
-* `[!parse! (DESTRUCTURING) = (INPUT)]` is a more general `[!set!]` which acts like a `let <XX> = <YY> else { panic!() }`. It takes a `()`-wrapped parse destructuring on the left and a token stream as input on the right. Any `#x` in the parse definition acts as a binding rather than as a substitution. Parsing will handled commas intelligently, and accept intelligent parse operations to do heavy-lifting for the user. Parse operations look like `[!OPERATION! DESTRUCTURING]` with the operation name in `UPPER_SNAKE_CASE`. Some examples might be:
+* `[!let! (DESTRUCTURING) = (INPUT)]` (or maybe `!let!` or `!set!`) is a more general `[!set!]` which acts like a `let <XX> = <YY> else { panic!() }`. It takes a `()`-wrapped parse destructuring on the left and a `()`-wrapped token stream as input on the right. Any `#x` in the parse definition acts as a binding rather than as a substitution. Parsing will handled commas intelligently, and accept intelligent parse operations to do heavy-lifting for the user. Parse operations look like `[!OPERATION! DESTRUCTURING]` with the operation name in `UPPER_SNAKE_CASE`. Some examples might be:
     * `[!FIELDS! { hello: #a, world?: #b }]` - which can be parsed in any order, cope with trailing commas, and forbid fields in the source stream which aren't in the destructuring.
     * `[!SUBFIELDS! { hello: #a, world?: #b }]` - which can parse fields in any order, cope with trailing commas, and allow fields in the source stream which aren't in the destructuring.
     * `[!ITEM! { #ident, #impl_generics, ... }]` - which calls syn's parse item on the token
     * `[!IDENT! #x]`, `[!LITERAL! #x]`, `[!TYPE! { tokens: #x, path: #y }]` and the like to parse idents / literals etc directly from the token stream (rather than token streams). These will either take just a variable to capture the full token stream, or support an optional-argument style binding, where the developer can request certain sub-patterns or mapped token streams.
     * More tailored examples, such as `[!GENERICS! { impl: #x, type: #y, where: #z }]` which uses syn to parse the generics, and then uses subfields on the result.
     * Possibly `[!GROUPED! #x]` to parse a group with no brackets, to avoid parser ambiguity in some cases
     * `[!OPTIONAL! ...]` might be supported, but other complex logic (loops, matching) is delayed lazily until interpretation time - which feels more intuitive.
-* `[!for! (DESTRUCTURING) in (INPUT) { ... }]` which operates like the rust `for` loop, and uses a parse destructuring on the left, and has support for optional commas between values
-* `[!match! (INPUT) => { (DESTRUCTURING_1) => { ... }, (DESTRUCTURING_2) => { ... }, (#fallback) => { ... } }]` which operates like a rust `match` expression, and can replace the function of the branches of declarative macro inputs.
-* `[!macro_rules! name!(DESTRUCTURING) = { ... }]` which can define a declarative macro, but just parses its inputs as a token stream, and uses preinterpret for its heavy lifting.
+* `[!for! (DESTRUCTURING) in [!split_at! INPUT ,] { ... }]` which operates like the rust `for` loop over token trees, allowing them to be parsed. Instead of the destructuring, you can also just use a variable e.g. `#x` Operates well with the `[!split_at! ..]` command.
+* `[!while! [!parse_start_from! #variable DESTRUCTURING] { ... }]` can be used to consume a destructuring from the start of #variable. `[!parse_start_from! ..]` returns true if the parse succeeded, false if the variable is an empty token stream, and errors otherwise.
+* `[!parse_loop! #variable as (DESTRUCTURING), { ... }]` where the `,` is a separator and cleverly handles trailing `,` or `;` - similar to `syn::Punctuated`
+* `[!match! (INPUT) => { (DESTRUCTURING_1) => { ... }, (DESTRUCTURING_2) => { ... }, #fallback => { ... } }]` which operates like a rust `match` expression, and can replace the function of the branches of declarative macro inputs.
+* `[!macro_rules! name!(DESTRUCTURING) = { ... }]` which can define a declarative macro, but just parses its inputs as a token stream, and uses preinterpret for its heavy lifting. This could alternatively exist as `preinterpret::define_macro!{ my_macro!(DESTRUCTURING) = { ... }}`.
 
 And then we can end up with syntax like the following:
 
@@ -381,7 +379,7 @@ And then we can end up with syntax like the following:
 // A simple macro can just take a token stream as input
 preinterpret::preinterpret! {
     [!macro_rules! my_macro!(#input) {
-        [!for! (#trait for #type) in (#input) {
+        [!parse_loop! #input as (#trait for #type), {
             impl #trait for #type
         }]
     }]
@@ -403,9 +401,9 @@ preinterpret::preinterpret! {
             punctuation?: #punct = ("!") // Default
         }]
     ) = {
-        [!for! (
+        [!parse_loop! #type_list as (
             #type [!GENERICS! { impl: #impl_generics, type: #type_generics }]
-        ) in (#type_list) {
+        ) {
             impl<#impl_generics> SuperDuper for #type #type_generics {
                 const Hello: &'static str = [!string! #hello " " #world #punct];
             }
@@ -414,95 +412,43 @@ preinterpret::preinterpret! {
 }
 ```
 
-### Possible extension: Integer commands
-
-Each of these commands functions in three steps:
-* Apply the interpreter to the token stream, which recursively executes preinterpret commands.
-* Iterate over each token (recursing into groups), expecting each to be an integer literal.
-* Apply some command-specific mapping to this stream of integer literals, and output a single integer literal without its type suffix. The suffix can be added back manually if required with a wrapper such as `[!literal! [!add! 1 2] u64]`.
+### Possible extension: Token stream commands
 
-Integer commands under consideration are:
+* `[!ungroup! #stream]` expects `#stream` to be a single group and unwraps it once
+* `[!flatten! #stream]` removes all singleton groups from `#stream`, leaving a token stream of idents, literals and punctuation
 
-* `[!add! 5u64 9 32]` outputs `46`. It takes any number of integers and outputs their sum. The calculation operates in `u128` space.
-* `[!sub! 64u32 1u32]` outputs `63`. It takes two integers and outputs their difference. The calculation operates in `i128` space.
-* `[!mod! $length 2]` outputs `0` if `$length` is even, else `1`. It takes two integers `a` and `b`, and outputs `a mod b`.
+We could support a postfix calling convention, such as the `[!pipe! ...]` special command: `[!pipe! #stream > #x [!index! #x[4]] > #x [!ungroup! #x]]` or something like a scala for comprehension. But honestly, just saving things to variables might be cleaner.
 
-We also support the following assignment commands:
+### Possible extension: Better performance via incremental parsing
 
-* `[!increment! #i]` is shorthand for `[!set! #i = [!add! #i 1]]` and outputs no tokens.
+Incremental parsing using a fork of syn ([see issue](https://github.com/dtolnay/syn/issues/1842)) would allow:
 
-Even better - we could even support calculator-style expression interpretation:
+* Cheaper conversions between variables and parsing.
+* `[!consume_from! #stream #x]` where `#x` is read as the first token tree from `#stream`
+* `[!consume_from! #stream (<PARSE_DESTRUCTURING>)]` where the parser is read greedily from `#stream`
 
-* `[!usize! (5 + 10) / mod(4, 2)]` outputs `7usize`
+Forking syn may also allow some parts to be made more performant.
 
 ### Possible extension: User-defined commands
 
-* `[!define! [!my_command! <PARSE_DESTRUCTURING>] { <OUTPUT> }]`
+* `[!define_command! [!my_command! <ARGUMENTS_DESTRUCTURING>] { <OUTPUT> }]`
+* Some ability to define and re-use commands across multiple invocations
+  without being too expensive. Still unsure how to make this work.
 
-### Possible extension: Boolean commands
+### Possible extension: Further utility commands
 
-Each of these commands functions in three steps:
-* Apply the interpreter to the token stream, which recursively executes preinterpret commands.
-* Expects to read exactly two token trees (unless otherwise specified)
-* Apply some command-specific comparison, and outputs the boolean literal `true` or `false`.
-
-Comparison commands under consideration are:
-* `[!eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example:
-  * `[!eq! (3 4) (3   4)]` outputs `true` because the token stream ignores spacing.
-  * `[!eq! 1u64 1]` outputs `false` because these are different literals.
-* `[!lt! #foo #bar]` outputs `true` if `#foo` is an integer literal and less than `#bar`
-* `[!gt! #foo #bar]` outputs `true` if `#foo` is an integer literal and greater than `#bar`
-* `[!lte! #foo #bar]` outputs `true` if `#foo` is an integer literal and less than or equal to `#bar`
-* `[!gte! #foo #bar]` outputs `true` if `#foo` is an integer literal and greater than or equal to `#bar`
-* `[!not! #foo]` expects a single boolean literal, and outputs the negation of `#foo`
+Other boolean commands could be possible, similar to numeric commands:
+* `[!tokens_eq! #foo #bar]` outputs `true` if `#foo` and `#bar` are exactly the same token tree, via structural equality. For example:
+  * `[!tokens_eq! (3 4) (3   4)]` outputs `true` because the token stream ignores spacing.
+  * `[!tokens_eq! 1u64 1]` outputs `false` because these are different literals.
+  * This can be effectively done already with `#(x.debug_string() == y.debug_string())`
+* `[!str_split! { input: Value<LitStr>, separator: Value<LitStr>, }]`
 * `[!str_contains! "needle" [!string! haystack]]` expects two string literals, and outputs `true` if the first string is a substring of the second string.
 
-### Possible extension: Token stream commands
-
-* `[!skip! 4 from [#stream]]` reads and drops the first 4 token trees from the stream, and outputs the rest
-* `[!ungroup! (#stream)]` outputs `#stream`. It expects to receive a single group (i.e. wrapped in brackets), and unwraps it.
-
-### Possible extension: Control flow commands
-
-#### If statement
-
-`[!if! #cond then { #a } else { #b }]` outputs `#a` if `#cond` is `true`, else `#b` if `#cond` is false.
-
-The `if` command works as follows:
-* It starts by only interpreting its first token tree, and expects to see a single `true` or `false` literal.
-* It then expects to reads an unintepreted `then` ident, following by a single `{ .. }` group, whose contents get interpreted and output only if the condition was `true`.
-* It optionally also reads an `else` ident and a by a single `{ .. }` group, whose contents get interpreted and output only if the condition was `false`.
-
-#### For loop
-
-* `[!for! #token_tree in [#stream] { ... }]`
-
-#### Goto and label
-
-* `[!label! loop_start]` - defines a label which can be returned to. Effectively, it takes a clones of the remaining token stream after the label in the interpreter.
-* `[!goto! loop_start]` - jumps to the last execution of `[!label! loop_start]`. It unrolls the preinterpret stack (dropping all unwritten token streams) until it finds a stackframe in which the interpreter has the defined label, and continues the token stream from there.
-
-```rust,ignore
-// Hypothetical future syntax - not yet implemented!
-preinterpret::preinterpret!{
-    [!set! #i = 0]
-    [!label! loop]
-    const [!ident! AB #i]: u8 = 0;
-    [!increment! #i]
-    [!if! [!lte! #i 100] then { [!goto! loop] }]
-}
-```
-
 ### Possible extension: Eager expansion of macros
 
 When [eager expansion of macros returning literals](https://github.com/rust-lang/rust/issues/90765) is stabilized, it would be nice to include a command to do that, which could be used to include code, for example: `[!expand_literal_macros! include!("my-poem.txt")]`.
 
-### Possible extension: Explicit parsing feature to enable syn
-
-The heavy `syn` library is (in basic preinterpret) only needed for literal parsing, and error conversion into compile errors.
-
-We could add a parsing feature to speed up compile times a lot for stacks which don't need the parsing functionality.
-
 ## License
 
 Licensed under either of the [Apache License, Version 2.0](LICENSE-APACHE)

book/.gitignore

@@ -0,0 +1,2 @@
+book
+bin

book/book.toml

@@ -0,0 +1,6 @@
+[book]
+authors = ["David Edey"]
+language = "en"
+multilingual = false
+src = "src"
+title = "The Preinterpret Guide"

book/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e pipefail
+
+# Ensure we're in the book directory
+cd "$(dirname "$0")"
+
+# Ensure mdbook has been downloaded
+/usr/bin/env bash ./download-mdbook.sh 
+
+./bin/mdbook build

book/download-mdbook.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -e pipefail
+
+# Ensure we're in the book directory
+cd "$(dirname "$0")"
+
+MDBOOK_VER="v0.4.31"
+
+## Check if the right version of mdbook is already downloaded
+if [ -f ./bin/mdbook ]; then
+    if ./bin/mdbook --version | grep $MDBOOK_VER > /dev/null; then
+        echo "bin/mdbook @ $MDBOOK_VER is already downloaded"
+        exit 0
+    fi
+    echo "bin/mdbook is already downloaded but the wrong version, so downloading the correct version"
+fi
+
+# Download and extract the mdbook binary into the bin folder
+mkdir -p bin && cd bin
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    TAR_FILE="mdbook-${MDBOOK_VER}-x86_64-unknown-linux-musl.tar.gz"
+else
+    TAR_FILE="mdbook-${MDBOOK_VER}-x86_64-apple-darwin.tar.gz"
+fi
+
+echo "Downloading $TAR_FILE..."
+curl -OL "https://github.com/rust-lang/mdBook/releases/download/${MDBOOK_VER}/${TAR_FILE}"
+tar -xf "${TAR_FILE}"
+rm "${TAR_FILE}"
+
+echo "Extracted to bin/mdbook"
+cd ..

book/src/SUMMARY.md

@@ -0,0 +1,16 @@
+# Summary
+
+- [Introduction](./introduction.md)
+  - [Quick Reference](./introduction/quick-reference.md)
+  - [Motivation](./introduction/motivation.md)
+  - [Examples](./introduction/examples.md)
+- [Concepts](./concepts.md)
+  - [Commands](./concepts/commands.md)
+  - [Variables](./concepts/variables.md)
+  - [Expressions](./concepts/expressions.md)
+  - [Parsing](./concepts/parsing.md)
+- [Command Reference](./command-reference.md)
+  - [Strings and Idents](./command-reference/strings.md)
+  - [Expressions](./command-reference/expressions.md)
+  - [Control Flow](./command-reference/control-flow.md)
+

book/src/command-reference.md

@@ -0,0 +1,3 @@
+# Command Reference
+
+Coming soon...

book/src/command-reference/control-flow.md

@@ -0,0 +1 @@
+# Control Flow

book/src/command-reference/expressions.md

@@ -0,0 +1 @@
+# Expressions

book/src/command-reference/strings.md

@@ -0,0 +1 @@
+# Strings and Idents

book/src/concepts.md

@@ -0,0 +1,3 @@
+# Concepts
+
+Coming soon...

book/src/concepts/commands.md

@@ -0,0 +1 @@
+# Commands

book/src/concepts/expressions.md

@@ -0,0 +1 @@
+# Expressions

book/src/concepts/parsing.md

@@ -0,0 +1 @@
+# Parsing

book/src/concepts/variables.md

@@ -0,0 +1 @@
+# Variables