diff --git a/docs/syntax/constant.md b/docs/syntax/constant.md index 4739a2a..56668f1 100644 --- a/docs/syntax/constant.md +++ b/docs/syntax/constant.md @@ -27,6 +27,11 @@ custom grammar: "Comments may be attached to values with a string following the definition" VALUE_B : constant[16] = 234 "These are attached to the constant definitions" + VALUE_C: constant[2] = 3 + """ + Multiline comments can be used for long descriptions. + Use triple quotes for these like with Python docstrings. + """ } ``` diff --git a/docs/syntax/docstrings.md b/docs/syntax/docstrings.md new file mode 100644 index 0000000..bf95710 --- /dev/null +++ b/docs/syntax/docstrings.md @@ -0,0 +1,66 @@ +Descriptions can be added to packtype definitions as shown below. +The documentation is attached to the code in a way that will allow automated generation of documentation, +as with Python docstrings. + +## Example + +Descriptions can be added with normal [Python docstrings]((https://peps.python.org/pep-0257/)) or with the Packtype custom grammar: + +=== "Python (.py)" + + ```python linenums="1" + import packtype + from packtype import Constant + + @packtype.package() + class MyPackage: + """ + Package decription, + using normal Python docstrings + """ + ... + + @MyPackage.enum() + class Fruit: + """ + Class description, + using normal Python docstrings + """ + APPLE : Constant + ORANGE : Constant + PEAR : Constant + BANANA : Constant + ``` + +=== "Packtype (.pt)" + + ```sv linenums="1" + package my_package { + """ + Package description + using multiline docstring syntax + """ + enum fruit_t { + """ + Class description + using multiline docstring syntax + """ + @prefix=FRUIT + APPLE : constant + "This is an apple" + ORANGE : constant + """ + Member descriptions can also be multiline. + Use triple quotes for these. + """ + PEAR : constant + "This is a pear" + BANANA : constant + "This is a banana" + } + + } + ``` + + +``` diff --git a/docs/syntax/package.md b/docs/syntax/package.md index fca14e4..dfe7fd5 100644 --- a/docs/syntax/package.md +++ b/docs/syntax/package.md @@ -22,7 +22,7 @@ custom grammar: ```sv linenums="1" package my_package { - "Description of what purpose this package serves" + """Description of what purpose this package serves""" // ... } ``` diff --git a/docs/syntax/scalar.md b/docs/syntax/scalar.md index 1ae37d2..75bd393 100644 --- a/docs/syntax/scalar.md +++ b/docs/syntax/scalar.md @@ -39,6 +39,10 @@ custom grammar: TypeB : Scalar[TYPE_B_W] "They can be queried from the digested result" TypeC : Scalar[7] + """ + Multiline comments can be used for long descriptions. + Use triple quotes for these like with Python docstrings. + """ } ``` diff --git a/mkdocs.yml b/mkdocs.yml index a69c1e1..419b26f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - Scalars: syntax/scalar.md - Structs: syntax/struct.md - Unions: syntax/union.md + - Docstrings: syntax/docstrings.md - Utilities: - Basic: utilities/basic.md - Constants: utilities/constant.md diff --git a/packtype/grammar/grammar.py b/packtype/grammar/grammar.py index 12ba463..0773b41 100644 --- a/packtype/grammar/grammar.py +++ b/packtype/grammar/grammar.py @@ -8,7 +8,7 @@ from pathlib import Path from lark import Lark -from lark.exceptions import UnexpectedToken +from lark.exceptions import UnexpectedToken, VisitError from ..common.logging import get_log from ..types.base import Base @@ -93,6 +93,9 @@ def parse_string( f"Failed to parse {source.name if source else 'input'} on line {exc.line}: " f"\n\n{exc.get_context(definition)}\n{exc}" ) from exc + except VisitError as exc: + raise exc.orig_exc from exc + # Gather declarations known_entities: dict[str, tuple[type[Base] | Constant, Position]] = {} diff --git a/packtype/grammar/packtype.lark b/packtype/grammar/packtype.lark index 86a5ebf..ac1731d 100644 --- a/packtype/grammar/packtype.lark +++ b/packtype/grammar/packtype.lark @@ -18,9 +18,11 @@ %import common.CNAME %import common.ESCAPED_STRING +%import common._STRING_ESC_INNER %import common.INT %import common.SIGNED_INT %import common.WS +%import common.NEWLINE %ignore WS // ============================================================================= @@ -42,7 +44,12 @@ dimension: "[" expr "]" dimensions: dimension+ ?name: CNAME -descr: ESCAPED_STRING + +// Multiline docstrings as with Python - start and end with three double-quotes +_DOCSTRING_QUOTES: "\"\"\"" +ESCAPED_MULTILINE_DOCSTRING: _DOCSTRING_QUOTES (_STRING_ESC_INNER|NEWLINE)* _DOCSTRING_QUOTES + +descr: (ESCAPED_STRING | ESCAPED_MULTILINE_DOCSTRING) modifier: "@" name "=" (name | ESCAPED_STRING | NUMERIC) diff --git a/packtype/grammar/transformer.py b/packtype/grammar/transformer.py index 8a46515..0895986 100644 --- a/packtype/grammar/transformer.py +++ b/packtype/grammar/transformer.py @@ -3,6 +3,7 @@ # import math +import textwrap from lark import Transformer, v_args @@ -85,7 +86,11 @@ def CNAME(self, body): # noqa: N802 return str(body) def descr(self, body): - return Description(str(body[0]).strip('"')) + """ + Take description and trim surrounding quotes, + then remove common indentation and remove leading and trailing newlines. + """ + return Description(textwrap.dedent(str(body[0]).strip('"')).strip("\n")) def modifier(self, body): return Modifier(*body) diff --git a/tests/grammar/test_descriptions.py b/tests/grammar/test_descriptions.py new file mode 100644 index 0000000..43ac11c --- /dev/null +++ b/tests/grammar/test_descriptions.py @@ -0,0 +1,304 @@ +# Copyright 2023-2025, Peter Birch, mailto:peter@intuity.io +# SPDX-License-Identifier: Apache-2.0 +# + +from packtype.grammar import parse_string +from tests.fixtures import reset_registry + +assert reset_registry + + +def test_multiline_docstring_package(): + """Test multiline docstring on package declaration""" + pkg = next( + parse_string( + ''' + package test_pkg { + """ + This is a multiline docstring + for the package. + + It can contain multiple lines + and preserve formatting. + """ + } + ''' + ) + ) + assert ( + pkg.__doc__ + == "This is a multiline docstring\nfor the package.\n\n" + + "It can contain multiple lines\nand preserve formatting." + ) + + +def test_multiline_docstring_constant(): + """Test multiline docstring on constant declaration""" + pkg = next( + parse_string( + ''' + package test_pkg { + MY_CONSTANT: constant = 42 + """ + This is a multiline docstring + for a constant. + + It explains what the constant + represents and its purpose. + """ + } + ''' + ) + ) + assert ( + pkg.MY_CONSTANT.__doc__ + == "This is a multiline docstring\nfor a constant.\n\n" + + "It explains what the constant\nrepresents and its purpose." + ) + + +def test_multiline_docstring_scalar(): + """Test multiline docstring on scalar declaration""" + pkg = next( + parse_string( + ''' + package test_pkg { + my_scalar: scalar[8] + """ + A scalar type with multiline + documentation. + + This describes the purpose + and usage of the scalar. + """ + } + ''' + ) + ) + assert ( + pkg.my_scalar.__doc__ + == "A scalar type with multiline\ndocumentation.\n\n" + + "This describes the purpose\nand usage of the scalar." + ) + + +def test_multiline_docstring_struct(): + """Test multiline docstring on struct declaration""" + pkg = next( + parse_string( + ''' + package test_pkg { + struct my_struct { + """ + A struct with multiline + documentation. + + This struct contains multiple + fields and has detailed + documentation. + """ + field1: scalar[8] + """ + Each field + can have a multiline docstring too + """ + field2: scalar[16] + """ + Including this one + :) + """ + } + } + ''' + ) + ) + assert ( + pkg.my_struct.__doc__ + == "A struct with multiline\ndocumentation.\n\n" + + "This struct contains multiple\nfields and has detailed\ndocumentation." + ) + + +def test_multiline_docstring_enum(): + """Test multiline docstring on enum declaration""" + pkg = next( + parse_string( + ''' + package test_pkg { + enum my_enum { + """ + An enumeration with multiline + documentation. + + This enum defines various + states and their meanings. + """ + STATE_A + STATE_B + STATE_C + } + } + ''' + ) + ) + assert ( + pkg.my_enum.__doc__ + == "An enumeration with multiline\ndocumentation.\n\n" + + "This enum defines various\nstates and their meanings." + ) + + +def test_multiline_docstring_union(): + """Test multiline docstring on union declaration""" + pkg = next( + parse_string( + ''' + package test_pkg { + """ + This is also + a multiline docstring + """ + union my_union { + """ + A union with multiline + documentation. + + This union can hold different + types of data structures. + """ + a: scalar[2] + b: scalar[2] + } + } + ''' + ) + ) + assert ( + pkg.my_union.__doc__ + == "A union with multiline\ndocumentation.\n\n" + + "This union can hold different\ntypes of data structures." + ) + + +def test_multiline_docstring_with_quotes(): + """Test multiline docstring containing quotes""" + pkg = next( + parse_string( + ''' + package test_pkg { + """ + This docstring contains "quoted text" + and 'single quotes' as well. + + It also has "nested quotes" within + the documentation. + """ + } + ''' + ) + ) + assert '"quoted text"' in pkg.__doc__ + assert "'single quotes'" in pkg.__doc__ + assert '"nested quotes"' in pkg.__doc__ + + +def test_multiline_docstring_empty(): + """Test empty multiline docstring""" + pkg = next( + parse_string( + ''' + package test_pkg { + """ + """ + } + ''' + ) + ) + assert pkg.__doc__ == "" + + +def test_multiline_docstring_single_line(): + """Test multiline docstring with single line content""" + pkg = next( + parse_string( + ''' + package test_pkg { + """ + Single line content + """ + } + ''' + ) + ) + assert pkg.__doc__ == "Single line content" + + +def test_multiline_docstring_with_special_characters(): + """Test multiline docstring with special characters and symbols""" + pkg = next( + parse_string( + ''' + package test_pkg { + """ + Special characters: @#$%^&*() + Math symbols: +-*/=<>! + Brackets: []{}() + Backslashes: \\ and forward slashes: / + Inline quotes: ' and " + """ + } + ''' + ) + ) + assert "@#$%^&*()" in pkg.__doc__ + assert "+-*/=<>!" in pkg.__doc__ + assert "[]{}()" in pkg.__doc__ + assert "\\" in pkg.__doc__ + assert "/" in pkg.__doc__ + assert "'" in pkg.__doc__ + assert '"' in pkg.__doc__ + + +def test_mixed_docstring_quotes_enum(): + """Test mixing single and triple quote docstrings on enum fields""" + pkg = next( + parse_string( + ''' + package test_pkg { + enum mixed_quotes_enum { + """ + An enum with mixed quote styles + for documentation. + """ + SINGLE_QUOTE + "Single line with single quotes" + TRIPLE_QUOTE + """ + Multi-line with triple quotes + + This has multiple lines + and formatting. + """ + ANOTHER_SINGLE + "Another single line docstring" + FINAL_TRIPLE + """ + Final field with triple quotes + and special characters: @#$% + """ + } + } + ''' + ) + ) + assert pkg.mixed_quotes_enum.__doc__ == "An enum with mixed quote styles\nfor documentation." + assert pkg.mixed_quotes_enum.SINGLE_QUOTE.__doc__ == "Single line with single quotes" + assert ( + pkg.mixed_quotes_enum.TRIPLE_QUOTE.__doc__ + == "Multi-line with triple quotes\n\nThis has multiple lines\nand formatting." + ) + assert pkg.mixed_quotes_enum.ANOTHER_SINGLE.__doc__ == "Another single line docstring" + assert ( + pkg.mixed_quotes_enum.FINAL_TRIPLE.__doc__ + == "Final field with triple quotes\nand special characters: @#$%" + ) diff --git a/vscode/packtype/language-configuration.json b/vscode/packtype/language-configuration.json index 6b619d0..63342a3 100644 --- a/vscode/packtype/language-configuration.json +++ b/vscode/packtype/language-configuration.json @@ -1,9 +1,8 @@ { "comments": { - // symbol used for single line comment. Remove this entry if your language does not support line comments + // C-style line comments "lineComment": "//", - // symbols used for start and end a block comment. Remove this entry if your language does not support block comments - "blockComment": [ "/*", "*/" ] + // blockComment not yet supported }, // symbols used as brackets "brackets": [ @@ -16,6 +15,7 @@ ["{", "}"], ["[", "]"], ["(", ")"], + ["\"\"\"", "\"\"\""], ["\"", "\""], ["'", "'"] ], diff --git a/vscode/packtype/syntaxes/packtype.tmLanguage.json b/vscode/packtype/syntaxes/packtype.tmLanguage.json index 2968b59..94fedd2 100644 --- a/vscode/packtype/syntaxes/packtype.tmLanguage.json +++ b/vscode/packtype/syntaxes/packtype.tmLanguage.json @@ -135,11 +135,25 @@ "match": "\\b(\\w+)\\b", "name": "entity.name.type.packtype" }, + { + "include": "#docstrings" + }, { "include": "#strings" } ], "repository": { + "docstrings": { + "name": "string.quoted.docstring.packtype", + "begin": "\"\"\"", + "end": "\"\"\"", + "patterns": [ + { + "name": "constant.character.escape.packtype", + "match": "\\\\." + } + ] + }, "strings": { "name": "string.quoted.double.packtype", "begin": "\"",