From f9ba7091fc01d895505de7c0d359047d254685dc Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 23 Apr 2022 21:38:53 -0400 Subject: [PATCH 01/15] Add readme --- tests/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..ede4b602 --- /dev/null +++ b/tests/README.md @@ -0,0 +1 @@ +# Harmony Tests From 5d55f956e9ee80bd885820f3b196f5a2b7219c6e Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 30 Apr 2022 14:58:59 -0400 Subject: [PATCH 02/15] Add e2e test framework --- tests/README.md | 10 ++++++++++ tests/e2e/test_compilation.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/e2e/test_compilation.py diff --git a/tests/README.md b/tests/README.md index ede4b602..7ef81d20 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1 +1,11 @@ # Harmony Tests + +Types of tests: + +- Compilation/Performance Tests +- Unit Tests + +## Compilation/Performance Tests + +## Unit Tests + diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py new file mode 100644 index 00000000..21805507 --- /dev/null +++ b/tests/e2e/test_compilation.py @@ -0,0 +1,29 @@ + +from datetime import timedelta +import pathlib +from typing import List, NamedTuple +import unittest + +from harmony_model_checker.compile import do_compile + +class Test(NamedTuple): + filename: pathlib.Path + max_time: timedelta + +def load_harmony_files() -> List[Test]: + code_dir = pathlib.Path(__file__).parent.parent.parent / "code" + return [ + Test( + filename=f, + max_time=timedelta(seconds=1) + ) for f in code_dir.glob("*.hny") + ] + +class TestCompilation(unittest.TestCase): + def test_compilation(self): + for h in load_harmony_files(): + with self.subTest(msg=f"Checking {str(h.filename)}"): + do_compile(str(h.filename), consts=[], mods=[], interface=None) + +if __name__ == "__main__": + unittest.main() From 4162a55aaaef959e231fadbf38946bd980d1a7c9 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 30 Apr 2022 15:18:44 -0400 Subject: [PATCH 03/15] Clear global values after each compilation --- tests/e2e/test_compilation.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index 21805507..a195a4e3 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -4,6 +4,7 @@ from typing import List, NamedTuple import unittest +import harmony_model_checker.harmony.harmony as legacy_harmony from harmony_model_checker.compile import do_compile class Test(NamedTuple): @@ -15,15 +16,24 @@ def load_harmony_files() -> List[Test]: return [ Test( filename=f, - max_time=timedelta(seconds=1) + max_time=timedelta(milliseconds=500) ) for f in code_dir.glob("*.hny") ] class TestCompilation(unittest.TestCase): def test_compilation(self): for h in load_harmony_files(): - with self.subTest(msg=f"Checking {str(h.filename)}"): + with self.subTest(msg=f"Compiling {str(h.filename)}"): do_compile(str(h.filename), consts=[], mods=[], interface=None) + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled + + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() + if __name__ == "__main__": unittest.main() From 6aaa1a4498e36686b851dd3a3563a196ec0b96b3 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 30 Apr 2022 15:51:46 -0400 Subject: [PATCH 04/15] Write tests with pytest --- tests/e2e/test_compilation.py | 44 ++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index a195a4e3..f098b631 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -1,39 +1,41 @@ from datetime import timedelta import pathlib +import time from typing import List, NamedTuple -import unittest +import pytest import harmony_model_checker.harmony.harmony as legacy_harmony from harmony_model_checker.compile import do_compile -class Test(NamedTuple): +class Params(NamedTuple): filename: pathlib.Path max_time: timedelta -def load_harmony_files() -> List[Test]: +def load_harmony_files() -> List[Params]: code_dir = pathlib.Path(__file__).parent.parent.parent / "code" return [ - Test( + Params( filename=f, max_time=timedelta(milliseconds=500) ) for f in code_dir.glob("*.hny") ] -class TestCompilation(unittest.TestCase): - def test_compilation(self): - for h in load_harmony_files(): - with self.subTest(msg=f"Compiling {str(h.filename)}"): - do_compile(str(h.filename), consts=[], mods=[], interface=None) - legacy_harmony.files.clear() # files that have been read already - legacy_harmony.modules.clear() # modules modified with -m - legacy_harmony.used_modules.clear() # modules modified and used - legacy_harmony.namestack.clear() # stack of module names being compiled - - legacy_harmony.imported.clear() - legacy_harmony.constants.clear() - legacy_harmony.used_constants.clear() - - -if __name__ == "__main__": - unittest.main() +@pytest.fixture(autouse=True) +def run_around_tests(): + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled + + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() + yield + +@pytest.mark.parametrize("param", load_harmony_files()) +def test_compilation(param): + start_time = time.perf_counter_ns() + do_compile(str(param.filename), consts=[], mods=[], interface=None) + end_time = time.perf_counter_ns() + assert end_time - start_time <= param.max_time.microseconds * 1000 From 9127f8ce20ca7ef379f66f148b0b9fbfdc46291c Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Mon, 2 May 2022 19:23:25 -0400 Subject: [PATCH 05/15] update test-related configurations in repo --- .gitignore | 2 ++ Makefile | 9 +++++++++ tests/e2e/test_compilation.py | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 62dae4d5..750ce00d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ book/pp.tex # Auto-generated by Makefile's [gen] target harmony_model_checker/__init__.py + +.coverage diff --git a/Makefile b/Makefile index 1674ae18..661ec875 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,16 @@ upload: dist twine upload dist/* clean: + # Harmony outputs in `code` directory rm -f code/*.htm code/*.hvm code/*.hco code/*.png code/*.hfa code/*.tla code/*.gv *.html + + # Harmony outputs in `modules` directory (cd harmony_model_checker/modules; rm -f *.htm *.hvm *.hco *.png *.hfa *.tla *.gv *.html) + rm -rf compiler_integration_results.md compiler_integration_results/ + + # Package publication related outputs rm -rf build/ dist/ harmony_model_checker.egg-info/ + + # Test coverage related outputs + rm -rf .coverage htmlcov diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index f098b631..6c560ffc 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -17,7 +17,7 @@ def load_harmony_files() -> List[Params]: return [ Params( filename=f, - max_time=timedelta(milliseconds=500) + max_time=timedelta(seconds=1) ) for f in code_dir.glob("*.hny") ] @@ -38,4 +38,5 @@ def test_compilation(param): start_time = time.perf_counter_ns() do_compile(str(param.filename), consts=[], mods=[], interface=None) end_time = time.perf_counter_ns() - assert end_time - start_time <= param.max_time.microseconds * 1000 + duration = end_time - start_time + assert duration <= param.max_time.total_seconds() * 1e9 From dcbfaa5ca819c5ba40329025636ae092eaec85ea Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 09:31:04 -0400 Subject: [PATCH 06/15] Add module/constants-parameterized and failing compilations --- tests/e2e/constants/base.hny | 3 ++ tests/e2e/errors/bad_elif_stmt.hny | 5 +++ tests/e2e/errors/def_label.hny | 2 + tests/e2e/errors/empty.hny | 0 tests/e2e/errors/eof_string.hny | 2 + tests/e2e/errors/missing_indent.hny | 3 ++ tests/e2e/errors/unclosed_paren.hny | 2 + tests/e2e/modules/base.hny | 4 ++ tests/e2e/modules/resources/math.hny | 3 ++ tests/e2e/test_compilation.py | 60 ++++++++++++++++++++++++---- 10 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/constants/base.hny create mode 100644 tests/e2e/errors/bad_elif_stmt.hny create mode 100644 tests/e2e/errors/def_label.hny create mode 100644 tests/e2e/errors/empty.hny create mode 100644 tests/e2e/errors/eof_string.hny create mode 100644 tests/e2e/errors/missing_indent.hny create mode 100644 tests/e2e/errors/unclosed_paren.hny create mode 100644 tests/e2e/modules/base.hny create mode 100644 tests/e2e/modules/resources/math.hny diff --git a/tests/e2e/constants/base.hny b/tests/e2e/constants/base.hny new file mode 100644 index 00000000..5a03ff14 --- /dev/null +++ b/tests/e2e/constants/base.hny @@ -0,0 +1,3 @@ + +const C = 23 +print C diff --git a/tests/e2e/errors/bad_elif_stmt.hny b/tests/e2e/errors/bad_elif_stmt.hny new file mode 100644 index 00000000..5cd1d427 --- /dev/null +++ b/tests/e2e/errors/bad_elif_stmt.hny @@ -0,0 +1,5 @@ + +if True: + pass +else if: + pass diff --git a/tests/e2e/errors/def_label.hny b/tests/e2e/errors/def_label.hny new file mode 100644 index 00000000..3860e451 --- /dev/null +++ b/tests/e2e/errors/def_label.hny @@ -0,0 +1,2 @@ + +def: x = 3 diff --git a/tests/e2e/errors/empty.hny b/tests/e2e/errors/empty.hny new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/errors/eof_string.hny b/tests/e2e/errors/eof_string.hny new file mode 100644 index 00000000..cc80910f --- /dev/null +++ b/tests/e2e/errors/eof_string.hny @@ -0,0 +1,2 @@ + +v = "Hello World diff --git a/tests/e2e/errors/missing_indent.hny b/tests/e2e/errors/missing_indent.hny new file mode 100644 index 00000000..7fcfa775 --- /dev/null +++ b/tests/e2e/errors/missing_indent.hny @@ -0,0 +1,3 @@ + +def ok(): +pass diff --git a/tests/e2e/errors/unclosed_paren.hny b/tests/e2e/errors/unclosed_paren.hny new file mode 100644 index 00000000..422e7aa3 --- /dev/null +++ b/tests/e2e/errors/unclosed_paren.hny @@ -0,0 +1,2 @@ + +a, b = (1, 5, \ No newline at end of file diff --git a/tests/e2e/modules/base.hny b/tests/e2e/modules/base.hny new file mode 100644 index 00000000..d1cb7cbe --- /dev/null +++ b/tests/e2e/modules/base.hny @@ -0,0 +1,4 @@ + +import math + +print math.sin(10) diff --git a/tests/e2e/modules/resources/math.hny b/tests/e2e/modules/resources/math.hny new file mode 100644 index 00000000..0ab098dd --- /dev/null +++ b/tests/e2e/modules/resources/math.hny @@ -0,0 +1,3 @@ + +def sin(x): + result = "0.12423" diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index 6c560ffc..3c02443a 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -4,6 +4,7 @@ import time from typing import List, NamedTuple import pytest +from harmony_model_checker.exception import HarmonyCompilerError, HarmonyCompilerErrorCollection import harmony_model_checker.harmony.harmony as legacy_harmony from harmony_model_checker.compile import do_compile @@ -11,16 +12,48 @@ class Params(NamedTuple): filename: pathlib.Path max_time: timedelta + modules: List[str] + constants: List[str] -def load_harmony_files() -> List[Params]: - code_dir = pathlib.Path(__file__).parent.parent.parent / "code" +def load_dir(dir: pathlib.Path, modules=None, constants=None): return [ Params( filename=f, - max_time=timedelta(seconds=1) - ) for f in code_dir.glob("*.hny") + max_time=timedelta(seconds=1), + modules=modules or [], + constants=constants or [], + ) for f in dir.glob("*.hny") ] +_DIR = pathlib.Path(__file__).parent + +def load_public_harmony_files(): + code_dir = _DIR.parent.parent / "code" + return load_dir(code_dir) + +def load_failing_harmony_files(): + code_dir = _DIR / "errors" + return load_dir(code_dir) + +def load_constant_defined_harmony_files(): + code_dir = _DIR / "constants" + return load_dir(code_dir, constants=['C=42']) + +def load_failing_constant_defined_harmony_files(): + code_dir = _DIR / "constants" + return load_dir(code_dir, constants=['C=']) \ + + load_dir(code_dir, constants=['C=42', 'A=12']) # unused constants + +def load_module_defined_harmony_files(): + code_dir = _DIR / "modules" + return load_dir(code_dir, modules=['math=resources/math']) + +def load_failing_module_defined_harmony_files(): + code_dir = _DIR / "modules" + return load_dir(code_dir, modules=['math=resources/matt']) \ + + load_dir(code_dir, modules=['math=resources/math', + 'numpy=resources/numpy']) # unused modules + @pytest.fixture(autouse=True) def run_around_tests(): legacy_harmony.files.clear() # files that have been read already @@ -33,10 +66,23 @@ def run_around_tests(): legacy_harmony.used_constants.clear() yield -@pytest.mark.parametrize("param", load_harmony_files()) -def test_compilation(param): +@pytest.mark.parametrize("param", load_public_harmony_files() + + load_constant_defined_harmony_files() + + load_module_defined_harmony_files()) +def test_compilation_success(param): + start_time = time.perf_counter_ns() + do_compile(str(param.filename), consts=param.constants, mods=param.modules, interface=None) + end_time = time.perf_counter_ns() + duration = end_time - start_time + assert duration <= param.max_time.total_seconds() * 1e9 + +@pytest.mark.parametrize("param", load_failing_harmony_files() + + load_failing_constant_defined_harmony_files() + + load_failing_module_defined_harmony_files()) +def test_compilation_error(param): start_time = time.perf_counter_ns() - do_compile(str(param.filename), consts=[], mods=[], interface=None) + with pytest.raises((HarmonyCompilerErrorCollection, HarmonyCompilerError)): + do_compile(str(param.filename), consts=param.constants, mods=param.modules, interface=None) end_time = time.perf_counter_ns() duration = end_time - start_time assert duration <= param.max_time.total_seconds() * 1e9 From 650db932257161cda45d3fb053b14ff41a061887 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 09:57:47 -0400 Subject: [PATCH 07/15] Add test for common pipeline --- tests/e2e/load_test_files.py | 49 ++++++++++++++++++++++++++++++++++ tests/e2e/test_compilation.py | 49 +--------------------------------- tests/e2e/test_gen_html.py | 50 +++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 48 deletions(-) create mode 100644 tests/e2e/load_test_files.py create mode 100644 tests/e2e/test_gen_html.py diff --git a/tests/e2e/load_test_files.py b/tests/e2e/load_test_files.py new file mode 100644 index 00000000..80577300 --- /dev/null +++ b/tests/e2e/load_test_files.py @@ -0,0 +1,49 @@ +from datetime import timedelta +import pathlib +from typing import List, NamedTuple + + +class Params(NamedTuple): + filename: pathlib.Path + max_time: timedelta + modules: List[str] + constants: List[str] + +def load_dir(dir: pathlib.Path, modules=None, constants=None): + return [ + Params( + filename=f, + max_time=timedelta(seconds=1), + modules=modules or [], + constants=constants or [], + ) for f in dir.glob("*.hny") + ] + +_DIR = pathlib.Path(__file__).parent + +def load_public_harmony_files(): + code_dir = _DIR.parent.parent / "code" + return load_dir(code_dir) + +def load_failing_harmony_files(): + code_dir = _DIR / "errors" + return load_dir(code_dir) + +def load_constant_defined_harmony_files(): + code_dir = _DIR / "constants" + return load_dir(code_dir, constants=['C=42']) + +def load_failing_constant_defined_harmony_files(): + code_dir = _DIR / "constants" + return load_dir(code_dir, constants=['C=']) \ + + load_dir(code_dir, constants=['C=42', 'A=12']) # unused constants + +def load_module_defined_harmony_files(): + code_dir = _DIR / "modules" + return load_dir(code_dir, modules=['math=resources/math']) + +def load_failing_module_defined_harmony_files(): + code_dir = _DIR / "modules" + return load_dir(code_dir, modules=['math=resources/matt']) \ + + load_dir(code_dir, modules=['math=resources/math', + 'numpy=resources/numpy']) # unused modules diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index 3c02443a..1808d4c1 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -1,59 +1,12 @@ +from load_test_files import * -from datetime import timedelta -import pathlib import time -from typing import List, NamedTuple import pytest from harmony_model_checker.exception import HarmonyCompilerError, HarmonyCompilerErrorCollection import harmony_model_checker.harmony.harmony as legacy_harmony from harmony_model_checker.compile import do_compile -class Params(NamedTuple): - filename: pathlib.Path - max_time: timedelta - modules: List[str] - constants: List[str] - -def load_dir(dir: pathlib.Path, modules=None, constants=None): - return [ - Params( - filename=f, - max_time=timedelta(seconds=1), - modules=modules or [], - constants=constants or [], - ) for f in dir.glob("*.hny") - ] - -_DIR = pathlib.Path(__file__).parent - -def load_public_harmony_files(): - code_dir = _DIR.parent.parent / "code" - return load_dir(code_dir) - -def load_failing_harmony_files(): - code_dir = _DIR / "errors" - return load_dir(code_dir) - -def load_constant_defined_harmony_files(): - code_dir = _DIR / "constants" - return load_dir(code_dir, constants=['C=42']) - -def load_failing_constant_defined_harmony_files(): - code_dir = _DIR / "constants" - return load_dir(code_dir, constants=['C=']) \ - + load_dir(code_dir, constants=['C=42', 'A=12']) # unused constants - -def load_module_defined_harmony_files(): - code_dir = _DIR / "modules" - return load_dir(code_dir, modules=['math=resources/math']) - -def load_failing_module_defined_harmony_files(): - code_dir = _DIR / "modules" - return load_dir(code_dir, modules=['math=resources/matt']) \ - + load_dir(code_dir, modules=['math=resources/math', - 'numpy=resources/numpy']) # unused modules - @pytest.fixture(autouse=True) def run_around_tests(): legacy_harmony.files.clear() # files that have been read already diff --git a/tests/e2e/test_gen_html.py b/tests/e2e/test_gen_html.py new file mode 100644 index 00000000..df10ca42 --- /dev/null +++ b/tests/e2e/test_gen_html.py @@ -0,0 +1,50 @@ +import pytest +from harmony_model_checker.main import handle_hny, handle_hvm, handle_hco +import harmony_model_checker.harmony.harmony as legacy_harmony + +from load_test_files import * + +class MockNS: + B = None + noweb = True + const = None + mods = None + intf = None + module = None + cf = [] + +def _replace_ext(p: pathlib.Path, ext: str): + p_ext = p.suffix + return str(p)[:-len(p_ext)] + '.' + ext + +@pytest.fixture(autouse=True) +def run_around_tests(): + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled + + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() + yield + +@pytest.mark.parametrize("param", load_public_harmony_files() + + load_constant_defined_harmony_files() + + load_module_defined_harmony_files()) +def test_gen_html(param: Params): + mock_ns = MockNS() + + output_files = { + "hfa": None, + "htm": _replace_ext(param.filename, 'htm'), + "hco": _replace_ext(param.filename, 'hco'), + "hvm": _replace_ext(param.filename, 'hvm'), + "png": None, + "tla": None, + "gv": None + } + + code, scope = handle_hny(mock_ns, output_files, False, str(param.filename)) + handle_hvm(mock_ns, output_files, False, code, scope) + # handle_hco(mock_ns, output_files) From 71a381fa1058e313be2b90c16e08ef7ea30ea2cd Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 10:34:07 -0400 Subject: [PATCH 08/15] Use subprocess instead of handle_* The model checker seems to have global variables, so any modifications to them persist across each run of the model checker with each test. To work around this, we directly call the harmony script instead. --- tests/e2e/test_gen_html.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test_gen_html.py b/tests/e2e/test_gen_html.py index df10ca42..78544398 100644 --- a/tests/e2e/test_gen_html.py +++ b/tests/e2e/test_gen_html.py @@ -1,9 +1,13 @@ +import logging +import subprocess import pytest -from harmony_model_checker.main import handle_hny, handle_hvm, handle_hco +from harmony_model_checker.main import handle_hco import harmony_model_checker.harmony.harmony as legacy_harmony from load_test_files import * +logger = logging.Logger(__file__) + class MockNS: B = None noweb = True @@ -12,6 +16,9 @@ class MockNS: intf = None module = None cf = [] + suppress = False + +_HARMONY_SCRIPT = pathlib.Path(__file__).parent.parent.parent / "harmony" def _replace_ext(p: pathlib.Path, ext: str): p_ext = p.suffix @@ -45,6 +52,15 @@ def test_gen_html(param: Params): "gv": None } - code, scope = handle_hny(mock_ns, output_files, False, str(param.filename)) - handle_hvm(mock_ns, output_files, False, code, scope) - # handle_hco(mock_ns, output_files) + try: + # If it takes longer than 3 seconds, just skip. + r = subprocess.run(args=[_HARMONY_SCRIPT, + str(param.filename), '--noweb'] + + [('-c' + c) for c in param.constants] + + [('-m' + m) for m in param.modules], + timeout=3) + assert r.returncode == 0 + except subprocess.TimeoutExpired: + logger.warning("TimeoutExpired for file %s.", str(param.filename)) + return + handle_hco(mock_ns, output_files) From 3420cad1e3738bfa578fb1870fcdd1eae5ddc5d3 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 10:36:41 -0400 Subject: [PATCH 09/15] Update BBsema code from older to newer syntax --- code/BBsema.hny | 17 +++++++++-------- code/BBsematest.hny | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/code/BBsema.hny b/code/BBsema.hny index 4490c5f2..19485830 100644 --- a/code/BBsema.hny +++ b/code/BBsema.hny @@ -1,7 +1,15 @@ -import synch; +from synch import Semaphore, P, V; const NSLOTS = 2; # size of bounded buffer +buf = { x:() for x in {1..NSLOTS} }; +b_in = 1; +b_out = 1; +l_in = Semaphore(1); +l_out = Semaphore(1); +n_full = Semaphore(0); +n_empty = Semaphore(NSLOTS); + def produce(item): P(?n_empty); P(?l_in); @@ -18,10 +26,3 @@ def consume(): V(?l_out); V(?n_empty); ; -buf = { x:() for x in {1..NSLOTS} }; -b_in = 1; -b_out = 1; -l_in = Semaphore(1); -l_out = Semaphore(1); -n_full = Semaphore(0); -n_empty = Semaphore(NSLOTS); diff --git a/code/BBsematest.hny b/code/BBsematest.hny index 24c02ade..46159eb3 100644 --- a/code/BBsematest.hny +++ b/code/BBsematest.hny @@ -1,4 +1,4 @@ -import BBsema; +from BBsema import produce, consume; const NPRODS = 3; # number of producers const NCONSS = 3; # number of consumers From 21fb2f03bbeab1f5d16f6822259798f35b499af9 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 10:57:58 -0400 Subject: [PATCH 10/15] Write tests using unittest --- tests/e2e/__init__.py | 0 tests/e2e/test_compilation.py | 70 ++++++++++++++++-------------- tests/e2e/test_gen_html.py | 80 +++++++++++++++++------------------ 3 files changed, 79 insertions(+), 71 deletions(-) create mode 100644 tests/e2e/__init__.py diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index 1808d4c1..82e1f89f 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -1,41 +1,49 @@ -from load_test_files import * +import unittest +from tests.e2e.load_test_files import * import time -import pytest from harmony_model_checker.exception import HarmonyCompilerError, HarmonyCompilerErrorCollection import harmony_model_checker.harmony.harmony as legacy_harmony from harmony_model_checker.compile import do_compile -@pytest.fixture(autouse=True) -def run_around_tests(): - legacy_harmony.files.clear() # files that have been read already - legacy_harmony.modules.clear() # modules modified with -m - legacy_harmony.used_modules.clear() # modules modified and used - legacy_harmony.namestack.clear() # stack of module names being compiled +class CompilationTestCase(unittest.TestCase): + def run_before_tests(self): + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled - legacy_harmony.imported.clear() - legacy_harmony.constants.clear() - legacy_harmony.used_constants.clear() - yield + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() -@pytest.mark.parametrize("param", load_public_harmony_files() - + load_constant_defined_harmony_files() - + load_module_defined_harmony_files()) -def test_compilation_success(param): - start_time = time.perf_counter_ns() - do_compile(str(param.filename), consts=param.constants, mods=param.modules, interface=None) - end_time = time.perf_counter_ns() - duration = end_time - start_time - assert duration <= param.max_time.total_seconds() * 1e9 + def test_compilation_success(self): + params = load_public_harmony_files() \ + + load_constant_defined_harmony_files() \ + + load_module_defined_harmony_files() + for param in params: + f = str(param.filename) + self.run_before_tests() + with self.subTest(f"Success compilation test: {f}"): + start_time = time.perf_counter_ns() + do_compile(f, consts=param.constants, mods=param.modules, interface=None) + end_time = time.perf_counter_ns() + duration = end_time - start_time + self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9) -@pytest.mark.parametrize("param", load_failing_harmony_files() - + load_failing_constant_defined_harmony_files() - + load_failing_module_defined_harmony_files()) -def test_compilation_error(param): - start_time = time.perf_counter_ns() - with pytest.raises((HarmonyCompilerErrorCollection, HarmonyCompilerError)): - do_compile(str(param.filename), consts=param.constants, mods=param.modules, interface=None) - end_time = time.perf_counter_ns() - duration = end_time - start_time - assert duration <= param.max_time.total_seconds() * 1e9 + def test_compilation_error(self): + params = load_failing_harmony_files() \ + + load_failing_constant_defined_harmony_files() \ + + load_failing_module_defined_harmony_files() + for param in params: + f = str(param.filename) + self.run_before_tests() + with self.subTest(f"Success compilation test: {f}"): + start_time = time.perf_counter_ns() + self.assertRaises( + (HarmonyCompilerErrorCollection, HarmonyCompilerError), + lambda: do_compile(f, consts=param.constants, mods=param.modules, interface=None)) + end_time = time.perf_counter_ns() + duration = end_time - start_time + self.assertLessEqual(duration, param.max_time.total_seconds() * 1e9) diff --git a/tests/e2e/test_gen_html.py b/tests/e2e/test_gen_html.py index 78544398..04bfe157 100644 --- a/tests/e2e/test_gen_html.py +++ b/tests/e2e/test_gen_html.py @@ -1,10 +1,10 @@ import logging import subprocess -import pytest +import unittest from harmony_model_checker.main import handle_hco import harmony_model_checker.harmony.harmony as legacy_harmony -from load_test_files import * +from tests.e2e.load_test_files import * logger = logging.Logger(__file__) @@ -24,43 +24,43 @@ def _replace_ext(p: pathlib.Path, ext: str): p_ext = p.suffix return str(p)[:-len(p_ext)] + '.' + ext -@pytest.fixture(autouse=True) -def run_around_tests(): - legacy_harmony.files.clear() # files that have been read already - legacy_harmony.modules.clear() # modules modified with -m - legacy_harmony.used_modules.clear() # modules modified and used - legacy_harmony.namestack.clear() # stack of module names being compiled +class GenHtmlTestCase(unittest.TestCase): + def run_before_tests(self): + legacy_harmony.files.clear() # files that have been read already + legacy_harmony.modules.clear() # modules modified with -m + legacy_harmony.used_modules.clear() # modules modified and used + legacy_harmony.namestack.clear() # stack of module names being compiled - legacy_harmony.imported.clear() - legacy_harmony.constants.clear() - legacy_harmony.used_constants.clear() - yield + legacy_harmony.imported.clear() + legacy_harmony.constants.clear() + legacy_harmony.used_constants.clear() -@pytest.mark.parametrize("param", load_public_harmony_files() - + load_constant_defined_harmony_files() - + load_module_defined_harmony_files()) -def test_gen_html(param: Params): - mock_ns = MockNS() - - output_files = { - "hfa": None, - "htm": _replace_ext(param.filename, 'htm'), - "hco": _replace_ext(param.filename, 'hco'), - "hvm": _replace_ext(param.filename, 'hvm'), - "png": None, - "tla": None, - "gv": None - } - - try: - # If it takes longer than 3 seconds, just skip. - r = subprocess.run(args=[_HARMONY_SCRIPT, - str(param.filename), '--noweb'] + - [('-c' + c) for c in param.constants] + - [('-m' + m) for m in param.modules], - timeout=3) - assert r.returncode == 0 - except subprocess.TimeoutExpired: - logger.warning("TimeoutExpired for file %s.", str(param.filename)) - return - handle_hco(mock_ns, output_files) + def test_gen_html(self): + params = load_public_harmony_files() \ + + load_constant_defined_harmony_files() \ + + load_module_defined_harmony_files() + mock_ns = MockNS() + for param in params: + output_files = { + "hfa": None, + "htm": _replace_ext(param.filename, 'htm'), + "hco": _replace_ext(param.filename, 'hco'), + "hvm": _replace_ext(param.filename, 'hvm'), + "png": None, + "tla": None, + "gv": None + } + with self.subTest(): + try: + # If it takes longer than 3 seconds, just skip. + r = subprocess.run( + args=[_HARMONY_SCRIPT, + str(param.filename), '--noweb'] + + [('-c' + c) for c in param.constants] + + [('-m' + m) for m in param.modules], + timeout=3) + self.assertEqual(r.returncode, 0) + except subprocess.TimeoutExpired: + logger.warning("TimeoutExpired for file %s.", str(param.filename)) + return + handle_hco(mock_ns, output_files) From 1d723f2bdc39f02b4850afd3990dbfb23eb9fe73 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 10:59:45 -0400 Subject: [PATCH 11/15] Add make test target --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 661ec875..d571e50b 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,11 @@ upload-test: dist upload: dist twine upload dist/* +test-e2e: + coverage run -m unittest discover tests/e2e + +test: test-e2e + clean: # Harmony outputs in `code` directory rm -f code/*.htm code/*.hvm code/*.hco code/*.png code/*.hfa code/*.tla code/*.gv *.html From f8281f67900e39d47e9e4886007994b5a1ba00ab Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 14 May 2022 11:01:56 -0400 Subject: [PATCH 12/15] Add coverage package to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5af3be68..8b4467a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ antlr-denter==1.3.1 antlr4-python3-runtime==4.9.3 automata-lib==5.0.0 +coverage==6.3.2 pydot==1.4.2 requests==2.27.1 twine==3.7.1 From 1c777830b46132c33ee1b768bfee9aec20b2d6bd Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Mon, 20 Jun 2022 12:34:33 -0700 Subject: [PATCH 13/15] Add Harmony AST tests --- Makefile | 5 +- tests/e2e/test_compilation.py | 2 +- tests/e2e/test_gen_html.py | 2 +- tests/harmony/test_ast.py | 111 ++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 tests/harmony/test_ast.py diff --git a/Makefile b/Makefile index d571e50b..3fbbb04f 100644 --- a/Makefile +++ b/Makefile @@ -42,10 +42,13 @@ upload-test: dist upload: dist twine upload dist/* +test-unit: + coverage run -m unittest discover tests/harmony + test-e2e: coverage run -m unittest discover tests/e2e -test: test-e2e +test: test-unit test-e2e clean: # Harmony outputs in `code` directory diff --git a/tests/e2e/test_compilation.py b/tests/e2e/test_compilation.py index 82e1f89f..8ad24e59 100644 --- a/tests/e2e/test_compilation.py +++ b/tests/e2e/test_compilation.py @@ -39,7 +39,7 @@ def test_compilation_error(self): for param in params: f = str(param.filename) self.run_before_tests() - with self.subTest(f"Success compilation test: {f}"): + with self.subTest(f"Failure compilation test: {f}"): start_time = time.perf_counter_ns() self.assertRaises( (HarmonyCompilerErrorCollection, HarmonyCompilerError), diff --git a/tests/e2e/test_gen_html.py b/tests/e2e/test_gen_html.py index 04bfe157..0d311252 100644 --- a/tests/e2e/test_gen_html.py +++ b/tests/e2e/test_gen_html.py @@ -62,5 +62,5 @@ def test_gen_html(self): self.assertEqual(r.returncode, 0) except subprocess.TimeoutExpired: logger.warning("TimeoutExpired for file %s.", str(param.filename)) - return + continue handle_hco(mock_ns, output_files) diff --git a/tests/harmony/test_ast.py b/tests/harmony/test_ast.py new file mode 100644 index 00000000..7d31a278 --- /dev/null +++ b/tests/harmony/test_ast.py @@ -0,0 +1,111 @@ +from unittest import TestCase +import unittest + +from harmony_model_checker.harmony.ast import * +from harmony_model_checker.harmony.code import Labeled_Op + +def create_token(value, file='test.hny', line=0, col=0): + return value, file, line, col + +class TestConstantAST(TestCase): + def create_constant_ast(self): + return [ + ConstantAST( + endtoken=create_token(const), + const=create_token(const), + ) for const in [12, True, "str"] + ] + + def test_is_constant(self): + for c in self.create_constant_ast(): + scope = Scope(None) + self.assertTrue(c.isConstant(scope)) + + def test_compile(self): + for c in self.create_constant_ast(): + scope = Scope(None) + code = Code() + c.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + + op: PushOp = lbled_op.op + self.assertIsInstance(op, PushOp) + self.assertEqual(op.constant, c.const) + + +class TestNameAST(TestCase): + def create_name_ast(self): + return [ + NameAST( + endtoken=create_token(name), + name=create_token(name), + ) for name in ['abc', 'foo', 'bar', 'harmony'] + ] + + def test_is_constant(self): + for n in self.create_name_ast(): + # default scope is global + scope = Scope(None) + self.assertFalse(n.isConstant(scope)) + + scope = Scope(None) + lexeme = n.name[0] + scope.names[lexeme] = ("constant", n.name) + self.assertTrue(n.isConstant(scope)) + + def test_compile(self): + for n in self.create_name_ast(): + lexeme = n.name[0] + + # test with global scope + scope = Scope(None) + code = Code() + n.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + op: LoadOp = lbled_op.op + self.assertIsInstance(op, LoadOp) + self.assertEqual(op.name, n.name) + + # test as a local variable + scope = Scope(None) + scope.names[lexeme] = ("local-var", n.name) + code = Code() + n.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + op: LoadVarOp = lbled_op.op + self.assertIsInstance(op, LoadVarOp) + self.assertEqual(op.v, n.name) + + # test as a constant + scope = Scope(None) + scope.names[lexeme] = ("constant", n.name) + code = Code() + n.compile(scope, code) + self.assertEqual(len(code.labeled_ops), 1) + lbled_op: Labeled_Op = code.labeled_ops[0] + self.assertIsInstance(lbled_op, Labeled_Op) + op: PushOp = lbled_op.op + self.assertIsInstance(op, PushOp) + self.assertEqual(op.constant, n.name) + + +class TestHarmonyAST(TestCase): + """Tests the creation and modification of Harmony AST classes + """ + def test_create(self): + pass + + def test_simple(self): + self.assertTrue(True) + pass + + +if __name__ == "__main__": + unittest.main() From 54256234b0b13c9f8540ec0fcaf93f3e4c90f410 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 24 Sep 2022 16:13:35 -0500 Subject: [PATCH 14/15] Fix bugs in ast --- harmony_model_checker/harmony/ast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harmony_model_checker/harmony/ast.py b/harmony_model_checker/harmony/ast.py index 9110c2f7..068d215e 100644 --- a/harmony_model_checker/harmony/ast.py +++ b/harmony_model_checker/harmony/ast.py @@ -1270,9 +1270,9 @@ def compile(self, scope, code, stmt): for var_or_cond in self.vars_and_conds: if var_or_cond[0] == 'var': (_, var, expr, tkn, endtkn, op) = var_or_cond - stmt = self.range(token, endtkn) + stmt = self.range(tkn, endtkn) expr.compile(ns, code, stmt) - code.append(StoreVarOp(var), token, op, stmt=stmt) + code.append(StoreVarOp(var), tkn, op, stmt=stmt) self.define(ns, var) elif var_or_cond[0] == 'cond': (_, cond, token, endtkn) = var_or_cond From 5b360ce5a4cd3ff0392a21f724af884df91f6877 Mon Sep 17 00:00:00 2001 From: Anthony Yang Date: Sat, 24 Sep 2022 16:14:18 -0500 Subject: [PATCH 15/15] Fix missing output file and increase timeout tolerance --- tests/e2e/load_test_files.py | 2 +- tests/e2e/test_gen_html.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/load_test_files.py b/tests/e2e/load_test_files.py index 80577300..68577faf 100644 --- a/tests/e2e/load_test_files.py +++ b/tests/e2e/load_test_files.py @@ -13,7 +13,7 @@ def load_dir(dir: pathlib.Path, modules=None, constants=None): return [ Params( filename=f, - max_time=timedelta(seconds=1), + max_time=timedelta(seconds=3), modules=modules or [], constants=constants or [], ) for f in dir.glob("*.hny") diff --git a/tests/e2e/test_gen_html.py b/tests/e2e/test_gen_html.py index 0d311252..1a044f11 100644 --- a/tests/e2e/test_gen_html.py +++ b/tests/e2e/test_gen_html.py @@ -46,6 +46,7 @@ def test_gen_html(self): "htm": _replace_ext(param.filename, 'htm'), "hco": _replace_ext(param.filename, 'hco'), "hvm": _replace_ext(param.filename, 'hvm'), + "hvb": _replace_ext(param.filename, 'hvb'), "png": None, "tla": None, "gv": None