From 244958fc69fc571fee6cf0185e1ad0480aae0c95 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 16 Apr 2012 20:45:44 -0400 Subject: [PATCH 001/167] basics --- LICENSE | 9 +++++++++ Makefile | 5 +++++ README.rst | 0 requirements.txt | 0 sample/__init__.py | 0 sample/core.py | 0 sample/helpers.py | 0 setup.py | 28 ++++++++++++++++++++++++++++ tests/context.py | 0 tests/test_advanced.py | 0 tests/test_basic.py | 0 11 files changed, 42 insertions(+) create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.rst create mode 100644 requirements.txt create mode 100644 sample/__init__.py create mode 100644 sample/core.py create mode 100644 sample/helpers.py create mode 100755 setup.py create mode 100644 tests/context.py create mode 100644 tests/test_advanced.py create mode 100644 tests/test_basic.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..791c001 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) 2012, Kenneth Reitz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3767188 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +init: + pip install -r requirements.txt --use-mirrors + +test: + nosetest tests/**.py \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/sample/__init__.py b/sample/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sample/core.py b/sample/core.py new file mode 100644 index 0000000..e69de29 diff --git a/sample/helpers.py b/sample/helpers.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..df5278d --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +import os + +from os.path import dirname, join +from setuptools import setup, find_packages, Command + +# Hack because logging + setuptools sucks. +import multiprocessing + +with open('README.rst') as f: + readme = f.read() + +setup( + name = 'oauthlib', + version = '0.0.1', + description = 'Python implementation of OAuth 1.0a', + long_description = fread('README.rst'), + author = '', + author_email = '', + url = 'https://github.com/idangazit/oauthlib', + license = fread('LICENSE'), + packages = find_packages(exclude=('tests', 'docs')), + test_suite = 'nose.collector', + tests_require=tests_require, + extras_require={'test': tests_require}, +) + diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_advanced.py b/tests/test_advanced.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..e69de29 From 8b76a382182e15dab2334f01431a9137c7bf0e28 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 16 Apr 2012 21:27:40 -0400 Subject: [PATCH 002/167] basics --- Makefile | 2 +- docs/Makefile | 153 ++++++++++++++++++++++++++++ docs/conf.py | 242 +++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 22 +++++ docs/make.bat | 190 +++++++++++++++++++++++++++++++++++ requirements.txt | 2 + sample/__init__.py | 1 + sample/core.py | 9 ++ setup.py | 31 +++--- 9 files changed, 633 insertions(+), 19 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat diff --git a/Makefile b/Makefile index 3767188..d766386 100644 --- a/Makefile +++ b/Makefile @@ -2,4 +2,4 @@ init: pip install -r requirements.txt --use-mirrors test: - nosetest tests/**.py \ No newline at end of file + nosetest tests \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..f828634 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/sample.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/sample.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/sample" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/sample" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..ac03bd0 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +# +# sample documentation build configuration file, created by +# sphinx-quickstart on Mon Apr 16 21:22:43 2012. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'sample' +copyright = u'2012, Kenneth Reitz' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = 'v0.0.1' +# The full version, including alpha/beta/rc tags. +release = 'v0.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'sampledoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'sample.tex', u'sample Documentation', + u'Kenneth Reitz', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'sample', u'sample Documentation', + [u'Kenneth Reitz'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'sample', u'sample Documentation', + u'Kenneth Reitz', 'sample', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..46f3fe8 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +.. sample documentation master file, created by + sphinx-quickstart on Mon Apr 16 21:22:43 2012. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to sample's documentation! +================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..deed45a --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,190 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\sample.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\sample.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/requirements.txt b/requirements.txt index e69de29..e4ea3e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,2 @@ +nose +sphinx \ No newline at end of file diff --git a/sample/__init__.py b/sample/__init__.py index e69de29..f4633fa 100644 --- a/sample/__init__.py +++ b/sample/__init__.py @@ -0,0 +1 @@ +from .core import hmm \ No newline at end of file diff --git a/sample/core.py b/sample/core.py index e69de29..2cea5dc 100644 --- a/sample/core.py +++ b/sample/core.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- + +def get_hmm(): + """Get a thought.""" + return 'hmmm...' + +def hmm(): + """Contemplation...""" + print get_hmm() \ No newline at end of file diff --git a/setup.py b/setup.py index df5278d..481bfb3 100755 --- a/setup.py +++ b/setup.py @@ -1,28 +1,23 @@ # -*- coding: utf-8 -*- -import os +from setuptools import setup, find_packages -from os.path import dirname, join -from setuptools import setup, find_packages, Command - -# Hack because logging + setuptools sucks. -import multiprocessing with open('README.rst') as f: readme = f.read() +with open('LICENSE') as f: + license = f.read() + setup( - name = 'oauthlib', - version = '0.0.1', - description = 'Python implementation of OAuth 1.0a', - long_description = fread('README.rst'), - author = '', - author_email = '', - url = 'https://github.com/idangazit/oauthlib', - license = fread('LICENSE'), - packages = find_packages(exclude=('tests', 'docs')), - test_suite = 'nose.collector', - tests_require=tests_require, - extras_require={'test': tests_require}, + name='sample', + version='0.0.1', + description='Sample package for Python-Guide.org', + long_description=readme, + author='Kenneth Reitz', + author_email='me@kennethreitz.com', + url='https://github.com/kennethreitz/samplemod', + license=license, + packages=find_packages(exclude=('tests', 'docs')) ) From 0fa2bd8aaeea7f673e71e54196131f849bf383fc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 16 Apr 2012 21:31:15 -0400 Subject: [PATCH 003/167] examples --- tests/context.py | 7 +++++++ tests/test_advanced.py | 16 ++++++++++++++++ tests/test_basic.py | 16 ++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/tests/context.py b/tests/context.py index e69de29..349ae8a 100644 --- a/tests/context.py +++ b/tests/context.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +import sys +import os +sys.path.insert(0, os.path.abspath('..')) + +import sample \ No newline at end of file diff --git a/tests/test_advanced.py b/tests/test_advanced.py index e69de29..5438ac3 100644 --- a/tests/test_advanced.py +++ b/tests/test_advanced.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from .context import sample + +import unittest + + +class AdvancedTestSuite(unittest.TestCase): + """Advanced test cases.""" + + def test_thoughts(self): + sample.hmm() + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py index e69de29..d3e4d5a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from .context import sample + +import unittest + + +class BasicTestSuite(unittest.TestCase): + """Basic test cases.""" + + def test_absolute_truth_and_meaning(self): + assert True + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 1c3e5b5eeaa2198d0d585ada672f846546cb8b2e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 16 Apr 2012 21:32:53 -0400 Subject: [PATCH 004/167] readme --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index e69de29..c92b928 100644 --- a/README.rst +++ b/README.rst @@ -0,0 +1,6 @@ +Sample Module Repository +======================== + +This simple project is an example repo for Python projects. + +`Learn more `_. From 052419685f13b1e654ac0b8c0bc29e53e8c099c0 Mon Sep 17 00:00:00 2001 From: chassing Date: Tue, 17 Apr 2012 10:47:03 +0300 Subject: [PATCH 005/167] fixed typo in nosetests command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d766386..8db3628 100644 --- a/Makefile +++ b/Makefile @@ -2,4 +2,4 @@ init: pip install -r requirements.txt --use-mirrors test: - nosetest tests \ No newline at end of file + nosetests tests \ No newline at end of file From cc5948759036eaee3a6151d466916381575531d2 Mon Sep 17 00:00:00 2001 From: Brian Forst Date: Wed, 20 Feb 2013 11:37:05 -0700 Subject: [PATCH 006/167] Added __init__.py to the tests directory so that relative package import works. --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From e65e415858deb9e366a4605639e8d81bf6b7a4ff Mon Sep 17 00:00:00 2001 From: Steeve Date: Fri, 3 Oct 2014 11:22:11 +0200 Subject: [PATCH 007/167] [readme] update `learn more` link the learn more url as changed, this fix the 404 error --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c92b928..6ed8b74 100644 --- a/README.rst +++ b/README.rst @@ -3,4 +3,4 @@ Sample Module Repository This simple project is an example repo for Python projects. -`Learn more `_. +`Learn more `_. From 35a7de920fc0e076aa468da9fb7bf2a28f749038 Mon Sep 17 00:00:00 2001 From: jones77 Date: Tue, 19 May 2015 14:51:06 -0400 Subject: [PATCH 008/167] Fixing link to blog post. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c92b928..6ed8b74 100644 --- a/README.rst +++ b/README.rst @@ -3,4 +3,4 @@ Sample Module Repository This simple project is an example repo for Python projects. -`Learn more `_. +`Learn more `_. From 8df82b72ba1e52932ebe0377d808ae7c54c770ef Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 4 Feb 2016 04:06:08 -0500 Subject: [PATCH 009/167] Update Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8db3628..997cfaa 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ init: - pip install -r requirements.txt --use-mirrors + pip install -r requirements.txt test: - nosetests tests \ No newline at end of file + nosetests tests From 48f4c8dba40cb2fe03a74a7a4d7d979892601ddc Mon Sep 17 00:00:00 2001 From: Vegard Nossum Date: Mon, 21 Mar 2016 10:11:43 +0100 Subject: [PATCH 010/167] tests: use __file__ to determine library path By using `os.path.dirname(__file__)` you can run the test from any directory/cwd and it will find the right path to the sample module. Without it you might accidentally import it from the wrong directory (i.e. if you have a development version of your project checked out, but you also have one installed in the system path, you may think that your tests pass when in fact they were testing the system installation rather than the development version). --- tests/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/context.py b/tests/context.py index 349ae8a..91de701 100644 --- a/tests/context.py +++ b/tests/context.py @@ -2,6 +2,6 @@ import sys import os -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import sample \ No newline at end of file +import sample From cb3a268aba4e76b52692dd35ea8d618934351db1 Mon Sep 17 00:00:00 2001 From: Dotan Nahum Date: Mon, 24 Oct 2016 19:23:04 +0300 Subject: [PATCH 011/167] add a Python gitignore --- .gitignore | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..37fc9d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,90 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +.venv/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject From 33ed50741218f5ec2ba7347aa9d83982316b6b64 Mon Sep 17 00:00:00 2001 From: Ivan Dmitrievsky Date: Wed, 21 Dec 2016 21:24:58 +0300 Subject: [PATCH 012/167] Change initial version to be 0.1.0 If it's intended to use semantic versioning then it's doesn't make sense to start from a bug fix release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 481bfb3..472f508 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='sample', - version='0.0.1', + version='0.1.0', description='Sample package for Python-Guide.org', long_description=readme, author='Kenneth Reitz', From 983ab0fe41071e4d6fc5e6431351471bfdf5a455 Mon Sep 17 00:00:00 2001 From: Simon Albinsson Date: Wed, 1 Feb 2017 13:20:02 +0100 Subject: [PATCH 013/167] Lets allow the helpers to be helpfull I had problems transissioning form python 2 to 3 due to imports failing when running tests. This shows (what i hope) is the correct way of dealing with imports. --- sample/core.py | 5 ++++- sample/helpers.py | 3 +++ tests/test_advanced.py | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sample/core.py b/sample/core.py index 2cea5dc..19bf06c 100644 --- a/sample/core.py +++ b/sample/core.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- +from . import helpers def get_hmm(): """Get a thought.""" return 'hmmm...' + def hmm(): """Contemplation...""" - print get_hmm() \ No newline at end of file + if helpers.get_answer(): + print(get_hmm()) diff --git a/sample/helpers.py b/sample/helpers.py index e69de29..63ab898 100644 --- a/sample/helpers.py +++ b/sample/helpers.py @@ -0,0 +1,3 @@ +def get_answer(): + """Get an answer.""" + return True diff --git a/tests/test_advanced.py b/tests/test_advanced.py index 5438ac3..cc48248 100644 --- a/tests/test_advanced.py +++ b/tests/test_advanced.py @@ -9,8 +9,8 @@ class AdvancedTestSuite(unittest.TestCase): """Advanced test cases.""" def test_thoughts(self): - sample.hmm() + self.assertIsNone(sample.hmm()) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 583b42bac3529211976d0ffdce94ff8666d4c6e6 Mon Sep 17 00:00:00 2001 From: Dotan Nahum Date: Sat, 11 Feb 2017 22:02:09 +0200 Subject: [PATCH 014/167] need to include the LICENSE file, otherwise pypi installs are broken (it is required on install) --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1aba38f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE From 8022bbc8e6cbb4667f751c58bff37105cb4fa982 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 30 May 2017 13:05:00 -0400 Subject: [PATCH 015/167] Update LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 791c001..3d3920d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012, Kenneth Reitz +Copyright (c) 2017, Kenneth Reitz All rights reserved. @@ -6,4 +6,4 @@ Redistribution and use in source and binary forms, with or without modification, Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 2b57a4fda4ff11f7910d056fa91d8d0764eabc03 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 13:31:05 -0400 Subject: [PATCH 016/167] Update setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 472f508..12af806 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +# Learn more: https://github.com/kennethreitz/setup.py + from setuptools import setup, find_packages From 5c3a87390e7192402246d8709b031a88941800ab Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 13:31:55 -0400 Subject: [PATCH 017/167] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 6ed8b74..9d3c5d7 100644 --- a/README.rst +++ b/README.rst @@ -4,3 +4,5 @@ Sample Module Repository This simple project is an example repo for Python projects. `Learn more `_. + +If you want to learn more about ``setup.py`` files, check out `this repository `_. From 91c947cd62f4ad50512e24e739897ceb3a628395 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 13:32:06 -0400 Subject: [PATCH 018/167] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 9d3c5d7..30f0e7f 100644 --- a/README.rst +++ b/README.rst @@ -5,4 +5,6 @@ This simple project is an example repo for Python projects. `Learn more `_. +--------------- + If you want to learn more about ``setup.py`` files, check out `this repository `_. From 9bf25dd447959b93248567ca561d7607ac97811d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 13:32:17 -0400 Subject: [PATCH 019/167] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 30f0e7f..b52938d 100644 --- a/README.rst +++ b/README.rst @@ -8,3 +8,5 @@ This simple project is an example repo for Python projects. --------------- If you want to learn more about ``setup.py`` files, check out `this repository `_. + +✨🍰✨ From 38431be82d202a2d8dd753086a5625a6e391b9c9 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 21 Nov 2017 01:45:03 -0500 Subject: [PATCH 020/167] initial commit --- LICENSE | 25 ++++++++++++++++++------ Makefile | 4 ++-- requirements.txt | 3 ++- simdem/__init__.py | 1 + simdem/core.py | 27 ++++++++++++++++++++++++++ simdem/executor.py | 36 ++++++++++++++++++++++++++++++++++ simdem/helpers.py | 3 +++ tests/context.py | 1 + tests/test_lexer.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_simdem.py | 34 ++++++++++++++++++++++++++++++++ 10 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 simdem/__init__.py create mode 100644 simdem/core.py create mode 100644 simdem/executor.py create mode 100644 simdem/helpers.py create mode 100644 tests/test_lexer.py create mode 100644 tests/test_simdem.py diff --git a/LICENSE b/LICENSE index 3d3920d..98682a0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,22 @@ -Copyright (c) 2017, Kenneth Reitz +MIT License -All rights reserved. +Copyright (c) 2017 Thomas Falgout -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile index 997cfaa..92fc0c1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ init: - pip install -r requirements.txt + pip3 install -r requirements.txt test: - nosetests tests + python3 setup.py nosetests diff --git a/requirements.txt b/requirements.txt index e4ea3e2..31ad10c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ nose -sphinx \ No newline at end of file +mistune +pexpect \ No newline at end of file diff --git a/simdem/__init__.py b/simdem/__init__.py new file mode 100644 index 0000000..0ef9e7c --- /dev/null +++ b/simdem/__init__.py @@ -0,0 +1 @@ +from .core import start,run_cmd,run_doc \ No newline at end of file diff --git a/simdem/core.py b/simdem/core.py new file mode 100644 index 0000000..fdb7989 --- /dev/null +++ b/simdem/core.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from . import helpers,executor +import mistune + +def start(): + exe = executor.Executor() + sh = exe.get_shell() + +def run_code_block(cmd): + # In the future, we'll want to split a code segment into individual lines + # For now, assume just one command in a block + return run_cmd(cmd) + +def run_cmd(cmd): + exe = executor.Executor() + return exe.run_cmd(cmd) + +def parse_doc(text): + blockLexer = mistune.BlockLexer() + return blockLexer.parse(text) + +def run_doc(text): + blocks = parse_doc(text) + for block in blocks: + print(block) + if block['type'] == 'code' and block['lang'] == 'shell': + run_code_block(block['text']) \ No newline at end of file diff --git a/simdem/executor.py b/simdem/executor.py new file mode 100644 index 0000000..61ce7f9 --- /dev/null +++ b/simdem/executor.py @@ -0,0 +1,36 @@ +import pexpect +import time +from pexpect import replwrap + +PEXPECT_PROMPT = u'[PEXPECT_PROMPT>' +PEXPECT_CONTINUATION_PROMPT = u'[PEXPECT_PROMPT+' + +class Executor(object): + _shell = None + _env = None + + def __init__(self): + pass + + + def run_cmd(self, command=None): + command = command.strip() + print("Execute command: '" + command + "'") + start_time = time.time() + response = self.get_shell().run_command(command) + end_time = time.time() + return response + + def get_shell(self): + """Gets or creates the shell in which to run commands for the + supplied demo + """ + if self._shell == None: + # Should we use spawn or spawnu? + child = pexpect.spawnu('/bin/bash', env=self._env, echo=False, timeout=None) + ps1 = PEXPECT_PROMPT[:5] + u'\[\]' + PEXPECT_PROMPT[5:] + ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + u'\[\]' + PEXPECT_CONTINUATION_PROMPT[5:] + prompt_change = u"PS1='{0}' PS2='{1}' PROMPT_COMMAND=''".format(ps1, ps2) + self._shell = pexpect.replwrap.REPLWrapper(child, u'\$', prompt_change) + return self._shell + diff --git a/simdem/helpers.py b/simdem/helpers.py new file mode 100644 index 0000000..63ab898 --- /dev/null +++ b/simdem/helpers.py @@ -0,0 +1,3 @@ +def get_answer(): + """Get an answer.""" + return True diff --git a/tests/context.py b/tests/context.py index 91de701..814dfdb 100644 --- a/tests/context.py +++ b/tests/context.py @@ -4,4 +4,5 @@ import os sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import simdem import sample diff --git a/tests/test_lexer.py b/tests/test_lexer.py new file mode 100644 index 0000000..299c077 --- /dev/null +++ b/tests/test_lexer.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +from .context import simdem +from mistune import Renderer, InlineGrammar, InlineLexer + +import mistune +import unittest + + +class LexerTestSuite(unittest.TestCase): + """Lexer test cases.""" + + def test_mistune(self): + res = mistune.markdown('I am using **mistune markdown parser**') + self.assertEquals(res, "

I am using mistune markdown parser

\n") + + def test_block_lexer(self): + blockLexer = mistune.BlockLexer() + res = blockLexer.parse('I am using **mistune markdown parser**') + self.assertEquals(res, [{'type': 'paragraph', 'text': 'I am using **mistune markdown parser**'}]) + + def test_block_lexer_cmd(self): + blockLexer = mistune.BlockLexer() + res = blockLexer.parse("```php\necho $foo```") + self.assertEquals(res, [{'lang': 'php', 'text': 'echo $foo', 'type': 'code'}]) + + def test_block_lexer_multiline(self): + blockLexer = mistune.BlockLexer() + res = blockLexer.parse("""this is text +```shell +echo $FOO``` +more text""") + self.assertEquals(res, [{'type': 'paragraph', 'text': 'this is text'}, + {'lang': 'shell', 'text': 'echo $FOO', 'type': 'code'}, + {'text': 'more text', 'type': 'paragraph'}]) + + ''' + Haven't figured this out yet. + def test_inline_lexer(self): + inlineLexer = mistune.InlineLexer() + res = inlineLexer.parse('I am using **mistune markdown parser**') + self.assertEquals(res, [{'type': 'paragraph', 'text': 'I am using **mistune markdown parser**'}]) + ''' + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_simdem.py b/tests/test_simdem.py new file mode 100644 index 0000000..506fefa --- /dev/null +++ b/tests/test_simdem.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +from .context import simdem + +import unittest +import os.path + +class SimDemTestSuite(unittest.TestCase): + """Advanced test cases.""" + + test_file = '/tmp/foo' + + def setUp(self): + os.remove(self.test_file) if os.path.exists(self.test_file) else None + + def test_init(self): + self.assertIsNone(simdem.start()) + + def test_run_cmd(self): + self.assertEquals("foobar\r\n", simdem.run_cmd('echo foobar')) + + def test_run_doc(self): + doc = """this is text +```shell +touch %(file)s``` +more text""" % { 'file' : self.test_file } + + self.assertFalse(os.path.exists(self.test_file)) + simdem.run_doc(doc) + self.assertTrue(os.path.exists(self.test_file)) + + +if __name__ == '__main__': + unittest.main() From 308b3b776f1c7a50a4faa4b4c4aecb2c25891de9 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 21 Nov 2017 01:47:22 -0500 Subject: [PATCH 021/167] Fix readme --- .gitignore | 3 +++ README.md | 3 +++ README.rst | 12 ------------ 3 files changed, 6 insertions(+), 12 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/.gitignore b/.gitignore index 37fc9d4..3d3a126 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,6 @@ ENV/ # Rope project settings .ropeproject + +# Other +.vscode diff --git a/README.md b/README.md new file mode 100644 index 0000000..d7398f4 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# SimDem2 + + diff --git a/README.rst b/README.rst deleted file mode 100644 index b52938d..0000000 --- a/README.rst +++ /dev/null @@ -1,12 +0,0 @@ -Sample Module Repository -======================== - -This simple project is an example repo for Python projects. - -`Learn more `_. - ---------------- - -If you want to learn more about ``setup.py`` files, check out `this repository `_. - -✨🍰✨ From e026a1034e2a1c44a6a17e61634b1390a2f98d43 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 22 Nov 2017 12:53:02 -0500 Subject: [PATCH 022/167] refactor towards classes --- simdem/__init__.py | 2 +- simdem/core.py | 41 +++++++++++++++++++++------------------ simdem/render/__init__.py | 1 + simdem/render/demo.py | 13 +++++++++++++ tests/test_simdem.py | 8 +++++--- 5 files changed, 42 insertions(+), 23 deletions(-) create mode 100644 simdem/render/__init__.py create mode 100644 simdem/render/demo.py diff --git a/simdem/__init__.py b/simdem/__init__.py index 0ef9e7c..a8d584a 100644 --- a/simdem/__init__.py +++ b/simdem/__init__.py @@ -1 +1 @@ -from .core import start,run_cmd,run_doc \ No newline at end of file +from .core import Core \ No newline at end of file diff --git a/simdem/core.py b/simdem/core.py index fdb7989..cd18d8c 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,27 +1,30 @@ # -*- coding: utf-8 -*- from . import helpers,executor import mistune +from .render import demo -def start(): - exe = executor.Executor() - sh = exe.get_shell() +class Core(object): -def run_code_block(cmd): - # In the future, we'll want to split a code segment into individual lines - # For now, assume just one command in a block - return run_cmd(cmd) + def start(self): + exe = executor.Executor() + sh = exe.get_shell() -def run_cmd(cmd): - exe = executor.Executor() - return exe.run_cmd(cmd) + def run_code_block(self, cmd): + # In the future, we'll want to split a code segment into individual lines + # For now, assume just one command in a block + return self.run_cmd(cmd) -def parse_doc(text): - blockLexer = mistune.BlockLexer() - return blockLexer.parse(text) + def run_cmd(self, cmd): + rend = demo.Demo() + return rend.run_cmd(cmd) -def run_doc(text): - blocks = parse_doc(text) - for block in blocks: - print(block) - if block['type'] == 'code' and block['lang'] == 'shell': - run_code_block(block['text']) \ No newline at end of file + def parse_doc(self, text): + blockLexer = mistune.BlockLexer() + return blockLexer.parse(text) + + def run_doc(self, text): + blocks = self.parse_doc(text) + for block in blocks: + print(block) + if block['type'] == 'code' and block['lang'] == 'shell': + self.run_code_block(block['text']) \ No newline at end of file diff --git a/simdem/render/__init__.py b/simdem/render/__init__.py new file mode 100644 index 0000000..3c2343c --- /dev/null +++ b/simdem/render/__init__.py @@ -0,0 +1 @@ +from .demo import Demo \ No newline at end of file diff --git a/simdem/render/demo.py b/simdem/render/demo.py new file mode 100644 index 0000000..8266948 --- /dev/null +++ b/simdem/render/demo.py @@ -0,0 +1,13 @@ +from .. import helpers,executor + +class Demo(object): + _shell = None + _env = None + exe = None + + def __init__(self): + self.exe = executor.Executor() + pass + + def run_cmd(self, cmd): + return self.exe.run_cmd(cmd) diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 506fefa..702b77e 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -9,15 +9,17 @@ class SimDemTestSuite(unittest.TestCase): """Advanced test cases.""" test_file = '/tmp/foo' + simdem = None def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None + self.simdem = simdem.Core() def test_init(self): - self.assertIsNone(simdem.start()) + self.assertIsNone(self.simdem.start()) def test_run_cmd(self): - self.assertEquals("foobar\r\n", simdem.run_cmd('echo foobar')) + self.assertEquals("foobar\r\n", self.simdem.run_cmd('echo foobar')) def test_run_doc(self): doc = """this is text @@ -26,7 +28,7 @@ def test_run_doc(self): more text""" % { 'file' : self.test_file } self.assertFalse(os.path.exists(self.test_file)) - simdem.run_doc(doc) + self.simdem.run_doc(doc) self.assertTrue(os.path.exists(self.test_file)) From 85e6cae592da52cdb36ec5a7a96a33d5c810e644 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 22 Nov 2017 13:53:17 -0500 Subject: [PATCH 023/167] Basic run via files works --- config.py | 19 +++++++++++++++++++ content/simple/README.md | 6 ++++++ main.py | 30 ++++++++++++++++++++++++++++++ simdem/core.py | 29 +++++++++++++++++++++-------- simdem/executor.py | 1 + simdem/render/demo.py | 3 ++- tests/test_simdem.py | 3 --- 7 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 config.py create mode 100644 content/simple/README.md create mode 100644 main.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..4719182 --- /dev/null +++ b/config.py @@ -0,0 +1,19 @@ +SIMDEM_VERSION = "2.0.0" + +# When in demo mode we insert a small random delay between characters. +# TYPING DELAY is the upper bound of this delay. +TYPING_DELAY = 0.2 + +# Prompt to use in the console +console_prompt = "$ " + +# ------------------------------------------------------------------ # +# Danger zone +# +# Do not change anything after this notice, +# unless you know what you are doing +# ------------------------------------------------------------------ # + +# Set is_debug to True if you want to run in debug mode. This setting +# can be overriden in the command like with the `--debug true` option. +is_debug = False \ No newline at end of file diff --git a/content/simple/README.md b/content/simple/README.md new file mode 100644 index 0000000..2b179bd --- /dev/null +++ b/content/simple/README.md @@ -0,0 +1,6 @@ +this is text + +```shell +echo foobar``` + +more text diff --git a/main.py b/main.py new file mode 100644 index 0000000..f00a193 --- /dev/null +++ b/main.py @@ -0,0 +1,30 @@ +from simdem import core +import os +import config +import optparse + +def main(): + p = optparse.OptionParser("%prog file", version=config.SIMDEM_VERSION) + p.add_option('--debug', '-d', default="False", + help="Turn on debug logging by setting to True.") + options, arguments = p.parse_args() + + validate_error = validate(options, arguments) + if validate_error: + print(validate_error) + exit(1) + + simdem = core.Core() + + file_path = arguments[0] + simdem.process_file(file_path) + +def validate(options, arguments): + if len(arguments) != 1: + return "Must provide one and only one argument: " + str(arguments) + + file_path = arguments[0] + if not os.path.isfile(file_path): + return "Unable to find file " + file_path + +main() \ No newline at end of file diff --git a/simdem/core.py b/simdem/core.py index cd18d8c..4f2fb33 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -5,9 +5,12 @@ class Core(object): - def start(self): - exe = executor.Executor() - sh = exe.get_shell() + rend = None + lexer = None + + def __init__(self): + self.rend = demo.Demo() + self.lexer = mistune.BlockLexer() def run_code_block(self, cmd): # In the future, we'll want to split a code segment into individual lines @@ -15,16 +18,26 @@ def run_code_block(self, cmd): return self.run_cmd(cmd) def run_cmd(self, cmd): - rend = demo.Demo() - return rend.run_cmd(cmd) + return self.rend.run_cmd(cmd) + + def process_file(self, file_path): + content = self.get_file_contents(file_path) + result = self.run_doc(content) + return result + + def get_file_contents(self, file_path): + f = open(file_path, 'r') + try: + content = f.read() + finally: + f.close() + return content def parse_doc(self, text): - blockLexer = mistune.BlockLexer() - return blockLexer.parse(text) + return self.lexer.parse(text) def run_doc(self, text): blocks = self.parse_doc(text) for block in blocks: - print(block) if block['type'] == 'code' and block['lang'] == 'shell': self.run_code_block(block['text']) \ No newline at end of file diff --git a/simdem/executor.py b/simdem/executor.py index 61ce7f9..ea94247 100644 --- a/simdem/executor.py +++ b/simdem/executor.py @@ -19,6 +19,7 @@ def run_cmd(self, command=None): start_time = time.time() response = self.get_shell().run_command(command) end_time = time.time() + print("Response: '" + response + "'") return response def get_shell(self): diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 8266948..5ed1180 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -10,4 +10,5 @@ def __init__(self): pass def run_cmd(self, cmd): - return self.exe.run_cmd(cmd) + res = self.exe.run_cmd(cmd) + return res diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 702b77e..3d591a2 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -15,9 +15,6 @@ def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None self.simdem = simdem.Core() - def test_init(self): - self.assertIsNone(self.simdem.start()) - def test_run_cmd(self): self.assertEquals("foobar\r\n", self.simdem.run_cmd('echo foobar')) From 6ab08ceaec85f9a29350f42054909905e603d432 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 22 Nov 2017 22:22:09 -0600 Subject: [PATCH 024/167] Add typing delay --- main.py | 6 ++++-- config.py => simdem/config.py | 7 ++++++- simdem/executor.py | 5 +++-- simdem/render/demo.py | 25 +++++++++++++++++++++++-- 4 files changed, 36 insertions(+), 7 deletions(-) rename config.py => simdem/config.py (85%) diff --git a/main.py b/main.py index f00a193..b8b4189 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,6 @@ -from simdem import core +from simdem import core,config import os -import config +import logging import optparse def main(): @@ -14,6 +14,8 @@ def main(): print(validate_error) exit(1) + logging.basicConfig(filename=config.LOG_FILE,level=config.LOG_LEVEL) + simdem = core.Core() file_path = arguments[0] diff --git a/config.py b/simdem/config.py similarity index 85% rename from config.py rename to simdem/config.py index 4719182..10d6314 100644 --- a/config.py +++ b/simdem/config.py @@ -1,11 +1,16 @@ SIMDEM_VERSION = "2.0.0" +# Logging +import logging +LOG_FILE = "simdem.log" +LOG_LEVEL = logging.DEBUG + # When in demo mode we insert a small random delay between characters. # TYPING DELAY is the upper bound of this delay. TYPING_DELAY = 0.2 # Prompt to use in the console -console_prompt = "$ " +CONSOLE_PROMPT = "$ " # ------------------------------------------------------------------ # # Danger zone diff --git a/simdem/executor.py b/simdem/executor.py index ea94247..280201e 100644 --- a/simdem/executor.py +++ b/simdem/executor.py @@ -1,5 +1,6 @@ import pexpect import time +import logging from pexpect import replwrap PEXPECT_PROMPT = u'[PEXPECT_PROMPT>' @@ -15,11 +16,11 @@ def __init__(self): def run_cmd(self, command=None): command = command.strip() - print("Execute command: '" + command + "'") + logging.debug("Execute command: '" + command + "'") start_time = time.time() response = self.get_shell().run_command(command) end_time = time.time() - print("Response: '" + response + "'") + logging.debug("Response: '" + response + "'") return response def get_shell(self): diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 5ed1180..4db6c2f 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -1,8 +1,10 @@ +import time +import random + from .. import helpers,executor +from .. import config class Demo(object): - _shell = None - _env = None exe = None def __init__(self): @@ -10,5 +12,24 @@ def __init__(self): pass def run_cmd(self, cmd): + self.type_command(cmd) res = self.exe.run_cmd(cmd) + self.display_result(res) return res + + def type_command(self, cmd): + """ + Displays the command on the screen + """ + + print(config.CONSOLE_PROMPT, end="", flush=True) + text = "" + for idx, char in enumerate(cmd): + if char != "\n": + delay = random.uniform(0.02, config.TYPING_DELAY) + time.sleep(delay) + print(char, end="", flush=True) + print("", flush=True) + + def display_result(self, res): + print(res) From 74444f03cbcf31b5153eb428dbe5a58a1b20f1c0 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 22 Nov 2017 22:55:13 -0600 Subject: [PATCH 025/167] Added stdout logging --- main.py | 23 +++++++++++++++++++---- simdem/config.py | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index b8b4189..1b82be7 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,13 @@ from simdem import core,config import os +import sys import logging import optparse def main(): p = optparse.OptionParser("%prog file", version=config.SIMDEM_VERSION) - p.add_option('--debug', '-d', default="False", - help="Turn on debug logging by setting to True.") + p.add_option('--debug', '-d', action="store_true", + help="Turn on logging to console") options, arguments = p.parse_args() validate_error = validate(options, arguments) @@ -14,7 +15,7 @@ def main(): print(validate_error) exit(1) - logging.basicConfig(filename=config.LOG_FILE,level=config.LOG_LEVEL) + setup_logging(options) simdem = core.Core() @@ -27,6 +28,20 @@ def validate(options, arguments): file_path = arguments[0] if not os.path.isfile(file_path): - return "Unable to find file " + file_path + return "Unable to find file " + file_pathu + +def setup_logging(options): + logFormatter = logging.Formatter(config.LOG_FORMAT) + rootLogger = logging.getLogger() + rootLogger.setLevel(config.LOG_LEVEL) + + fileHandler = logging.FileHandler(config.LOG_FILE) + fileHandler.setFormatter(logFormatter) + rootLogger.addHandler(fileHandler) + + if options.debug: + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(logFormatter) + rootLogger.addHandler(consoleHandler) main() \ No newline at end of file diff --git a/simdem/config.py b/simdem/config.py index 10d6314..476ac5c 100644 --- a/simdem/config.py +++ b/simdem/config.py @@ -4,6 +4,7 @@ import logging LOG_FILE = "simdem.log" LOG_LEVEL = logging.DEBUG +LOG_FORMAT = "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s" # When in demo mode we insert a small random delay between characters. # TYPING DELAY is the upper bound of this delay. From 931173fc0ddb470c9fd19f7f1cd9dacb04b51a20 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 22 Nov 2017 23:38:54 -0600 Subject: [PATCH 026/167] refactor config --- content/config/demo.ini | 16 ++++++++++++++++ main.py | 26 ++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 content/config/demo.ini diff --git a/content/config/demo.ini b/content/config/demo.ini new file mode 100644 index 0000000..4b438c2 --- /dev/null +++ b/content/config/demo.ini @@ -0,0 +1,16 @@ +[DEFAULT] +SIMDEM_VERSION = 2.0.0 + +# Logging +[LOG] +FILE = simdem.log +LEVEL = DEBUG +FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s + +[RENDER] +# When in demo mode we insert a small random delay between characters. +# TYPING DELAY is the upper bound of this delay. +TYPING_DELAY = 0.2 + +# Prompt to use in the console +CONSOLE_PROMPT = "$ " \ No newline at end of file diff --git a/main.py b/main.py index 1b82be7..b2d033d 100644 --- a/main.py +++ b/main.py @@ -1,13 +1,16 @@ -from simdem import core,config +from simdem import core import os import sys import logging import optparse +import configparser def main(): - p = optparse.OptionParser("%prog file", version=config.SIMDEM_VERSION) + p = optparse.OptionParser("%prog file", version="%prog 1.0") p.add_option('--debug', '-d', action="store_true", help="Turn on logging to console") + p.add_option('--config-file', '-f', default="content/config/demo.ini", + help="Config file to use") options, arguments = p.parse_args() validate_error = validate(options, arguments) @@ -15,7 +18,10 @@ def main(): print(validate_error) exit(1) - setup_logging(options) + config = configparser.ConfigParser() + config.read(options.config_file) + + setup_logging(config, options) simdem = core.Core() @@ -28,14 +34,18 @@ def validate(options, arguments): file_path = arguments[0] if not os.path.isfile(file_path): - return "Unable to find file " + file_pathu + return "Unable to find file " + file_path + + if not os.path.isfile(options.config_file): + return "Unable to find config file " + options.config_file + -def setup_logging(options): - logFormatter = logging.Formatter(config.LOG_FORMAT) +def setup_logging(config, options): + logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() - rootLogger.setLevel(config.LOG_LEVEL) + rootLogger.setLevel(config.get('LOG', 'LEVEL')) - fileHandler = logging.FileHandler(config.LOG_FILE) + fileHandler = logging.FileHandler(config.get('LOG', 'FILE')) fileHandler.setFormatter(logFormatter) rootLogger.addHandler(fileHandler) From c9915dd3ed613aa9930e13dca6b9f810dee83c46 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 00:04:05 -0600 Subject: [PATCH 027/167] refactor config --- content/config/unit_test.ini | 16 ++++++++++++++++ main.py | 2 +- simdem/core.py | 4 +++- tests/test_simdem.py | 10 +++++++++- 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 content/config/unit_test.ini diff --git a/content/config/unit_test.ini b/content/config/unit_test.ini new file mode 100644 index 0000000..4b438c2 --- /dev/null +++ b/content/config/unit_test.ini @@ -0,0 +1,16 @@ +[DEFAULT] +SIMDEM_VERSION = 2.0.0 + +# Logging +[LOG] +FILE = simdem.log +LEVEL = DEBUG +FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s + +[RENDER] +# When in demo mode we insert a small random delay between characters. +# TYPING DELAY is the upper bound of this delay. +TYPING_DELAY = 0.2 + +# Prompt to use in the console +CONSOLE_PROMPT = "$ " \ No newline at end of file diff --git a/main.py b/main.py index b2d033d..5dab52f 100644 --- a/main.py +++ b/main.py @@ -23,7 +23,7 @@ def main(): setup_logging(config, options) - simdem = core.Core() + simdem = core.Core(config) file_path = arguments[0] simdem.process_file(file_path) diff --git a/simdem/core.py b/simdem/core.py index 4f2fb33..83d81b7 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -7,8 +7,10 @@ class Core(object): rend = None lexer = None + config = None - def __init__(self): + def __init__(self, config): + self.config = config self.rend = demo.Demo() self.lexer = mistune.BlockLexer() diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 3d591a2..cd71f3d 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -4,6 +4,7 @@ import unittest import os.path +import configparser class SimDemTestSuite(unittest.TestCase): """Advanced test cases.""" @@ -13,7 +14,10 @@ class SimDemTestSuite(unittest.TestCase): def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None - self.simdem = simdem.Core() + config = configparser.ConfigParser() + config.read("../content/config/demo.ini") + + self.simdem = simdem.Core(config) def test_run_cmd(self): self.assertEquals("foobar\r\n", self.simdem.run_cmd('echo foobar')) @@ -28,6 +32,10 @@ def test_run_doc(self): self.simdem.run_doc(doc) self.assertTrue(os.path.exists(self.test_file)) + def test_process_file(self): + res = self.simdem.process_file("./content/simple/README.md") + self.assertTrue("$ echo foobar\nfoobar\n") + if __name__ == '__main__': unittest.main() From ac0483045680083f4a3b55b62ab017272c90943d Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 00:05:42 -0600 Subject: [PATCH 028/167] cleanup dead code --- sample/__init__.py | 1 - sample/core.py | 12 ------------ sample/helpers.py | 3 --- tests/context.py | 1 - tests/test_advanced.py | 16 ---------------- tests/test_basic.py | 16 ---------------- 6 files changed, 49 deletions(-) delete mode 100644 sample/__init__.py delete mode 100644 sample/core.py delete mode 100644 sample/helpers.py delete mode 100644 tests/test_advanced.py delete mode 100644 tests/test_basic.py diff --git a/sample/__init__.py b/sample/__init__.py deleted file mode 100644 index f4633fa..0000000 --- a/sample/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .core import hmm \ No newline at end of file diff --git a/sample/core.py b/sample/core.py deleted file mode 100644 index 19bf06c..0000000 --- a/sample/core.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -from . import helpers - -def get_hmm(): - """Get a thought.""" - return 'hmmm...' - - -def hmm(): - """Contemplation...""" - if helpers.get_answer(): - print(get_hmm()) diff --git a/sample/helpers.py b/sample/helpers.py deleted file mode 100644 index 63ab898..0000000 --- a/sample/helpers.py +++ /dev/null @@ -1,3 +0,0 @@ -def get_answer(): - """Get an answer.""" - return True diff --git a/tests/context.py b/tests/context.py index 814dfdb..398dc56 100644 --- a/tests/context.py +++ b/tests/context.py @@ -5,4 +5,3 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import simdem -import sample diff --git a/tests/test_advanced.py b/tests/test_advanced.py deleted file mode 100644 index cc48248..0000000 --- a/tests/test_advanced.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- - -from .context import sample - -import unittest - - -class AdvancedTestSuite(unittest.TestCase): - """Advanced test cases.""" - - def test_thoughts(self): - self.assertIsNone(sample.hmm()) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_basic.py b/tests/test_basic.py deleted file mode 100644 index d3e4d5a..0000000 --- a/tests/test_basic.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- - -from .context import sample - -import unittest - - -class BasicTestSuite(unittest.TestCase): - """Basic test cases.""" - - def test_absolute_truth_and_meaning(self): - assert True - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file From 049799d53592ad82ffcd15832063d4a63438b416 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 10:59:41 -0600 Subject: [PATCH 029/167] Added dependency injection --- main.py | 25 ++++++++++++++++++++++--- simdem/core.py | 8 +++----- simdem/render/demo.py | 1 - tests/context.py | 1 + tests/test_simdem.py | 5 +++-- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 5dab52f..c036a4a 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,8 @@ from simdem import core +from simdem.render import demo import os import sys +import mistune import logging import optparse import configparser @@ -11,6 +13,10 @@ def main(): help="Turn on logging to console") p.add_option('--config-file', '-f', default="content/config/demo.ini", help="Config file to use") + p.add_option('--render', '-r', default="demo", + help="Render class to use") + p.add_option('--lexer', '-l', default="mistune.BlockLexer", + help="Lexer class to use") options, arguments = p.parse_args() validate_error = validate(options, arguments) @@ -23,7 +29,7 @@ def main(): setup_logging(config, options) - simdem = core.Core(config) + simdem = core.Core(config, get_render(options), get_lexer(options)) file_path = arguments[0] simdem.process_file(file_path) @@ -34,11 +40,24 @@ def validate(options, arguments): file_path = arguments[0] if not os.path.isfile(file_path): - return "Unable to find file " + file_path + return "Unable to find file: " + file_path if not os.path.isfile(options.config_file): - return "Unable to find config file " + options.config_file + return "Unable to find config file: " + options.config_file + if options.render not in ['demo']: + return "Unknown Render: " + options.render + + if options.lexer not in ['mistune.BlockLexer']: + return "Unknown Lexer: " + options.lexer + +def get_render(options): + if options.render == 'demo': + return demo.Demo() + +def get_lexer(options): + if options.lexer == 'mistune.BlockLexer': + return mistune.BlockLexer() def setup_logging(config, options): logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) diff --git a/simdem/core.py b/simdem/core.py index 83d81b7..b127b81 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- from . import helpers,executor -import mistune -from .render import demo class Core(object): @@ -9,10 +7,10 @@ class Core(object): lexer = None config = None - def __init__(self, config): + def __init__(self, config, rend, lexer): self.config = config - self.rend = demo.Demo() - self.lexer = mistune.BlockLexer() + self.rend = rend + self.lexer = lexer def run_code_block(self, cmd): # In the future, we'll want to split a code segment into individual lines diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 4db6c2f..a7101bc 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -23,7 +23,6 @@ def type_command(self, cmd): """ print(config.CONSOLE_PROMPT, end="", flush=True) - text = "" for idx, char in enumerate(cmd): if char != "\n": delay = random.uniform(0.02, config.TYPING_DELAY) diff --git a/tests/context.py b/tests/context.py index 398dc56..eb61ab6 100644 --- a/tests/context.py +++ b/tests/context.py @@ -5,3 +5,4 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import simdem +from simdem.render import demo \ No newline at end of file diff --git a/tests/test_simdem.py b/tests/test_simdem.py index cd71f3d..ba023da 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- -from .context import simdem +from .context import simdem, demo import unittest import os.path import configparser +import mistune class SimDemTestSuite(unittest.TestCase): """Advanced test cases.""" @@ -17,7 +18,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("../content/config/demo.ini") - self.simdem = simdem.Core(config) + self.simdem = simdem.Core(config, demo.Demo(), mistune.BlockLexer()) def test_run_cmd(self): self.assertEquals("foobar\r\n", self.simdem.run_cmd('echo foobar')) From b44e10dec9df78e1794c435faa998a2be807436d Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 12:18:52 -0600 Subject: [PATCH 030/167] Added more robust testcases --- content/simple-variable/README.md | 6 ++++ content/simple-variable/expected_result.out | 2 ++ content/simple/expected_result.out | 2 ++ main.py | 1 + setup.py | 2 +- simdem/executor.py | 4 +++ simdem/render/demo.py | 2 +- tests/test_lexer.py | 8 ----- tests/test_simdem.py | 7 +--- tests/test_system.py | 37 +++++++++++++++++++++ 10 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 content/simple-variable/README.md create mode 100644 content/simple-variable/expected_result.out create mode 100644 content/simple/expected_result.out mode change 100644 => 100755 main.py create mode 100644 tests/test_system.py diff --git a/content/simple-variable/README.md b/content/simple-variable/README.md new file mode 100644 index 0000000..9234ff8 --- /dev/null +++ b/content/simple-variable/README.md @@ -0,0 +1,6 @@ +this is text + +```shell +echo blarg``` + +more text diff --git a/content/simple-variable/expected_result.out b/content/simple-variable/expected_result.out new file mode 100644 index 0000000..1043b2e --- /dev/null +++ b/content/simple-variable/expected_result.out @@ -0,0 +1,2 @@ +$ echo blarg +blarg diff --git a/content/simple/expected_result.out b/content/simple/expected_result.out new file mode 100644 index 0000000..c4664ae --- /dev/null +++ b/content/simple/expected_result.out @@ -0,0 +1,2 @@ +$ echo foobar +foobar diff --git a/main.py b/main.py old mode 100644 new mode 100755 index c036a4a..dfa4a4e --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +#!/usr/local/bin/python3 from simdem import core from simdem.render import demo import os diff --git a/setup.py b/setup.py index 12af806..6eb17ea 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages -with open('README.rst') as f: +with open('README.md') as f: readme = f.read() with open('LICENSE') as f: diff --git a/simdem/executor.py b/simdem/executor.py index 280201e..0a27853 100644 --- a/simdem/executor.py +++ b/simdem/executor.py @@ -20,6 +20,10 @@ def run_cmd(self, command=None): start_time = time.time() response = self.get_shell().run_command(command) end_time = time.time() + # https://pexpect.readthedocs.io/en/stable/overview.html#find-the-end-of-line-cr-lf-conventions + # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. This might get tricky if we start supporting windows + # This is because to easily write expected testcase output files, most unix-ish text editors write with \n + response = response.replace("\r\n", "\n") logging.debug("Response: '" + response + "'") return response diff --git a/simdem/render/demo.py b/simdem/render/demo.py index a7101bc..b893cef 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -31,4 +31,4 @@ def type_command(self, cmd): print("", flush=True) def display_result(self, res): - print(res) + print(res, end="", flush=True) diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 299c077..bb180b1 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -34,13 +34,5 @@ def test_block_lexer_multiline(self): {'lang': 'shell', 'text': 'echo $FOO', 'type': 'code'}, {'text': 'more text', 'type': 'paragraph'}]) - ''' - Haven't figured this out yet. - def test_inline_lexer(self): - inlineLexer = mistune.InlineLexer() - res = inlineLexer.parse('I am using **mistune markdown parser**') - self.assertEquals(res, [{'type': 'paragraph', 'text': 'I am using **mistune markdown parser**'}]) - ''' - if __name__ == '__main__': unittest.main() diff --git a/tests/test_simdem.py b/tests/test_simdem.py index ba023da..3d48607 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -21,7 +21,7 @@ def setUp(self): self.simdem = simdem.Core(config, demo.Demo(), mistune.BlockLexer()) def test_run_cmd(self): - self.assertEquals("foobar\r\n", self.simdem.run_cmd('echo foobar')) + self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) def test_run_doc(self): doc = """this is text @@ -33,10 +33,5 @@ def test_run_doc(self): self.simdem.run_doc(doc) self.assertTrue(os.path.exists(self.test_file)) - def test_process_file(self): - res = self.simdem.process_file("./content/simple/README.md") - self.assertTrue("$ echo foobar\nfoobar\n") - - if __name__ == '__main__': unittest.main() diff --git a/tests/test_system.py b/tests/test_system.py new file mode 100644 index 0000000..e8cdd71 --- /dev/null +++ b/tests/test_system.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from .context import simdem, demo + +import unittest +import os.path +import configparser +import mistune +import sys + +class SimDemSystemTestSuite(unittest.TestCase): + """Advanced test cases.""" + + test_file = '/tmp/foo' + simdem = None + + def setUp(self): + os.remove(self.test_file) if os.path.exists(self.test_file) else None + config = configparser.ConfigParser() + config.read("../content/config/demo.ini") + + self.simdem = simdem.Core(config, demo.Demo(), mistune.BlockLexer()) + + + def test_process_file(self): + self.simdem.process_file("./content/simple/README.md") + res = sys.stdout.getvalue() + exp_res = open('./content/simple/expected_result.out', 'r').read() + self.assertEquals(open('./content/simple/expected_result.out', 'r').read(), res) + + def test_process_file2(self): + self.simdem.process_file("./content/simple-variable/README.md") + res = sys.stdout.getvalue() + self.assertEquals(open('./content/simple-variable/expected_result.out', 'r').read(), res) + +if __name__ == '__main__': + unittest.main() From 512b341862d240cb10a2d0bccdefb7a06d6c117a Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 21:17:42 -0600 Subject: [PATCH 031/167] Simple variable works --- content/config/unit_test.ini | 2 +- content/simple-variable/README.md | 5 ++++- content/simple-variable/expected_result.out | 5 +++-- simdem/render/demo.py | 5 +++-- tests/test_system.py | 6 +++--- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/content/config/unit_test.ini b/content/config/unit_test.ini index 4b438c2..f64768a 100644 --- a/content/config/unit_test.ini +++ b/content/config/unit_test.ini @@ -10,7 +10,7 @@ FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s [RENDER] # When in demo mode we insert a small random delay between characters. # TYPING DELAY is the upper bound of this delay. -TYPING_DELAY = 0.2 +TYPING_DELAY = 0 # Prompt to use in the console CONSOLE_PROMPT = "$ " \ No newline at end of file diff --git a/content/simple-variable/README.md b/content/simple-variable/README.md index 9234ff8..5eeec84 100644 --- a/content/simple-variable/README.md +++ b/content/simple-variable/README.md @@ -1,6 +1,9 @@ this is text ```shell -echo blarg``` +FOO=bar``` + +```shell +echo $FOO``` more text diff --git a/content/simple-variable/expected_result.out b/content/simple-variable/expected_result.out index 1043b2e..2b21e48 100644 --- a/content/simple-variable/expected_result.out +++ b/content/simple-variable/expected_result.out @@ -1,2 +1,3 @@ -$ echo blarg -blarg +$ FOO=bar +$ echo $FOO +bar diff --git a/simdem/render/demo.py b/simdem/render/demo.py index b893cef..345d268 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -25,8 +25,9 @@ def type_command(self, cmd): print(config.CONSOLE_PROMPT, end="", flush=True) for idx, char in enumerate(cmd): if char != "\n": - delay = random.uniform(0.02, config.TYPING_DELAY) - time.sleep(delay) + if config.TYPING_DELAY: + delay = random.uniform(0.02, config.TYPING_DELAY) + time.sleep(delay) print(char, end="", flush=True) print("", flush=True) diff --git a/tests/test_system.py b/tests/test_system.py index e8cdd71..1584d6a 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -17,18 +17,18 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() - config.read("../content/config/demo.ini") + config.read("../content/config/unit_test.ini") self.simdem = simdem.Core(config, demo.Demo(), mistune.BlockLexer()) - + # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer def test_process_file(self): self.simdem.process_file("./content/simple/README.md") res = sys.stdout.getvalue() exp_res = open('./content/simple/expected_result.out', 'r').read() self.assertEquals(open('./content/simple/expected_result.out', 'r').read(), res) - def test_process_file2(self): + def test_process_file_variable(self): self.simdem.process_file("./content/simple-variable/README.md") res = sys.stdout.getvalue() self.assertEquals(open('./content/simple-variable/expected_result.out', 'r').read(), res) From 2da9fafc8ad2de7a45d0b436b2390ed41eeef0e3 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 21:24:53 -0600 Subject: [PATCH 032/167] Add cmd block --- content/simple/README.md | 3 ++- content/simple/expected_result.out | 6 ++++-- simdem/core.py | 5 +++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/content/simple/README.md b/content/simple/README.md index 2b179bd..6318a3e 100644 --- a/content/simple/README.md +++ b/content/simple/README.md @@ -1,6 +1,7 @@ this is text ```shell -echo foobar``` +echo foo +echo bar``` more text diff --git a/content/simple/expected_result.out b/content/simple/expected_result.out index c4664ae..ac40788 100644 --- a/content/simple/expected_result.out +++ b/content/simple/expected_result.out @@ -1,2 +1,4 @@ -$ echo foobar -foobar +$ echo foo +foo +$ echo bar +bar diff --git a/simdem/core.py b/simdem/core.py index b127b81..af60d95 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -12,10 +12,11 @@ def __init__(self, config, rend, lexer): self.rend = rend self.lexer = lexer - def run_code_block(self, cmd): + def run_code_block(self, cmd_block): # In the future, we'll want to split a code segment into individual lines # For now, assume just one command in a block - return self.run_cmd(cmd) + for cmd in cmd_block.split("\n"): + self.run_cmd(cmd) def run_cmd(self, cmd): return self.rend.run_cmd(cmd) From ab34357f682190764b434f8426f5e7800db2e668 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 21:27:47 -0600 Subject: [PATCH 033/167] Enhance simple tests --- content/simple/README.md | 7 ++++++- content/simple/expected_result.out | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/content/simple/README.md b/content/simple/README.md index 6318a3e..21bf98c 100644 --- a/content/simple/README.md +++ b/content/simple/README.md @@ -4,4 +4,9 @@ this is text echo foo echo bar``` -more text +more text + +```shell +echo baz``` + +even more text diff --git a/content/simple/expected_result.out b/content/simple/expected_result.out index ac40788..166ed98 100644 --- a/content/simple/expected_result.out +++ b/content/simple/expected_result.out @@ -2,3 +2,5 @@ $ echo foo foo $ echo bar bar +$ echo baz +baz From 48c9630b858454321a070d98197ef4b66fe50d99 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 21:47:18 -0600 Subject: [PATCH 034/167] Remove duplicate config + refactor --- content/config/demo.ini | 2 +- content/config/unit_test.ini | 2 +- main.py | 6 +++--- simdem/config.py | 25 ------------------------- simdem/render/demo.py | 14 +++++++++----- tests/test_simdem.py | 4 ++-- tests/test_system.py | 5 ++--- 7 files changed, 18 insertions(+), 40 deletions(-) delete mode 100644 simdem/config.py diff --git a/content/config/demo.ini b/content/config/demo.ini index 4b438c2..c50d152 100644 --- a/content/config/demo.ini +++ b/content/config/demo.ini @@ -13,4 +13,4 @@ FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s TYPING_DELAY = 0.2 # Prompt to use in the console -CONSOLE_PROMPT = "$ " \ No newline at end of file +CONSOLE_PROMPT = $ \ No newline at end of file diff --git a/content/config/unit_test.ini b/content/config/unit_test.ini index f64768a..e1f8af5 100644 --- a/content/config/unit_test.ini +++ b/content/config/unit_test.ini @@ -13,4 +13,4 @@ FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s TYPING_DELAY = 0 # Prompt to use in the console -CONSOLE_PROMPT = "$ " \ No newline at end of file +CONSOLE_PROMPT = $ \ No newline at end of file diff --git a/main.py b/main.py index dfa4a4e..54eb18f 100755 --- a/main.py +++ b/main.py @@ -30,7 +30,7 @@ def main(): setup_logging(config, options) - simdem = core.Core(config, get_render(options), get_lexer(options)) + simdem = core.Core(config, get_render(options, config), get_lexer(options)) file_path = arguments[0] simdem.process_file(file_path) @@ -52,9 +52,9 @@ def validate(options, arguments): if options.lexer not in ['mistune.BlockLexer']: return "Unknown Lexer: " + options.lexer -def get_render(options): +def get_render(options, config): if options.render == 'demo': - return demo.Demo() + return demo.Demo(config) def get_lexer(options): if options.lexer == 'mistune.BlockLexer': diff --git a/simdem/config.py b/simdem/config.py deleted file mode 100644 index 476ac5c..0000000 --- a/simdem/config.py +++ /dev/null @@ -1,25 +0,0 @@ -SIMDEM_VERSION = "2.0.0" - -# Logging -import logging -LOG_FILE = "simdem.log" -LOG_LEVEL = logging.DEBUG -LOG_FORMAT = "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s" - -# When in demo mode we insert a small random delay between characters. -# TYPING DELAY is the upper bound of this delay. -TYPING_DELAY = 0.2 - -# Prompt to use in the console -CONSOLE_PROMPT = "$ " - -# ------------------------------------------------------------------ # -# Danger zone -# -# Do not change anything after this notice, -# unless you know what you are doing -# ------------------------------------------------------------------ # - -# Set is_debug to True if you want to run in debug mode. This setting -# can be overriden in the command like with the `--debug true` option. -is_debug = False \ No newline at end of file diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 345d268..4e24b37 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -2,12 +2,13 @@ import random from .. import helpers,executor -from .. import config class Demo(object): exe = None + config = None - def __init__(self): + def __init__(self, config): + self.config = config self.exe = executor.Executor() pass @@ -22,11 +23,14 @@ def type_command(self, cmd): Displays the command on the screen """ - print(config.CONSOLE_PROMPT, end="", flush=True) + # Must add ' ' when typing command because whitespaces are removed from configparser + # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure + print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ', end="", flush=True) for idx, char in enumerate(cmd): if char != "\n": - if config.TYPING_DELAY: - delay = random.uniform(0.02, config.TYPING_DELAY) + typing_delay = float(self.config.get('RENDER', 'TYPING_DELAY')) + if typing_delay: + delay = random.uniform(0.02, typing_delay) time.sleep(delay) print(char, end="", flush=True) print("", flush=True) diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 3d48607..b28ae4f 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -16,9 +16,9 @@ class SimDemTestSuite(unittest.TestCase): def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() - config.read("../content/config/demo.ini") + config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(), mistune.BlockLexer()) + self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer()) def test_run_cmd(self): self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) diff --git a/tests/test_system.py b/tests/test_system.py index 1584d6a..699c370 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -17,9 +17,8 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() - config.read("../content/config/unit_test.ini") - - self.simdem = simdem.Core(config, demo.Demo(), mistune.BlockLexer()) + config.read("content/config/unit_test.ini") + self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer()) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer def test_process_file(self): From 92c115394b11201da3ed66f8ce920b75d36c3294 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 23 Nov 2017 21:58:00 -0600 Subject: [PATCH 035/167] Made adding new system tests easier --- tests/test_system.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/test_system.py b/tests/test_system.py index 699c370..ed19e44 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -3,34 +3,30 @@ from .context import simdem, demo import unittest -import os.path import configparser import mistune import sys +from ddt import ddt,data +@ddt class SimDemSystemTestSuite(unittest.TestCase): """Advanced test cases.""" - test_file = '/tmp/foo' simdem = None def setUp(self): - os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer()) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - def test_process_file(self): - self.simdem.process_file("./content/simple/README.md") + @data('simple', 'simple-variable') + def test_process(self, dir): + self.simdem.process_file('./content/' + dir + '/README.md') res = sys.stdout.getvalue() - exp_res = open('./content/simple/expected_result.out', 'r').read() - self.assertEquals(open('./content/simple/expected_result.out', 'r').read(), res) + exp_res = open('./content/' + dir + '/expected_result.out', 'r').read() + self.assertEquals(exp_res, res) - def test_process_file_variable(self): - self.simdem.process_file("./content/simple-variable/README.md") - res = sys.stdout.getvalue() - self.assertEquals(open('./content/simple-variable/expected_result.out', 'r').read(), res) if __name__ == '__main__': unittest.main() From 0fd3ba93d37707fc626ec898478a8c94ed42ce8f Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 28 Nov 2017 12:19:43 -0600 Subject: [PATCH 036/167] update requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 31ad10c..828b457 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ nose mistune -pexpect \ No newline at end of file +pexpect +ddt From 29532506d2d188895943719824f8f9a76939322e Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 29 Nov 2017 13:58:39 -0600 Subject: [PATCH 037/167] Add "result" processing. --- content/results-block-fail/README.md | 16 +++++ .../results-block-fail/expected_result.out | 4 ++ content/results-block/README.md | 12 ++++ content/results-block/expected_result.out | 4 ++ requirements.txt | 1 + simdem/core.py | 66 +++++++++++++++++-- tests/test_system.py | 11 +++- 7 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 content/results-block-fail/README.md create mode 100644 content/results-block-fail/expected_result.out create mode 100644 content/results-block/README.md create mode 100644 content/results-block/expected_result.out diff --git a/content/results-block-fail/README.md b/content/results-block-fail/README.md new file mode 100644 index 0000000..8e455f1 --- /dev/null +++ b/content/results-block-fail/README.md @@ -0,0 +1,16 @@ +this is text + +```shell +echo foo +echo bar``` + +Results: + +```shell +barr``` + +After this, we should not pass +```shell +echo YOU SHALL NOT PASS!``` + +even more text diff --git a/content/results-block-fail/expected_result.out b/content/results-block-fail/expected_result.out new file mode 100644 index 0000000..ac40788 --- /dev/null +++ b/content/results-block-fail/expected_result.out @@ -0,0 +1,4 @@ +$ echo foo +foo +$ echo bar +bar diff --git a/content/results-block/README.md b/content/results-block/README.md new file mode 100644 index 0000000..ba15443 --- /dev/null +++ b/content/results-block/README.md @@ -0,0 +1,12 @@ +this is text + +```shell +echo foo +echo bar``` + +Results: + +```shell +bar``` + +even more text diff --git a/content/results-block/expected_result.out b/content/results-block/expected_result.out new file mode 100644 index 0000000..ac40788 --- /dev/null +++ b/content/results-block/expected_result.out @@ -0,0 +1,4 @@ +$ echo foo +foo +$ echo bar +bar diff --git a/requirements.txt b/requirements.txt index 828b457..41f28db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ nose mistune pexpect ddt +difflib diff --git a/simdem/core.py b/simdem/core.py index af60d95..1d64142 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from . import helpers,executor +import difflib +import logging class Core(object): @@ -15,8 +17,12 @@ def __init__(self, config, rend, lexer): def run_code_block(self, cmd_block): # In the future, we'll want to split a code segment into individual lines # For now, assume just one command in a block + # Returning the latest result so we can validate the result. + # We might want to validate the result of the entire block, but for now, validate just the last run command + result_latest = None for cmd in cmd_block.split("\n"): - self.run_cmd(cmd) + result_latest = self.run_cmd(cmd) + return result_latest def run_cmd(self, cmd): return self.rend.run_cmd(cmd) @@ -39,6 +45,58 @@ def parse_doc(self, text): def run_doc(self, text): blocks = self.parse_doc(text) - for block in blocks: - if block['type'] == 'code' and block['lang'] == 'shell': - self.run_code_block(block['text']) \ No newline at end of file + results_latest = None + for idx in range(len(blocks)): + if self.is_result_block(blocks, idx): + is_passable = self.is_result_passable(blocks[idx]['text'], results_latest) + if not is_passable: + logging.error("Result did not pass") + return + elif self.is_runable_block(blocks[idx]): + results_latest = self.run_code_block(blocks[idx]['text']) + + def is_runable_block(self, block): + if block['type'] == 'code' and block['lang'] == 'shell': + return True + return False + + def is_result_block(self, blocks, idx): + block = blocks[idx] + block_prev = blocks[idx-1] + if block['type'] == 'code' and block['lang'] == 'shell' and \ + block_prev['type'] == 'paragraph' and block_prev['text'].lower().startswith('results:'): + return True + return False + + + def is_result_passable(self, expected_results, actual_results, expected_similarity = 1.0): + """Checks to see if a command execution passes. + If actual results compared to expected results is within + the expected similarity level then it's considered a pass. + + expected_similarity = 1.0 could be a breaking change for older SimDem scripts. + explicit fails > implicit passes + Ross may disagree with me. Let's see how this story unfolds. + """ + + if not actual_results: + logging.error("is_result_passable(): actual_results is empty.") + return False + + logging.debug("is_result_passable(" + expected_results + "," + actual_results + "," + str(expected_similarity) + ")") + + expected_results_str = expected_results.rstrip() + actual_results_str = actual_results.rstrip() + logging.debug("is_result_passable(" + expected_results_str + "," + actual_results_str + "," + str(expected_similarity) + ")") + seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", actual_results_str, expected_results_str) + + is_pass = seq.ratio() >= expected_similarity + + if is_pass: + logging.info("is_result_passable passed") + else: + logging.error("is_result_passable failed") + logging.error("actual_results = " + actual_results) + logging.error("expected_results = " + expected_results) + + return is_pass \ No newline at end of file diff --git a/tests/test_system.py b/tests/test_system.py index ed19e44..add99ab 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -5,6 +5,7 @@ import unittest import configparser import mistune +import logging import sys from ddt import ddt,data @@ -19,8 +20,16 @@ def setUp(self): config.read("content/config/unit_test.ini") self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer()) + logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + rootLogger = logging.getLogger() + rootLogger.setLevel(config.get('LOG', 'LEVEL')) + fileHandler = logging.FileHandler(config.get('LOG', 'FILE')) + fileHandler.setFormatter(logFormatter) + rootLogger.addHandler(fileHandler) + # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable') +# @data('simple', 'simple-variable', 'results-block', 'results-block) + @data( 'results-block-fail') def test_process(self, dir): self.simdem.process_file('./content/' + dir + '/README.md') res = sys.stdout.getvalue() From 9ba73f0b9f69f4c99004d269901ed97d598d87be Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 29 Nov 2017 14:03:32 -0600 Subject: [PATCH 038/167] delete helper file --- simdem/core.py | 2 +- simdem/helpers.py | 3 --- simdem/render/demo.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 simdem/helpers.py diff --git a/simdem/core.py b/simdem/core.py index 1d64142..3846b7a 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from . import helpers,executor +from . import executor import difflib import logging diff --git a/simdem/helpers.py b/simdem/helpers.py deleted file mode 100644 index 63ab898..0000000 --- a/simdem/helpers.py +++ /dev/null @@ -1,3 +0,0 @@ -def get_answer(): - """Get an answer.""" - return True diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 4e24b37..3a1ca7b 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -1,7 +1,7 @@ import time import random -from .. import helpers,executor +from .. import executor class Demo(object): exe = None From 3525ebd61a25843c93fc84fc5f0ee0afebf84d9a Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 29 Nov 2017 14:05:07 -0600 Subject: [PATCH 039/167] fix test' --- tests/test_system.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_system.py b/tests/test_system.py index add99ab..671fc3a 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -28,8 +28,7 @@ def setUp(self): rootLogger.addHandler(fileHandler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer -# @data('simple', 'simple-variable', 'results-block', 'results-block) - @data( 'results-block-fail') + @data('simple', 'simple-variable', 'results-block', 'results-block-fail') def test_process(self, dir): self.simdem.process_file('./content/' + dir + '/README.md') res = sys.stdout.getvalue() From c3a3cca8b1500e41366d9f46cbbf9404a76090a6 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 11 Dec 2017 11:14:58 -0600 Subject: [PATCH 040/167] Add parser class and pre-req --- simdem/__init__.py | 3 ++- simdem/core.py | 38 +++++++++++++++++--------------------- simdem/parser.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_lexer.py | 16 ++++++++++++++++ tests/test_parser.py | 27 +++++++++++++++++++++++++++ tests/test_simdem.py | 7 ++++--- tests/test_system.py | 2 +- 7 files changed, 103 insertions(+), 26 deletions(-) create mode 100644 simdem/parser.py create mode 100644 tests/test_parser.py diff --git a/simdem/__init__.py b/simdem/__init__.py index a8d584a..54ef264 100644 --- a/simdem/__init__.py +++ b/simdem/__init__.py @@ -1 +1,2 @@ -from .core import Core \ No newline at end of file +from .core import Core +from .parser import Parser \ No newline at end of file diff --git a/simdem/core.py b/simdem/core.py index 3846b7a..7141a85 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,18 +1,21 @@ # -*- coding: utf-8 -*- -from . import executor +from . import executor,parser import difflib import logging +import re class Core(object): rend = None lexer = None config = None + parser = None - def __init__(self, config, rend, lexer): + def __init__(self, config, rend, lexer, parser): self.config = config self.rend = rend self.lexer = lexer + self.parser = parser def run_code_block(self, cmd_block): # In the future, we'll want to split a code segment into individual lines @@ -29,7 +32,8 @@ def run_cmd(self, cmd): def process_file(self, file_path): content = self.get_file_contents(file_path) - result = self.run_doc(content) + blocks = self.parse_doc(content) + result = self.run_blocks(blocks) return result def get_file_contents(self, file_path): @@ -43,32 +47,24 @@ def get_file_contents(self, file_path): def parse_doc(self, text): return self.lexer.parse(text) - def run_doc(self, text): - blocks = self.parse_doc(text) + def run_blocks(self, blocks): results_latest = None for idx in range(len(blocks)): - if self.is_result_block(blocks, idx): + if self.parser.is_prerequisite_block(blocks[idx]): + # TODO: We need to skip processing the next block if it's a preqreq. + # we might need to refactor the looping mechanism to do so. Fight for a different day + preqreq_file = parse_ref_from_text(blocks[idx+1]['text']) + if preqreq_file: + self.process_file(prereq_file) + if self.parser.is_result_block(blocks, idx): is_passable = self.is_result_passable(blocks[idx]['text'], results_latest) if not is_passable: logging.error("Result did not pass") return - elif self.is_runable_block(blocks[idx]): + elif self.parser.is_runable_block(blocks[idx]): results_latest = self.run_code_block(blocks[idx]['text']) - - def is_runable_block(self, block): - if block['type'] == 'code' and block['lang'] == 'shell': - return True - return False - - def is_result_block(self, blocks, idx): - block = blocks[idx] - block_prev = blocks[idx-1] - if block['type'] == 'code' and block['lang'] == 'shell' and \ - block_prev['type'] == 'paragraph' and block_prev['text'].lower().startswith('results:'): - return True - return False - + def is_result_passable(self, expected_results, actual_results, expected_similarity = 1.0): """Checks to see if a command execution passes. If actual results compared to expected results is within diff --git a/simdem/parser.py b/simdem/parser.py new file mode 100644 index 0000000..de1393c --- /dev/null +++ b/simdem/parser.py @@ -0,0 +1,36 @@ +import re +import logging + +class Parser(object): + + def is_prerequisite_block(self, block): + # Example: {'level': 1, 'text': 'Prerequisites', 'type': 'heading'} + if 'prerequisite' in block['text'].lower() and block['type'] == 'heading': + return True + return False + + def is_runable_block(self, block): + if block['type'] == 'code' and block['lang'] == 'shell': + return True + return False + + def is_result_block(self, blocks, idx): + block = blocks[idx] + block_prev = blocks[idx-1] + if block['type'] == 'code' and block['lang'] == 'shell' and \ + block_prev['type'] == 'paragraph' and block_prev['text'].lower().startswith('results:'): + return True + return False + + # Assuming just one for now + def parse_ref_from_text(self, text): + # Does mistune allow us to parse this? Would be nice. + pattern = re.compile('.*\[(.*)\]\((.*)\).*') + match = pattern.match(text) + if match: + title = match.groups()[0].strip() + href = match.groups()[1] + logging.debug("Found prereq: " + href) + return href + return None + diff --git a/tests/test_lexer.py b/tests/test_lexer.py index bb180b1..7532b2e 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -24,6 +24,22 @@ def test_block_lexer_cmd(self): res = blockLexer.parse("```php\necho $foo```") self.assertEquals(res, [{'lang': 'php', 'text': 'echo $foo', 'type': 'code'}]) + def test_block_lexer_prerequisite(self): + blockLexer = mistune.BlockLexer() + res = blockLexer.parse("""foo +# Prerequisites +We should be able to run [nested prerequisites](./nested_prereq.md). + +# Do stuff here +```echo foo +```""") + self.assertEquals(res, + [{'text': 'foo', 'type': 'paragraph'}, + {'level': 1, 'text': 'Prerequisites', 'type': 'heading'}, + {'text': 'We should be able to run [nested prerequisites](./nested_prereq.md).', 'type': 'paragraph'}, + {'level': 1, 'text': 'Do stuff here', 'type': 'heading'}, + {'text': '```echo foo\n```', 'type': 'paragraph'}]) + def test_block_lexer_multiline(self): blockLexer = mistune.BlockLexer() res = blockLexer.parse("""this is text diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..bd0ede8 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from .context import simdem, demo + +import unittest +import os.path +import configparser +import mistune + +class SimDemTestSuite(unittest.TestCase): + """Advanced test cases.""" + + test_file = '/tmp/foo' + parser = None + + def setUp(self): + os.remove(self.test_file) if os.path.exists(self.test_file) else None + config = configparser.ConfigParser() + config.read("content/config/unit_test.ini") + + self.parser = simdem.Parser() + + def test_parse_ref_from_text(self): + self.assertEquals('./nested_prereq.md', self.parser.parse_ref_from_text('We should be able to run [nested prerequisites](./nested_prereq.md).')) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_simdem.py b/tests/test_simdem.py index b28ae4f..65d7922 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -18,19 +18,20 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer()) + self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer(), simdem.Parser()) def test_run_cmd(self): self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) - def test_run_doc(self): + def test_run_blocks(self): doc = """this is text ```shell touch %(file)s``` more text""" % { 'file' : self.test_file } self.assertFalse(os.path.exists(self.test_file)) - self.simdem.run_doc(doc) + blocks = self.simdem.parse_doc(doc) + self.simdem.run_blocks(blocks) self.assertTrue(os.path.exists(self.test_file)) if __name__ == '__main__': diff --git a/tests/test_system.py b/tests/test_system.py index 671fc3a..0ae9996 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -18,7 +18,7 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer()) + self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer(), simdem.Parser()) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() From c4638dc7bcd51f34f48d4adab7ee4b9dc66d2d15 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 11 Dec 2017 12:11:51 -0600 Subject: [PATCH 041/167] add prereq --- simdem/core.py | 27 ++++++++++++--------------- simdem/parser.py | 25 +++++++++++++++++++++++++ tests/test_parser.py | 2 +- tests/test_simdem.py | 7 ++++--- tests/test_system.py | 4 ++-- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/simdem/core.py b/simdem/core.py index 7141a85..fb9c135 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -7,14 +7,12 @@ class Core(object): rend = None - lexer = None config = None parser = None - def __init__(self, config, rend, lexer, parser): + def __init__(self, config, rend, parser): self.config = config self.rend = rend - self.lexer = lexer self.parser = parser def run_code_block(self, cmd_block): @@ -31,21 +29,20 @@ def run_cmd(self, cmd): return self.rend.run_cmd(cmd) def process_file(self, file_path): - content = self.get_file_contents(file_path) - blocks = self.parse_doc(content) + content = self.parser.get_file_contents(file_path) + blocks = self.parser.parse_doc(content) + self.process_prereqs(blocks) result = self.run_blocks(blocks) return result - def get_file_contents(self, file_path): - f = open(file_path, 'r') - try: - content = f.read() - finally: - f.close() - return content - - def parse_doc(self, text): - return self.lexer.parse(text) + def process_prereqs(self, blocks): + prereqs = self.parser.get_prereqs(blocks) + for prereq in prereqs: + prereq_content = self.get_file_contents(prereq) + preqreq_blocks = self.parse.parse_doc(prereq_content) + preqreq_validation = self.parser.get_validation_block(preqreq_blocks) + if not self.is_prereq_required(preqreq_validation): + self.process_file(prereq) def run_blocks(self, blocks): results_latest = None diff --git a/simdem/parser.py b/simdem/parser.py index de1393c..c60026f 100644 --- a/simdem/parser.py +++ b/simdem/parser.py @@ -3,6 +3,11 @@ class Parser(object): + lexer = None + + def __init__(self, lexer): + self.lexer = lexer + def is_prerequisite_block(self, block): # Example: {'level': 1, 'text': 'Prerequisites', 'type': 'heading'} if 'prerequisite' in block['text'].lower() and block['type'] == 'heading': @@ -34,3 +39,23 @@ def parse_ref_from_text(self, text): return href return None + def get_prereqs(self, blocks): + # WTF: Filter changed b/w python 2 -> 3? Returns an object now? There's no stack overflow on this plane + res = [] + for block in blocks: + if self.is_prerequisite_block(block): + res.append(block) +# pre_reqs = filter(lambda(block): self.is_prerequisite_block(block), blocks) + return res + + def get_file_contents(self, file_path): + f = open(file_path, 'r') + try: + content = f.read() + finally: + f.close() + return content + + def parse_doc(self, text): + return self.lexer.parse(text) + diff --git a/tests/test_parser.py b/tests/test_parser.py index bd0ede8..a7f7f96 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -18,7 +18,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.parser = simdem.Parser() + self.parser = simdem.Parser(mistune.BlockLexer()) def test_parse_ref_from_text(self): self.assertEquals('./nested_prereq.md', self.parser.parse_ref_from_text('We should be able to run [nested prerequisites](./nested_prereq.md).')) diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 65d7922..520770a 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -12,13 +12,14 @@ class SimDemTestSuite(unittest.TestCase): test_file = '/tmp/foo' simdem = None + parser = None def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - - self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer(), simdem.Parser()) + self.parser = simdem.Parser(mistune.BlockLexer()) + self.simdem = simdem.Core(config, demo.Demo(config), self.parser) def test_run_cmd(self): self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) @@ -30,7 +31,7 @@ def test_run_blocks(self): more text""" % { 'file' : self.test_file } self.assertFalse(os.path.exists(self.test_file)) - blocks = self.simdem.parse_doc(doc) + blocks = self.parser.parse_doc(doc) self.simdem.run_blocks(blocks) self.assertTrue(os.path.exists(self.test_file)) diff --git a/tests/test_system.py b/tests/test_system.py index 0ae9996..d0318fa 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -18,7 +18,7 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), mistune.BlockLexer(), simdem.Parser()) + self.simdem = simdem.Core(config, demo.Demo(config), simdem.Parser(mistune.BlockLexer())) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() @@ -28,7 +28,7 @@ def setUp(self): rootLogger.addHandler(fileHandler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable', 'results-block', 'results-block-fail') + @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'create-file') def test_process(self, dir): self.simdem.process_file('./content/' + dir + '/README.md') res = sys.stdout.getvalue() From 6b27b2dd1053109bd6fb0356818400fcd6b3a931 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 14 Dec 2017 20:14:19 -0800 Subject: [PATCH 042/167] add create-file test --- content/create-file/README.md | 3 +++ content/create-file/expected_result.out | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 content/create-file/README.md create mode 100644 content/create-file/expected_result.out diff --git a/content/create-file/README.md b/content/create-file/README.md new file mode 100644 index 0000000..caf2360 --- /dev/null +++ b/content/create-file/README.md @@ -0,0 +1,3 @@ +```shell +echo 'abc' > scratch/foo +cat scratch/foo``` \ No newline at end of file diff --git a/content/create-file/expected_result.out b/content/create-file/expected_result.out new file mode 100644 index 0000000..ae4b462 --- /dev/null +++ b/content/create-file/expected_result.out @@ -0,0 +1,3 @@ +$ echo 'abc' > scratch/foo +$ cat scratch/foo +abc From 2c96df55b8bf8997f5f79c7734993f223c9aadbc Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 15 Dec 2017 15:46:45 -0600 Subject: [PATCH 043/167] Add prereq. Very basic right now --- content/prerequisite-run/README.md | 9 +++++++ content/prerequisite-run/expected_result.out | 4 +++ content/prerequisite-run/prereq.md | 7 +++++ simdem/core.py | 27 +++++++++++--------- simdem/parser.py | 27 +++++++++++++------- tests/test_system.py | 5 +++- 6 files changed, 57 insertions(+), 22 deletions(-) create mode 100644 content/prerequisite-run/README.md create mode 100644 content/prerequisite-run/expected_result.out create mode 100644 content/prerequisite-run/prereq.md diff --git a/content/prerequisite-run/README.md b/content/prerequisite-run/README.md new file mode 100644 index 0000000..6017ec6 --- /dev/null +++ b/content/prerequisite-run/README.md @@ -0,0 +1,9 @@ +# Prerequisite + +content/prerequisite-run/prereq.md + +# Main + +```shell +echo post-prereq-run +``` \ No newline at end of file diff --git a/content/prerequisite-run/expected_result.out b/content/prerequisite-run/expected_result.out new file mode 100644 index 0000000..d5991b2 --- /dev/null +++ b/content/prerequisite-run/expected_result.out @@ -0,0 +1,4 @@ +$ echo running-prereq +running-prereq +$ echo post-prereq-run +post-prereq-run diff --git a/content/prerequisite-run/prereq.md b/content/prerequisite-run/prereq.md new file mode 100644 index 0000000..c6c9648 --- /dev/null +++ b/content/prerequisite-run/prereq.md @@ -0,0 +1,7 @@ +This text should be run + +```shell +echo running-prereq +``` + +End running text diff --git a/simdem/core.py b/simdem/core.py index fb9c135..5d0b062 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -29,36 +29,39 @@ def run_cmd(self, cmd): return self.rend.run_cmd(cmd) def process_file(self, file_path): + logging.info("process_file():file_path=" + file_path) content = self.parser.get_file_contents(file_path) + logging.info("process_file():content=" + str(content)) blocks = self.parser.parse_doc(content) + logging.info("process_file():blocks=" + str(blocks)) self.process_prereqs(blocks) + logging.info("process_file():completed process_prereqs()") result = self.run_blocks(blocks) return result def process_prereqs(self, blocks): prereqs = self.parser.get_prereqs(blocks) - for prereq in prereqs: - prereq_content = self.get_file_contents(prereq) - preqreq_blocks = self.parse.parse_doc(prereq_content) - preqreq_validation = self.parser.get_validation_block(preqreq_blocks) - if not self.is_prereq_required(preqreq_validation): - self.process_file(prereq) + logging.info("process_preqreqs():" + str(prereqs)) + for prereq_file in prereqs: +# prereq_content = self.parser.get_file_contents(prereq_file) +# preqreq_blocks = self.parser.parse_doc(prereq_content) +# preqreq_validation = self.parser.get_validation_block(preqreq_blocks) +# if not self.is_prereq_required(preqreq_validation): + self.process_file(prereq_file) def run_blocks(self, blocks): + logging.info("run_blocks():blocks=" + str(blocks)) results_latest = None for idx in range(len(blocks)): - if self.parser.is_prerequisite_block(blocks[idx]): - # TODO: We need to skip processing the next block if it's a preqreq. - # we might need to refactor the looping mechanism to do so. Fight for a different day - preqreq_file = parse_ref_from_text(blocks[idx+1]['text']) - if preqreq_file: - self.process_file(prereq_file) + logging.info("run_blocks():processing " + str(blocks[idx])) if self.parser.is_result_block(blocks, idx): + logging.info("run_blocks():is_result_block") is_passable = self.is_result_passable(blocks[idx]['text'], results_latest) if not is_passable: logging.error("Result did not pass") return elif self.parser.is_runable_block(blocks[idx]): + logging.info("run_blocks():is_runable_block") results_latest = self.run_code_block(blocks[idx]['text']) diff --git a/simdem/parser.py b/simdem/parser.py index c60026f..4b5e082 100644 --- a/simdem/parser.py +++ b/simdem/parser.py @@ -4,7 +4,7 @@ class Parser(object): lexer = None - + def __init__(self, lexer): self.lexer = lexer @@ -41,21 +41,30 @@ def parse_ref_from_text(self, text): def get_prereqs(self, blocks): # WTF: Filter changed b/w python 2 -> 3? Returns an object now? There's no stack overflow on this plane +# logging.debug("get_prereqs: " + str(blocks)) res = [] - for block in blocks: + # Is there a better way to do this? Probably so. I'm on a plane and can't research + for idx in range(len(blocks)): + block = blocks[idx] if self.is_prerequisite_block(block): - res.append(block) + # We want the text block after the prereq heading + res.append(blocks[idx+1]['text']) # pre_reqs = filter(lambda(block): self.is_prerequisite_block(block), blocks) +# logging.debug("get_prereqs: res= " + str(res)) return res def get_file_contents(self, file_path): +# logging.debug("get_file_contents: " + file_path) f = open(file_path, 'r') - try: - content = f.read() - finally: - f.close() + content = f.read() + f.close() return content def parse_doc(self, text): - return self.lexer.parse(text) - +# logging.debug("parse_doc: text=" + text) + # https://github.com/lepture/mistune/issues/147 + # Stoopid non-idempotent parser. + self.lexer.tokens = [] + res = self.lexer.parse(text) + logging.debug("parse_doc: res=" + str(res)) + return res diff --git a/tests/test_system.py b/tests/test_system.py index d0318fa..6ec94b6 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -3,6 +3,7 @@ from .context import simdem, demo import unittest +import os import configparser import mistune import logging @@ -16,6 +17,7 @@ class SimDemSystemTestSuite(unittest.TestCase): simdem = None def setUp(self): + config = configparser.ConfigParser() config.read("content/config/unit_test.ini") self.simdem = simdem.Core(config, demo.Demo(config), simdem.Parser(mistune.BlockLexer())) @@ -28,7 +30,8 @@ def setUp(self): rootLogger.addHandler(fileHandler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'create-file') +# @data('prerequisite-run') + @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'create-file', 'prerequisite-run') def test_process(self, dir): self.simdem.process_file('./content/' + dir + '/README.md') res = sys.stdout.getvalue() From 8bedd501bf9f029fc0322c71216f049bb4a12e6e Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 15 Dec 2017 20:59:29 -0600 Subject: [PATCH 044/167] refactor parser --- scratch/.gitignore | 1 + simdem/core.py | 10 ++++++---- simdem/parser.py | 40 +++++++++++++++++++++++++++++++++------- tests/test_parser.py | 26 +++++++++++++++++++++++++- 4 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 scratch/.gitignore diff --git a/scratch/.gitignore b/scratch/.gitignore new file mode 100644 index 0000000..72e8ffc --- /dev/null +++ b/scratch/.gitignore @@ -0,0 +1 @@ +* diff --git a/simdem/core.py b/simdem/core.py index 5d0b062..1d1eb72 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -54,15 +54,17 @@ def run_blocks(self, blocks): results_latest = None for idx in range(len(blocks)): logging.info("run_blocks():processing " + str(blocks[idx])) - if self.parser.is_result_block(blocks, idx): + block = blocks[idx] + block_prev = blocks[idx-1] + if self.parser.is_result_block(block, block_prev): logging.info("run_blocks():is_result_block") - is_passable = self.is_result_passable(blocks[idx]['text'], results_latest) + is_passable = self.is_result_passable(block['text'], results_latest) if not is_passable: logging.error("Result did not pass") return - elif self.parser.is_runable_block(blocks[idx]): + elif self.parser.is_command_block(block): logging.info("run_blocks():is_runable_block") - results_latest = self.run_code_block(blocks[idx]['text']) + results_latest = self.run_code_block(block['text']) def is_result_passable(self, expected_results, actual_results, expected_similarity = 1.0): diff --git a/simdem/parser.py b/simdem/parser.py index 4b5e082..25e4bcf 100644 --- a/simdem/parser.py +++ b/simdem/parser.py @@ -14,14 +14,12 @@ def is_prerequisite_block(self, block): return True return False - def is_runable_block(self, block): + def is_command_block(self, block): if block['type'] == 'code' and block['lang'] == 'shell': return True return False - def is_result_block(self, blocks, idx): - block = blocks[idx] - block_prev = blocks[idx-1] + def is_result_block(self, block, block_prev): if block['type'] == 'code' and block['lang'] == 'shell' and \ block_prev['type'] == 'paragraph' and block_prev['text'].lower().startswith('results:'): return True @@ -40,7 +38,6 @@ def parse_ref_from_text(self, text): return None def get_prereqs(self, blocks): - # WTF: Filter changed b/w python 2 -> 3? Returns an object now? There's no stack overflow on this plane # logging.debug("get_prereqs: " + str(blocks)) res = [] # Is there a better way to do this? Probably so. I'm on a plane and can't research @@ -48,11 +45,12 @@ def get_prereqs(self, blocks): block = blocks[idx] if self.is_prerequisite_block(block): # We want the text block after the prereq heading - res.append(blocks[idx+1]['text']) -# pre_reqs = filter(lambda(block): self.is_prerequisite_block(block), blocks) + for line in blocks[idx+1]['text'].split("\n"): + res.append(line) # logging.debug("get_prereqs: res= " + str(res)) return res + def get_file_contents(self, file_path): # logging.debug("get_file_contents: " + file_path) f = open(file_path, 'r') @@ -60,11 +58,39 @@ def get_file_contents(self, file_path): f.close() return content + def parse_doc2(self, text): +# logging.debug("parse_doc: text=" + text) + # https://github.com/lepture/mistune/issues/147 + # Stoopid non-idempotent parser. + self.lexer.tokens = [] + blocks = self.lexer.parse(text) + return { + 'prerequisites': self.get_prereqs(blocks), + 'commands': self.get_commands(blocks) + } + + def get_commands(self, blocks): + res = [] + for idx in range(len(blocks)): + logging.debug("get_commands():processing " + str(blocks[idx])) + block = blocks[idx] + block_prev = blocks[idx-1] + if self.is_result_block(block, block_prev): + logging.debug("get_commands():is_result_block") + elif self.is_command_block(block): + logging.debug("get_commands():is_command_block") + for line in block['text'].split("\n"): + res.append(line) + else: + logging.debug("get_commands():unknown_block") + return res + def parse_doc(self, text): # logging.debug("parse_doc: text=" + text) # https://github.com/lepture/mistune/issues/147 # Stoopid non-idempotent parser. self.lexer.tokens = [] res = self.lexer.parse(text) + logging.debug("parse_doc: res=" + str(res)) return res diff --git a/tests/test_parser.py b/tests/test_parser.py index a7f7f96..0ae5c4a 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -22,6 +22,30 @@ def setUp(self): def test_parse_ref_from_text(self): self.assertEquals('./nested_prereq.md', self.parser.parse_ref_from_text('We should be able to run [nested prerequisites](./nested_prereq.md).')) - + + def test_parse_doc2(self): + doc = """foo +# Prerequisites + +prereq.md +prereq-2.md + +# Do stuff here +```shell +echo foo +echo bar +``` + +# Do more stuff here +```shell +echo baz +```""" + exp_res = { + 'prerequisites': ['prereq.md', 'prereq-2.md'], + 'commands': ['echo foo', 'echo bar', 'echo baz'] + } + res = self.parser.parse_doc2(doc) + self.assertEquals(res, exp_res) + if __name__ == '__main__': unittest.main() From bdb6a6631c480d03dc9ba5f7d41dab3464d79e75 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 15 Dec 2017 23:48:45 -0600 Subject: [PATCH 045/167] ready to replace parse_doc2 --- content/results-block-fail/README.md | 2 +- content/results-block/README.md | 2 +- simdem/core.py | 2 +- simdem/parser.py | 15 ++++++++------- tests/test_parser.py | 15 +++++++++++++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/content/results-block-fail/README.md b/content/results-block-fail/README.md index 8e455f1..ae2bf70 100644 --- a/content/results-block-fail/README.md +++ b/content/results-block-fail/README.md @@ -6,7 +6,7 @@ echo bar``` Results: -```shell +```result barr``` After this, we should not pass diff --git a/content/results-block/README.md b/content/results-block/README.md index ba15443..e1c0ea5 100644 --- a/content/results-block/README.md +++ b/content/results-block/README.md @@ -6,7 +6,7 @@ echo bar``` Results: -```shell +```result bar``` even more text diff --git a/simdem/core.py b/simdem/core.py index 1d1eb72..980500a 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -56,7 +56,7 @@ def run_blocks(self, blocks): logging.info("run_blocks():processing " + str(blocks[idx])) block = blocks[idx] block_prev = blocks[idx-1] - if self.parser.is_result_block(block, block_prev): + if self.parser.is_result_block(block): logging.info("run_blocks():is_result_block") is_passable = self.is_result_passable(block['text'], results_latest) if not is_passable: diff --git a/simdem/parser.py b/simdem/parser.py index 25e4bcf..47da965 100644 --- a/simdem/parser.py +++ b/simdem/parser.py @@ -19,9 +19,10 @@ def is_command_block(self, block): return True return False - def is_result_block(self, block, block_prev): - if block['type'] == 'code' and block['lang'] == 'shell' and \ - block_prev['type'] == 'paragraph' and block_prev['text'].lower().startswith('results:'): + def is_result_block(self, block): + # This is different than previous SimDem because it didn't require a language for the result. + # I believe this approach is more declarative. + if block['type'] == 'code' and block['lang'] == 'result': return True return False @@ -74,15 +75,15 @@ def get_commands(self, blocks): for idx in range(len(blocks)): logging.debug("get_commands():processing " + str(blocks[idx])) block = blocks[idx] - block_prev = blocks[idx-1] - if self.is_result_block(block, block_prev): + if self.is_result_block(block): logging.debug("get_commands():is_result_block") + res[len(res) - 1]['expected_result'] = block['text'] elif self.is_command_block(block): logging.debug("get_commands():is_command_block") for line in block['text'].split("\n"): - res.append(line) + res.append({ 'command': line }) else: - logging.debug("get_commands():unknown_block") + logging.info("get_commands():unknown_block. Ignoring") return res def parse_doc(self, text): diff --git a/tests/test_parser.py b/tests/test_parser.py index 0ae5c4a..c9c2e6f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -39,10 +39,21 @@ def test_parse_doc2(self): # Do more stuff here ```shell echo baz -```""" +``` + +Results: +```result +baz +``` + + +""" exp_res = { 'prerequisites': ['prereq.md', 'prereq-2.md'], - 'commands': ['echo foo', 'echo bar', 'echo baz'] + 'commands': [ + { 'command': 'echo foo' }, + { 'command': 'echo bar' }, + { 'command': 'echo baz', 'expected_result': 'baz' } ] } res = self.parser.parse_doc2(doc) self.assertEquals(res, exp_res) From c129ad44a60a9e6f2afbcd29bb46cfac7944e6a5 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sat, 16 Dec 2017 00:10:34 -0600 Subject: [PATCH 046/167] MUCH better parser/executor --- simdem/core.py | 45 ++++++++++++++++++-------------------------- simdem/parser.py | 14 ++------------ tests/test_lexer.py | 2 +- tests/test_parser.py | 16 +++++++++++----- tests/test_simdem.py | 2 +- 5 files changed, 33 insertions(+), 46 deletions(-) diff --git a/simdem/core.py b/simdem/core.py index 980500a..cf00b70 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -34,40 +34,31 @@ def process_file(self, file_path): logging.info("process_file():content=" + str(content)) blocks = self.parser.parse_doc(content) logging.info("process_file():blocks=" + str(blocks)) - self.process_prereqs(blocks) + self.process_prereqs(blocks['prerequisites']) logging.info("process_file():completed process_prereqs()") - result = self.run_blocks(blocks) + result = self.run_blocks(blocks['commands']) return result - def process_prereqs(self, blocks): - prereqs = self.parser.get_prereqs(blocks) + def process_prereqs(self, prereqs): logging.info("process_preqreqs():" + str(prereqs)) for prereq_file in prereqs: -# prereq_content = self.parser.get_file_contents(prereq_file) -# preqreq_blocks = self.parser.parse_doc(prereq_content) -# preqreq_validation = self.parser.get_validation_block(preqreq_blocks) -# if not self.is_prereq_required(preqreq_validation): self.process_file(prereq_file) - def run_blocks(self, blocks): - logging.info("run_blocks():blocks=" + str(blocks)) + def run_blocks(self, commands): + logging.info("run_blocks():blocks=" + str(commands)) results_latest = None - for idx in range(len(blocks)): - logging.info("run_blocks():processing " + str(blocks[idx])) - block = blocks[idx] - block_prev = blocks[idx-1] - if self.parser.is_result_block(block): - logging.info("run_blocks():is_result_block") - is_passable = self.is_result_passable(block['text'], results_latest) - if not is_passable: + for command in commands: + logging.info("run_blocks():processing " + str(command)) + result = self.run_code_block(command['command']) + if 'expected_result' in command: + if self.is_result_valid(command['expected_result'], result): + logging.info("Result passed") + else: logging.error("Result did not pass") return - elif self.parser.is_command_block(block): - logging.info("run_blocks():is_runable_block") - results_latest = self.run_code_block(block['text']) - def is_result_passable(self, expected_results, actual_results, expected_similarity = 1.0): + def is_result_valid(self, expected_results, actual_results, expected_similarity = 1.0): """Checks to see if a command execution passes. If actual results compared to expected results is within the expected similarity level then it's considered a pass. @@ -78,22 +69,22 @@ def is_result_passable(self, expected_results, actual_results, expected_similari """ if not actual_results: - logging.error("is_result_passable(): actual_results is empty.") + logging.error("is_result_valid(): actual_results is empty.") return False - logging.debug("is_result_passable(" + expected_results + "," + actual_results + "," + str(expected_similarity) + ")") + logging.debug("is_result_valid(" + expected_results + "," + actual_results + "," + str(expected_similarity) + ")") expected_results_str = expected_results.rstrip() actual_results_str = actual_results.rstrip() - logging.debug("is_result_passable(" + expected_results_str + "," + actual_results_str + "," + str(expected_similarity) + ")") + logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + "," + str(expected_similarity) + ")") seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", actual_results_str, expected_results_str) is_pass = seq.ratio() >= expected_similarity if is_pass: - logging.info("is_result_passable passed") + logging.info("is_result_valid passed") else: - logging.error("is_result_passable failed") + logging.error("is_result_valid failed") logging.error("actual_results = " + actual_results) logging.error("expected_results = " + expected_results) diff --git a/simdem/parser.py b/simdem/parser.py index 47da965..3e350e9 100644 --- a/simdem/parser.py +++ b/simdem/parser.py @@ -59,7 +59,7 @@ def get_file_contents(self, file_path): f.close() return content - def parse_doc2(self, text): + def parse_doc(self, text): # logging.debug("parse_doc: text=" + text) # https://github.com/lepture/mistune/issues/147 # Stoopid non-idempotent parser. @@ -84,14 +84,4 @@ def get_commands(self, blocks): res.append({ 'command': line }) else: logging.info("get_commands():unknown_block. Ignoring") - return res - - def parse_doc(self, text): -# logging.debug("parse_doc: text=" + text) - # https://github.com/lepture/mistune/issues/147 - # Stoopid non-idempotent parser. - self.lexer.tokens = [] - res = self.lexer.parse(text) - - logging.debug("parse_doc: res=" + str(res)) - return res + return res \ No newline at end of file diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 7532b2e..0d105f3 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -7,7 +7,7 @@ import unittest -class LexerTestSuite(unittest.TestCase): +class SimDemLexerTestSuite(unittest.TestCase): """Lexer test cases.""" def test_mistune(self): diff --git a/tests/test_parser.py b/tests/test_parser.py index c9c2e6f..4a3d9ee 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -7,14 +7,12 @@ import configparser import mistune -class SimDemTestSuite(unittest.TestCase): +class SimDemParserTestSuite(unittest.TestCase): """Advanced test cases.""" - test_file = '/tmp/foo' parser = None def setUp(self): - os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") @@ -31,17 +29,25 @@ def test_parse_doc2(self): prereq-2.md # Do stuff here + +We want to execute this because the code type is shell + ```shell echo foo echo bar ``` # Do more stuff here + ```shell echo baz ``` -Results: +# Results + +The only thing that makes it a result is the code type is result. +We assume the result is for the last command of the last code block + ```result baz ``` @@ -55,7 +61,7 @@ def test_parse_doc2(self): { 'command': 'echo bar' }, { 'command': 'echo baz', 'expected_result': 'baz' } ] } - res = self.parser.parse_doc2(doc) + res = self.parser.parse_doc(doc) self.assertEquals(res, exp_res) if __name__ == '__main__': diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 520770a..5c636df 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -32,7 +32,7 @@ def test_run_blocks(self): self.assertFalse(os.path.exists(self.test_file)) blocks = self.parser.parse_doc(doc) - self.simdem.run_blocks(blocks) + self.simdem.run_blocks(blocks['commands']) self.assertTrue(os.path.exists(self.test_file)) if __name__ == '__main__': From 24eecfca3d176d540ef5dbac932cef67d60031f8 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 20 Dec 2017 08:45:43 -0600 Subject: [PATCH 047/167] rm all SimDem v1 files --- .circleci/config.yml | 38 - Dockerfile_cli | 49 -- Dockerfile_novnc | 65 -- GETTING_STARTED.md | 25 - README.md | 341 +------- cli.py | 471 ---------- config.py | 28 - demo.py | 801 ------------------ demo_scripts/README.md | 19 - demo_scripts/env.json | 3 - demo_scripts/simdem/README.md | 48 -- demo_scripts/simdem/building/README.md | 13 - demo_scripts/simdem/demo/README.md | 74 -- demo_scripts/simdem/env.json | 3 - demo_scripts/simdem/env.local.json | 3 - demo_scripts/simdem/modes/README.md | 95 --- demo_scripts/simdem/multipart/README.md | 106 --- demo_scripts/simdem/prerequisites/README.md | 96 --- demo_scripts/simdem/prerequisites/remote.md | 4 - demo_scripts/simdem/running/README.md | 56 -- .../simdem/special_commands/README.md | 37 - demo_scripts/simdem/syntax/README.md | 120 --- demo_scripts/simdem/test/README.md | 89 -- demo_scripts/simdem/tutorial/README.md | 46 - demo_scripts/simdem/variables/README.md | 168 ---- demo_scripts/simdem/variables/env.json | 3 - demo_scripts/simdem/variables/env.local.json | 3 - demo_scripts/test/README.md | 157 ---- demo_scripts/test/directory/README.md | 26 - demo_scripts/test/env.json | 4 - demo_scripts/test/env.local.json | 3 - demo_scripts/test/env.test.json | 3 - demo_scripts/test/environment_test.md | 195 ----- demo_scripts/test/prerequisites/README.md | 33 - .../test/prerequisites/nested_prereq.md | 25 - .../prerequisites/passing_prerequisite.md | 28 - demo_scripts/test/remote/README.md | 33 - demo_scripts/test/test_plan.txt | 4 - env.json | 3 - env.local.json | 4 - environment.py | 155 ---- js/common.js | 19 - js/console.js | 22 - js/control.js | 70 -- main.py | 145 ---- novnc/.XDefaults | 15 - novnc/.config/autostart/xterm.desktop | 10 - novnc/Desktop/Terminal Emulator.desktop | 10 - pre-commit.sh | 33 - release_process.md | 67 -- requirements.txt | 5 - scripts/build.sh | 41 - scripts/install.sh | 86 -- scripts/publish.sh | 40 - scripts/run.sh | 96 --- style/common.css | 8 - style/console.css | 27 - style/speaker.css | 47 - templates/console.html | 29 - templates/index.html | 44 - web.py | 232 ----- 61 files changed, 5 insertions(+), 4518 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 Dockerfile_cli delete mode 100644 Dockerfile_novnc delete mode 100644 GETTING_STARTED.md delete mode 100644 cli.py delete mode 100644 config.py delete mode 100644 demo.py delete mode 100644 demo_scripts/README.md delete mode 100644 demo_scripts/env.json delete mode 100644 demo_scripts/simdem/README.md delete mode 100644 demo_scripts/simdem/building/README.md delete mode 100644 demo_scripts/simdem/demo/README.md delete mode 100644 demo_scripts/simdem/env.json delete mode 100644 demo_scripts/simdem/env.local.json delete mode 100644 demo_scripts/simdem/modes/README.md delete mode 100644 demo_scripts/simdem/multipart/README.md delete mode 100644 demo_scripts/simdem/prerequisites/README.md delete mode 100644 demo_scripts/simdem/prerequisites/remote.md delete mode 100644 demo_scripts/simdem/running/README.md delete mode 100644 demo_scripts/simdem/special_commands/README.md delete mode 100644 demo_scripts/simdem/syntax/README.md delete mode 100644 demo_scripts/simdem/test/README.md delete mode 100644 demo_scripts/simdem/tutorial/README.md delete mode 100644 demo_scripts/simdem/variables/README.md delete mode 100644 demo_scripts/simdem/variables/env.json delete mode 100644 demo_scripts/simdem/variables/env.local.json delete mode 100644 demo_scripts/test/README.md delete mode 100644 demo_scripts/test/directory/README.md delete mode 100644 demo_scripts/test/env.json delete mode 100644 demo_scripts/test/env.local.json delete mode 100644 demo_scripts/test/env.test.json delete mode 100644 demo_scripts/test/environment_test.md delete mode 100644 demo_scripts/test/prerequisites/README.md delete mode 100644 demo_scripts/test/prerequisites/nested_prereq.md delete mode 100644 demo_scripts/test/prerequisites/passing_prerequisite.md delete mode 100644 demo_scripts/test/remote/README.md delete mode 100644 demo_scripts/test/test_plan.txt delete mode 100644 env.json delete mode 100644 env.local.json delete mode 100644 environment.py delete mode 100644 js/common.js delete mode 100644 js/console.js delete mode 100644 js/control.js delete mode 100644 main.py delete mode 100644 novnc/.XDefaults delete mode 100644 novnc/.config/autostart/xterm.desktop delete mode 100755 novnc/Desktop/Terminal Emulator.desktop delete mode 100755 pre-commit.sh delete mode 100644 release_process.md delete mode 100644 requirements.txt delete mode 100755 scripts/build.sh delete mode 100755 scripts/install.sh delete mode 100755 scripts/publish.sh delete mode 100755 scripts/run.sh delete mode 100644 style/common.css delete mode 100644 style/console.css delete mode 100644 style/speaker.css delete mode 100644 templates/console.html delete mode 100644 templates/index.html delete mode 100644 web.py diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e2d2176..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/python:3.6.1 - - working_directory: ~/simdem - - steps: - - checkout - - restore_cache: - keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - run: - name: install dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip install -r requirements.txt - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} - - - run: - name: run tests - command: | - . venv/bin/activate - python main.py test test - - - store_artifacts: - path: test-reports - destination: test-reports - diff --git a/Dockerfile_cli b/Dockerfile_cli deleted file mode 100644 index b08a1f8..0000000 --- a/Dockerfile_cli +++ /dev/null @@ -1,49 +0,0 @@ -FROM ubuntu:16.04 - -ENV HOME /home/simdem -ENV TERM xterm -WORKDIR $HOME - -RUN apt-get update - -# Not really needed, but used in the SimDem demo script -RUN apt-get install tree -y - -# Azure CLI -RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ wheezy main" | tee /etc/apt/sources.list.d/azure-cli.list -RUN apt-get install apt-transport-https -y -RUN apt-get update -RUN apt-key adv --keyserver packages.microsoft.com --recv-keys 417A0893 -RUN apt-get install azure-cli -y --allow-unauthenticated - -# Python -RUN apt-get install python3-pip -y - -# Create SimDem User -RUN apt-get install sudo -y -RUN apt-get install whois -y -RUN useradd simdem -u 1984 -p `mkpasswd password` -RUN usermod -aG sudo simdem -RUN echo "simdem ALL=NOPASSWD: ALL" >> /etc/sudoers -RUN mkdir -p $HOME && chown -R 1984 $HOME -RUN mkdir -p $HOME/.azure && chown -R 1984 $HOME/.azure -RUN mkdir -p $HOME/.ssh && chown -R 1984 $HOME/.ssh -RUN mkdir -p $HOME/demo_scripts && chown -R 1984 $HOME/demo_scripts - -# SimDem -COPY ./env.json $/env.json -COPY ./requirements.txt requirements.txt -RUN pip3 install -r requirements.txt -RUN mkdir /usr/local/bin/simdem_cli -COPY *.py /usr/local/bin/simdem_cli/ -RUN chmod +x /usr/local/bin/simdem_cli/main.py -RUN ln -s /usr/local/bin/simdem_cli/main.py /usr/local/bin/simdem - -# Demo Scripts -COPY demo_scripts/simdem demo_scripts - -USER 1984 - -ENTRYPOINT [ "simdem" ] - - diff --git a/Dockerfile_novnc b/Dockerfile_novnc deleted file mode 100644 index 3ceff8e..0000000 --- a/Dockerfile_novnc +++ /dev/null @@ -1,65 +0,0 @@ -FROM consol/ubuntu-xfce-vnc - -# FIXME: For consistency with CLI we should use /home/simdem, currently using /headless because that's what comes with novnc container -ENV HOME /headless -WORKDIR $HOME - -### VNC config -ENV NO_VNC_PORT 8080 -ENV VNC_COL_DEPTH 24 -ENV VNC_RESOLUTION 1024x768 -ENV VNC_PW vncpassword - -USER 0 - -RUN apt-get update - -# Not really needed but handy when debugging -RUN apt-get install emacs -y - -# Not really needed, but used in the SimDem demo script -RUN apt-get install tree -y - -# Azure CLI -RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ wheezy main" | tee /etc/apt/sources.list.d/azure-cli.list -RUN apt-get install apt-transport-https -y -RUN apt-get update -RUN apt-key adv --keyserver packages.microsoft.com --recv-keys 417A0893 -RUN apt-get install azure-cli -y --allow-unauthenticated - -# Python -RUN apt-get install python3-pip -y - -# Desktop -USER 1984 -COPY ./novnc/ /headless/ -USER 0 -RUN rm .config/bg_sakuli.png -RUN rm Desktop/chromium-browser.desktop - -# Create SimDem User -USER 0 -RUN apt-get install sudo -y -RUN apt-get install whois -y -RUN useradd simdem -u 1984 -p `mkpasswd password` -d $HOME -s /bin/bash -RUN usermod -aG sudo simdem -RUN echo "simdem ALL=NOPASSWD: ALL" >> /etc/sudoers -RUN mkdir -p $HOME && chown -R 1984 $HOME -RUN mkdir -p $HOME/.azure && chown -R 1984 $HOME/.azure -RUN mkdir -p $HOME/.ssh && chown -R 1984 $HOME/.ssh -RUN mkdir -p $HOME/demo_scripts && chown -R 1984 $HOME/demo_scripts - -# SimDem -COPY ./requirements.txt requirements.txt -RUN pip3 install -r requirements.txt -RUN mkdir /usr/local/bin/simdem_cli -COPY *.py /usr/local/bin/simdem_cli/ -RUN chmod +x /usr/local/bin/simdem_cli/main.py -RUN ln -s /usr/local/bin/simdem_cli/main.py /usr/local/bin/simdem - -# Demo Scripts -COPY demo_scripts/simdem demo_scripts - -USER 1984 - -COPY demo_scripts demo_scripts diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md deleted file mode 100644 index bbfe255..0000000 --- a/GETTING_STARTED.md +++ /dev/null @@ -1,25 +0,0 @@ -Assuming you have Docker installed… - -``` -docker run -it rgardler/simdem -``` - -If you don’t have Docker then: - -``` -git clone git@github.com:rgardler/simdem.git -cd simdem -./scripts/install.sh -simdem -``` - -Should tell you everything you need to get started, including show you -how to build a hello world script. - -If you prefer to read via markdown see -https://github.com/rgardler/simdem/tree/master/demo_scripts/simdem - -For the hello world tutorial see -https://github.com/rgardler/simdem/tree/master/demo_scripts/simdem/tutorial - -Feedback / bug reports / pull requests welcome diff --git a/README.md b/README.md index aec25fe..4da1f3d 100644 --- a/README.md +++ b/README.md @@ -1,340 +1,9 @@ -[![CircleCI](https://circleci.com/gh/Azure/simdem.svg?style=svg)](https://circleci.com/gh/Azure/simdem) +# SimDem -This project provides ways to write tutorials in markdown that then -become interactive demo's and automated tests. You can run in a number -of different modes: +This project provides ways to write tutorials in markdown that then become interactive demo's and automated tests. - * Tutorial: Displays the descriptive text of the tutorial and pauses - at code blocks to allow user interaction. - * Simulate: Does not display the descriptive text, but pauses at each - code block. When the user hits a key the command is "typed", a - second keypress executes the command. - * Test: Runs the commands and then verifies that the output is - sufficiently similar to the expected results (recorded in the - markdown file) to be considered correct. - * Script: Creates an executable bash script from the document - * Auto: allows any of the above modes to be run but without user - interaction +# History -The application can be run in either a CLI mode, which is ideal for -console based demo's and tutorials, or it can be run using NoVNC for a -browser based desktop experience - that is, using only a browser you -can have a full desktop experience. +SimDem v2 is a complete rewrite of SimDem v1. The latest commit for v1 can be found at: +https://github.com/Azure/simdem/tree/cb1caf17fd684e125789c26817f43eeae0e1c523 -# Try it Out - -The easiest way to try SimDem out is with a Docker container and work -through the embedded tutorial. The following command will run the -latest developer version of the code (i.e. there may be errors). - -``` -docker run -it rgardler/simdem -``` - -This will start SimDem in CLI mode. - -## Experimental browser mode - -In addition to the web NoVNC container noted above there is an -experimental web mode available using the `--webui true` option shown -below. Once started point your browser at port 8080 on your host. - -``` -docker run -it -p 8080:8080 rgardler/simdem --webui true -``` - -### Presenter view - -When using the Web UI it is possible to open a view on the demo that -shows only the console. This is ideal for use when delivering -presentations. Have this view visible to the audience while using the -speaker view, with full descriptive text, on the presenters machine. - -To open this in a separate window click the -'Console View' button (or browse to http://HOST:8080/console). - -## Python - -The most flexible way to run SimDem is to use the Python code -directly. This is generally best for developers of SimDem, so we -provide minimal documentation here. - -# Writing a Script - -The above command will walk you through the SimDem documentation. Pay -particular attention to the Syntax page, but the short version is that -you are writing markdown with code blocks. For more information see -`demo_scripts/simdem/tutorial/README.md`. - -## Running in a Docker Container - -There are two containers available, the 'cli' version and the 'novnc' -version. The first is command line only, the latter provides a browser -based Linux desktop envirnment in which the CLI is availale. The NoVNC -version makes it easy to do demo's with browser based steps without -having to install any software (other than Docker) on your client. - -We provide scripts that make it easy to run the container and to load -custom scripts into it. - -### CLI Container - -The CLI container can be run in four modes: - -Tutorial : in which full textual descriptions are provided -Learn : similar to Tutorial mode, but users are expected to type the commands -Demo : in which no textual descriptions are shown and commands are "typed" -Test : run the tests - -#### Tutorial Mode - -It's easier to explain through action, so just run the container and -work through the interactive tutorial that is included - -``` -./scripts/run.sh cli -``` - -If you want to start execution in a different place, or load in your -own scripts provide the path as the second parameter. For example, the -following example skips the introductory text and runs the demo script -provided in the SimDem GitHub repository. - -``` -./scripts/run.sh cli demo_scripts/simdem -``` - -#### Learn mode - -Learn mode is similar to tutorial mode, but the user is expected to -type the commands after being provided instructions. - -``` -./scripts/run.sh cli learn -``` - -If you want to start execution in a different place, or load in your -own scripts provide the path as the second parameter. For example, the -following example skips the introductory text and runs the demo script -provided in the SimDem GitHub repository. - -``` -./scripts/run.sh cli demo_scripts/simdem learn -``` - -#### Demo mode - -To run the same file as a demo (that is without explanatory text and -with simulated typing) simply add a third paramater with the value -`demo` as folows: - -``` -./scripts/run.sh cli demo_scripts/simdem demo -``` - -#### Preparation mode - -In this mode only the preparation (prerequisite) steps are -executed. This is useful for setting up the environment for a -demo. Next time the demo is run all prepration steps will be -skipped. This means that steps that take a long time can be -pre-baked. - -To use this mode: - -``` -./scripts/run.sh cli demo_scripts/simdem prep -``` - -#### Test mode - -To run the same file as a series of tests use a third parameter value -of `test` as follows: - -``` -./scripts/run.sh cli demo_scripts/simdem test -``` - -Test mode is very useful in a continuous integration environment. For -example, you can configure your scripts to always use the latest -versions of tooling they depend upon and get early warning when a -change in one of those tools breaks your -scripts. The -[Azure Container Service demos GitHub repository](http://github.com/Azure/acs-demos) shows -this technique in use. - -### NoVNC Container - -When running in NoVNC mode a lightweight Linux desktop is run inside -the container you can then access that container using a browser. To -run the container use: - -``` -./scripts/run.sh -``` - -Now connect using the URL http://HOSTNAME_OR_IP:8080/?password=vncpassword - -Open a terminal and type: - -``` -simdem --help -``` - -To load your own demo scripts into this container use: - -``` -./scripts/run.sh novnc /path/to/scripts -``` - -## Python - -You can run the Python source without a Docker container. To learn -more install Python 3 and pip, then type the following commands: - -``` -sudo pip3 install -r requirements.txt -python3 main.py --help -``` - -## Azure Cloud Shell - -The CLI version of SimDem works fine in Azure Cloud Shell, but you -need to install it manually at this time. Here's how. - - -NOTE: in the current code (at the time of writing) the HEAD version -does not work because of the need to install some dependencies that -CloudShell does not like. We are therefore going to use an earlier -version of SimDem, this means some of the more recent features will -not be available. See issues https://github.com/Azure/simdem/issues/19 -and https://github.com/Azure/simdem/issues/22. - - -``` -git clone https://github.com/Azure/simdem.git -cd simdem -git checkout tags/CloudShell - -pip install -r requirements.txt - -mkdir -p ~/bin/simdem-dev -cp -r * ~/bin/simdem-dev -chmod +x ~/bin/simdem-dev/main.py - -echo 'export PATH=$PATH:~/bin/simdem-dev' >> ~/.bashrc -ln -s ~/bin/simdem-dev/main.py ~/bin/simdem -``` - -Now that you have the code and enviornment setup you will first need -to acivite the Python Virtual Environment. - -``` -source simdem-env/bin/activate -``` - -Now you can run the Simdem CLI: - -``` -simdem --version -``` - -# Hacking Guide - -If you make changes to the code the easiest way to build and redeploy -the container is with the scripts in `scripts` directory. These -scripts pull the current version number from the config.py (see -`SIMDEM_VERSION=x.y.z`). This version number is used as the defaiult -for the scripts in this folder. - -## Writing a SimDem Script - - - -## Building SimDem in Containers - -./scripts/build.sh Builds the noVNC (browser based) version of -the container with the default tag of `rgardler/simdem_novnc:x.y.z` - -## Running - -Use `./script/run.sh ` to execture the container -using two volume containers (see below). The `FLAVOR` is either -`novnc` (for the browser based version) or `cli` fo rthe command line -version.. - -`azure_data` volume container is used to maintain details of your -Azure Subscription (including login details). - -`simdem_VERSION_scripts` volume container has the scripts to execute -in the container, that is the contents in `SCRIPTS_DIR` (or `./demo_scripts` if no value provided). - -### Running the NoVNC container - -`./scripts/run.sh novnc ` runs an instance of the noVNC -container - -Once the container is running you can connect to it at `http://HOST:8080?password=vncpassword`. Open a terminal and run: - -`simdem` to run the default script - -Use `simdem --help` for more information. - -### Running the CLI container - -`./scripts/run.sh cli ` runs an instance of the CLI -container - -## Contributing - -This is an open source project. Don't keep your code improvements, -features and cool ideas to yourself. Please issue pull requests -against our [GitHub repo](http://github.com/rgardler/simdem). - -Be sure to use our Git pre-commit script to test your contributions -before committing, simply run the following command: - -``` -ln -s ../../pre-commit.sh .git/hooks/pre-commit -``` - -This project welcomes contributions and suggestions. Most -contributions require you to agree to a Contributor License Agreement -(CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit -https://cla.microsoft.com. - -When you submit a pull request, a CLA-bot will automatically determine -whether you need to provide a CLA and decorate the PR appropriately -(e.g., label, comment). Simply follow the instructions provided by the -bot. You will only need to do this once across all repos using our -CLA. - -This project has adopted -the -[Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see -the -[Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with -any additional questions or comments. - -## Publishing - -The `latest` version is built from source on each commit. To publish a -version tagged image use `./scripts/publish.sh ` script. This -will publish both the CLI and NoVNC containers if no `FLAVOR` is -provided. - -Don't forget to bump the version number after using this script. To do -this open simdem.py and find and edit the following line (somewhere -near the top of the file): - -`SIMDEM_VERSION = "0.4.1"` - -# Learn more - -If you want to learn more before running the container then why not -read the interactive tutorial as -a -[markdown page on GitHub](https://github.com/rgardler/simdem/blob/master/demo_scripts/simdem/README.md). diff --git a/cli.py b/cli.py deleted file mode 100644 index 670a370..0000000 --- a/cli.py +++ /dev/null @@ -1,471 +0,0 @@ -# A console based UI for SimDem. - -import difflib -import os -import pexpect -import pexpect.replwrap -import random -import re -import time -import sys -import colorama -import config -colorama.init(strip=None) - -PEXPECT_PROMPT = u'[PEXPECT_PROMPT>' -PEXPECT_CONTINUATION_PROMPT = u'[PEXPECT_PROMPT+' - -class Ui(object): - _shell = None - demo = None - execution_log = "" - - def __init__(self): - pass - - def prompt(self): - """Display the prompt for the user. This is intended to indicate that - the user is expected to take an action at this point. - """ - self.display(config.console_prompt, colorama.Fore.WHITE) - - def command(self, text): - """Display a command, or a part of a command tp be executed.""" - self.display(text, colorama.Fore.WHITE + colorama.Style.BRIGHT) - - def results(self, text): - """Display the results of a command execution""" - self.display(text, colorama.Fore.GREEN + colorama.Style.BRIGHT, True) - - def heading(self, text): - """Display a heading""" - self.display(text, colorama.Fore.CYAN + colorama.Style.BRIGHT, True) - self.new_line() - - def description(self, text): - """Display some descriptive text. Usually this is text from the demo - document itself. - - """ - self.display(text, colorama.Fore.CYAN) - - def information(self, text, new_line = False): - """Display some informative text. Usually this is content generated by - SimDem. Do not print a new line unless new_line == True. - - """ - self.display(text, colorama.Fore.WHITE, new_line) - - def prep_step(self, step): - """Displays a preparation step item. - """ - self.display(step["title"], colorama.Fore.MAGENTA, True) - - def next_step(self, index, title): - """Displays a next step item with an index (the number to be entered -to select it) and a title (to be displayed). - """ - self.display(index, colorama.Fore.CYAN) - self.display(title, colorama.Fore.CYAN, True) - - def instruction(self, text): - """Display an instruction for the user. - """ - self.display(text, colorama.Fore.MAGENTA, True) - - def warning(self, text): - """Display a warning to the user. - """ - self.display(text, colorama.Fore.RED + colorama.Style.BRIGHT, True) - - def new_para(self): - """Starts a new paragraph.""" - self.new_line() - self.new_line() - - def new_line(self): - """Move to the next line""" - self.display("", colorama.Fore.WHITE, True) - - def horizontal_rule(self): - self.display("\n\n============================================\n\n", colorama.Fore.WHITE) - - def clear(self): - """Clears the screen ready for anew section of the script.""" - if self.demo.is_simulation: - self.demo.current_command = "clear" - self.simulate_command() - else: - self.run_command("clear") - - def display(self, text, color, new_line=False): - """Display some text in a given color. Do not print a new line unless - new_line is set to True. - - """ - self.execution_log += color - if self.demo.output_format == "log": - print(color, end="") - - self.execution_log += text - if self.demo.output_format == "log": - print(text, end="", flush=True) - - if new_line: - self.execution_log += colorama.Style.RESET_ALL + "\n" - if self.demo.output_format == "log": - print(colorama.Style.RESET_ALL) - else: - self.execution_log += colorama.Style.RESET_ALL - if self.demo.output_format == "log": - print(colorama.Style.RESET_ALL, end="") - - def log(self, level, text): - if config.is_debug: - print(level.upper() + " : " + text) - - def request_input(self, text): - """Displays text that is intended to propmt the user for - input and then waits for input. - - """ - print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="") - print(text) - print(colorama.Style.RESET_ALL, end="") - return self.input_string().lower() - - def input_interactive_variable(self, name): - """ - Gets a value from stdin for a variable. - """ - print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="") - print("\n\nEnter a value for ", end="") - print(colorama.Fore.YELLOW + colorama.Style.BRIGHT, end="") - print("$" + name, end="") - print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="") - print(": ", end="") - print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="") - value = input() - return value - - def type_command(self): - """ - Displays the command on the screen - If simulation == True then it will look like someone is typing the command - """ - - text = "" - end_of_var = 0 - current_command, undefined_var_list, defined_var_list = self.demo.get_current_command() - for idx, char in enumerate(current_command): - if char != "\n": - text += char - - if self.demo.is_simulation: - for char in text: - delay = random.uniform(0.02, config.TYPING_DELAY) - time.sleep(delay) - self.command(char) - else: - self.command(text) - - - def simulate_command(self, silent = False): - """Types the current command on the screen, executes it and outputs - the results if simulation == True then system will make the - "typing" look real and will wait for keyboard entry before - proceeding to the next command. - - If silent = True then the command and its results will not be - ouptut. - """ - - self.log("debug", "Simulating command: '" + self.demo.current_command + "'") - if not self.demo.is_learning or self.demo.current_command.strip() == "clear": - self.type_command() - _, undefined_var_list, defined_var_list = self.demo.get_current_command() - - # Get values for unknown variables - for var_name in undefined_var_list: - if (self.demo.is_testing): - var_value = "Dummy value for test" - else: - var_value = self.input_interactive_variable(var_name) - if not var_name.startswith("SIMDEM_"): - self.demo.env.set(var_name, var_value) - self.run_command(var_name + '="' + var_value + '"') - - # Log values if in debug mode - if config.is_debug: - self.information("\n") - for var_name in undefined_var_list: - self.log("debug", "$" + var_name + " = " + self.demo.env.get(var_name)) - for var_name in defined_var_list: - self.log("debug", "$" + var_name + " = " + self.demo.env.get(var_name)) - - output = self.run_command() - self.demo.last_command = self.demo.current_command - self.demo.current_command = "" - else: - done = False - while not done: - print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="") - print("\nType the command '", end = "") - print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="") - print(self.demo.current_command.strip(), end = "") - print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="") - print("'") - print("\t- type 'auto' (or 'a') to automatically type the command") - print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="") - print("\n$ ", end = "", flush = True) - typed_command = input() - if typed_command.lower() == "a" or typed_command.lower() == "auto": - self.demo.is_learning = False - output = self.simulate_command() - self.demo.is_learning = True - done = True - elif typed_command == self.demo.current_command.strip(): - self.demo.is_learning = False - output = self.simulate_command() - self.demo.is_learning = True - done = True - else: - print(colorama.Fore.RED, end="") - print("You have a typo there") - - self.log("debug", "Output: '" + output +"'") - return output - - def input_string(self): - """ Get a string from the user.""" - return input() - - def get_shell(self): - """Gets or creates the shell in which to run commands for the - supplied demo - """ - if self._shell == None: - child = pexpect.spawnu('/bin/bash', env=self.demo.env.get(), echo=False, timeout=None) - ps1 = PEXPECT_PROMPT[:5] + u'\[\]' + PEXPECT_PROMPT[5:] - ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + u'\[\]' + PEXPECT_CONTINUATION_PROMPT[5:] - prompt_change = u"PS1='{0}' PS2='{1}' PROMPT_COMMAND=''".format(ps1, ps2) - self._shell = pexpect.replwrap.REPLWrapper(child, u'\$', prompt_change) - return self._shell - - def run_command(self, command=None, silent = False): - """ - Run the self.demo.curent_command unless command is passed in, in - which case run the supplied command in the current demo - environment. Return the output of the command. - - A small number of commands are intercepted and handled as - special cases, see `run_special_command` - """ - if not command: - command = self.demo.current_command - - command = command.strip() - self.new_line(); - - self.log("debug", "Execute command: '" + command + "'") - start_time = time.time() - - response = self.run_special_command(command) - if response: - pass - else: - response = self.get_shell().run_command(command) - end_time = time.time() - - if not silent: - self.results(response) - - if self.demo.is_testing: - self.information("--- %s seconds execution time ---" % (end_time - start_time), True) - - return response - - def run_special_command(self, command): - """Test to see if the command is a spcial command that needs to be - handled diferently, these include: - - `xdg-open $URL` - intercepted and converted to a curl for headless CLI - `az acs create ...` - if we have a service principle set in environment variables 'SERVICE_PRINCIPAL_ID' and 'SERVICE_PRINCIPAL_SECRET_KEY' then add them to the command (assuming that we don't have a device login active) - - Returns the response from the command if it was handled by this function, - otherwise returns False. - - """ - orig_command = command - if command.startswith("xdg-open "): - self.warning("Since you are running in headless CLI mode it is not possible to execute xdg-open commands.") - - command = "curl -I " + command[9:] + " --connect-timeout 90" - - self.warning("Converting to `" + command + "`") - self.warning("Note that this may break tests.") - - if command.startswith('az acs create '): - if "--orchestrator-type=kubernetes" in command and not "--service-principal" in command: - if os.getenv('SERVICE_PRINCIPAL_ID'): - command += " --service-principal ${SERVICE_PRINCIPAL_ID} --client-secret ${SERVICE_PRINCIPAL_SECRET_KEY}" - - if orig_command != command: - self.log("INFO", "Running special command " + orig_command + " as " + command) - response = self.get_shell().run_command(command) - return response - else: - return False - - def expand_vars(self, command): - """Expand the variables in the supplied command by replacing them - with the value they carry in the Environment. This is used by some special commands because the shell doesn't expand them (e.g. copying a $URL into a browser window using xdg-open)""" - - self.log("debug", "Expanding vars in " + command) - var_pattern = re.compile(".*?(?<=\$)\(?{?(\w*)(?=[\W|\$|\s|\\\"]?)\)?(?!\$).*") - matches = var_pattern.findall(command) - if matches: - for var in matches: - value = self.demo.env.get(var) - self.log("Debug", "Expanding variable " + var + " to value " + value) - command = command.replace("$" + var, value) - return command - - def get_help(self): - help = [] - help.append("SimDem Help") - help.append("===========") - help.append("") - help.append("Pressing any key other than those listed below will result in the script progressing") - help.append("") - help.append("b - break out of the script and accept a command from user input") - help.append("b -> CTRL-C - stop the script") - help.append("d - (redisplay the description that precedes the current command then resume from this point)") - help.append("r - repeat the previous command") - help.append("h - displays this help message") - help.append("") - return help - - def check_for_interactive_command(self): - """Wait for a key to be pressed. - - Most keys result in the script - progressing, but a few have special meaning. See the - documentation or code for a description of the special keys. - """ - if not self.demo.is_automated: - if not self.demo.is_simulation: - self.instruction("Press a command key to proceed (h for help)") - key = self.get_instruction_key() - - if key == 'h': - text = self.get_help() - for line in text: - self.information(line, True) - - self.check_for_interactive_command() - elif key == 'b': - print("shell> ", end='') - command = input() - if command != "": - self.run_command(command) - self.prompt() - self.check_for_interactive_command() - elif key == 'd': - print("") - print(colorama.Fore.CYAN) - print(self.demo.current_description); - print(colorama.Style.RESET_ALL) - self.prompt() - print(self.demo.current_command, end="", flush=True) - self.check_for_interactive_command() - elif key == 'r': - if not self.demo.last_command == "": - self.demo.current_command = self.demo.last_command - self.simulate_command() - self.prompt() - self.check_for_interactive_command() - - def get_instruction_key(self): - """Waits for a single keypress on stdin. - - This is a silly function to call if you need to do it a lot because it has - to store stdin's current setup, setup stdin for reading single keystrokes - then read the single keystroke then revert stdin back after reading the - keystroke. - - Returns the character of the key that was pressed (zero on - KeyboardInterrupt which can happen when a signal gets handled) - - This method is licensed under cc by-sa 3.0 - Thanks to mheyman http://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key\ - """ - import termios, fcntl, sys, os - fd = sys.stdin.fileno() - # save old state - flags_save = fcntl.fcntl(fd, fcntl.F_GETFL) - attrs_save = termios.tcgetattr(fd) - # make raw - the way to do this comes from the termios(3) man page. - attrs = list(attrs_save) # copy the stored version to update - # iflag - attrs[0] &= ~(termios.IGNBRK | termios.BRKINT | termios.PARMRK - | termios.ISTRIP | termios.INLCR | termios. IGNCR - | termios.ICRNL | termios.IXON ) - # oflag - attrs[1] &= ~termios.OPOST - # cflag - attrs[2] &= ~(termios.CSIZE | termios. PARENB) - attrs[2] |= termios.CS8 - # lflag - attrs[3] &= ~(termios.ECHONL | termios.ECHO | termios.ICANON - | termios.ISIG | termios.IEXTEN) - termios.tcsetattr(fd, termios.TCSANOW, attrs) - # turn off non-blocking - fcntl.fcntl(fd, fcntl.F_SETFL, flags_save & ~os.O_NONBLOCK) - # read a single keystroke - try: - ret = sys.stdin.read(1) # returns a single character - except KeyboardInterrupt: - ret = 0 - finally: - # restore old state - termios.tcsetattr(fd, termios.TCSAFLUSH, attrs_save) - fcntl.fcntl(fd, fcntl.F_SETFL, flags_save) - return ret - - def test_results(self, results): - """Display the test results for a single test - """ - if results["passed"]: - return - else: - print("\n\n=============================\n\n") - print(colorama.Fore.RED + colorama.Style.BRIGHT) - print("FAILED") - print(colorama.Style.RESET_ALL) - print("Similarity ratio: " + str(results["similarity"])) - print("Expected Similarity: " + str(results["required_similarity"])) - print("\n\n=============================\n\n") - print("Expected results:") - print(colorama.Fore.GREEN + colorama.Style.BRIGHT) - print(results["expected_results"]) - print(colorama.Style.RESET_ALL) - print("Actual results:") - print(colorama.Fore.RED + colorama.Style.BRIGHT) - print(results["results"]) - print(colorama.Style.RESET_ALL) - - print("\n\n=============================\n\n") - print(colorama.Style.RESET_ALL) - - def get_command(self, commands): - cmd = self.request_input("What mode do you want to run in? (default 'tutorial')") - - if cmd == "": - cmd = "tutorial" - while not cmd in commands: - cmd = self.get_command(commands) - return cmd - - def set_demo(self, demo): - self.demo = demo diff --git a/config.py b/config.py deleted file mode 100644 index 38d66ec..0000000 --- a/config.py +++ /dev/null @@ -1,28 +0,0 @@ -SIMDEM_VERSION = "0.8.2-dev" -SIMDEM_TEMP_DIR = "~/.simdem/tmp" - -# When in demo mode we insert a small random delay between characters. -# TYPING DELAY is the upper bound of this delay. -TYPING_DELAY = 0.2 - -# Prompt to use in the console -console_prompt = "$ " - -# Port for web server when running with '--webui true' optios -port = 8080 - -# ------------------------------------------------------------------ # -# Danger zone -# -# Do not change anything after this notice, -# unless you know what you are doing -# ------------------------------------------------------------------ # - -# Set is_debug to True if you want to run in debug mode. This setting -# can be overriden in the command like with the `--debug true` option. -is_debug = False - -# Available modes of execution -modes = [ "tutorial", "demo", "learn", "test", "script", "prep" ] - - diff --git a/demo.py b/demo.py deleted file mode 100644 index 669e5fe..0000000 --- a/demo.py +++ /dev/null @@ -1,801 +0,0 @@ -# This class represents a Demo to be executed in SimDem. - -import datetime -import difflib -from itertools import tee, islice, zip_longest -import json -import os -import re -import sys -import urllib.request -from environment import Environment - -from cli import Ui -import config - -def get_next(some_iterable, window=1): - items, nexts = tee(some_iterable, 2) - nexts = islice(nexts, window, None) - return zip_longest(items, nexts) - -class Demo(object): - def __init__(self, is_running_in_docker, script_dir="demo_scripts", filename="README.md", is_simulation=True, is_automated=False, is_testing=False, is_fast_fail=True,is_learning = False, parent_script_dir = None, is_prep_only = False, is_prerequisite = False, output_format="log"): - """ - is_running_in_docker should be set to true is we are running inside a Docker container - script_dir is the location to look for scripts - filename is the filename of the script this demo represents - is_simulation should be set to true if we want to simulate a human running the commands - is_automated should be set to true if we don't want to wait for an operator to indicate it's time to execute the next command - is_testing is set to true if we want to compare actual results with expected results, by default execution will stop if any test fails (see is_fast_fail) - is_fast_fail should be set to true if we want to contnue running tests even after a failure - is_learning should be set to true if we want a human to type in the commands - parent_script_dir should be the directory of the script that calls this one, or None if this is the root script - is_prep_only should be set to true if we want to stop execution after all prerequisites are satsified - is_prerequisite indicates whether this is a prerequisite or not. It is used to decide behaviour with respect to simulation etc. - """ - self.mode = None - self.is_docker = is_running_in_docker - self.filename = filename - self.script_dir = "" - self.set_script_dir(script_dir) - self.is_simulation = is_simulation - self.is_automated = is_automated - self.is_testing = is_testing - self.is_fast_fail = is_fast_fail - self.is_learning = is_learning - self.current_command = "" - self.current_description = "" - self.last_command = "" - self.is_prep_only = is_prep_only - self.parent_script_dir = parent_script_dir - if self.parent_script_dir: - self.env = Environment(self.parent_script_dir, is_test = self.is_testing) - else: - self.env = Environment(self.script_dir, is_test = self.is_testing) - self.is_prerequisite = is_prerequisite - self.output_format = output_format - self.all_results = [] - self.completed_validation_steps = [] - - def set_script_dir(self, script_dir, base_dir = None): - if base_dir is not None and not base_dir.endswith(os.sep): - base_dir += os.sep - elif base_dir is None: - base_dir = "" - - self.script_dir = os.path.abspath(os.path.join(base_dir, script_dir)) - - def get_current_command(self): - """ - Return a tuple of the current command and a list of environment - variables that haven't been set and a list that have been set.. - """ - - # If the command sets a variable put it in our env copy - pattern = re.compile("^(\w*)=(.*)$") - match = pattern.match(self.current_command) - if match: - key = match.groups()[0] - val = match.groups()[1] - self.env.set(key, val) - - # Get all the vars, check to see if they are uninitialized - var_pattern = re.compile(".*?(?<=\$)\(?{?(\w*)(?=[\W|\$|\s|\\\"]?)\)?(?!\$).*") - matches = var_pattern.findall(self.current_command) - undefined_var_list = [] - defined_var_list = [] - if matches: - for var in matches: - have_value = False - if self.env: - for item in self.env.get(): - if var == item: - have_value = True - defined_var_list.append(var) - break - if len(var) > 0 and not have_value and not '$(' + var in self.current_command: - value = self.ui.get_shell().run_command("echo $" + var).strip() - if len(value) == 0: - undefined_var_list.append(var) - else: - defined_var_list.append(var) - return self.current_command, undefined_var_list, defined_var_list - - def get_scripts(self, directory): - """ - Starting with the supplied directory find all `README.md` files - and return them as a list of scripts available to this execution. - We will not return multiple `README.md` files from each part of - the tree. It is assumed that the highest level `README.md` files - contains an index of scripts in that directory. - - """ - lines = [] - for dirpath, dirs, files in os.walk(directory): - for file in files: - if file == "README.md" or file == "script.md": - lines.append(os.path.join(dirpath[len(directory):], file) + "\n") - - return lines - - def generate_toc(self): - toc = {} - lines = [] - lines.append("# Welcome to Simdem\n") - lines.append("Below is an autogenerated list of scripts available in `" + self.script_dir + "` and its subdirectories. You can execute any of them from here.\n\n") - lines.append("# Next Steps\n") - - scripts = self.get_scripts(self.script_dir) - - for script in scripts: - script = script.strip() - with open(os.path.join(self.script_dir, script)) as f: - title = f.readline().strip() - title = title[2:] - demo = { "title": title, "path": script } - - name, _ = os.path.split(script) - if not name.endswith(".md") and name not in toc: - toc[name] = [ demo ] - elif name in toc: - demos = toc[name] - demos.append(demo) - toc[name] = demos - - idx = 1 - for item in sorted(toc): - demos = toc[item] - for demo in demos: - if not item == "": - lines.append(" " + str(idx) + ". [" + item + " / " + demo["title"] + "](" + demo["path"] + ")\n") - else: - lines.append(" " + str(idx) + ". [" + demo["title"] + "](" + demo["path"] + ")\n") - idx += 1 - - return lines - - def run(self, mode = None): - """ - Reads a README.md file in the indicated directoy and runs the - commands contained within. If simulation == True then human - entry will be simulated (looks like typing and waits for - keyboard input before proceeding to the next command). This is - useful if you want to run a fully automated demo. - - The README.md file will be parsed as follows: - - ``` marks the start or end of a code block - - Each line in a code block will be treated as a separate command. - All other lines will be ignored - """ - if self.ui is None: - raise Exception("Attempt to run a demo before ui is configured") - - if mode is None: - mode = self.ui.get_command(config.modes) - self.mode = mode - - self.ui.log("debug", "Running script in " + self.mode + " mode") - - if mode == "script": - print(self.get_bash_script()) - return - elif mode == "demo": - # we automate the prereq steps so start in auto mode without simulation - # we'll switch these two values once prereqs are complete - self.is_simulation = False - self.is_automated = True - elif mode == "test": - self.is_testing = True - self.is_automated = True - elif mode == "learn": - self.is_learning = True - elif mode == "prep": - self.is_prep_only = True - self.is_testing = True - self.is_automated = True - elif mode == "run" or mode == "tutorial": - pass - else: - raise Exception("Unkown mode: '" + mode + "'") - - self.env = Environment(self.script_dir, is_test = self.is_testing) - - self.ui.log("debug", (str(self.env))) - - self.ui.log("debug", "Running script called '" + self.filename + "' in '" + self.script_dir +"'") - - classified_lines = self.classify_lines() - failed_tests, passed_tests = self.execute(classified_lines) - - if self.is_prep_only: - if failed_tests == 0: - self.ui.information("Preparation steps for '" + self.script_dir + "' complete", True) - else: - self.ui.error("Preparation steps for '" + self.script_dir + "' failed", True) - elif self.is_testing: - self.ui.horizontal_rule() - self.ui.heading("Test Results") - if failed_tests > 0: - self.ui.warning("Failed Tests: " + str(failed_tests)) - self.ui.information("Passed Tests: " + str(passed_tests)) - self.ui.new_para() - else: - self.ui.information("No failed tests.", True) - self.ui.information("Passed Tests: " + str(passed_tests)) - self.ui.new_para() - if failed_tests > 0: - self.ui.instruction("View failure reports in context in the above output.") - if self.is_fast_fail: - self.output_results() - - if not self.is_simulation and not self.is_testing and not self.is_prep_only: - next_steps = [] - for line in classified_lines: - if line["type"] == "next_step" and len(line["text"].strip()) > 0: - pattern = re.compile('.*\[.*\]\((.*)\/(.*)\).*') - match = pattern.match(line["text"]) - if match: - next_steps.append(line) - - if len(next_steps) > 0: - if self.parent_script_dir: - return - in_string = "" - in_value = 0 - self.ui.instruction("Would you like to move on to one of the next steps listed above?") - - while in_value < 1 or in_value > len(next_steps): - in_string = self.ui.request_input("Enter a value between 1 and " + str(len(next_steps)) + " or 'quit'") - if in_string.lower() == "quit" or in_string.lower() == "q": - return - try: - in_value = int(in_string) - except ValueError: - pass - - self.ui.log("debug", "Selected next step: " + str(next_steps[in_value -1])) - pattern = re.compile('.*\[.*\]\((.*)\/(.*)\).*') - match = pattern.match(next_steps[in_value -1]["text"]) - self.set_script_dir(match.groups()[0], self.script_dir) - self.filename = match.groups()[1] - self.run(self.mode) - self.output_results() - - def output_results(self): - """Output the results of the run in the format requested. Note that -if `--output` is `log` (or undefined) we will have been outputing the -logs throughout execution.""" - passed = True - if self.output_format == "json": - output = [] - else: - output = "" - - for result in self.all_results: - timestamp = datetime.datetime.utcnow().strftime("%Y%m%d - %H:%M") - test_name = os.path.join(self.script_dir, self.filename) - is_success = result["passed"] - if is_success: - failure_message = None - else: - failure_message = result - test_type = "SimDem" - resource_group = self.env.get("SIMDEM_RESOURCE_GROUP") - region = self.env.get("SIMDEM_LOCATION") - orchestrator = self.env.get("SIMDEM_ORCHESTRATOR") - - if self.output_format == "summary": - if is_success: - meta = "Succesful test" - else: - meta = "Failed test:\t" + json.dumps(failure_message) - meta += "\nTime (UTC):\t" + timestamp - meta += "\nTest Name:\t" + test_name - meta += "\nOrchestrator:\t" + orchestrator - meta += "\nResource Group:\t" + resource_group - meta += "\nRegion:\t\t" + region - meta += "\n\n" - - output += meta - elif self.output_format == "json": - meta = { - "TimeStampUTC": timestamp, - "TestName": test_name, - "TestType": test_type, - "ResourceGroup": resource_group, - "Region": region, - "Orchestrator": orchestrator, - "Success": is_success, - "FailureStr": failure_message - } - output.append(meta) - elif self.output_format != "log": - sys.exit("Invalid option for '--output', see 'simdem --help' for available options") - else: - message = "" # logs were output during execution - - if not is_success: - passed = False - - if len(output) == 0: - output = "Completed run succesfully" - - if passed: - if self.parent_script_dir: - # This is a prereq script so we don't output summary results for this, but we'll log them - if self.output_format == "json": - self.ui.log("debug", json.dumps(output)) - else: - self.ui.log("debug", output) - else: - if self.output_format == "json": - print(json.dumps(output)) - else: - print(output) - - else: - if self.output_format == "json": - sys.exit(json.dumps(output)) - else: - sys.exit(output) - - def classify_lines(self): - lines = None - - # Only run through test plan for the first script - if self.is_testing and self.parent_script_dir is None: - test_file = os.path.join(self.script_dir, "test_plan.txt") - if os.path.isfile(test_file): - self.ui.log("info", "Executing test plan in " + test_file) - plan_lines = list(open(test_file)) - lines = [] - for line in plan_lines: - line = line.strip() - if not line == "" and not line.startswith("#"): - # not a comment or whitespace so should be a path to a script with tests - self.ui.log("debug", "Including " + line + " in tests.") - before = len(lines) - file = os.path.join(self.script_dir, line) - lines.append("START TEST FILE: " + file) - lines = lines + list(open(file)) - lines.append("END TEST FILE: " + file) - after = len(lines) - self.ui.log("debug", "Added " + str(after - before) + " lines.") - - if lines is None: - file = os.path.join(self.script_dir, self.filename) - self.ui.log("info", "Reading lines from " + file) - - if file.startswith("http"): - # FIXME: Error handling - response = urllib.request.urlopen(file) - data = response.read().decode("utf-8") - lines = data.splitlines(True) - else: - if not lines and os.path.isfile(file): - lines = list(open(file)) - elif not lines: - if self.parent_script_dir != "": - # If we have a parent then this is a preqiusite and therefore it should exist - # if it doesn't then it may be that we are using relative paths - # from the script location and that is different from self.script_dir - exit("Missing prerequisite script: " + self.filename + " in " + self.script_dir) - else: - lines = self.generate_toc() - - test_file_path = None - in_code_block = False - in_results_section = False - in_next_steps = False - in_prerequisites = False - in_validation_section = False - executed_code_in_this_section = False - - classified_lines = [] - - for line in lines: - if line.startswith("START TEST FILE: "): - test_file_path = line[17:] - self.ui.log("debug", "Entering test file: " + test_file_path) - classified_lines.append({"type": "start_test_file", - "file": test_file_path}) - elif line.startswith("END TEST FILE: "): - test_file_path = line[15:] - self.ui.log("debug", "Exiting test file: " + test_file_path) - classified_lines.append({"type": "end_test_file", - "file": test_file_path}) - test_file_path = None - elif line.lower().startswith("results:"): - # Entering results section - in_results_section = True - elif line.startswith("```") and not in_code_block: - # Entering a code block, - in_code_block = True - pos = line.lower().find("expected_similarity=") - if pos >= 0: - pos = pos + len("expected_similarity=") - similarity = line[pos:] - expected_similarity = float(similarity) - else: - expected_similarity = 0.66 - elif line.startswith("```") and in_code_block: - # Finishing code block - in_code_block = False - in_results_section = False - in_validation_section = False - elif in_results_section and in_code_block: - classified_lines.append({"type": "result", - "expected_similarity": expected_similarity, - "text": line}) - elif in_code_block and not in_results_section: - # Executable line - if line.startswith("#"): - # comment - pass - else: - classified_lines.append({"type": "executable", - "text": line}) - elif line.startswith("#") and not in_code_block and not in_results_section: - # Heading in descriptive text, indicating a new section - if line.lower().strip().endswith("# next steps"): - in_next_steps = True - in_prerequisites = False - in_validation_section = False - elif line.lower().strip().endswith("# prerequisites"): - self.ui.log("debug", "Found a prerequisites section") - in_prerequisites = True - in_validation_section = False - in_next_steps = False - elif line.lower().strip().startswith("# validation"): - # Entering validation section - self.ui.log("debug", "Entering Validation Section") - in_validation_section = True - in_prerequisites = False - in_next_steps = False - else: - in_prerequisites = False - in_validation_section = False - in_next_steps = False - classified_lines.append({"type": "heading", - "text": line}) - else: - if in_next_steps: - classified_lines.append({"type": "next_step", - "text": line}) - elif in_prerequisites and len(line.strip()) > 0: - if test_file_path: - source_file_path = test_file_path - else: - source_file_path = os.path.join(self.script_dir, self.filename) - classified_lines.append({"type": "prerequisite", - "text": line, - "source_file_path": source_file_path}) - elif in_validation_section: - classified_lines.append({"type": "validation", - "text": line}) - else: - classified_lines.append({"type": "description", - "text": line}) - - is_first_line = False - - classified_lines.append({"type": "EOF", - "text": ""}) - - if config.is_debug: - self.ui.log("debug", "Classified lines: ") - for line in classified_lines: - self.ui.log("debug", str(line)) - - return classified_lines - - def execute(self, lines): - """Execute the script found in the lines. Return the number of failed - tests and the number of passed tests.""" - - source_file_directory = None - is_first_line = True - in_results = False - expected_results = "" - actual_results = "" - failed_tests = 0 - passed_tests = 0 - done_prerequisites = False - in_validation = False - executed_code_in_this_section = False - next_steps = [] - - self.ui.clear() - for line, next_line in get_next(lines): - if line["type"] == "start_test_file": - source_file_directory = os.path.dirname(line["file"]) - self.ui.get_shell().run_command("pushd " + source_file_directory) - done_prerequisites = False - elif line["type"] == "end_test_file": - source_file_directory = None - self.ui.get_shell().run_command("popd") - elif line["type"] == "result": - if not in_results: - in_results = True - expected_results = "" - expected_results += line["text"] - expected_similarity = line["expected_similarity"] - elif line["type"] != "result" and in_results: - # Finishing results section - if self.is_testing: - ansi_escape = re.compile(r'\x1b[^m]*m') - results = self.is_pass(expected_results, self.strip_ansi(actual_results), expected_similarity) - self.ui.test_results(results) - self.all_results.append(results) - if results["passed"]: - passed_tests += 1 - else: - failed_tests += 1 - if self.is_fast_fail: - break - expected_results = "" - actual_results = "" - in_results = False - elif line["type"] == "prerequisite": - if not done_prerequisites: - self.ui.heading(line["text"]) - self.check_prerequisites(lines, source_file_directory) - done_prerequisites = True - if self.is_prep_only: - return failed_tests, passed_tests - - if self.mode == "demo": - # prereqs are now complete, so switch to simulated demo mode - self.is_simulation = True - self.is_automated = False - elif line["type"] == "executable": - if line["text"].strip() == "": - break - if not self.is_learning: - self.ui.prompt() - self.ui.check_for_interactive_command() - self.current_command = line["text"] - actual_results = self.ui.simulate_command() - executed_code_in_this_section = True - self.current_description = "" - if not self.is_automated and not next_line["type"] == "executable": - self.ui.check_for_interactive_command() - elif line["type"] == "heading": - if not is_first_line and not self.is_simulation: - self.ui.check_for_interactive_command() - if not self.is_simulation: - self.ui.clear() - self.ui.heading(line["text"]) - else: - if not self.is_simulation and (line["type"] == "description" or line["type"] == "validation"): - # Descriptive text - if not self.is_testing: - self.ui.description(line["text"]) - self.current_description += line["text"] - if line["type"] == "next_step" and not self.is_simulation: - pattern = re.compile('(.*)\[(.*)\]\(.*\).*') - match = pattern.match(line["text"]) - if match: - self.ui.next_step(match.groups()[0], match.groups()[1]) - else: - self.ui.description(line["text"]) - - is_first_line = False - - return failed_tests, passed_tests - - def check_prerequisites(self, lines, source_file_directory = None): - """Check that all prerequisites have been satisfied by iterating - through them and running the validation steps. If the - validatin tests pass then move on, if they do not then execute - the prerequisite script. If running in test mode assume that - this is the case (pre-requisites should be handled in the - test_plan). - - When running in test mode the script_dir may be different from - the location of the prerequisite script file. In these cases the - 'source_file_directory' should container the directory in which - the prequisite script is located. - """ - if source_file_directory is None: - source_file_directory = self.script_dir - steps = [] - for line in lines: - step = {} - if line["type"] == "prerequisite" and len(line["text"].strip()) > 0: - if source_file_directory and line["source_file_path"].startswith(source_file_directory): - self.ui.log("debug", "Looking for prereq file in line: " + line["text"]) - self.ui.description(line["text"]) - pattern = re.compile('.*\[(.*)\]\((.*)\).*') - match = pattern.match(line["text"]) - if match: - step["title"] = match.groups()[0].strip() - href = match.groups()[1] - if not href.endswith(".md"): - if not href.endswith("/"): - href = href + "/" - href = href + "README.md" - step["href"] = href - steps.append(step) - self.ui.log("debug", "Found prereq: " + str(step)) - - for step in steps: - path, filename = os.path.split(step["href"]) - self.ui.log("debug", "Source file directory is " + source_file_directory) - if (step["href"].startswith(".")): - if source_file_directory is not None: - new_dir = os.path.join(self.script_dir, source_file_directory, path) - else: - new_dir = os.path.join(self.script_dir, path) - else: - new_dir = path - - new_dir = os.path.abspath(new_dir) - full_path = os.path.join(new_dir, filename) - if full_path in self.completed_validation_steps: - self.ui.log("debug", "Already validated / executed script in " + full_path) - continue - - self.ui.new_para() - - self.ui.log("debug", "Execute prerequisite step in " + filename + " in " + new_dir) - demo = Demo(self.is_docker, new_dir, filename, self.is_simulation, self.is_automated, self.is_testing, self.is_fast_fail, self.is_learning, self.script_dir, is_prerequisite = True, output_format=self.output_format) - demo.mode = self.mode - demo.set_ui(self.ui) - demo.run_if_validation_fails(self.mode) - self.ui.get_shell().run_command("popd ") # set_ui runs pushd - self.ui.set_demo(self) # demo.set_ui(...) assigns new demo to ui, this reverts after prereq execution - - self.completed_validation_steps.append(full_path) - self.ui.check_for_interactive_command() - - def run_if_validation_fails(self, mode = None): - self.ui.information("Validating pre-requisite of '" + self.parent_script_dir + "' in '" + os.path.abspath(os.path.join(self.script_dir, self.filename)) + "'") - self.ui.new_para() - lines = self.classify_lines() - self.check_prerequisites(lines, self.script_dir) - if self.validate(lines): - self.ui.information("Validation passed.", True) - else: - self.ui.information("Validation failed. Running prerequisite steps in '" + os.path.abspath(os.path.join(self.script_dir, self.filename)) + "'", True) - self.ui.new_para() - self.ui.check_for_interactive_command() - self.run(mode) - self.ui.clear() - - def validate(self, lines): - """Run through the supplied lines, executing and testing any that are -found in the validation section. - - """ - self.ui.prompt() - result = True - in_validation = False - has_validation_steps = False - in_results = False - expected_results = "" - for line in lines: - if line["type"] == "validation": - in_validation = True - has_validation_steps = True - elif line["type"] == "heading": - in_validation = False - elif in_validation and line["type"] == "executable": - self.current_command = line["text"] - self.ui.log("debug", "Execute validation command: " + self.current_command) - actual_results = self.ui.simulate_command(not config.is_debug) - expected_results = "" - elif in_validation and line["type"] == "result": - if not in_results: - in_results = True - expected_results = "" - expected_results += line["text"] - expected_similarity = line["expected_similarity"] - elif (line["type"] != "result" and in_results): - # Finishing results section - test_results = self.is_pass(expected_results, self.strip_ansi(actual_results), expected_similarity) - if not test_results["passed"]: - self.ui.log("debug", "validation expected results: '" + expected_results + "'") - self.ui.log("debug", "validation actual results: '" + actual_results + "'") - result = False - expected_results = "" - actual_results = "" - in_results = False - - return result and has_validation_steps - - def strip_ansi(self, text): - """ Strip ANSI codes from a string.""" - ansi_escape = re.compile(r'\x1b[^m]*m') - return ansi_escape.sub('', text) - - def is_pass(self, expected_results, actual_results, expected_similarity = 0.66): - """Checks to see if a command execution passes. - If actual results compared to expected results is within - the expected similarity level then it's considered a pass. - - Returns a dictionary containing the results: - { - "passed": boolean, - "command": "the command executed", - "results": "Results returned", - "expected_results": "Expected results", - "similarity": float, - "required_similarity": float - } - - """ - differ = difflib.Differ() - comparison = differ.compare(actual_results, expected_results) - diff = differ.compare(actual_results, expected_results) - seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", actual_results, expected_results) - - is_pass = seq.ratio() >= expected_similarity - - self.ui.log("debug", "Similarity is: " + str(seq.ratio())) - - message = { - "passed": is_pass, - "command": self.last_command, - "results": actual_results, - "expected_results": expected_results, - "similarity": seq.ratio(), - "required_similarity": expected_similarity - } - - return message - - def __str__( self ): - s = "Demo directory: " + self.script_dir + "\n" - s += "Demo filename: " + self.filename + "\n" - if self.is_docker: - s += "Running in a Docker container\n" - else: - s += "Not running in a Docker container\n" - s += "Simulation mode: {0}\n".format(self.is_simulation) - s += "Automatic mode: {0}\n".format(self.is_automated) - s += "Learn mode: {0}\n".format(self.is_learning) - s += "Test mode: {0}\n".format(self.is_testing) - if self.is_testing: - s += "Fast fail test mode: {0}\n".format(self.is_fast_fail) - - return s - - def set_ui(self, ui): - self.ui = ui - ui.set_demo(self) - self.ui.get_shell().run_command("pushd " + self.script_dir) - self.ui.log("debug", str(self)) - - def get_bash_script(self): - """Reads a README.md file in the indicated directoy and builds an - executable bash script from the commands contained within. - - """ - script = "" - for key, value in env.items(): - script += key + "='" + value + "'\n" - - in_code_block = False - in_results_section = False - lines = list(open(self.script_dir + "README.md")) - for line in lines: - if line.startswith("Results:"): - # Entering results section - in_results_section = True - elif line.startswith("```") and not in_code_block: - # Entering a code block, if in_results_section = True then it's a results block - in_code_block = True - elif line.startswith("```") and in_code_block: - # Finishing code block - in_results_section = False - in_code_block = False - elif in_code_block and not in_results_section: - # Executable line - script += line - elif line.startswith("#") and not in_code_block and not in_results_section: - # Heading in descriptive text - script += "\n" - return script diff --git a/demo_scripts/README.md b/demo_scripts/README.md deleted file mode 100644 index 9e3b09a..0000000 --- a/demo_scripts/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Welcome to SimDem - -Is it: - - * Documentation - * An interactive tutorial - * A live demo - * An automated test script - * A Shell script - -## Simdem is Documentation, Tutorials, Demo's and Tests - -It's all of them! - -# Next Steps - - 1. [Hello World Demo](simdem/demo/README.md) - 2. [SimDem Documentation](simdem/README.md) - diff --git a/demo_scripts/env.json b/demo_scripts/env.json deleted file mode 100644 index 6fcf682..0000000 --- a/demo_scripts/env.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "PARENT_TEST": "Hello from the parent" -} diff --git a/demo_scripts/simdem/README.md b/demo_scripts/simdem/README.md deleted file mode 100644 index 8bb26dc..0000000 --- a/demo_scripts/simdem/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Welcome to SimDem - -Is it: - - * Documentation - * An interactive tutorial - * A live demo - * An automated test script - * A Shell script - -## Simdem is Documentation, Tutorials, Demo's and Tests - -It's all of them! - -Simdem allows you to wite a tutorial in markdown format and then run -the commands within it as a simulated demo, interactive tutorial or -even a test script. You can also generate executable shell scripts. - -SimDem reads a script, written in the form of a human readable -Markdown file, and executes the commands within this script on your -behalf. It will even make it look like you are really typing the -commands, which is great if you want to concentrate on explaining what -you are doing but still run the demo live. - -It's easier to describe if you see it working. In fact you are already -in a SimDem. Press a key (other than 'b', we'll look at that shortly) -to "type" a command, once the command has been "typed" hit -a key to execute the command. - -# Next Steps - -Tutorials can branch too, for example you can choose any of the -following paths next: - - 1. [Modes of operation](modes/README.md) - 2. [Hello World Demo](demo/README.md) - 3. [Build a Hello World script](tutorial/README.md) - 4. [Write SimDem documents](syntax/README.md) - 5. [Special Commands](special_commands/README.md) - 6. [Configure your scripts through variables](variables/README.md) - 7. [Write multi-part documents](multipart/README.md) - 8. [Use your documents as interactive tutorials or demos](running/README.md) - 9. [Use your documents as automated tests](test/README.md) - 10. [Build an SimDem container](building/README.md) - - - - diff --git a/demo_scripts/simdem/building/README.md b/demo_scripts/simdem/building/README.md deleted file mode 100644 index 2837b1f..0000000 --- a/demo_scripts/simdem/building/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Building your own Demo Container - -Create a Dockerfile and add (at least) the following: - - ``` - FROM rgardler/simdem - - COPY my_script_dir demo_scripts - ``` - -# Next Steps - - 1. [SimDem Index](../README.md) diff --git a/demo_scripts/simdem/demo/README.md b/demo_scripts/simdem/demo/README.md deleted file mode 100644 index c46c51a..0000000 --- a/demo_scripts/simdem/demo/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Hello World SimDem Demo - -This script is intended to be used to demonstrate the key features of -Simdem. - -When you hit 'spacebar' a command will be displayed and -executed. Actually you can hit almost any key but we recommend -'spacebar' here because we've not told you about the special keys yet -and spacebar is not one of them. - -``` -echo "Hello World" -``` - -That's cool, lets try again: - -``` -echo "It might look like this was typed into the terminal (even more so if you ran SimDem with the '--style simulate' flag, that simulates a person typing), bit it really comes from a markdown file." -``` - -The date command will show that these commands are being executed in real time. - -``` -date -``` - -Results: - -```expected_similarity=0.3 -Sat Mar 12 08:59:01 UTC 2016 -``` - -You can run almost any shell command this way. - -# Special keys - -Although we said "spacebar" above, in reality you can hit almost any -key. There are a few exceptions though: - -## 'd' for description - -Hitting 'd' will print all the text since the last command, that is it -will print the description of the next command to be executed. - -``` -echo "Hitting 'd' now will display the description for this command." -``` - -## 'b' for break - -Hitting 'b' will "break" from the current script. This allows you to -type in commands that are not part of the script. This is particularly -useful when running in demo mode as it alllows you to respond to -questions by entering an unscripted command. - -``` -echo "Give it a go, why not hit 'b' and type 'ls', or some other command" -``` - -NOTE: at the time of writing it is not possible to use interactive -commands or commands. - -# Next Steps - -It's possible to provide a branching point a the end of a script. The -user can select one of a selection of options or they can enter "quit" -(or just "q") to exit SimDem. - - 1. [Write SimDem documents](../syntax/README.md) - 2. [SimDem Index](../README.md) - 3. [Modes of operation](../modes/README.md) - - - diff --git a/demo_scripts/simdem/env.json b/demo_scripts/simdem/env.json deleted file mode 100644 index 7a7485b..0000000 --- a/demo_scripts/simdem/env.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "TEST": "hello-world" -} diff --git a/demo_scripts/simdem/env.local.json b/demo_scripts/simdem/env.local.json deleted file mode 100644 index e6c7c04..0000000 --- a/demo_scripts/simdem/env.local.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "LOCAL_TEST": "A warm local hello" -} diff --git a/demo_scripts/simdem/modes/README.md b/demo_scripts/simdem/modes/README.md deleted file mode 100644 index 9afa34a..0000000 --- a/demo_scripts/simdem/modes/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# Modes of Operation - -SimDem demos are interactive and can be run in a number of different -modes: - - * Tutorial: Displays the descriptive text of the tutorial and pauses - at code blocks to allow user interaction. - * Learn: similar to Tutorial mode, but users are expected to type - the commands themselves. - * Simulate: Does not display the descriptive text, but pauses at each - code block. When the user hits a key the command is "typed", a - second keypress executes the command. - * Test: Runs the commands and then verifies that the output is - sufficiently similar to the expected results (recorded in the - markdown file) to be considered correct. - * Script: Creates an executable bash script from the document - * Auto: allows any of the above modes to be run but without user - interaction - -## Tutorial Mode - -Tutorial mode is ideal if you are using this as a learning or teaching -tool (see also learn mode below, which suits some learning styles -better. In this mode a description of what you are about to do is -shown on the screen, hit a key to see the command, hit another key to -execute the command. Tutorial mode is the default. - -## Learn mode - -Learn mode is similar to tutorial mode above, however, in learn mode -the user is expected to type each command themselves. Some people find -that this aids recall. - -## Demo (or Simulation) mode - -Demo mode is ideal if you are using this to teach or demonstrate how -to achive the goal. In this mode no descriptive text is shown, instead -when you press a key the next command is "typed", pressing another key -will execute the command. The idea is that you describe what is -happening as the application "types" the command for you. To run in -demo mode use the `--style simulate` command line switch with any -other mode. There is also a default demo mode configuration avilable -with the `demo` command" - -``` -simdem demo -``` - -#### Preparation mode - -In this mode only the preparation (prerequisite) steps are -executed. This is useful for setting up the environment for a -demo. Next time the demo is run all prepration steps will be -skipped. This means that steps that take a long time can be -pre-baked. - -To use this mode use the command 'mode' - -``` -simdem prep -``` - -## Test Mode - -Test mode runs the commands and then verifies that the output is -sufficiently similar to the expected results (recorded in the markdown -file) to be considered correct. To run in test mode use the `--test -yes` switch. For convenience you can use the command `test` to execute -tests with the optimal configuration for automated testing.. - -## Script mode - -Script mode does not execute any of the commands, instead is outputs -an executable bash script that can be run without SimDem. Use the -command `script` to generate the executable script. - -# Unnattended (Auto) Mode - -Each of these modes can be run in auto mode too. This means that the -program does not wait for a keypress before proceeding. This can be -useful if you want to runthe complete script unattended. To run in -automated or unnattended mode use the `--auto true` command line -switch. - -Manual mode is ideal if you would like to manually type the commands, -many people find this helps them remember. It can be useful in the -first few runs, but we still recommend using "demo" mode when doing -live demo's - it's much harder to make a mistake this way. - -# Next Steps - - 1. [Hello World Demo](../demo/README.md) - 2. [SimDem Index](../README.md) - 3. [Write SimDem documents](../syntax/README.md) - 4. [Build a SimDem container](../building/README.md) diff --git a/demo_scripts/simdem/multipart/README.md b/demo_scripts/simdem/multipart/README.md deleted file mode 100644 index 18ab03b..0000000 --- a/demo_scripts/simdem/multipart/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# Multi-Part Demo's - -Most tutorials's will consist of at least three parts, preparation, -main body and cleanup. Many will have multiple staged in the main part -of the tutorial. SimDem is able to provide an interactive menu -allowing users to select which part of the tutorial to work through -next. This is achieved by providing a final section with the title `# -Next Steps`. This section should include a list in which each item -provides a link to a markdown document that contain SimDem scripts -that the user may want to work through next. - -For example, this file is one part of a multi-part document. - -``` -cat $SIMDEM_CWD/README.md -``` - -When executed using SimDem this results in the user being prompted to -select a "next step" (or hit 'q' to quit). If the user selects one of -the scripts it will be executed. - -# Prerequisites - -It is assumed that you have a basic understanding of the various -SimDem execution [modes](../modes/README.md). You should also ensure -you understand the SimDem [document syntax](../syntax). - -# Directory structure - -SimDem projects consist of a root directory and one or more -sub-directories. Project directories will contain at least a -`README.md` file that will be used by default when SimDem is run -against the project. Therefore the minimum directory structure for a -simple tutorial is: - -` -My_SimDem_Tutorial -└── README.md -` - -# Multi-part tutorials - -A more complex project will contain a number of sub-directories -containing tutorials. Tutorial sub-directories will contain at least a -`README.md` file, this is the main file for that tutorial. For example: - -` -My_Complex_SimDem_Tutorial -├── README.md -├── Tutorial_1 -│ └── README.md -├── Tutorial_2 -│ └── README.md -└── Tutorial_3 - └── README.md -` - -## Auto Table of Contents - -If the root of the demo scripts directory does not contain a -`README.md` file then SimDem will create a Table of Contents from all -sub-directories that contain a `README.md` file. This ToC will use the -first line (which should be a heading marked with '# ' at the start) -as the text for the link to the script. This ToC will be displayed as -a 'Next Steps' section, thus users will be able to step into any area -of the available demo's. - -# Other files - -Tutorials may also provide an `env.json` and/or an `env.local.json` -and/or an `env.test.json` file to define environment variables to use -when executing in demo or test mode. - -# Demo Scripts example - -The directory structure for the SimDem demo scripts is: - -``` -tree $SIMDEM_CWD/.. -``` - -Results: Expected Similarity: 0.5 - -``` -demo_scripts/ -├── env.json -├── env.local.json -├── README.md -├── simdem -│ ├── env.json -│ ├── env.local.json -│ └── README.md -└── test - ├── env.json - └── README.md -``` - -# Next Steps - - 1. [Configure your scripts through variables](../variables/README.md) - 2. [SimDem Index](../README.md) - 3. [Use your documents as interactive tutorials or demos](../running/README.md) - 4. [Use your documents as automated tests](../test/README.md) - 5. [Build a SimDem container](../building/README.md) - - diff --git a/demo_scripts/simdem/prerequisites/README.md b/demo_scripts/simdem/prerequisites/README.md deleted file mode 100644 index d0bd278..0000000 --- a/demo_scripts/simdem/prerequisites/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Understanding Prerequisites - -Prerequisite scripts are scripts that must be run in order for another -script to work. When SimDem finds a pre-requisite section it will test -whether the steps have been completed (see validation below). If the -validation tests fail then the code blocks in the pre-requisite script -are executed. - -There aren't really any pre-requisites for this tutorial / -demo. Howeve,r this document is inserted as a pre-requisite so that we -can see how they work. - -# Prerequisites Syntax - -The prerequisites section starts with a heading of `# prerequisites`. - -The body of this section will contain 0 or more links to a script that -should be run ahead of the current one. - -The scripts should appear in the order of required exection in the body. - -# Automatically validating Pre-requisites - -Some pre-requisite steps can take a long time to execute. For this -reason it is possible to provide some validation checks to see if the -pre-requisite step has been completed. These are defined in a section -towards the end of the script, before the next steps section (if one -exists). The validation steps will be executed by SimDem *before* -running the pre-requisite steps, if the tests in that section pass -then there is no need to run the pre-requisites. - -It's easier to explain with an example. - -Imagine we have a prerequisite step that takes 5 seconds, we don't -want to wait 5 seconds only to find that we already completed that -pre-requisite (OK, we know 5 seconds is not long, but it's long enough -to serve for this demo). For this example we will merely sleep for 5 -seconds then touch a file. To validate this prequisite has been -satisfied we will test the modified date of the file, if it has been -modified in the last 5 minutes then the pre-requisite has been -satisfied. - -``` -sleep 5 -echo $SIMDEM_TEMP_DIR -mkdir -p $SIMDEM_TEMP_DIR -touch $SIMDEM_TEMP_DIR/this_file_must_be_modfied_every_minute.txt -``` - -Now we have a set of commands that should be executed as part of this -pre-requisite. In order to use them we simply add a reference to this -file in the pre-requisites section of any other script. - -Any code in a section headed with '# Validation' will be used by -SimDem to test whether the pre-requisites have been satisfied. If -validation tests pass the pre-requisite step will be skipped over, -otherwise the other commands in the script will be executed. - -# Validation - -In order to continue with our example we include some vlaidation steps -in this script. If you have not run through the commands above less -than one minute ago this validation stage will fail. If you are -working through this tutorial now you just executed the above -statements and so the tests here will pass, but if you include this -file as pre-requisite again it may well fail and thus automatically -execute this script. - -For this pre-requisite we need to ensure that the test.txt file has -been updated in the last 5 minutes. If not then we need to run the -commands in this document. If you are running through this document in -SimDem itself then it might be worth going back to the page that calls -this as a pre-requisite, as long as you do this in the next five -minutes you won't come back here. You can do this by selecting -"Understanding SimDem Syntax" in the next steps section. - -``` -find $SIMDEM_TEMP_DIR -name "this_file_must_be_modfied_every_minute.txt" -newermt "1 minutes ago" -``` - -Results: - -``` -/home//.simdem/tmp/this_file_must_be_modfied_every_minute.txt -``` - -# Next Steps - - 1. [Understanding SimDem Syntax](../syntax/script.md) - 2. [Configure your scripts through variables](../variables/script.md) - 3. [Build a Hello World script](../tutorial/script.md) - 4. [SimDem Index](../script.md) - 5. [Write multi-part documents](../multipart/script.md) - 6. [Use your documents as interactive tutorials or demos](../running/script.md) - - diff --git a/demo_scripts/simdem/prerequisites/remote.md b/demo_scripts/simdem/prerequisites/remote.md deleted file mode 100644 index 2c3adc8..0000000 --- a/demo_scripts/simdem/prerequisites/remote.md +++ /dev/null @@ -1,4 +0,0 @@ -# A remote Prerequsite - -This files is really only for test purposes. See the main -prerequisites script for more details. diff --git a/demo_scripts/simdem/running/README.md b/demo_scripts/simdem/running/README.md deleted file mode 100644 index c430b44..0000000 --- a/demo_scripts/simdem/running/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Running SimDem - -SimDem is packaged as a container, you run it with: - -`docker run -it rgardler/simdem` - -This will run the demo script you are working through now. - -## Adding Your Own Demo Script - -You will likely want to add your own demo script. To do this you can -either build your own container or you can mount a volume which -contains your demo scripts. To mount a volume run: - -`docker run -it -v ~/my_demo_dir:/demo_scripts rgardler/simdem` - -When mounting a directory the script found in the first folder within -the mounted folder will be run. If you want to run a specific demo -within that folder add `run SCRIPT_NAME` to the command, where -SCRIPT_NAME is replaced with the name of the folder containing the -script you want to run. For example: - -`docker run -it -v ~/my_demo_dir:/demo_scripts rgardler/simdem run myscript` - -# Going Off-Script - -You can go off-script if you want to. This is where you should hit 'b' -(for break). You will now be able to type a command to be -run. However, note that at the time of writing the parser for this -command is not very smart, so some commands do not work. In addition -you can't run fully interactive commands this way (so no editors for -example). Go ahead and try it, hit 'b' and type a command, e.g. 'ls'. - -Note that when you hit 'b' you will not see any change in the output, -but you can now start typing freely. - -``` -echo "This is a dummy code block to ensure SimDem pauses in interactive mode" -``` - -# Repeating Commands - -Sometimes a command will need to be run a number of times, for example -it might be monitoring the state of an operation. The easiest way to -repeat a command is simply to press 'r'. - -``` -echo "This is a dummy code block to ensure SimDem pauses in interactive mode" -``` - -# Next Steps - - 1. [Use your documents as automated tests](../test/README.md) - 2. [SimDem Index](../README.md) - 3. [Modes of operation](../modes/README.md) - 4. [Build a SimDem container](../building/README.md) diff --git a/demo_scripts/simdem/special_commands/README.md b/demo_scripts/simdem/special_commands/README.md deleted file mode 100644 index f9dbcb6..0000000 --- a/demo_scripts/simdem/special_commands/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Special Commands - -Some commands will be intercepted by SimDem and handled as special -commands. For example, we might interccept a command to open a browser -at a specific page and handle it differently in a headless CLI -environment to how it is handled in a Web UI environment. - -In fact lets look at that use case as an example. - -## Opening a Browser Tab - -On linux the command `xdg-open` is the accepted way of opening a -browser, therefore it is the accepted way of having such a command in -SimDem script. However, this poses a problem, behoviour of this -command will be different in different UI environments. - -For example, on a headliess CLI environment it will attempt to open -"lynx" or similar text based browsers. Since these are interactive -programs they will not work in SimDem. If running in a desktop -environment it will attempt to open the preferred browser. - -SimDem will intercept this command and handle it appropriately. That -is, in a headless CLI environment it will convert the command to a -"curl -I" command, this at least allows us to ensure that there is a -resposne from the URL provided. When running in a Web UI it will open -a new browser tab (at least at the time of writing, we may decide to -integrate this with the Web UI at some point). - -### In Action - -The command block below contains the `xdg-open` command, depending on -whether you are running in the Web UI or the CLI you will see -different behaviour, as described above. - -``` -xdg-open http://bing.com -``` diff --git a/demo_scripts/simdem/syntax/README.md b/demo_scripts/simdem/syntax/README.md deleted file mode 100644 index 8098683..0000000 --- a/demo_scripts/simdem/syntax/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Document syntax - -For the most part SimDem uses standard -[Markdown syntx](https://daringfireball.net/projects/markdown/syntax). There -are a few special strings that can be used to influence how SimDem -works, but even these are intended to be human readable, thus -preventing the need to maintain separate documents for the different -use cases (documentation, tutorial, demo, test and script). - -For example, lets take a look at the start of this file this file: - -``` -head -n 25 README.md -``` - -# Prerequisites - -It is common for a tutorial or demo to have a number of -prerequisites. For example, it's a good idea to understand how these -work, so take a look at our [prerequisites](../prerequisites/README.md) -before proceeding. - -# SimDem Syntax - -There are a small number of SimDem specific items that you should be -aware of. They are detailed in the next few sections. - -## Code Blocks - -In SimDem a code block is marked in exactly the same way it is in -Markdown, that is with three backticks (``````). Unless a Code Block -is marked as a Results block (see next section) it is assumed that -this is executable code. SimDem will execute each line individually. - -For example: - -``` -# This is a code block, this comment will be ignored by SimDem -echo "This command will be 'typed' and executed." -``` - -### Command Limitations - -At the time of writing it is not possible to have interactive -commands. If you try to include such a command SimDem will "hang" as -it waits, silently, for input. - -## Result Blocks - -Result blocks serve two main purposes: - - 1. Help readers of the markdown content understand expected behaviour - 2. Enable SimDem documents to be used as tests - -### Including results blocks for readablity and testing - -When using the script as a web page or printout it is likley that you -will want to include the results. However, when you are running in a -simulation or tutorial mode you will want to rely on the real results -from the current run. You can include a "Results:" section after any -code block. The first code block after this text will be ignored when -running in an interactive mode (such as tutorial or simulation). That -is, in the example below, the `date -u` command will be run -interactively but the `Sat Mar 12 10:09:12 UTC 2016` will only be -included in a static form of the script. - -``` -date -u -``` - -Results: - -```expected_similarity=0.4 -Sat Mar 12 10:09:12 UTC 2016 -``` - -### Modifying Test Accuracy - -When running a script as a test the outputs of the command are -compared to the result block associated with the code block. By -default a similarity of 66%, meaning at least 66% of the characters -are the same, is considered a pass. However, in some cases this is too -high or too low. - -The expected similarity between the command output and the contents of -the result block can be set in the `Results:` header by adding -`Expected similarity: 0.2`, where `0.2` is the similarity desired. - -This can be used to ensure things that have low similarity in the results will pass tests, for example, outputing a date will always result in a different date and thus a much lower expected similarity. - -The date command will prove this is running in real time. - -``` -date -``` - -Results: - -```expected_similarity=0.4 -Sat Mar 12 08:59:01 UTC 2016 -``` - -## Defining Next Steps - -When running in interactive mode it is possible to provide optional -paths for the user to take next. These appear in a section with the -heading "# Next Steps". Note, for this to work as an interactive set -of options this must be the last section in the document. if it is not -the last section then it will be treated like any other heading. - -For example, this document offers next steps options. - -# Next Steps - - 1. [Special Commands](special_commands/README.md) - 2. [Configure your scripts through variables](../variables/README.md) - 3. [Build a Hello World script](../tutorial/README.md) - 4. [SimDem Index](../README.md) - 5. [Write multi-part documents](../multipart/README.md) - 6. [Use your documents as interactive tutorials or demos](../running/README.md) diff --git a/demo_scripts/simdem/test/README.md b/demo_scripts/simdem/test/README.md deleted file mode 100644 index 0f00162..0000000 --- a/demo_scripts/simdem/test/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Automated Testing - -When running with the `--test` flag or using the `test` command SimDem -will verify that the output of each command is as expected. It does -this by comparing the output of the command with the `Results:` -section in the script. - -Test mode is very useful in a continuous integration environment. For -example, you can configure your scripts to always use the latest -versions of tooling they depend upon and get early warning when a -change in one of those tools breaks your -scripts. The -[Azure Container Service demos GitHub repository](http://github.com/Azure/acs-demos) shows -this technique in use. - -## Testing in practice - -First, lets take a look at the source of this file. - -``` -cat $SIMDEM_CWD/README.md -``` - -## Running in test mode - -Running in demo mode will not check the results against -expectations. However, running with the `test` command will do so. - -``` -echo "This test is expected to fail" -``` - -Results: - -``` -It fails because the results we have in the script are significantly -different to the output of the command. -``` - -### Similarity tests - -By default a 66% or more match indicates a pass. However, in some -cases a much lower similarity is expected, for example, the output of -`date` will vary considerably each time it is run. In these situations -you can provide an expected similarity as part of the three backticks -that start a code block, for example ```expected_similarity=0.2 which -is low enough for the test to be recorded as a pass. Note, it is -important that you do not insert any spaces in this notation. - -``` -date -``` - -Results: - -```Expected_Similarity=0.2 -Tue Jun 6 15:23:53 UTC 2017 -``` - -## Fast Fail - -The default setting is for SimDem to stop the test run on the first -test failure. This can be overridden by setting the command line flag -`--fastfail` to any value other than `True`. - -## Test Plans - -It is often a good idea to split tests into separate files. SimDem -will allow you to do this by providing a `test_plan.txt` file. Each -line in this file is either a comment (lines starting with '#') or a -filename for a SimDem script to be used in testing. Each of these -files will be concatenated together to create a complete test plan. - -For example, the following example `test_plan.txt` will run all the -code and tests in `preparation/README.md` followed by those in -`main/README.md` and finally those in `cleanup/README.md`. - -` -preparation/README.md -main/README.md -cleanup/README.md -` - -# Next Steps - - 1. [SimDem Index](../README.md) - 2. [Build a Hello World script](../tutorial/README.md) - 3. [Write SimDem documents](../syntax/README.md) - 4. [Configure your scripts through variables](../variables/README.md) diff --git a/demo_scripts/simdem/tutorial/README.md b/demo_scripts/simdem/tutorial/README.md deleted file mode 100644 index e458dbc..0000000 --- a/demo_scripts/simdem/tutorial/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Writing a SimDem script - -This document desribes how to write a SimDem script. - -## Install SimDem - -These commands can't be run from within SimDem - danger of turning the -whole world into a recursive simulation of itself ;-). Therefore you -should execute them on your client (Linux, Mac or Windows Subsystem -for Linux). - -`git clone git@github.com:rgardler/simdem.git` -`pushd simdem` -`./install.sh` -`simdem` - -This last command will launch you into the SimDem documentation. If -you haven't reviewed it already you should do so, of particular -importance is the Syntax section. - -## Hello World Script - -Let's build a hello world script: - -``` -mkdir -p hello_world -echo "# Hello World Script" > hello_world/README.md -``` - -## Run the script - -You can't run SimDem within SimDem so if you are reading this from -within SimDem you will need to exit and run `simdem -p hello_world` to -see this in action. - -# Next Steps - -Now you have a working hello world script you are on your own (not -really ask questions / report bugs via the -http://github.com/rgardler/simdem issue tracker). If you feel a little -lost then try one of these documents for guidance: - - 1. [Review SimDem Syntax](../syntx/README.md) - 2. [Understand how to parameterize scripts](../variables/README.md) - 3. [Understand how to make a script a test](../test/README.md) - diff --git a/demo_scripts/simdem/variables/README.md b/demo_scripts/simdem/variables/README.md deleted file mode 100644 index 62f4bd2..0000000 --- a/demo_scripts/simdem/variables/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Environment Variables - -In order to use environment variables, you can define one or more -files. These variables are available in every command that is -executed. - -Tutorials can carry `env.json` files in the directory the simdem -command was run and/or in tutorial sub-directories. Files in tutorial -sub-directories will overwrite settings pulled from the project -directory. - -For example, this tutorial defines an 'env.json' in the `simdem` -parent folder and in the `variables` subdirectory that contains this -script. Here is the content from the test subdirectory. - -``` -cat $SIMDEM_CWD/env.json -``` - -Results: - -``` -{ - "NAME": "Hello from the variables subdirectory" -} -``` - -It also defines an 'env.json' file in the `SimDem` root -folder. Assuming you executed the `simdem` command from within that -folder the followin command will display it's content. - -``` -cat env.json -``` - -Results: - -``` -{ - "TEST": "A local hello from the current working directory (where the simdem command was executed)" -} -``` - -Finally, a project may define an `env.local.json` file in the -directory from which the `simdem` command is run. This file is the -last to be loaded and overrides all other values. - -Values are loaded in the following order, the last file to define a - vlaue is the one that "wins". - - - PARENT_OF_SCRIPT_DIR/env.json - - SCRIPT_DIR/env.json - - PARENT_OF_SCRIPT_DIR/env.local.json - - SCRIPT_DIR/env.local.json - - CWD/env.json - - CWD/env.local.json - - -## Interactive Variables - -If you include an environment variable that isn't set, SimDem will -prompt you to give it a value and will add it to the running -environment. If you are running in test mode the variable will be -given a value of 'Dummy vlaue for test'. - -``` -echo $NEW_VARIABLE -``` - -Results: - -``` Expected_Similarity=0 -Enter a value for $NEW_VARIABLE: SimDem - -SimDem - -``` - -### Defining variables can be important - -Because SimDem will interactively ask for values for undefined -variables it is sometimes necessary to first declare a variable to -prevent this action. For example: - -``` -i=0 -for i in {0..4}; do echo "Welcome $i times"; done -``` - -Results: - -``` -Welcome 0 times -Welcome 1 times -Welcome 2 times -Welcome 3 times -Welcome 4 times -``` - -## User provided environment - -Since it is helpful to provide configuration files in published -scripts SimDem also provides a way for users to provide user specific -configurations. So that users can setup their demo's to use private -keys etc. These files are provided in the same way as `env.json` files -(i.e. in the project and tutorial sub-directories) but are called -`env.local.json`. These files take precedence over both project and -tutorial provided files. - -For example, this project provides a local files in both the project -and this tutorial sub-directories. Note that in this case we have -checked them into version control as they are part of the example, -normally they would be added to your local '.gitignore' or equivalent. - -``` -cat $SIMDEM_CWD/../env.local.json -``` - -Results: - -``` -{ - "LOCAL_TEST": "Hello from the local project config" -} -``` - -It also defines an 'env.json' file in the tutorial folder: - -``` -cat $SIMDEM_CWD/env.local.json -``` - -Results: - -``` -{ - "LOCAL_TEST": "A warm local hello" -} -``` - -The file that "wins" is the most local one, that is the one in the tutorial: - -``` -echo $LOCAL_TEST -``` - -Results: - -``` -A warm local hello -``` - -## SimDem Environment Variables - -SimDem provides some information about itself in environment -variables. These are all nameed `SIMDEM_*`. At present the available -variables are: - -``` -env | grep "SIMDEM_" -``` - -# Next Steps - - 1. [Build a SimDem container](../building/README.md) - 2. [SimDem Index](../README.md) - 3. [Use your documents as interactive tutorials or demos](../running/README.md) - 4. [Use your documents as automated tests](../test/README.md) diff --git a/demo_scripts/simdem/variables/env.json b/demo_scripts/simdem/variables/env.json deleted file mode 100644 index b76ccd5..0000000 --- a/demo_scripts/simdem/variables/env.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "NAME": "Hello from the variables subdirectory" -} diff --git a/demo_scripts/simdem/variables/env.local.json b/demo_scripts/simdem/variables/env.local.json deleted file mode 100644 index e6c7c04..0000000 --- a/demo_scripts/simdem/variables/env.local.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "LOCAL_TEST": "A warm local hello" -} diff --git a/demo_scripts/test/README.md b/demo_scripts/test/README.md deleted file mode 100644 index bfe00c2..0000000 --- a/demo_scripts/test/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# SimDem Test Script - -This is a simple test script. It runs a number of commands in -succession. This script also lists commands known not to work. - -# Setup - -Ensure the test environment is correctly setup. - -## SimDem version check - -``` -echo $SIMDEM_VERSION -``` - -Results: - -```expected_similarity=0.8 -0.8.2-dev -``` - -## Clean test working files - -Ensure that our working files folder exists and that there are no -residual files from previus test runs. - -``` -echo $SIMDEM_TEMP_DIR -mkdir -p $SIMDEM_TEMP_DIR/test -rm -Rf $SIMDEM_TEMP_DIR/test/* -``` - -# Prerequisites - -Test to see if our prerequesites work. In the setup we cleaned out our -test files. The [prerequisite test script](./prerequisites/README.md) -validates whether the file exists and, if it doesn't it will execute -and create it. - -Each [prerequisite](./prerequisites/README.md) will only be run once, -so even though this partucular prereq appears twice it will only -execute once. This is important when building multi-part tutorials/ -demos where a prereq may be included in more than one part. - -## Validate prerequisite ran - -The prerequisite script should have run and created a `prereq_ran` -file. - -``` -ls $SIMDEM_TEMP_DIR/test -``` - -Results: - -``` -prereq_ran -``` - - -# Directory Check - -``` -head -n 1 README.md -``` - -Results: - -``` Expected_Similarity=0.8 -# SimDem Test Script -``` - -# Simple Echo - -``` -echo "Hello world" -``` - -Results: - -``` -Hello world -``` - -# Code comments - -``` -# This is a comment and should be ignored -echo "This output should be displayed, the comment before this line should be ignored" -``` - -Results: - -``` -This output should be displayed, the comment before this line should be ignored -``` - -# Expected different results - -When we know the results will be different and we want to use them in -tests we need to override the similarity expected by adding -`expected_similarity=x.y` in the start line of the results block: - -``` -date -``` - -Results: - -```expected_Similarity=0.2 -Tue Jun 6 15:23:53 UTC 2017 -``` - -# For Loop - -Because SimDem will interactively ask for values for undefined -variables it is sometimes necessary to first declare a variable to -prevent this action. For example: - -``` -i=0 -for i in {0..10}; do echo "Welcome $i times"; done -``` - -Results: - -``` -Welcome 0 times -Welcome 1 times -Welcome 2 times -Welcome 3 times -Welcome 4 times -Welcome 5 times -Welcome 6 times -Welcome 7 times -Welcome 8 times -Welcome 9 times -Welcome 10 times -``` - -# Stripping ANSI escape sequances - -To make it easier to write scripts we don't want to include ANSI -escape sequences (such as colors and text deocration) in the results -section. SimDem automatically strips these when capturing the results. - -``` -printf "Normal \e[4mUnderlined\e[24m Normal" -``` - -Results: - -```expected_similarity=0.9 -Normal Underlined Normal -``` - - diff --git a/demo_scripts/test/directory/README.md b/demo_scripts/test/directory/README.md deleted file mode 100644 index a8443dc..0000000 --- a/demo_scripts/test/directory/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Directory Check - -In this test ensures that the currrent working directory is set -correctly when a test file is loaded as part of the test_plan.txt -file, see [Issue #70](https://github.com/Azure/simdem/issues/70). - -Frst lets check the current working directory, this is useful for -debugging if the test fails. - -``` -pwd -``` - -Since we don't know exactly where this will be stored we need to check -that we can open this file in the test. - -``` -head -n 1 README.md -``` - -Results: - -``` Expected_Similarity=0.8 -# Directory Check -``` - diff --git a/demo_scripts/test/env.json b/demo_scripts/test/env.json deleted file mode 100644 index 8bfd5a8..0000000 --- a/demo_scripts/test/env.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "TEST": "Hello from the test script", - "DIR_IN_HOME": "~/should/be/expanded" -} diff --git a/demo_scripts/test/env.local.json b/demo_scripts/test/env.local.json deleted file mode 100644 index 3d9ee80..0000000 --- a/demo_scripts/test/env.local.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "TEST": "A local hello from the current working directory (where the simdem command was executed)" -} diff --git a/demo_scripts/test/env.test.json b/demo_scripts/test/env.test.json deleted file mode 100644 index 948b4f7..0000000 --- a/demo_scripts/test/env.test.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "TEST_VALUE": "Test value for the test script" -} diff --git a/demo_scripts/test/environment_test.md b/demo_scripts/test/environment_test.md deleted file mode 100644 index c7bde24..0000000 --- a/demo_scripts/test/environment_test.md +++ /dev/null @@ -1,195 +0,0 @@ -# Environment tests - -We should be able to retrieve environment variables from the directory -in which the command was given. Note that SimDem provides the -environment variable `SIMDEM_EXEC_DIR` which provides access to this -folder in SimDem scripts should it be necessary. - -``` -cat $SIMDEM_EXEC_DIR/./env.json -``` - -Results: - -``` -{ - "TEST": "Hello from the current working directory (where the simdem command was executed)" -} -``` - -We should also be able to retrieve locallay defined environment -variables from the directory in which the command was given: - - -``` -cat env.local.json -``` - -Results: - -``` -{ - "TEST": "A local hello from the current working directory (where the simdem command was executed)" -} -``` - -There should also be environment variables in the the directory in -which the current script resides. - -``` -cat env.json -``` - -Results: - -``` -{ - "TEST": "Hello from the test script", - "DIR_IN_HOME": "~/should/be/expanded" -} -``` - -Local variables can also be found in the the directory in which the -current script resides. - -``` -cat env.local.json -``` - -Results: - -``` -{ - "TEST": "A local hello from the current working directory (where the simdem command was executed)" -} -``` - -For the `TEST` variable we should have the `env.local.json` value from -the directory in which the application was executed. - -``` -echo $TEST -``` - -Results: - -``` -A local hello from the current working directory (where the simdem command was executed) -``` - -There should be variable definitions in the parent of the -current script directory: - -``` -cat ../env.json -``` - -Results: - -``` -{ - "PARENT_TEST": "Hello from the parent" -} -``` - -Since the value of `PARENT_TEST` is only defined in this file we -should have the value from there: - -``` -echo $PARENT_TEST -``` - -Results: - -``` -Hello from the parent -``` - -## Test Environment - -We can also provide values in `env.test.json` in either the script -directory or the parent of the script directory. If available these -will be loaded first and overwritten by subsequent `env.json` and -`env.local.json` files. For this reason if you want to dorce the user -to provide a value for an environment variable it is important that -you define it as an empty string in `env.json` if a value has been -provided in `env.test.json`. - - -``` -echo $TEST_VALUE -``` - -Results: - -``` -Test value for the test script -``` - -# Replacing variables in special commands - -Some commands cannot be run in headless mode, e.g. `xdg-open`. These -commands will be replaced with an appropriate alternative, -e.g. `curl`. Variables included in such commands will be expanded at -execution time as expected. - -``` -url=http://bing.com -xdg-open $url -``` - -Results: - -```expected_similarity=0.2 -HTTP/1.1 405 Method Not Allowed -Content-Length: 0 -Server: Microsoft-IIS/10.0 -X-MSEdge-Ref: Ref A: D6871D12117C436FAE3762BC8BCA0C29 Ref B: CO1EDGE0415 Ref C: 2017-08-17T15:37:59Z -Date: Thu, 17 Aug 2017 15:37:58 GMT -``` - -Note: this test will only pass when running in headless mode as the -`xdg-open` command will be executed in other environments. - -# Setting new variables in script - -If a script sets a variable during execution this will be recorded in -the SimDem environment. This includes setting to an empty string, this -will prevent SimDem interactively requesting a value for the variable -(or setting a dummt value in test mode). - -``` -new_var="" -echo $new_var -``` - -Results: - -```expected_similarity=0.2 -``` - -# Capturing the output of commands - -``` -CAPTURED_OUTPUT=$(echo foo | sed 's/foo/bar/') -``` - -Captured value is: - -``` -echo $CAPTURED_OUTPUT -``` - -Results: - -``` -bar -``` - -# Processing of Environment Variables - -'~' should be expanded to a home directory (no way to test this). - -``` -echo $DIR_IN_HOME -``` diff --git a/demo_scripts/test/prerequisites/README.md b/demo_scripts/test/prerequisites/README.md deleted file mode 100644 index 969792d..0000000 --- a/demo_scripts/test/prerequisites/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Test Prerequisites - -This script is not included as part of the test plan, but it should be -executed as part of the root `script.md`. Therefore, because of this -prerequisite there should be a file called `prereq_ran` and another -called `nested_prereq_ran` in the temp directory. - -# Prerequisites - -We should be able to run [nested prerequisites](./nested_prereq.md). - -# Create the test file - -``` -touch $SIMDEM_TEMP_DIR/test/prereq_ran -``` - -# Validation - -If the `prereq_ran` file exists then we don't need to run this -script. In our tests the setup phase removes this file so the -validation test should always fail. - -``` -ls $SIMDEM_TEMP_DIR/test -``` - -Results: - -``` -nested_prereq_ran -prereq_ran -``` diff --git a/demo_scripts/test/prerequisites/nested_prereq.md b/demo_scripts/test/prerequisites/nested_prereq.md deleted file mode 100644 index 2876fcf..0000000 --- a/demo_scripts/test/prerequisites/nested_prereq.md +++ /dev/null @@ -1,25 +0,0 @@ -# Nested Prerequisites - -This prerequisite is executed from the main prerequisite test file. - -# Create the test file - -``` -touch $SIMDEM_TEMP_DIR/test/nested_prereq_ran -``` - -# Validation - -If the `prereq_ran` file exists then we don't need to run this -script. In our tests the setup phase removes this file so the -validation test should always fail. - -``` -ls $SIMDEM_TEMP_DIR/test -``` - -Results: - -``` -nested_prereq_ran -``` diff --git a/demo_scripts/test/prerequisites/passing_prerequisite.md b/demo_scripts/test/prerequisites/passing_prerequisite.md deleted file mode 100644 index 3dcd455..0000000 --- a/demo_scripts/test/prerequisites/passing_prerequisite.md +++ /dev/null @@ -1,28 +0,0 @@ -# Passing Prerequisite - -This is a dummy prerequisite file that contains a validation step that -will always pass and thus the body of this script will never be -executed. To ensure this is the case when we run tests we have placed -a failing test in the body. - -``` -echo "This test will always fail" -``` - -Results: - -``` -So we can ensure it never runs (the validation step will always pass) -``` - -# Validation - -``` -echo "This validation step is designed to always pass" -``` - -Results: - -``` -This validation step is designed to always pass -``` diff --git a/demo_scripts/test/remote/README.md b/demo_scripts/test/remote/README.md deleted file mode 100644 index f3647df..0000000 --- a/demo_scripts/test/remote/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# A remote script - -This file will be pulled from GitHub as a remote file to ensure that -execution of remote files works. It is pulled into the test suite in -two ways: - - 1. As a remote prerequisite - 2. AS a remote file in a test plan - -# Create the test file - -``` -mkdir -p $SIMDEM_TEMP_DIR/test -touch $SIMDEM_TEMP_DIR/test/remote_prereq_ran -``` - -# Validation - -If the `remote_prereq_ran` file exists then we don't need to run this -script. In our tests the setup phase removes this file so the -validation test should always fail. - -``` -ls $SIMDEM_TEMP_DIR/test -``` - -Results: - -``` -nested_prereq_ran -prereq_ran -remote_prereq_ran who -``` diff --git a/demo_scripts/test/test_plan.txt b/demo_scripts/test/test_plan.txt deleted file mode 100644 index d482ad4..0000000 --- a/demo_scripts/test/test_plan.txt +++ /dev/null @@ -1,4 +0,0 @@ -# SimDem Test Plan -README.md -directory/README.md -environment_test.md diff --git a/env.json b/env.json deleted file mode 100644 index 3d9ee80..0000000 --- a/env.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "TEST": "A local hello from the current working directory (where the simdem command was executed)" -} diff --git a/env.local.json b/env.local.json deleted file mode 100644 index bbcace7..0000000 --- a/env.local.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "TEST": "A local hello from the current working directory (where the simdem command was executed)" -} - diff --git a/environment.py b/environment.py deleted file mode 100644 index 06cbb62..0000000 --- a/environment.py +++ /dev/null @@ -1,155 +0,0 @@ -# For managing the environment in which a SimDem demo executes. - -import os -import sys -import json - -import config - -class Environment(object): - def __init__(self, directory, copy_env=True, is_test=False): - """Initialize the environment""" - if copy_env: - self.env = os.environ.copy() - else: - self.env = {} - self.is_test = is_test - self.read_simdem_environment(directory) - self.set("SIMDEM_VERSION", config.SIMDEM_VERSION) - self.set("SIMDEM_CWD", directory) - self.set("SIMDEM_EXEC_DIR", os.getcwd()) - temp_dir = os.path.expanduser(config.SIMDEM_TEMP_DIR) - self.set("SIMDEM_TEMP_DIR", temp_dir) - - def read_simdem_environment(self, directory): - """Populates each shell environment with a set of environment vars - loaded via env.json and/or env.local.json files. Variables are - loaded in order first from the parent of the current script - directory, then the current scriptdir itself and finally from - the directory in which the `simdem` command was executed (the - CWD). - - Values are loaded in the following order, the last file to - define a vlaue is the one that "wins". - - - PARENT_OF_SCRIPT_DIR/env.json - - SCRIPT_DIR/env.json - - PARENT_OF_SCRIPT_DIR/env.local.json - - SCRIPT_DIR/env.local.json - - CWD/env.json - - CWD/env.local.json - - Note that it is possible to supply test values in an - `env.test.json` file stored in the SCRIPT_DIR, its parent or - the current working directory. If we are running in test mode - then the following three files will be loaded, if they exist, - in the following order at the end of the initialization - procedure. This means they will take precedence over - everything else. - - - PARENT_OF_SCRIPT_DIR/env.test.json - - SCRIPT_DIR/env.test.json - - CWD/env.json - - """ - env = {} - - if not directory.endswith('/'): - directory = directory + "/" - - filename = directory + "../env.json" - if os.path.isfile(directory + "../env.json"): - with open(filename) as env_file: - app_env = self.process_env(json.load(env_file)) - env.update(app_env) - - filename = directory + "env.json" - if os.path.isfile(filename): - with open(filename) as env_file: - script_env = self.process_env(json.load(env_file)) - env.update(script_env) - - filename = directory + "../env.local.json" - if os.path.isfile(filename): - with open(filename) as env_file: - local_env = self.process_env(json.load(env_file)) - env.update(local_env) - - filename = directory + "env.local.json" - if os.path.isfile(filename): - with open(filename) as env_file: - local_env = self.process_env(json.load(env_file)) - env.update(local_env) - - filename = os.getcwd() + "env.json" - if os.path.isfile(filename): - with open(filename) as env_file: - local_env = self.process_env(json.load(env_file)) - env.update(local_env) - - filename = os.getcwd() + "env.local.json" - if os.path.isfile(filename): - with open(filename) as env_file: - local_env = self.process_env(json.load(env_file)) - env.update(local_env) - - if self.is_test: - filename = directory + "../env.test.json" - if os.path.isfile(filename): - with open(filename) as env_file: - script_env = self.process_env(json.load(env_file)) - env.update(script_env) - - filename = directory + "env.test.json" - if os.path.isfile(filename): - with open(filename) as env_file: - script_env = self.process_env(json.load(env_file)) - env.update(script_env) - - filename = os.getcwd() + "/env.test.json" - if os.path.isfile(filename): - with open(filename) as env_file: - local_env = self.process_env(json.load(env_file)) - env.update(local_env) - - self.env.update(env) - - def process_env(self, new_env): - """ - Takes an environment definition and processes it for use. - For example, expand '~' to home directory. - """ - for key in new_env: - val = new_env[key] - if val.startswith('~'): - new_env[key] = os.path.expanduser(val) - return new_env - - def set(self, var, value): - """Sets a new variable to the environment""" - self.env[var] = value - - def get(self, key=None): - """Returns a either a value for a supplied key or, if key is None, a - dictionary containing the current environment""" - if key: - if key not in self.env: - return "UNDEFINED" - else: - return self.env[key] - else: - return self.env - - def dump_env(self): - """ - Prints the environment to the console. - """ - for item in self.env.items(): - print(str(item)) - - def __str__(self): - s = "" - for item in self.env.items(): - s += str(item) - s += "\n" - return s diff --git a/js/common.js b/js/common.js deleted file mode 100644 index 99e2bcd..0000000 --- a/js/common.js +++ /dev/null @@ -1,19 +0,0 @@ -function sleep(delay) { - var start = new Date().getTime(); - while (new Date().getTime() < start + delay); -} - -function log(type, msg) { - $('#log').prepend('
' + $('
').text(new Date() + " : " + type + " : " + msg).html()); -} - -function open_tab(url) { - var win = window.open(url, '_blank'); - if (win) { - //Browser has allowed it to be opened - win.focus(); - } else { - //Browser has blocked it - alert('Please allow popups for this website'); - } -} diff --git a/js/console.js b/js/console.js deleted file mode 100644 index 3b965c3..0000000 --- a/js/console.js +++ /dev/null @@ -1,22 +0,0 @@ -function init_console() { - var socket = io.connect('http://' + document.domain + ':' + location.port + '/console'); - - socket.on('update_console', function(msg) { - $('#console').append(msg); - $('#console').animate({ - scrollTop: $('#console')[0].scrollHeight}, 500) - log("CONSOLE", msg); - }); - - socket.on('clear', function(msg) { - $('#console').html(''); - log("CONSOLE", "clear") - }); - - socket.on('open_tab', function(url) { - open_tab(url) - log("CONSOLE", "Open tab for " + url) - }); -} - - diff --git a/js/control.js b/js/control.js deleted file mode 100644 index dc96e9f..0000000 --- a/js/control.js +++ /dev/null @@ -1,70 +0,0 @@ -function init_control() { - var socket = io('http://' + document.domain + ':' + location.port + '/control'); - var command_key = "" - var keypress_interval; - - $("#btn_launch_console_window").click(function() { - window.open('/console', '_blank', 'toolbar=0,location=0,menubar=0'); - }) - - socket.on('update_info', function(msg) { - $('#info').append(msg); - $('#info').animate({ - scrollTop: $('#info')[0].scrollHeight}, 500) - log("INFO", msg) - }); - - socket.on('get_command_key', function(msg) { - input = $('').text("Press a command key (h for help)"); - $('#info').append(input); - - $(document).keypress(function(event) { - command_key = String.fromCharCode(event.which) - window.clearInterval(keypress_interval); - $(document).off("keypress") - $('#input').remove() - log("GET_COMMAND_KEY", "Got '" + command_key + "'") - socket.emit('command_key', command_key) - }) - - keypress_interval = window.setInterval(function () { - log("GET_COMAND_KEY", "Waiting for input") - }, 1000); - }); - - socket.on('input_string', function(msg) { - in_str = "" - $('#input_string_wrap').remove() - in_element = $(''); - div = $('
').append(in_element); - $('#info').append(div) - - in_element.on('change', function() { - in_str = $('#input_string').val() - log("GET_INPUT", "Got '" + in_str + "'") - socket.emit('input_string', in_str) - }); - - in_element.keypress(function(event) { - if (event.which == 13) { - in_element.change() - } - }) - - in_element.blur(function(){ - $('#input_string_wrap').remove() - }); - - in_element.focus(); - }); - - socket.on('clear', function(msg) { - $('#info').html(''); - log("INFO", "clear") - }); - - socket.on('log', function(msg) { - log("LOG", msg) - }); -} - diff --git a/main.py b/main.py deleted file mode 100644 index 2d6c2a4..0000000 --- a/main.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 - -# This is a python script that emulates a terminal session and runs -# commands from a supplied markdown file.. - -import optparse -import os -import sys -import time - -from cli import Ui -from web import WebUi -import config -from demo import Demo -from environment import Environment - -def get_bash_script(script_dir, is_simulation = True, is_automated=False, is_testing=False): - """ - Reads a README.md file in the indicated directoy and builds an - executable bash script from the commands contained within. - """ - if not script_dir.endswith('/'): - script_dir = script_dir + "/" - - script = "" - env = Environment(script_dir, False).get() - for key, value in env.items(): - script += key + "='" + value + "'\n" - - filename = env.get_script_file_name(script_dir) - in_code_block = False - in_results_section = False - lines = list(open(script_dir + filename)) - for line in lines: - if line.startswith("Results:"): - # Entering results section - in_results_section = True - elif line.startswith("```") and not in_code_block: - # Entering a code block, if in_results_section = True then it's a results block - in_code_block = True - elif line.startswith("```") and in_code_block: - # Finishing code block - in_results_section = False - in_code_block = False - elif in_code_block and not in_results_section: - # Executable line - script += line - elif line.startswith("#") and not in_code_block and not in_results_section and not is_automated: - # Heading in descriptive text - script += "\n" - return script - -def main(): - """SimDem CLI interpreter""" - - commands = config.modes - command_string = "" - for command in commands: - command_string = command_string + command + "|" - command_string = command_string[0:len(command_string)-1] - - p = optparse.OptionParser("%prog [" + command_string + "] DEMO_NAME", version=config.SIMDEM_VERSION) - p.add_option('--style', '-s', default="tutorial", - help="The style of simulation you want to run. 'tutorial' (the default) will print out all text and pause for user input before running commands. 'simulate' will not print out the text but will still pause for input.") - p.add_option('--path', '-p', default="demo_scripts/", - help="The Path to the demo scripts directory.") - p.add_option('--auto', '-a', default="False", - help="Set to 'true' (or 'yes') to prevent the application waiting for user keypresses between commands. Set to 'no' when running in test mode to allow users to step through each test.") - p.add_option('--test', '-t', default="False", - help="If set to anything other than False the output of the command will be compared to the expected results in the sript. Any failures will be reported") - p.add_option('--fastfail', default="True", - help="If set to anything other than True test execution has will stop on the first failure. This has no affect if running in any mode other than 'test'.") - p.add_option('--debug', '-d', default="False", - help="Turn on debug logging by setting to True.") - p.add_option('--webui', '-w', default="False", - help="If set to anything other than False will interact with the user through a Web UI rather than the CLI.") - p.add_option('--output', '-o', default="log", - help="Format of the output. The default is `log` which will output all stdout data. Other options are `summary` which provides a summary of the execution status and `json`") - - options, arguments = p.parse_args() - - if not options.path.endswith("/"): - options.path += "/" - - if options.auto == "False": - is_automatic = False - else: - is_automatic = True - - if options.test == "False": - is_test = False - else: - is_test = True - - if options.fastfail == "True": - is_fast_fail= True - else: - is_fast_fail= False - - if options.style == "simulate": - simulate = True - elif options.style == 'tutorial': - simulate = False - else: - print("Unknown style (--style, -s): " + options.style) - exit(1) - - if options.debug.lower() == "true": - config.is_debug = True - - if len(arguments) == 2: - script_dir = options.path + arguments[1] - else: - script_dir = options.path - - cmd = None - if len(arguments) > 0: - cmd = arguments[0] - # 'run' is deprecated in the CLI, but not yet removed from code - if cmd == "tutorial": - cmd = "run" - if cmd == "test": - is_test = True - is_auto = True - - filename = "README.md" - is_docker = os.path.isfile('/.dockerenv') - demo = Demo(is_docker, script_dir, filename, simulate, is_automatic, is_test, is_fast_fail, output_format=options.output); - - if options.webui == "False": - ui = Ui() - else: - ui = WebUi(config.port) - print("Server started. Listening on port " + str(ui.port)) - print("Point your browser at " + str(ui.port)) - print() - while not ui.ready: - time.sleep(0.25) - print("Waiting for client connection") - cmd = None - - demo.set_ui(ui) - demo.run(cmd) - -main() diff --git a/novnc/.XDefaults b/novnc/.XDefaults deleted file mode 100644 index 1c21c22..0000000 --- a/novnc/.XDefaults +++ /dev/null @@ -1,15 +0,0 @@ -XTerm.VT100.background: Black -XTerm.VT100.color0: Black -XTerm.VT100.color1: Red -XTerm.VT100.color2: Green -XTerm.VT100.color3: Yellow -XTerm.VT100.color4: CornflowerBlue -XTerm.VT100.color5: Magenta -XTerm.VT100.color6: Cyan -XTerm.VT100.color7: White -XTerm.VT100.colorBD: White -XTerm.VT100.colorBDMode: true -XTerm.VT100.colorUL: Yellow -XTerm.VT100.colorULMode: true -XTerm.VT100.cursorColor: Red -XTerm.VT100.foreground: White diff --git a/novnc/.config/autostart/xterm.desktop b/novnc/.config/autostart/xterm.desktop deleted file mode 100644 index 11cf322..0000000 --- a/novnc/.config/autostart/xterm.desktop +++ /dev/null @@ -1,10 +0,0 @@ -[Desktop Entry] -Version=1.0 -Type=Application -Name=Terminal -Comment=Use the command line -Exec=xterm -Icon=xterm-color -Path= -Terminal=false -StartupNotify=false diff --git a/novnc/Desktop/Terminal Emulator.desktop b/novnc/Desktop/Terminal Emulator.desktop deleted file mode 100755 index 11cf322..0000000 --- a/novnc/Desktop/Terminal Emulator.desktop +++ /dev/null @@ -1,10 +0,0 @@ -[Desktop Entry] -Version=1.0 -Type=Application -Name=Terminal -Comment=Use the command line -Exec=xterm -Icon=xterm-color -Path= -Terminal=false -StartupNotify=false diff --git a/pre-commit.sh b/pre-commit.sh deleted file mode 100755 index 542a69f..0000000 --- a/pre-commit.sh +++ /dev/null @@ -1,33 +0,0 @@ -#! /bin/sh -# script to run tests on what is to be committed -# Thanks Torek, -# https://stackoverflow.com/questions/20479794/how-do-i-properly-git-stash-pop-in-pre-commit-hooks-to-get-a-clean-working-tree - -# First, stash index and work dir, keeping only the -# to-be-committed changes in the working directory. -echo "Stash changes not ready to be committed" -old_stash=$(git rev-parse -q --verify refs/stash) -git stash save -q --keep-index -new_stash=$(git rev-parse -q --verify refs/stash) - -# If there were no changes (e.g., `--amend` or `--allow-empty`) -# then nothing was stashed, and we should skip everything, -# including the tests themselves. (Presumably the tests passed -# on the previous commit, so there is no need to re-run them.) -if [ "$old_stash" = "$new_stash" ]; then - echo "pre-commit script: no changes to test" - sleep 1 # XXX hack, editor may erase message - exit 0 -fi - -echo "Run the Simdem tests" -python3 main.py -p demo_scripts/test test -RESULT=$? - -echo "Restore unstaged changes" -git reset --hard -q && git stash apply --index -q && git stash drop -q - -# Exit with status from test-run: nonzero prevents commit -exit $RESULT - - diff --git a/release_process.md b/release_process.md deleted file mode 100644 index 036d8d1..0000000 --- a/release_process.md +++ /dev/null @@ -1,67 +0,0 @@ -# Releasig Simdem - -This document describes the process for releasing SimDem. - -# Ensure we are building master - -``` -git checkout master -git pull upstream master -git push -``` - -# Test - -``` -./script/install.sh -simdem -p demo_scripts/test test -``` - -# Remove '-dev' from the version number - -The '-dev' prefix is used to indicate unreleased version, therefore it -should be removed. - -In `config.py` change the line that starts with `SIMDEM_VERSION = `. - -# Docker Containers - -## Build containers - -``` -./scripts/build.sh -``` - -## Publish the containers - -``` -./scripts/publish.sh -``` - -# Tag Git - -```` -git tag -a $SIMDEM_VERSION -m "Build and publish $SIMDEM_VERSION" -git push origin 0.8.1 -git push upstream 0.8.1 -``` - -# Bump the version number - -Increment the version number as appropriate, adding '-dev' to indicate -this is an unreleased version. In `config.py` change the line that -starts with `SIMDEM_VERSION = `. - -## Update version number in tests - -In `demo_sctipts/test/README.md` update the expected version number in -the first test.` - -# Commit the new version number - -``` -git add config.py -git add demo_scripts/test/README.md -git commit -m "bump version number" -git push -``` diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c464dc7..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -colorama -flask -flask-socketio -pexpect - diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100755 index 3c8c417..0000000 --- a/scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Builds a SimDem container. -# -# Usage: -# -# build.sh - builds both containers (cli and novnc) -# build.sh novnc - builds the novnc version of the container -# build.sh cli - builds the CLI version of the container - -REPOSITORY=rgardler -FLAVOR=${1:-} -IMAGE_NAME_PREFIX=simdem_ - -VERSION=`grep SIMDEM_VERSION config.py | awk '{print $3}' | tr -d '"'` - -build_container() { - docker build -f Dockerfile_$1 -t $REPOSITORY/${IMAGE_NAME_PREFIX}$1:$VERSION . - - if [ $? -eq 0 ]; then - echo "Built $REPOSITORY/${IMAGE_NAME_PREFIX}$1:$VERSION" - else - echo "Failed to build $REPOSITORY/${IMAGE_NAME_PREFIX}$1:$VERSION" - return 0 - fi -} - -if [[ $FLAVOR == "novnc" ]]; then - build_container novnc -elif [[ $FLAVOR == "cli" ]]; then - build_container cli -else - build_container cli - if [ $? -ne 0 ]; then - echo "Building container failed. Exiting" - return 1 - fi - build_container novnc -fi - -exit $? diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100755 index 8e00993..0000000 --- a/scripts/install.sh +++ /dev/null @@ -1,86 +0,0 @@ -function removeEarlier() { -# Remove < 0.7.0 version - if [ -f ~/bin/simdem.py ]; then - echo "Since version 0.7.0 Simdem is installed in a different way, we need to remove the old Simdem version." - rm ~/bin/simdem.py - rm ~/bin/simdem - fi - - if [ -f /usr/local/bin/simdem.py ]; then - echo "Since version 0.7.0 Simdem is installed in a different way, we need to remove the old Simdem version." - sudo rm /usr/local/bin/simdem.py - sudo rm /usr/local/bin/simdem - fi - - if [ -f /.dockerenv ]; then - echo "Running in a Docker container" - IS_DOCKER=true - INSTALL_DIR=~/bin/simdem-dev/ - else - echo "Not running in a Docker container" - IS_DOCKER=false - INSTALL_DIR=/usr/local/bin/simdem-dev/ - fi -} - -function installLinuxDependencies() { - if [ "$IS_DOCKER" = true ]; then - apt update - apt-get install -y python3-pip - else - sudo apt update - sudo apt-get install -y python3-pip - fi -} - -if [ -f /.dockerenv ]; then - echo "Running in a Docker container" - IS_DOCKER=true - BIN_DIR=~/bin -else - echo "Not running in a Docker container" - IS_DOCKER=false - BIN_DIR=/usr/local/bin -fi - -INSTALL_DIR=$BIN_DIR/simdem-dev/ -MAIN_FILE=main.py -SYMLINK=simdem - -unameOut="$(uname -s)" -case "${unameOut}" in - Linux*) installLinuxDependencies;; - Darwin*) brew install python3;; - *) echo "Unsupported OS: ${unameOut}" -esac - -virtualenv simdem-env -source simdem-env/bin/activate - -pip3 install -r requirements.txt - -if [ "$IS_DOCKER" = true ]; then - mkdir -p $INSTALL_DIR - - cp -r * $INSTALL_DIR - chmod +x $INSTALL_DIR$MAIN_FILE - - echo 'export PATH=$PATH:'$BIN_DIR >> ~/.bashrc - - if [ ! -L $INSTALL_DIR../$SYMLINK ]; then - ln -s $INSTALL_DIR$MAIN_FILE $INSTALL_DIR../$SYMLINK - fi -else - echo "Make install directory at $INSTALL_DIR" - sudo mkdir -p $INSTALL_DIR - - echo "Copy source into install dir" - sudo cp -r * $INSTALL_DIR - echo "Make main file executable" - sudo chmod +x $INSTALL_DIR$MAIN_FILE - - if [ ! -L $INSTALL_DIR../$SYMLINK ]; then - echo "Create symlink to main.py file" - sudo ln -s $INSTALL_DIR$MAIN_FILE $INSTALL_DIR../$SYMLINK - fi -fi diff --git a/scripts/publish.sh b/scripts/publish.sh deleted file mode 100755 index f26d7f3..0000000 --- a/scripts/publish.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Publishes SimDem containers. -# -# Usage: -# -# publish.sh - publiches both containers (cli and novnc) -# publish.sh novnc - publishes the novnc version of the container -# publish.sh cli - publishes the CLI version of the container - -REPOSITORY=rgardler -FLAVOR=${1:-} -CONTAINERNAME=simdem_$FLAVOR - -VERSION=`grep -Po '(?<=SIMDEM_VERSION = \")(.*)(?=\")' config.py` - -print_result() { - if [ $1 -eq 0 ]; then - echo "Published $2" - else - echo "Failed to publish $2" - exit 1 - fi -} - - -if [[ $FLAVOR == "novnc" ]]; then - docker push $REPOSITORY/$CONTAINERNAME:$VERSION - print_result $? "$REPOSITORY/$CONTAINERNAME:$VERSION" -elif [[ $FLAVOR == "cli" ]]; then - docker push $REPOSITORY/$CONTAINERNAME:$VERSION - print_result $? "$REPOSITORY/$CONTAINERNAME:$VERSION" - -else - docker push $REPOSITORY/${CONTAINERNAME}cli:$VERSION - print_result $? "$REPOSITORY/${CONTAINERNAME}cli:$VERSION" - - docker push $REPOSITORY/${CONTAINERNAME}novnc:$VERSION - print_result $? "$REPOSITORY/${CONTAINERNAME}novnc:$VERSION" -fi diff --git a/scripts/run.sh b/scripts/run.sh deleted file mode 100755 index 9e6886c..0000000 --- a/scripts/run.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# Runs either a command line or headless VNC Docker container with -# Simdem installed. -# -# Usage: run.sh [FLAVOR] [SCRIPT_DIR] [MODE] -# -# FLAVOR is an optional parameter to define which container to run, -# either the `novnc` or the `cli` versions. If not specified the -# `novnc` version is used -# -# SCRIPT_DIR is an optional parameter to define a directory containing -# SimDem scripts that will be mounted into the final container. Default -# value is `./demo_scripts`. -# -# MODE is an optional parameter that is only used by the CLI -# container, the novnc container will ignore it. MODE defines the mode -# of execution. It can take the values 'tutorial' (default, run with -# full instructional text), 'demo' (run with no instructional text and -# simulate typed commnds', 'test' (run in auto mode and test results -# against expected results) -# -# Connect with a browser at http://YOUR_DOCKER_HOST:8080/?password=vncpassword -# -# Based on https://github.com/ConSol/docker-headless-vnc-container - - -# Configuration options: -# You can configure the following variables to change behaviour of -# the container. -# -# VNC config -# ========== -VNC_COL_DEPTH='24' -VNC_RESOLUTION='1024x768' -VNC_PW='vncpassword' - -NO_VNC_PORT=8080 - -FLAVOR=${1:-novnc} -SCRIPTS_DIR=${2:-`pwd`/demo_scripts} -MODE=${3:-tutorial} -REPOSITORY=rgardler -CONTAINER_NAME=simdem_$FLAVOR -SCRIPTS_VOLUME=${CONTAINER_NAME}_scripts -AZURE_VOLUME=${AZURE_VOLUME:-$HOME/.azure} -SSH_VOLUME=${SSH_VOLUME:-$HOME/.ssh} - -if [[ $FLAVOR == "novnc" ]]; then - HOME="/headless" -else - HOME="/home/simdem" -fi - -VERSION=`grep SIMDEM_VERSION config.py | awk '{print $3}' | tr -d '"'` - -echo Running $REPOSITORY/$CONTAINER_NAME:$VERSION - -echo Stopping and removing pre-existing containers -docker stop $CONTAINER_NAME -docker rm $CONTAINER_NAME -docker stop $SCRIPTS_VOLUME -docker rm $SCRIPTS_VOLUME - -echo Creating scripts data container named $SCRIPTS_VOLUME containing the scripts in $SCRIPTS_DIR -docker create -v $HOME/demo_scripts --name $SCRIPTS_VOLUME ubuntu /bin/true -docker cp $SCRIPTS_DIR/. $SCRIPTS_VOLUME:$HOME/demo_scripts/ - -echo starting the $CONTAINER_NAME container in mode $MODE - -if [[ $MODE == "tutorial" ]]; then - COMMAND="tutorial" -elif [[ $MODE == "demo" ]]; then - COMMAND="run --style simulate" -elif [[ $MODE == "test" ]]; then - COMMAND="test" -fi - -if [[ $FLAVOR == "novnc" ]]; then - docker run -d -p 5901:5901 -p 8080:$NO_VNC_PORT --name $CONTAINER_NAME \ - --volume $AZURE_VOLUME:$HOME/.azure \ - --volume $SSH_VOLUME:$HOME/.ssh \ - --volumes-from $SCRIPTS_VOLUME \ - -e VNC_COL_DEPTH=$VNC_COL_DEPTH \ - -e VNC_RESOLUTION=$VNC_RESOLUTION \ - -e VNC_PW=$VNC_PW \ - $REPOSITORY/$CONTAINER_NAME:$VERSION -else - docker run -it \ - --volume $AZURE_VOLUME:$HOME/.azure \ - --volume $SSH_VOLUME:$HOME/.ssh \ - --volumes-from $SCRIPTS_VOLUME \ - --name $CONTAINER_NAME \ - $REPOSITORY/$CONTAINER_NAME:$VERSION \ - $COMMAND -fi diff --git a/style/common.css b/style/common.css deleted file mode 100644 index c818889..0000000 --- a/style/common.css +++ /dev/null @@ -1,8 +0,0 @@ -#container { - margin: 0 auto; -} - -.twrap { - overflow: hidden; - margin: 4px 4px 4px 4px -} diff --git a/style/console.css b/style/console.css deleted file mode 100644 index d6de5d9..0000000 --- a/style/console.css +++ /dev/null @@ -1,27 +0,0 @@ -.console { - background-color: black; - color: white; - font-family: "Lucida Console", Monaco, monospace; - width: 48%; - height: 80vh; - overflow: auto; - float: right -} - -.full_screen { - width: 100%; - height: 100vh; -} - -span.prompt { - color: White -} - -span.command { - color: white -} - -span.results { - color: LightGreen -} - diff --git a/style/speaker.css b/style/speaker.css deleted file mode 100644 index 918b840..0000000 --- a/style/speaker.css +++ /dev/null @@ -1,47 +0,0 @@ -.info { - background-color: black; - color: white; - font-family: "Lucida Console", Monaco, monospace; - width: 48%; - height: 80vh; - overflow: auto; - float: left; -} - -.debug { - width: 100%; - clear: both -} - -span.console_input { - display: block; - background-color: white; - color: black; - margin: 2px 2px 2px 2px; -} - -span.heading { - color: Cyan; -} - -span.description { - color: Lime; -} - -span.next_step { - color: RebeccaPurple; -} - -span.warning { - color: Red; -} - -span.request_input { - color: Purple -} - -.console_input { - width: 100% -} - - diff --git a/templates/console.html b/templates/console.html deleted file mode 100644 index 225c831..0000000 --- a/templates/console.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - SimDem Console View - - - - - - - - - - - - - - -
-
- {{ console }} -
-
- - diff --git a/templates/index.html b/templates/index.html deleted file mode 100644 index 3ccf1c2..0000000 --- a/templates/index.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - SimDem Control Center - - - - - - - - - - - - - - - -
-
- {{ console }} -
-
-
- -
-

Control Panel

- -
- -
-

Log

-
-
-
-
- - diff --git a/web.py b/web.py deleted file mode 100644 index c7f2fd0..0000000 --- a/web.py +++ /dev/null @@ -1,232 +0,0 @@ -from flask import Flask, send_from_directory -from flask import render_template -from flask_socketio import SocketIO -import threading -import time - -from cli import Ui -import config - -ui = None -app = Flask(__name__) -app.config['SECRET_KEY'] = 'secret!' -socketio = SocketIO(app) -thread = None -command_key = None -in_string = None - -def background_thread(): - while True: - socketio.sleep(10) - if ui.ready: - text = "Server connection alive." - - socketio.emit('log', - text, - namespace='/control') - -@socketio.on('connect', namespace='/control') -def connect(): - global thread - global ui - while ui is None: - time.sleep(0.25) - - ui.clear() - - if thread is None: - thread = socketio.start_background_task(target=background_thread) - ui.ready = True - - print("Connection in /control namespace") - -@socketio.on('command_key', namespace='/control') -def got_command_key(key): - global command_key - command_key = key - -@socketio.on('input_string', namespace='/control') -def got_input_String(in_str): - global in_string - in_string = in_str - -@app.route('/js/') -def send_js(filename): - return send_from_directory('js', filename) - -@app.route('/style/') -def send_style(filename): - return send_from_directory('style', filename) - -@app.route('/') -def index(): - return render_template('index.html', console = "Initializing...") - -@app.route('/console') -def console(): - return render_template('console.html', console= "$") - -class WebUi(Ui): - def __init__(self, port=8080): - global ui - import logging - logging.basicConfig(filename='error.log',level=logging.DEBUG) - ui = self - self.port = port - self.ready = False - t = threading.Thread(target=socketio.run, args=(app, '0.0.0.0', port)) - t.start() - - def prompt(self): - """Display the prompt for the user. This is intended to indicate that - the user is expected to take an action at this point. - """ - self._send_to_console(config.console_prompt, "prompt") - - def command(self, text): - """Display a command, or a part of a command tp be executed.""" - self._send_to_console(text, "command", False) - - def results(self, text): - """Display the results of a command execution""" - self._send_to_console(self.demo.strip_ansi(text), "results", True) - - def clear(self): - """Clears the console and info panel ready for a new section of the script.""" - socketio.emit('clear', - namespace='/console') - socketio.emit('clear', - namespace='/control') - self.prompt() - - def heading(self, text): - """Display a heading""" - self._send_to_info(text, "heading", True) - self.new_line() - - def description(self, text): - """Display some descriptive text. Usually this is text from the demo - document itself. - - """ - # fixme: color self.display(text, colorama.Fore.CYAN) - self._send_to_info(text, "description", True) - - def next_step(self, index, title): - """Displays a next step item with an index (the number to be entered -to select it) and a title (to be displayed). - """ - self._send_to_info(str(index) + " " + title, "next_step", True) - - def instruction(self, text): - """Display an instruction for the user. - """ - self._send_to_info(text, "instruction", True) - - def warning(self, text): - """Display a warning to the user. - """ - self._send_to_info(text, "warning", True) - - def new_para(self, div = "console"): - """Starts a new paragraph in the indicated div.""" - self.new_line(div) - self.new_line(div) - - def new_line(self, div = "console"): - """Send a single new line to te indicated div""" - if div == "console": - self._send_to_console("
") - elif div == "info": - self._send_to_info("
") - - def horizontal_rule(self): - self._send_to_info("

============================================

") - - def display(self, text, color, new_line=False): - """Display some text in a given color. Do not print a new line unless - new_line is set to True. - - """ - self._send_to_info(text, color, new_line) - - def request_input(self, text): - """Displays text that is intended to propmt the user for input and - then waits for input.""" - self._send_to_info(text, "request_input", True) - return self.input_string().lower() - - def _send_to_console(self, text, css_class = "description", new_line = False): - """ Send a string to the console. If new_line is set to true then also send a
""" - text = text.replace("\n", "
") - html = "" + text + "" - if new_line: - html += "
" - - socketio.emit('update_console', - html, - namespace='/console') - - def _send_to_info(self, text, css_class = "description", new_line = False): - """ Send a string to the console. If new_line is set to true then also send a
""" - html = "" + text + "" - if new_line: - html += "
" - - socketio.emit('update_info', - html, - namespace='/control') - - def get_instruction_key(self): - """Gets an instruction from the user. See get_help() for details of - relevant keys to respond with. - - """ - global command_key - command_key = None - socketio.emit('get_command_key', - namespace='/control') - while command_key is None: - pass - - return command_key - - def input_string(self): - """ Get a string from the user.""" - global in_string - in_string = None - socketio.emit('input_string', - namespace='/control') - while in_string is None: - pass - return in_string - - def run_special_command(self, command): - """Test to see if the command is a spcial command that needs to be - handled diferently, these include: - - `xdg-open $URL` - intercepted and converted to an instruction to open a new window - - Returns the response from the command if it was handled by this function, - otherwise returns False. - - """ - orig_command = command - if command.startswith("xdg-open "): - self.warning("Since you are running in Web UI mode it is not possible to execute xdg-open commands.") - self.warning("Attempting to open a new browser tab instead.") - self.warning("Note that this may break tests.") - - command = self.expand_vars(command) - url = command[9:] - - socketio.emit('open_tab', - url, - namespace='/console') - - return "" - else: - return False - - - From b530414b9393ccfc6139a65f5e225e8f971e5416 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 20 Dec 2017 09:10:58 -0600 Subject: [PATCH 048/167] Update setup.py --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 6eb17ea..2a0fb08 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Learn more: https://github.com/kennethreitz/setup.py +# Created from https://github.com/kennethreitz/setup.py from setuptools import setup, find_packages @@ -14,11 +14,11 @@ setup( name='sample', version='0.1.0', - description='Sample package for Python-Guide.org', + description='SimDem', long_description=readme, - author='Kenneth Reitz', - author_email='me@kennethreitz.com', - url='https://github.com/kennethreitz/samplemod', + author='Tommy Falgout/Ross Gardler', + author_email='thfalgou@microsoft.com', + url='https://github.com/Azure/simdem', license=license, packages=find_packages(exclude=('tests', 'docs')) ) From 5b091bceb1cdf67be47db1a1aadc8623127aa544 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 21 Dec 2017 19:48:58 -0600 Subject: [PATCH 049/167] First attempt at migration to mistletoe --- content/complete-features/README.md | 29 +++ simdem/mistletoe/parser.py | 95 +++++++++ .../mistletoe/simdem_renderer.latex.base.py | 162 ++++++++++++++ simdem/mistletoe/simdem_renderer.py | 197 ++++++++++++++++++ tests/context.py | 3 +- tests/test_mistletoe.py | 63 ++++++ tests/test_mistletoe_parser.py | 37 ++++ 7 files changed, 585 insertions(+), 1 deletion(-) create mode 100644 content/complete-features/README.md create mode 100644 simdem/mistletoe/parser.py create mode 100644 simdem/mistletoe/simdem_renderer.latex.base.py create mode 100644 simdem/mistletoe/simdem_renderer.py create mode 100644 tests/test_mistletoe.py create mode 100644 tests/test_mistletoe_parser.py diff --git a/content/complete-features/README.md b/content/complete-features/README.md new file mode 100644 index 0000000..ea7c9b7 --- /dev/null +++ b/content/complete-features/README.md @@ -0,0 +1,29 @@ +# Prerequisites + +* [prereq-1](prereq.md) +* [prereq-2](prereq-2.md) + +# Do stuff here + +We want to execute this because the code type is shell + +```shell +echo foo +echo bar +``` + +# Do more stuff here + +```shell +echo baz +``` + +# Results + +The only thing that makes it a result is the code type is result. +We assume the result is for the last command of the last code block + +```result +baz +``` + diff --git a/simdem/mistletoe/parser.py b/simdem/mistletoe/parser.py new file mode 100644 index 0000000..581c2cb --- /dev/null +++ b/simdem/mistletoe/parser.py @@ -0,0 +1,95 @@ +import re +import logging +import mistletoe.ast_renderer as renderer +import mistletoe.block_token as token +import pprint + +class MistletoeParser(object): + + def __init__(self): + return + + def is_command_block(self, block): + if block['type'] == 'BlockCode' and block['language'] == 'shell': + return True + return False + + def is_result_block(self, block): + # This is different than previous SimDem because it didn't require a language for the result. + # I believe this approach is more declarative. + if block['type'] == 'BlockCode' and block['language'] == 'result': + return True + return False + + # Assuming just one for now + def parse_ref_from_text(self, text): + # Does mistune allow us to parse this? Would be nice. + pattern = re.compile('.*\[(.*)\]\((.*)\).*') + match = pattern.match(text) + if match: + title = match.groups()[0].strip() + href = match.groups()[1] + logging.debug("Found prereq: " + href) + return href + return None + + """ + I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now + Will look like this coming out of AST + {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], + 'level': 1, + 'type': 'Heading'}, + """ + def is_prerequisite_block(self, block): + if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ + and 'prerequisite' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': + return True + return False + + def get_prereqs(self, doc): + logging.debug("get_prereqs: ") + pprint.pprint(doc) + res = [] + # Is there a better way to do this? Probably so. I'm on a plane and can't research + for idx in range(len(doc['children'])): + block = doc['children'][idx] + logging.debug("get_prereqs():processing " + str(block)) + if self.is_prerequisite_block(block): + logging.debug("get_prereqs():found preqreq block") + res = [x['children'][0]['target'] for x in doc['children'][idx+1]['children']] + logging.debug(res) + return res + + + def get_file_contents(self, file_path): +# logging.debug("get_file_contents: " + file_path) + f = open(file_path, 'r') + content = f.read() + f.close() + return content + + def parse_file(self, file_path): + with open(file_path, 'r') as fin: + ast = renderer.get_ast(token.Document(fin)) + + return { + 'prerequisites': self.get_prereqs(ast), + 'commands': self.get_commands(ast) + } + + def get_commands(self, doc): + res = [] + for idx in range(len(doc['children'])): + logging.debug("get_commands():processing " + str(doc['children'][idx])) + block = doc['children'][idx] + if self.is_result_block(block): + logging.debug("get_commands():is_result_block") + res[len(res) - 1]['expected_result'] = block['children'][0]['content'] + elif self.is_command_block(block): + logging.debug("get_commands():is_command_block") + for line in block['children'][0]['content'].split("\n"): + if line: + res.append({ 'command': line }) + else: + logging.info("get_commands():unknown_block. Ignoring") + return res \ No newline at end of file diff --git a/simdem/mistletoe/simdem_renderer.latex.base.py b/simdem/mistletoe/simdem_renderer.latex.base.py new file mode 100644 index 0000000..6c7ad8a --- /dev/null +++ b/simdem/mistletoe/simdem_renderer.latex.base.py @@ -0,0 +1,162 @@ +""" +LaTeX renderer for mistletoe. +""" + +from itertools import chain +import mistletoe.latex_token as latex_token +from mistletoe.base_renderer import BaseRenderer + +class SimdemRenderer(BaseRenderer): + def __init__(self, *extras): + """ + Args: + extras (list): allows subclasses to add even more custom tokens. + """ + tokens = self._tokens_from_module(latex_token) + super().__init__(*chain(tokens, extras)) + + def render_strong(self, token): + return self.render_inner(token) + + def render_emphasis(self, token): + return '\\textit{{{}}}'.format(self.render_inner(token)) + + def render_inline_code(self, token): + print("INLINE CODE") + print(token) + return '\\verb|{}|'.format(self.render_inner(token)) + + def render_strikethrough(self, token): + return '\\sout{{{}}}'.format(self.render_inner(token)) + + @staticmethod + def render_image(token): + return '\n\\includegraphics{{{}}}\n'.format(token.src) + + def render_footnote_image(self, token): + maybe_src = self.footnotes.get(token.src.key, '') + src = maybe_src.split(' "', 1)[0] + return '\n\\includegraphics{{{}}}\n'.format(src) + + def render_link(self, token): + template = '\\href{{{target}}}{{{inner}}}' + inner = self.render_inner(token) + return template.format(target=token.target, inner=inner) + + def render_footnote_link(self, token): + template = '\\href{{{target}}}{{{inner}}}' + inner = self.render_inner(token) + target = self.footnotes.get(token.target.key, '') + return template.format(target=target, inner=inner) + + @staticmethod + def render_auto_link(token): + return '\\url{{{}}}'.format(token.target) + + @staticmethod + def render_math(token): + return token.content + + def render_escape_sequence(self, token): + return self.render_inner(token) + + @staticmethod + def render_raw_text(token): + return (token.content.replace('$', '\$').replace('#', '\#') + .replace('{', '\{').replace('}', '\}') + .replace('&', '\&')) + + def render_heading(self, token): + inner = self.render_inner(token) + if token.level == 1: + return '\n\\section{{{}}}\n'.format(inner) + elif token.level == 2: + return '\n\\subsection{{{}}}\n'.format(inner) + return '\n\\subsubsection{{{}}}\n'.format(inner) + + def render_quote(self, token): + template = '\\begin{{displayquote}}\n{inner}\\end{{displayquote}}\n' + return template.format(inner=self.render_inner(token)) + + def render_paragraph(self, token): + return {'type' : 'paragraph', 'text' : self.render_inner(token) } + + def render_block_code(self, token): + print("RENDER_BLOCK_CODE") + template = ('\n\\begin{{lstlisting}}[language={}]\n' + '{}' + '\\end{{lstlisting}}\n') + inner = self.render_inner(token) + return template.format(token.language, inner) + + def render_list(self, token): + template = '\\begin{{{tag}}}\n{inner}\\end{{{tag}}}\n' + tag = 'enumerate' if token.start is not None else 'itemize' + inner = self.render_inner(token) + return template.format(tag=tag, inner=inner) + + def render_list_item(self, token): + inner = self.render_inner(token) + return '\\item {}\n'.format(inner) + + def render_table(self, token): + def render_align(column_align): + if column_align != [None]: + cols = [get_align(col) for col in token.column_align] + return '{{{}}}'.format(' '.join(cols)) + else: + return '' + + def get_align(col): + if col is None: + return 'l' + elif col == 0: + return 'c' + elif col == 1: + return 'r' + raise RuntimeError('Unrecognized align option: ' + col) + + template = ('\\begin{{tabular}}{align}\n' + '{inner}' + '\\end{{tabular}}\n') + if token.has_header: + head_template = '{inner}\\hline\n' + header = next(token.children) + head_inner = self.render_table_row(header) + head_rendered = head_template.format(inner=head_inner) + else: head_rendered = '' + inner = self.render_inner(token) + align = render_align(token.column_align) + return template.format(inner=head_rendered+inner, align=align) + + def render_table_row(self, token): + cells = [self.render(child) for child in token.children] + return ' & '.join(cells) + '\n' + + def render_table_cell(self, token): + return self.render_inner(token) + + @staticmethod + def render_separator(token): + return '' + + def render_document(self, token): + return self.render_inner_list(token) + + def render_inner(self, token): + """ + Recursively renders child tokens. Joins the rendered + strings with no space in between. + If newlines / spaces are needed between tokens, add them + in their respective templates, or override this function + in the renderer subclass, so that whitespace won't seem to + appear magically for anyone reading your program. + Arguments: + token: a branch node who has children attribute. + """ + rendered = [self.render(child) for child in token.children] + return ''.join(rendered) + + def render_inner_list(self, token): + rendered = [self.render(child) for child in token.children] + return rendered diff --git a/simdem/mistletoe/simdem_renderer.py b/simdem/mistletoe/simdem_renderer.py new file mode 100644 index 0000000..b5ef291 --- /dev/null +++ b/simdem/mistletoe/simdem_renderer.py @@ -0,0 +1,197 @@ +""" +HTML renderer for mistletoe. +""" + +import html +from itertools import chain +import mistletoe.html_token as html_token +from mistletoe.base_renderer import BaseRenderer + +class SimdemRenderer(BaseRenderer): + """ + Simdem renderer class. + + See mistletoe.base_renderer module for more info. + """ + def __init__(self, *extras): + """ + Args: + extras (list): allows subclasses to add even more custom tokens. + """ + tokens = self._tokens_from_module(html_token) + super().__init__(*chain(tokens, extras)) + + def render_strong(self, token): + template = '{}' + return template.format(self.render_inner(token)) + + def render_emphasis(self, token): + template = '{}' + return template.format(self.render_inner(token)) + + def render_inline_code(self, token): + template = '{}' + return template.format(self.render_inner(token)) + + def render_strikethrough(self, token): + template = '{}' + return template.format(self.render_inner(token)) + + def render_image(self, token): + template = '{}' + inner = self.render_inner(token) + return template.format(token.src, token.title, inner) + + def render_footnote_image(self, token): + template = '{inner}' + maybe_src = self.footnotes.get(token.src.key, '') + if maybe_src.find('"') != -1: + src = maybe_src[:maybe_src.index(' "')] + title = maybe_src[maybe_src.index(' "')+2:-1] + else: + src = maybe_src + title = '' + inner = self.render_inner(token) + return template.format(src=src, title=title, inner=inner) + + def render_link(self, token): + template = '{inner}' + target = escape_url(token.target) + inner = self.render_inner(token) + return template.format(target=target, inner=inner) + + def render_footnote_link(self, token): + template = '{inner}' + raw_target = self.footnotes.get(token.target.key, '') + target = escape_url(raw_target) + inner = self.render_inner(token) + return template.format(target=target, inner=inner) + + def render_auto_link(self, token): + template = '{inner}' + target = escape_url(token.target) + inner = self.render_inner(token) + return template.format(target=target, inner=inner) + + def render_escape_sequence(self, token): + return self.render_inner(token) + + @staticmethod + def render_raw_text(token): + return html.escape(token.content) + + @staticmethod + def render_html_span(token): + return token.content + + def render_heading(self, token): + template = '{inner}\n' + inner = self.render_inner(token) + return template.format(level=token.level, inner=inner) + + def render_quote(self, token): + template = '
\n{inner}
\n' + return template.format(inner=self.render_inner(token)) + + def render_paragraph(self, token): + return { 'type': 'paragraph', 'text' : self.render_inner(token) } + #return '{}'.format(self.render_inner(token)) + + def render_block_code(self, token): + print('foo' ) + print(self.render_inner(token).split("\n") ) + print('bar' ) + text = [x for x in self.render_inner(token).split("\n") if x ] + template = { 'type': 'code', 'text': text } + if token.language: + template['lang'] = token.language + return template + + def render_list(self, token): + template = '<{tag}{attr}>\n{inner}\n' + if token.start: + tag = 'ol' + attr = ' start="{}"'.format(token.start) + else: + tag = 'ul' + attr = '' + inner = self.render_inner(token) + return template.format(tag=tag, attr=attr, inner=inner) + + def render_list_item(self, token): + return '
  • {}
    • \n'.format(self.render_inner(token)) + + def render_table(self, token): + # This is actually gross and I wonder if there's a better way to do it. + # + # The primary difficulty seems to be passing down alignment options to + # reach individual cells. + template = '\n{inner}
    \n' + if token.has_header: + head_template = '\n{inner}\n' + header = next(token.children) + head_inner = self.render_table_row(header, True) + head_rendered = head_template.format(inner=head_inner) + else: head_rendered = '' + body_template = '\n{inner}\n' + body_inner = self.render_inner(token) + body_rendered = body_template.format(inner=body_inner) + return template.format(inner=head_rendered+body_rendered) + + def render_table_row(self, token, is_header=False): + template = '\n{inner}\n' + inner = ''.join([self.render_table_cell(child, is_header) + for child in token.children]) + return template.format(inner=inner) + + def render_table_cell(self, token, in_header=False): + template = '<{tag}{attr}>{inner}\n' + tag = 'th' if in_header else 'td' + if token.align is None: + align = 'left' + elif token.align == 0: + align = 'center' + elif token.align == 1: + align = 'right' + attr = ' align="{}"'.format(align) + inner = self.render_inner(token) + return template.format(tag=tag, attr=attr, inner=inner) + + @staticmethod + def render_separator(token): + return '
    \n' + + @staticmethod + def render_html_block(token): + return token.content + + def render_document(self, token): + self.footnotes.update(token.footnotes) + return self.render_inner_list(token) + + def render_inner_string(self, token): + """ + Recursively renders child tokens. Joins the rendered + strings with no space in between. + If newlines / spaces are needed between tokens, add them + in their respective templates, or override this function + in the renderer subclass, so that whitespace won't seem to + appear magically for anyone reading your program. + Arguments: + token: a branch node who has children attribute. + """ + rendered = [self.render(child) for child in token.children] + return ''.join(rendered) + + def render_inner_list(self, token): + rendered = [self.render(child) for child in token.children] + return rendered + + + +def escape_url(raw): + """ + Escape urls to prevent code injection craziness. (Hopefully.) + """ + from urllib.parse import quote + return quote(raw, safe='/#:') \ No newline at end of file diff --git a/tests/context.py b/tests/context.py index eb61ab6..43d9cbd 100644 --- a/tests/context.py +++ b/tests/context.py @@ -5,4 +5,5 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import simdem -from simdem.render import demo \ No newline at end of file +from simdem.render import demo +from simdem.mistletoe import simdem_renderer,parser \ No newline at end of file diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py new file mode 100644 index 0000000..4ab92a8 --- /dev/null +++ b/tests/test_mistletoe.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +from .context import simdem_renderer + +import mistletoe +import unittest + +import mistletoe.block_token as token +from mistletoe.ast_renderer import ASTRenderer +import mistletoe.ast_renderer as renderer + + + +class SimDemMistletoeTestSuite(unittest.TestCase): + """Lexer test cases.""" + + def test_basic(self): + doc = 'Nothing special here. Move along' + res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) + exp_res = [{'type': 'paragraph', 'text': 'Nothing special here. Move along'}] + self.assertEquals(res, exp_res) + + + def test_strong(self): + doc = 'This is **strong text**' + res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) + exp_res = [{'type': 'paragraph', 'text': 'This is strong text'}] + self.assertEquals(res, exp_res) + + ''' + def test_block_code(self): + doc = ("```php", "echo $foo", 'echo bar', "```") + print(doc) + #res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) + res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) + exp_res = [{'lang': 'php', 'text': 'echo $foo', 'type': 'code'}] + self.assertEquals(res, exp_res) + ''' + + def test_block_code_file(self): + file_path = '/tmp/foo4.md' + with open(file_path, 'r') as fin: + res = mistletoe.markdown(fin, simdem_renderer.SimdemRenderer) + exp_res = [{'lang': 'php', 'text': ['echo $foo', 'echo $bar'], 'type': 'code'}] + self.assertEquals(res, exp_res) + ''' + def test_ast(self): + self.maxDiff = None + file_path = 'content/simple/README.md' + file_path = '/tmp/foo3.md' + with open(file_path, 'r') as fin: + res = renderer.get_ast(token.Document(fin)) + + self.assertEqual(res, {}) + ''' + +# with open('content/simple/README.md', 'r') as fin: +# output = renderer.get_ast(fin) + + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_mistletoe_parser.py b/tests/test_mistletoe_parser.py new file mode 100644 index 0000000..538f83a --- /dev/null +++ b/tests/test_mistletoe_parser.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from .context import simdem, parser + +import unittest +import os.path +import configparser +import mistune + +class MistletoeParserTestSuite(unittest.TestCase): + """Advanced test cases.""" + + parser = None + + def setUp(self): + self.maxDiff = None + config = configparser.ConfigParser() + config.read("content/config/unit_test.ini") + + self.parser = parser.MistletoeParser() + + def test_full(self): + file_path = 'content/complete-features/README.md' + res = self.parser.parse_file(file_path) + exp_res = { + 'prerequisites': ['prereq.md', 'prereq-2.md'], + 'commands': [ + { 'command': 'echo foo' }, + { 'command': 'echo bar' }, + { 'command': 'echo baz', 'expected_result': 'baz\n' } ] + } + + self.assertEqual(res, exp_res) + + +if __name__ == '__main__': + unittest.main() From 7c33b32163563ab7063291db1ce10b9fad40f2a7 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 21 Dec 2017 23:37:42 -0600 Subject: [PATCH 050/167] completed migration from mistune to mistletoe --- content/create-file/README.md | 3 +- content/prerequisite-run/README.md | 4 +- content/results-block-fail/README.md | 9 +- content/results-block/README.md | 6 +- content/simple-variable/README.md | 6 +- content/simple/README.md | 6 +- simdem/__init__.py | 3 +- simdem/core.py | 7 +- simdem/mistletoe/__init__.py | 1 + simdem/mistletoe/parser.py | 83 +++----- .../mistletoe/simdem_renderer.latex.base.py | 162 -------------- simdem/mistletoe/simdem_renderer.py | 197 ------------------ simdem/parser.py | 87 -------- tests/context.py | 2 +- tests/test_lexer.py | 54 ----- tests/test_mistletoe.py | 60 ++---- tests/test_mistletoe_parser.py | 2 +- tests/test_parser.py | 68 ------ tests/test_simdem.py | 11 +- tests/test_system.py | 2 +- 20 files changed, 80 insertions(+), 693 deletions(-) create mode 100644 simdem/mistletoe/__init__.py delete mode 100644 simdem/mistletoe/simdem_renderer.latex.base.py delete mode 100644 simdem/mistletoe/simdem_renderer.py delete mode 100644 simdem/parser.py delete mode 100644 tests/test_lexer.py delete mode 100644 tests/test_parser.py diff --git a/content/create-file/README.md b/content/create-file/README.md index caf2360..f3abbb6 100644 --- a/content/create-file/README.md +++ b/content/create-file/README.md @@ -1,3 +1,4 @@ ```shell echo 'abc' > scratch/foo -cat scratch/foo``` \ No newline at end of file +cat scratch/foo +``` diff --git a/content/prerequisite-run/README.md b/content/prerequisite-run/README.md index 6017ec6..b8fd504 100644 --- a/content/prerequisite-run/README.md +++ b/content/prerequisite-run/README.md @@ -1,9 +1,9 @@ # Prerequisite -content/prerequisite-run/prereq.md +* [prereq-1](content/prerequisite-run/prereq.md) # Main ```shell echo post-prereq-run -``` \ No newline at end of file +``` diff --git a/content/results-block-fail/README.md b/content/results-block-fail/README.md index ae2bf70..0025574 100644 --- a/content/results-block-fail/README.md +++ b/content/results-block-fail/README.md @@ -2,15 +2,18 @@ this is text ```shell echo foo -echo bar``` +echo bar +``` Results: ```result -barr``` +barr +``` After this, we should not pass ```shell -echo YOU SHALL NOT PASS!``` +echo YOU SHALL NOT PASS! +``` even more text diff --git a/content/results-block/README.md b/content/results-block/README.md index e1c0ea5..f41762e 100644 --- a/content/results-block/README.md +++ b/content/results-block/README.md @@ -2,11 +2,13 @@ this is text ```shell echo foo -echo bar``` +echo bar +``` Results: ```result -bar``` +bar +``` even more text diff --git a/content/simple-variable/README.md b/content/simple-variable/README.md index 5eeec84..4d74318 100644 --- a/content/simple-variable/README.md +++ b/content/simple-variable/README.md @@ -1,9 +1,11 @@ this is text ```shell -FOO=bar``` +FOO=bar +``` ```shell -echo $FOO``` +echo $FOO +``` more text diff --git a/content/simple/README.md b/content/simple/README.md index 21bf98c..0f2a643 100644 --- a/content/simple/README.md +++ b/content/simple/README.md @@ -2,11 +2,13 @@ this is text ```shell echo foo -echo bar``` +echo bar +``` more text ```shell -echo baz``` +echo baz +``` even more text diff --git a/simdem/__init__.py b/simdem/__init__.py index 54ef264..a8d584a 100644 --- a/simdem/__init__.py +++ b/simdem/__init__.py @@ -1,2 +1 @@ -from .core import Core -from .parser import Parser \ No newline at end of file +from .core import Core \ No newline at end of file diff --git a/simdem/core.py b/simdem/core.py index cf00b70..9de0455 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from . import executor,parser +from . import executor import difflib import logging import re @@ -29,10 +29,7 @@ def run_cmd(self, cmd): return self.rend.run_cmd(cmd) def process_file(self, file_path): - logging.info("process_file():file_path=" + file_path) - content = self.parser.get_file_contents(file_path) - logging.info("process_file():content=" + str(content)) - blocks = self.parser.parse_doc(content) + blocks = self.parser.parse_file(file_path) logging.info("process_file():blocks=" + str(blocks)) self.process_prereqs(blocks['prerequisites']) logging.info("process_file():completed process_prereqs()") diff --git a/simdem/mistletoe/__init__.py b/simdem/mistletoe/__init__.py new file mode 100644 index 0000000..a01a345 --- /dev/null +++ b/simdem/mistletoe/__init__.py @@ -0,0 +1 @@ +from .parser import MistletoeParser \ No newline at end of file diff --git a/simdem/mistletoe/parser.py b/simdem/mistletoe/parser.py index 581c2cb..cc12025 100644 --- a/simdem/mistletoe/parser.py +++ b/simdem/mistletoe/parser.py @@ -6,9 +6,6 @@ class MistletoeParser(object): - def __init__(self): - return - def is_command_block(self, block): if block['type'] == 'BlockCode' and block['language'] == 'shell': return True @@ -21,18 +18,6 @@ def is_result_block(self, block): return True return False - # Assuming just one for now - def parse_ref_from_text(self, text): - # Does mistune allow us to parse this? Would be nice. - pattern = re.compile('.*\[(.*)\]\((.*)\).*') - match = pattern.match(text) - if match: - title = match.groups()[0].strip() - href = match.groups()[1] - logging.debug("Found prereq: " + href) - return href - return None - """ I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now Will look like this coming out of AST @@ -46,50 +31,44 @@ def is_prerequisite_block(self, block): return True return False - def get_prereqs(self, doc): - logging.debug("get_prereqs: ") - pprint.pprint(doc) - res = [] - # Is there a better way to do this? Probably so. I'm on a plane and can't research - for idx in range(len(doc['children'])): - block = doc['children'][idx] - logging.debug("get_prereqs():processing " + str(block)) - if self.is_prerequisite_block(block): - logging.debug("get_prereqs():found preqreq block") - res = [x['children'][0]['target'] for x in doc['children'][idx+1]['children']] - logging.debug(res) - return res - - - def get_file_contents(self, file_path): -# logging.debug("get_file_contents: " + file_path) - f = open(file_path, 'r') - content = f.read() - f.close() - return content - def parse_file(self, file_path): with open(file_path, 'r') as fin: ast = renderer.get_ast(token.Document(fin)) - - return { - 'prerequisites': self.get_prereqs(ast), - 'commands': self.get_commands(ast) + + res = { + 'prerequisites': [], + 'commands': [] } - def get_commands(self, doc): - res = [] - for idx in range(len(doc['children'])): - logging.debug("get_commands():processing " + str(doc['children'][idx])) - block = doc['children'][idx] - if self.is_result_block(block): - logging.debug("get_commands():is_result_block") - res[len(res) - 1]['expected_result'] = block['children'][0]['content'] + idx = 0 + blocks = ast['children'] + #logging.debug("parse_file():processing " + str(blocks) + while idx < len(blocks): + block = blocks[idx] + logging.debug("parse_file():processing " + str(block)) + + if self.is_prerequisite_block(block): + logging.debug("parse_file():found preqreq block") + res['prerequisites'] = [x['children'][0]['target'] for x in blocks[idx+1]['children']] + # No need to process the next block since that's the prereqs + idx = idx + 1 + + elif self.is_result_block(block): + # Assume that the result block is for the previous command block + logging.debug("parse_file():is_result_block") + content = block['children'][0]['content'].rstrip() + if content: + res['commands'][len(res['commands']) - 1]['expected_result'] = content + elif self.is_command_block(block): - logging.debug("get_commands():is_command_block") + logging.debug("parse_file():is_command_block") for line in block['children'][0]['content'].split("\n"): if line: - res.append({ 'command': line }) + res['commands'].append({ 'command': line }) + else: logging.info("get_commands():unknown_block. Ignoring") - return res \ No newline at end of file + + idx = idx + 1 + #logging.debug(res) + return res diff --git a/simdem/mistletoe/simdem_renderer.latex.base.py b/simdem/mistletoe/simdem_renderer.latex.base.py deleted file mode 100644 index 6c7ad8a..0000000 --- a/simdem/mistletoe/simdem_renderer.latex.base.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -LaTeX renderer for mistletoe. -""" - -from itertools import chain -import mistletoe.latex_token as latex_token -from mistletoe.base_renderer import BaseRenderer - -class SimdemRenderer(BaseRenderer): - def __init__(self, *extras): - """ - Args: - extras (list): allows subclasses to add even more custom tokens. - """ - tokens = self._tokens_from_module(latex_token) - super().__init__(*chain(tokens, extras)) - - def render_strong(self, token): - return self.render_inner(token) - - def render_emphasis(self, token): - return '\\textit{{{}}}'.format(self.render_inner(token)) - - def render_inline_code(self, token): - print("INLINE CODE") - print(token) - return '\\verb|{}|'.format(self.render_inner(token)) - - def render_strikethrough(self, token): - return '\\sout{{{}}}'.format(self.render_inner(token)) - - @staticmethod - def render_image(token): - return '\n\\includegraphics{{{}}}\n'.format(token.src) - - def render_footnote_image(self, token): - maybe_src = self.footnotes.get(token.src.key, '') - src = maybe_src.split(' "', 1)[0] - return '\n\\includegraphics{{{}}}\n'.format(src) - - def render_link(self, token): - template = '\\href{{{target}}}{{{inner}}}' - inner = self.render_inner(token) - return template.format(target=token.target, inner=inner) - - def render_footnote_link(self, token): - template = '\\href{{{target}}}{{{inner}}}' - inner = self.render_inner(token) - target = self.footnotes.get(token.target.key, '') - return template.format(target=target, inner=inner) - - @staticmethod - def render_auto_link(token): - return '\\url{{{}}}'.format(token.target) - - @staticmethod - def render_math(token): - return token.content - - def render_escape_sequence(self, token): - return self.render_inner(token) - - @staticmethod - def render_raw_text(token): - return (token.content.replace('$', '\$').replace('#', '\#') - .replace('{', '\{').replace('}', '\}') - .replace('&', '\&')) - - def render_heading(self, token): - inner = self.render_inner(token) - if token.level == 1: - return '\n\\section{{{}}}\n'.format(inner) - elif token.level == 2: - return '\n\\subsection{{{}}}\n'.format(inner) - return '\n\\subsubsection{{{}}}\n'.format(inner) - - def render_quote(self, token): - template = '\\begin{{displayquote}}\n{inner}\\end{{displayquote}}\n' - return template.format(inner=self.render_inner(token)) - - def render_paragraph(self, token): - return {'type' : 'paragraph', 'text' : self.render_inner(token) } - - def render_block_code(self, token): - print("RENDER_BLOCK_CODE") - template = ('\n\\begin{{lstlisting}}[language={}]\n' - '{}' - '\\end{{lstlisting}}\n') - inner = self.render_inner(token) - return template.format(token.language, inner) - - def render_list(self, token): - template = '\\begin{{{tag}}}\n{inner}\\end{{{tag}}}\n' - tag = 'enumerate' if token.start is not None else 'itemize' - inner = self.render_inner(token) - return template.format(tag=tag, inner=inner) - - def render_list_item(self, token): - inner = self.render_inner(token) - return '\\item {}\n'.format(inner) - - def render_table(self, token): - def render_align(column_align): - if column_align != [None]: - cols = [get_align(col) for col in token.column_align] - return '{{{}}}'.format(' '.join(cols)) - else: - return '' - - def get_align(col): - if col is None: - return 'l' - elif col == 0: - return 'c' - elif col == 1: - return 'r' - raise RuntimeError('Unrecognized align option: ' + col) - - template = ('\\begin{{tabular}}{align}\n' - '{inner}' - '\\end{{tabular}}\n') - if token.has_header: - head_template = '{inner}\\hline\n' - header = next(token.children) - head_inner = self.render_table_row(header) - head_rendered = head_template.format(inner=head_inner) - else: head_rendered = '' - inner = self.render_inner(token) - align = render_align(token.column_align) - return template.format(inner=head_rendered+inner, align=align) - - def render_table_row(self, token): - cells = [self.render(child) for child in token.children] - return ' & '.join(cells) + '\n' - - def render_table_cell(self, token): - return self.render_inner(token) - - @staticmethod - def render_separator(token): - return '' - - def render_document(self, token): - return self.render_inner_list(token) - - def render_inner(self, token): - """ - Recursively renders child tokens. Joins the rendered - strings with no space in between. - If newlines / spaces are needed between tokens, add them - in their respective templates, or override this function - in the renderer subclass, so that whitespace won't seem to - appear magically for anyone reading your program. - Arguments: - token: a branch node who has children attribute. - """ - rendered = [self.render(child) for child in token.children] - return ''.join(rendered) - - def render_inner_list(self, token): - rendered = [self.render(child) for child in token.children] - return rendered diff --git a/simdem/mistletoe/simdem_renderer.py b/simdem/mistletoe/simdem_renderer.py deleted file mode 100644 index b5ef291..0000000 --- a/simdem/mistletoe/simdem_renderer.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -HTML renderer for mistletoe. -""" - -import html -from itertools import chain -import mistletoe.html_token as html_token -from mistletoe.base_renderer import BaseRenderer - -class SimdemRenderer(BaseRenderer): - """ - Simdem renderer class. - - See mistletoe.base_renderer module for more info. - """ - def __init__(self, *extras): - """ - Args: - extras (list): allows subclasses to add even more custom tokens. - """ - tokens = self._tokens_from_module(html_token) - super().__init__(*chain(tokens, extras)) - - def render_strong(self, token): - template = '{}' - return template.format(self.render_inner(token)) - - def render_emphasis(self, token): - template = '{}' - return template.format(self.render_inner(token)) - - def render_inline_code(self, token): - template = '{}' - return template.format(self.render_inner(token)) - - def render_strikethrough(self, token): - template = '{}' - return template.format(self.render_inner(token)) - - def render_image(self, token): - template = '{}' - inner = self.render_inner(token) - return template.format(token.src, token.title, inner) - - def render_footnote_image(self, token): - template = '{inner}' - maybe_src = self.footnotes.get(token.src.key, '') - if maybe_src.find('"') != -1: - src = maybe_src[:maybe_src.index(' "')] - title = maybe_src[maybe_src.index(' "')+2:-1] - else: - src = maybe_src - title = '' - inner = self.render_inner(token) - return template.format(src=src, title=title, inner=inner) - - def render_link(self, token): - template = '{inner}' - target = escape_url(token.target) - inner = self.render_inner(token) - return template.format(target=target, inner=inner) - - def render_footnote_link(self, token): - template = '{inner}' - raw_target = self.footnotes.get(token.target.key, '') - target = escape_url(raw_target) - inner = self.render_inner(token) - return template.format(target=target, inner=inner) - - def render_auto_link(self, token): - template = '{inner}' - target = escape_url(token.target) - inner = self.render_inner(token) - return template.format(target=target, inner=inner) - - def render_escape_sequence(self, token): - return self.render_inner(token) - - @staticmethod - def render_raw_text(token): - return html.escape(token.content) - - @staticmethod - def render_html_span(token): - return token.content - - def render_heading(self, token): - template = '{inner}\n' - inner = self.render_inner(token) - return template.format(level=token.level, inner=inner) - - def render_quote(self, token): - template = '
    \n{inner}
    \n' - return template.format(inner=self.render_inner(token)) - - def render_paragraph(self, token): - return { 'type': 'paragraph', 'text' : self.render_inner(token) } - #return '{}'.format(self.render_inner(token)) - - def render_block_code(self, token): - print('foo' ) - print(self.render_inner(token).split("\n") ) - print('bar' ) - text = [x for x in self.render_inner(token).split("\n") if x ] - template = { 'type': 'code', 'text': text } - if token.language: - template['lang'] = token.language - return template - - def render_list(self, token): - template = '<{tag}{attr}>\n{inner}\n' - if token.start: - tag = 'ol' - attr = ' start="{}"'.format(token.start) - else: - tag = 'ul' - attr = '' - inner = self.render_inner(token) - return template.format(tag=tag, attr=attr, inner=inner) - - def render_list_item(self, token): - return '
  • {}
    • \n'.format(self.render_inner(token)) - - def render_table(self, token): - # This is actually gross and I wonder if there's a better way to do it. - # - # The primary difficulty seems to be passing down alignment options to - # reach individual cells. - template = '\n{inner}
    \n' - if token.has_header: - head_template = '\n{inner}\n' - header = next(token.children) - head_inner = self.render_table_row(header, True) - head_rendered = head_template.format(inner=head_inner) - else: head_rendered = '' - body_template = '\n{inner}\n' - body_inner = self.render_inner(token) - body_rendered = body_template.format(inner=body_inner) - return template.format(inner=head_rendered+body_rendered) - - def render_table_row(self, token, is_header=False): - template = '\n{inner}\n' - inner = ''.join([self.render_table_cell(child, is_header) - for child in token.children]) - return template.format(inner=inner) - - def render_table_cell(self, token, in_header=False): - template = '<{tag}{attr}>{inner}\n' - tag = 'th' if in_header else 'td' - if token.align is None: - align = 'left' - elif token.align == 0: - align = 'center' - elif token.align == 1: - align = 'right' - attr = ' align="{}"'.format(align) - inner = self.render_inner(token) - return template.format(tag=tag, attr=attr, inner=inner) - - @staticmethod - def render_separator(token): - return '
    \n' - - @staticmethod - def render_html_block(token): - return token.content - - def render_document(self, token): - self.footnotes.update(token.footnotes) - return self.render_inner_list(token) - - def render_inner_string(self, token): - """ - Recursively renders child tokens. Joins the rendered - strings with no space in between. - If newlines / spaces are needed between tokens, add them - in their respective templates, or override this function - in the renderer subclass, so that whitespace won't seem to - appear magically for anyone reading your program. - Arguments: - token: a branch node who has children attribute. - """ - rendered = [self.render(child) for child in token.children] - return ''.join(rendered) - - def render_inner_list(self, token): - rendered = [self.render(child) for child in token.children] - return rendered - - - -def escape_url(raw): - """ - Escape urls to prevent code injection craziness. (Hopefully.) - """ - from urllib.parse import quote - return quote(raw, safe='/#:') \ No newline at end of file diff --git a/simdem/parser.py b/simdem/parser.py deleted file mode 100644 index 3e350e9..0000000 --- a/simdem/parser.py +++ /dev/null @@ -1,87 +0,0 @@ -import re -import logging - -class Parser(object): - - lexer = None - - def __init__(self, lexer): - self.lexer = lexer - - def is_prerequisite_block(self, block): - # Example: {'level': 1, 'text': 'Prerequisites', 'type': 'heading'} - if 'prerequisite' in block['text'].lower() and block['type'] == 'heading': - return True - return False - - def is_command_block(self, block): - if block['type'] == 'code' and block['lang'] == 'shell': - return True - return False - - def is_result_block(self, block): - # This is different than previous SimDem because it didn't require a language for the result. - # I believe this approach is more declarative. - if block['type'] == 'code' and block['lang'] == 'result': - return True - return False - - # Assuming just one for now - def parse_ref_from_text(self, text): - # Does mistune allow us to parse this? Would be nice. - pattern = re.compile('.*\[(.*)\]\((.*)\).*') - match = pattern.match(text) - if match: - title = match.groups()[0].strip() - href = match.groups()[1] - logging.debug("Found prereq: " + href) - return href - return None - - def get_prereqs(self, blocks): -# logging.debug("get_prereqs: " + str(blocks)) - res = [] - # Is there a better way to do this? Probably so. I'm on a plane and can't research - for idx in range(len(blocks)): - block = blocks[idx] - if self.is_prerequisite_block(block): - # We want the text block after the prereq heading - for line in blocks[idx+1]['text'].split("\n"): - res.append(line) -# logging.debug("get_prereqs: res= " + str(res)) - return res - - - def get_file_contents(self, file_path): -# logging.debug("get_file_contents: " + file_path) - f = open(file_path, 'r') - content = f.read() - f.close() - return content - - def parse_doc(self, text): -# logging.debug("parse_doc: text=" + text) - # https://github.com/lepture/mistune/issues/147 - # Stoopid non-idempotent parser. - self.lexer.tokens = [] - blocks = self.lexer.parse(text) - return { - 'prerequisites': self.get_prereqs(blocks), - 'commands': self.get_commands(blocks) - } - - def get_commands(self, blocks): - res = [] - for idx in range(len(blocks)): - logging.debug("get_commands():processing " + str(blocks[idx])) - block = blocks[idx] - if self.is_result_block(block): - logging.debug("get_commands():is_result_block") - res[len(res) - 1]['expected_result'] = block['text'] - elif self.is_command_block(block): - logging.debug("get_commands():is_command_block") - for line in block['text'].split("\n"): - res.append({ 'command': line }) - else: - logging.info("get_commands():unknown_block. Ignoring") - return res \ No newline at end of file diff --git a/tests/context.py b/tests/context.py index 43d9cbd..10b6fba 100644 --- a/tests/context.py +++ b/tests/context.py @@ -6,4 +6,4 @@ import simdem from simdem.render import demo -from simdem.mistletoe import simdem_renderer,parser \ No newline at end of file +from simdem.mistletoe import parser \ No newline at end of file diff --git a/tests/test_lexer.py b/tests/test_lexer.py deleted file mode 100644 index 0d105f3..0000000 --- a/tests/test_lexer.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -from .context import simdem -from mistune import Renderer, InlineGrammar, InlineLexer - -import mistune -import unittest - - -class SimDemLexerTestSuite(unittest.TestCase): - """Lexer test cases.""" - - def test_mistune(self): - res = mistune.markdown('I am using **mistune markdown parser**') - self.assertEquals(res, "

    I am using mistune markdown parser

    \n") - - def test_block_lexer(self): - blockLexer = mistune.BlockLexer() - res = blockLexer.parse('I am using **mistune markdown parser**') - self.assertEquals(res, [{'type': 'paragraph', 'text': 'I am using **mistune markdown parser**'}]) - - def test_block_lexer_cmd(self): - blockLexer = mistune.BlockLexer() - res = blockLexer.parse("```php\necho $foo```") - self.assertEquals(res, [{'lang': 'php', 'text': 'echo $foo', 'type': 'code'}]) - - def test_block_lexer_prerequisite(self): - blockLexer = mistune.BlockLexer() - res = blockLexer.parse("""foo -# Prerequisites -We should be able to run [nested prerequisites](./nested_prereq.md). - -# Do stuff here -```echo foo -```""") - self.assertEquals(res, - [{'text': 'foo', 'type': 'paragraph'}, - {'level': 1, 'text': 'Prerequisites', 'type': 'heading'}, - {'text': 'We should be able to run [nested prerequisites](./nested_prereq.md).', 'type': 'paragraph'}, - {'level': 1, 'text': 'Do stuff here', 'type': 'heading'}, - {'text': '```echo foo\n```', 'type': 'paragraph'}]) - - def test_block_lexer_multiline(self): - blockLexer = mistune.BlockLexer() - res = blockLexer.parse("""this is text -```shell -echo $FOO``` -more text""") - self.assertEquals(res, [{'type': 'paragraph', 'text': 'this is text'}, - {'lang': 'shell', 'text': 'echo $FOO', 'type': 'code'}, - {'text': 'more text', 'type': 'paragraph'}]) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index 4ab92a8..ac4c780 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -1,62 +1,36 @@ # -*- coding: utf-8 -*- -from .context import simdem_renderer - import mistletoe import unittest import mistletoe.block_token as token -from mistletoe.ast_renderer import ASTRenderer import mistletoe.ast_renderer as renderer - - class SimDemMistletoeTestSuite(unittest.TestCase): """Lexer test cases.""" - def test_basic(self): - doc = 'Nothing special here. Move along' - res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) - exp_res = [{'type': 'paragraph', 'text': 'Nothing special here. Move along'}] - self.assertEquals(res, exp_res) - - - def test_strong(self): - doc = 'This is **strong text**' - res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) - exp_res = [{'type': 'paragraph', 'text': 'This is strong text'}] - self.assertEquals(res, exp_res) - - ''' - def test_block_code(self): - doc = ("```php", "echo $foo", 'echo bar', "```") - print(doc) - #res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) - res = mistletoe.markdown(doc, simdem_renderer.SimdemRenderer) - exp_res = [{'lang': 'php', 'text': 'echo $foo', 'type': 'code'}] - self.assertEquals(res, exp_res) - ''' - - def test_block_code_file(self): - file_path = '/tmp/foo4.md' - with open(file_path, 'r') as fin: - res = mistletoe.markdown(fin, simdem_renderer.SimdemRenderer) - exp_res = [{'lang': 'php', 'text': ['echo $foo', 'echo $bar'], 'type': 'code'}] - self.assertEquals(res, exp_res) - ''' def test_ast(self): self.maxDiff = None file_path = 'content/simple/README.md' - file_path = '/tmp/foo3.md' with open(file_path, 'r') as fin: res = renderer.get_ast(token.Document(fin)) - - self.assertEqual(res, {}) - ''' - -# with open('content/simple/README.md', 'r') as fin: -# output = renderer.get_ast(fin) - + exp_res = {'children': [{'children': [{'content': 'this is text', 'type': 'RawText'}], + 'type': 'Paragraph'}, + {'children': [{'content': 'echo foo\necho bar\n', + 'type': 'RawText'}], + 'language': 'shell', + 'type': 'BlockCode'}, + {'children': [{'content': 'more text', 'type': 'RawText'}], + 'type': 'Paragraph'}, + {'children': [{'content': 'echo baz\n', 'type': 'RawText'}], + 'language': 'shell', + 'type': 'BlockCode'}, + {'children': [{'content': 'even more text', 'type': 'RawText'}], + 'type': 'Paragraph'}], + 'footnotes': {}, + 'type': 'Document'} + + self.assertEqual(res, exp_res) if __name__ == '__main__': diff --git a/tests/test_mistletoe_parser.py b/tests/test_mistletoe_parser.py index 538f83a..046e52c 100644 --- a/tests/test_mistletoe_parser.py +++ b/tests/test_mistletoe_parser.py @@ -27,7 +27,7 @@ def test_full(self): 'commands': [ { 'command': 'echo foo' }, { 'command': 'echo bar' }, - { 'command': 'echo baz', 'expected_result': 'baz\n' } ] + { 'command': 'echo baz', 'expected_result': 'baz' } ] } self.assertEqual(res, exp_res) diff --git a/tests/test_parser.py b/tests/test_parser.py deleted file mode 100644 index 4a3d9ee..0000000 --- a/tests/test_parser.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -from .context import simdem, demo - -import unittest -import os.path -import configparser -import mistune - -class SimDemParserTestSuite(unittest.TestCase): - """Advanced test cases.""" - - parser = None - - def setUp(self): - config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") - - self.parser = simdem.Parser(mistune.BlockLexer()) - - def test_parse_ref_from_text(self): - self.assertEquals('./nested_prereq.md', self.parser.parse_ref_from_text('We should be able to run [nested prerequisites](./nested_prereq.md).')) - - def test_parse_doc2(self): - doc = """foo -# Prerequisites - -prereq.md -prereq-2.md - -# Do stuff here - -We want to execute this because the code type is shell - -```shell -echo foo -echo bar -``` - -# Do more stuff here - -```shell -echo baz -``` - -# Results - -The only thing that makes it a result is the code type is result. -We assume the result is for the last command of the last code block - -```result -baz -``` - - -""" - exp_res = { - 'prerequisites': ['prereq.md', 'prereq-2.md'], - 'commands': [ - { 'command': 'echo foo' }, - { 'command': 'echo bar' }, - { 'command': 'echo baz', 'expected_result': 'baz' } ] - } - res = self.parser.parse_doc(doc) - self.assertEquals(res, exp_res) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 5c636df..089e9a6 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -10,7 +10,7 @@ class SimDemTestSuite(unittest.TestCase): """Advanced test cases.""" - test_file = '/tmp/foo' + test_file = 'scratch/foo' simdem = None parser = None @@ -18,20 +18,15 @@ def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.parser = simdem.Parser(mistune.BlockLexer()) + self.parser = simdem.mistletoe.MistletoeParser() self.simdem = simdem.Core(config, demo.Demo(config), self.parser) def test_run_cmd(self): self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) def test_run_blocks(self): - doc = """this is text -```shell -touch %(file)s``` -more text""" % { 'file' : self.test_file } - self.assertFalse(os.path.exists(self.test_file)) - blocks = self.parser.parse_doc(doc) + blocks = self.parser.parse_file('content/create-file/README.md') self.simdem.run_blocks(blocks['commands']) self.assertTrue(os.path.exists(self.test_file)) diff --git a/tests/test_system.py b/tests/test_system.py index 6ec94b6..47ba9da 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -20,7 +20,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), simdem.Parser(mistune.BlockLexer())) + self.simdem = simdem.Core(config, demo.Demo(config), simdem.mistletoe.MistletoeParser()) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() From ed477695b9cdf945362e4a9a929bcfa076847a8a Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 21 Dec 2017 23:39:02 -0600 Subject: [PATCH 051/167] add scent.py --- scent.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 scent.py diff --git a/scent.py b/scent.py new file mode 100644 index 0000000..e6afa02 --- /dev/null +++ b/scent.py @@ -0,0 +1,34 @@ +from sniffer.api import * # import the really small API +import os, termstyle + +# you can customize the pass/fail colors like this +pass_fg_color = termstyle.green +pass_bg_color = termstyle.bg_default +fail_fg_color = termstyle.red +fail_bg_color = termstyle.bg_default + +# All lists in this variable will be under surveillance for changes. +watch_paths = ['.'] + +# this gets invoked on every file that gets changed in the directory. Return +# True to invoke any runnable functions, False otherwise. +# +# This fires runnables only if files ending with .py extension and not prefixed +# with a period. +@file_validator +def py_files(filename): + return (filename.endswith('.py') or filename.endswith('.md')) and not os.path.basename(filename).startswith('.') + +# This gets invoked for verification. This is ideal for running tests of some sort. +# For anything you want to get constantly reloaded, do an import in the function. +# +# sys.argv[0] and any arguments passed via -x prefix will be sent to this function as +# it's arguments. The function should return logically True if the validation passed +# and logicially False if it fails. +# +# This example simply runs nose. +@runnable +def execute_nose(*args): + import nose + return nose.run(argv=list(args)) + From c7d242c2cc22d4d8890c6c4b9b6c97df0d5dbe09 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 22 Dec 2017 10:09:01 -0600 Subject: [PATCH 052/167] Further optimizations from mistletoe --- content/complete-features/README.md | 7 +-- content/prerequisite-run/README.md | 4 +- simdem/core.py | 13 ++--- simdem/mistletoe/__init__.py | 2 +- simdem/mistletoe/parser.py | 74 ----------------------------- simdem/mistletoe/renderer.py | 36 ++++++++++++++ tests/context.py | 2 +- tests/test_mistletoe_parser.py | 9 ++-- tests/test_simdem.py | 8 ++-- tests/test_system.py | 2 +- 10 files changed, 61 insertions(+), 96 deletions(-) delete mode 100644 simdem/mistletoe/parser.py create mode 100644 simdem/mistletoe/renderer.py diff --git a/content/complete-features/README.md b/content/complete-features/README.md index ea7c9b7..bf941ae 100644 --- a/content/complete-features/README.md +++ b/content/complete-features/README.md @@ -1,8 +1,9 @@ # Prerequisites -* [prereq-1](prereq.md) -* [prereq-2](prereq-2.md) - +```prerequisites +prereq.md +prereq-2.md +``` # Do stuff here We want to execute this because the code type is shell diff --git a/content/prerequisite-run/README.md b/content/prerequisite-run/README.md index b8fd504..5b1088d 100644 --- a/content/prerequisite-run/README.md +++ b/content/prerequisite-run/README.md @@ -1,6 +1,8 @@ # Prerequisite -* [prereq-1](content/prerequisite-run/prereq.md) +```prerequisites +content/prerequisite-run/prereq.md +``` # Main diff --git a/simdem/core.py b/simdem/core.py index 9de0455..47fc9c1 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -8,12 +8,12 @@ class Core(object): rend = None config = None - parser = None + markdown_parser = None - def __init__(self, config, rend, parser): + def __init__(self, config, rend, markdown_parser): self.config = config self.rend = rend - self.parser = parser + self.markdown_parser = markdown_parser def run_code_block(self, cmd_block): # In the future, we'll want to split a code segment into individual lines @@ -29,10 +29,11 @@ def run_cmd(self, cmd): return self.rend.run_cmd(cmd) def process_file(self, file_path): - blocks = self.parser.parse_file(file_path) + blocks = self.markdown_parser.render_file(file_path) logging.info("process_file():blocks=" + str(blocks)) - self.process_prereqs(blocks['prerequisites']) - logging.info("process_file():completed process_prereqs()") + if 'prerequisites' in blocks: + self.process_prereqs(blocks['prerequisites']) + logging.info("process_file():completed process_prereqs()") result = self.run_blocks(blocks['commands']) return result diff --git a/simdem/mistletoe/__init__.py b/simdem/mistletoe/__init__.py index a01a345..1fd7755 100644 --- a/simdem/mistletoe/__init__.py +++ b/simdem/mistletoe/__init__.py @@ -1 +1 @@ -from .parser import MistletoeParser \ No newline at end of file +from .renderer import SimdemMistletoeRenderer \ No newline at end of file diff --git a/simdem/mistletoe/parser.py b/simdem/mistletoe/parser.py deleted file mode 100644 index cc12025..0000000 --- a/simdem/mistletoe/parser.py +++ /dev/null @@ -1,74 +0,0 @@ -import re -import logging -import mistletoe.ast_renderer as renderer -import mistletoe.block_token as token -import pprint - -class MistletoeParser(object): - - def is_command_block(self, block): - if block['type'] == 'BlockCode' and block['language'] == 'shell': - return True - return False - - def is_result_block(self, block): - # This is different than previous SimDem because it didn't require a language for the result. - # I believe this approach is more declarative. - if block['type'] == 'BlockCode' and block['language'] == 'result': - return True - return False - - """ - I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now - Will look like this coming out of AST - {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], - 'level': 1, - 'type': 'Heading'}, - """ - def is_prerequisite_block(self, block): - if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ - and 'prerequisite' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': - return True - return False - - def parse_file(self, file_path): - with open(file_path, 'r') as fin: - ast = renderer.get_ast(token.Document(fin)) - - res = { - 'prerequisites': [], - 'commands': [] - } - - idx = 0 - blocks = ast['children'] - #logging.debug("parse_file():processing " + str(blocks) - while idx < len(blocks): - block = blocks[idx] - logging.debug("parse_file():processing " + str(block)) - - if self.is_prerequisite_block(block): - logging.debug("parse_file():found preqreq block") - res['prerequisites'] = [x['children'][0]['target'] for x in blocks[idx+1]['children']] - # No need to process the next block since that's the prereqs - idx = idx + 1 - - elif self.is_result_block(block): - # Assume that the result block is for the previous command block - logging.debug("parse_file():is_result_block") - content = block['children'][0]['content'].rstrip() - if content: - res['commands'][len(res['commands']) - 1]['expected_result'] = content - - elif self.is_command_block(block): - logging.debug("parse_file():is_command_block") - for line in block['children'][0]['content'].split("\n"): - if line: - res['commands'].append({ 'command': line }) - - else: - logging.info("get_commands():unknown_block. Ignoring") - - idx = idx + 1 - #logging.debug(res) - return res diff --git a/simdem/mistletoe/renderer.py b/simdem/mistletoe/renderer.py new file mode 100644 index 0000000..4c8f04b --- /dev/null +++ b/simdem/mistletoe/renderer.py @@ -0,0 +1,36 @@ +from mistletoe.base_renderer import BaseRenderer +from mistletoe import Document +from collections import defaultdict +import logging + +# Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 + +class SimdemMistletoeRenderer(BaseRenderer): + def __init__(self): + super().__init__() + self.output = defaultdict(list) + + def render_block_code(self, token): + lines = token.children[0].content.splitlines() + if 'prerequisite' in token.language: + self.output['prerequisites'].extend(lines) + elif 'result' in token.language: + self.output['commands'][-1]['expected_result'] = lines[0] + elif 'shell' in token.language: + self.output['commands'].extend({'command': line} for line in lines) + else: + logging.info("get_commands():unknown_block. Ignoring") + return '' + + def render_document(self, token): + self.render_inner(token) + return dict(self.output) + + def __getattr__(self, name): + return lambda token: '' + + def render_file(self, file_path): + with open(file_path, 'r') as fin: + with SimdemMistletoeRenderer() as renderer: + rendered = renderer.render(Document(fin)) + return rendered \ No newline at end of file diff --git a/tests/context.py b/tests/context.py index 10b6fba..ae6ed8b 100644 --- a/tests/context.py +++ b/tests/context.py @@ -6,4 +6,4 @@ import simdem from simdem.render import demo -from simdem.mistletoe import parser \ No newline at end of file +from simdem.mistletoe import renderer \ No newline at end of file diff --git a/tests/test_mistletoe_parser.py b/tests/test_mistletoe_parser.py index 046e52c..fdf3f90 100644 --- a/tests/test_mistletoe_parser.py +++ b/tests/test_mistletoe_parser.py @@ -1,27 +1,26 @@ # -*- coding: utf-8 -*- -from .context import simdem, parser +from .context import simdem, renderer import unittest import os.path import configparser -import mistune class MistletoeParserTestSuite(unittest.TestCase): """Advanced test cases.""" - parser = None + renderer = None def setUp(self): self.maxDiff = None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.parser = parser.MistletoeParser() + self.renderer = renderer.SimdemMistletoeRenderer() def test_full(self): file_path = 'content/complete-features/README.md' - res = self.parser.parse_file(file_path) + res = self.renderer.render_file(file_path) exp_res = { 'prerequisites': ['prereq.md', 'prereq-2.md'], 'commands': [ diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 089e9a6..d5a4641 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -12,21 +12,21 @@ class SimDemTestSuite(unittest.TestCase): test_file = 'scratch/foo' simdem = None - parser = None + markdown_parser = None def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.parser = simdem.mistletoe.MistletoeParser() - self.simdem = simdem.Core(config, demo.Demo(config), self.parser) + self.markdown_parser = simdem.mistletoe.SimdemMistletoeRenderer() + self.simdem = simdem.Core(config, demo.Demo(config), self.markdown_parser) def test_run_cmd(self): self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) def test_run_blocks(self): self.assertFalse(os.path.exists(self.test_file)) - blocks = self.parser.parse_file('content/create-file/README.md') + blocks = self.markdown_parser.render_file('content/create-file/README.md') self.simdem.run_blocks(blocks['commands']) self.assertTrue(os.path.exists(self.test_file)) diff --git a/tests/test_system.py b/tests/test_system.py index 47ba9da..1529b95 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -20,7 +20,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), simdem.mistletoe.MistletoeParser()) + self.simdem = simdem.Core(config, demo.Demo(config), simdem.mistletoe.SimdemMistletoeRenderer()) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() From 05394a08949a1459806ebbf26d3919c259e10b65 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 1 Jan 2018 20:12:09 -0600 Subject: [PATCH 053/167] Rename to parser Signed-off-by: Tommy Falgout --- simdem/core.py | 2 +- simdem/mistletoe/__init__.py | 1 - simdem/parser/__init__.py | 1 + simdem/{mistletoe/renderer.py => parser/codeblock.py} | 6 +++--- tests/context.py | 2 +- tests/test_mistletoe_parser.py | 6 +++--- tests/test_simdem.py | 4 ++-- tests/test_system.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 simdem/mistletoe/__init__.py create mode 100644 simdem/parser/__init__.py rename simdem/{mistletoe/renderer.py => parser/codeblock.py} (89%) diff --git a/simdem/core.py b/simdem/core.py index 47fc9c1..899a060 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -29,7 +29,7 @@ def run_cmd(self, cmd): return self.rend.run_cmd(cmd) def process_file(self, file_path): - blocks = self.markdown_parser.render_file(file_path) + blocks = self.markdown_parser.parse_file(file_path) logging.info("process_file():blocks=" + str(blocks)) if 'prerequisites' in blocks: self.process_prereqs(blocks['prerequisites']) diff --git a/simdem/mistletoe/__init__.py b/simdem/mistletoe/__init__.py deleted file mode 100644 index 1fd7755..0000000 --- a/simdem/mistletoe/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .renderer import SimdemMistletoeRenderer \ No newline at end of file diff --git a/simdem/parser/__init__.py b/simdem/parser/__init__.py new file mode 100644 index 0000000..4ee6abc --- /dev/null +++ b/simdem/parser/__init__.py @@ -0,0 +1 @@ +from .codeblock import CodeBlock \ No newline at end of file diff --git a/simdem/mistletoe/renderer.py b/simdem/parser/codeblock.py similarity index 89% rename from simdem/mistletoe/renderer.py rename to simdem/parser/codeblock.py index 4c8f04b..e823af2 100644 --- a/simdem/mistletoe/renderer.py +++ b/simdem/parser/codeblock.py @@ -5,7 +5,7 @@ # Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 -class SimdemMistletoeRenderer(BaseRenderer): +class CodeBlock(BaseRenderer): def __init__(self): super().__init__() self.output = defaultdict(list) @@ -29,8 +29,8 @@ def render_document(self, token): def __getattr__(self, name): return lambda token: '' - def render_file(self, file_path): + def parse_file(self, file_path): with open(file_path, 'r') as fin: - with SimdemMistletoeRenderer() as renderer: + with CodeBlock() as renderer: rendered = renderer.render(Document(fin)) return rendered \ No newline at end of file diff --git a/tests/context.py b/tests/context.py index ae6ed8b..e034c83 100644 --- a/tests/context.py +++ b/tests/context.py @@ -6,4 +6,4 @@ import simdem from simdem.render import demo -from simdem.mistletoe import renderer \ No newline at end of file +from simdem.parser import codeblock \ No newline at end of file diff --git a/tests/test_mistletoe_parser.py b/tests/test_mistletoe_parser.py index fdf3f90..1b1c997 100644 --- a/tests/test_mistletoe_parser.py +++ b/tests/test_mistletoe_parser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from .context import simdem, renderer +from .context import simdem, codeblock import unittest import os.path @@ -16,11 +16,11 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.renderer = renderer.SimdemMistletoeRenderer() + self.renderer = codeblock.CodeBlock() def test_full(self): file_path = 'content/complete-features/README.md' - res = self.renderer.render_file(file_path) + res = self.renderer.parse_file(file_path) exp_res = { 'prerequisites': ['prereq.md', 'prereq-2.md'], 'commands': [ diff --git a/tests/test_simdem.py b/tests/test_simdem.py index d5a4641..4e66845 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -18,7 +18,7 @@ def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.markdown_parser = simdem.mistletoe.SimdemMistletoeRenderer() + self.markdown_parser = simdem.parser.CodeBlock() self.simdem = simdem.Core(config, demo.Demo(config), self.markdown_parser) def test_run_cmd(self): @@ -26,7 +26,7 @@ def test_run_cmd(self): def test_run_blocks(self): self.assertFalse(os.path.exists(self.test_file)) - blocks = self.markdown_parser.render_file('content/create-file/README.md') + blocks = self.markdown_parser.parse_file('content/create-file/README.md') self.simdem.run_blocks(blocks['commands']) self.assertTrue(os.path.exists(self.test_file)) diff --git a/tests/test_system.py b/tests/test_system.py index 1529b95..ed07e96 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -20,7 +20,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), simdem.mistletoe.SimdemMistletoeRenderer()) + self.simdem = simdem.Core(config, demo.Demo(config), simdem.parser.CodeBlock()) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() From 3117719209cc3cc664ac171bd3a0914800451581 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 1 Jan 2018 20:59:33 -0600 Subject: [PATCH 054/167] Add codeblock and context to live together in harmony Signed-off-by: Tommy Falgout --- .../{README.md => codeblock.md} | 0 content/complete-features/context.md | 29 ++++++++ simdem/parser/__init__.py | 3 +- simdem/parser/codeblock.py | 4 +- simdem/parser/context.py | 74 +++++++++++++++++++ tests/context.py | 2 +- ...toe_parser.py => test_parser_codeblock.py} | 4 +- tests/test_parser_context.py | 37 ++++++++++ tests/test_simdem.py | 2 +- tests/test_system.py | 2 +- 10 files changed, 149 insertions(+), 8 deletions(-) rename content/complete-features/{README.md => codeblock.md} (100%) create mode 100644 content/complete-features/context.md create mode 100644 simdem/parser/context.py rename tests/{test_mistletoe_parser.py => test_parser_codeblock.py} (87%) create mode 100644 tests/test_parser_context.py diff --git a/content/complete-features/README.md b/content/complete-features/codeblock.md similarity index 100% rename from content/complete-features/README.md rename to content/complete-features/codeblock.md diff --git a/content/complete-features/context.md b/content/complete-features/context.md new file mode 100644 index 0000000..ea7c9b7 --- /dev/null +++ b/content/complete-features/context.md @@ -0,0 +1,29 @@ +# Prerequisites + +* [prereq-1](prereq.md) +* [prereq-2](prereq-2.md) + +# Do stuff here + +We want to execute this because the code type is shell + +```shell +echo foo +echo bar +``` + +# Do more stuff here + +```shell +echo baz +``` + +# Results + +The only thing that makes it a result is the code type is result. +We assume the result is for the last command of the last code block + +```result +baz +``` + diff --git a/simdem/parser/__init__.py b/simdem/parser/__init__.py index 4ee6abc..b03f17d 100644 --- a/simdem/parser/__init__.py +++ b/simdem/parser/__init__.py @@ -1 +1,2 @@ -from .codeblock import CodeBlock \ No newline at end of file +from .codeblock import CodeBlockParser +from .context import ContextParser \ No newline at end of file diff --git a/simdem/parser/codeblock.py b/simdem/parser/codeblock.py index e823af2..1dd5202 100644 --- a/simdem/parser/codeblock.py +++ b/simdem/parser/codeblock.py @@ -5,7 +5,7 @@ # Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 -class CodeBlock(BaseRenderer): +class CodeBlockParser(BaseRenderer): def __init__(self): super().__init__() self.output = defaultdict(list) @@ -31,6 +31,6 @@ def __getattr__(self, name): def parse_file(self, file_path): with open(file_path, 'r') as fin: - with CodeBlock() as renderer: + with CodeBlockParser() as renderer: rendered = renderer.render(Document(fin)) return rendered \ No newline at end of file diff --git a/simdem/parser/context.py b/simdem/parser/context.py new file mode 100644 index 0000000..07d57a8 --- /dev/null +++ b/simdem/parser/context.py @@ -0,0 +1,74 @@ +import re +import logging +import mistletoe.ast_renderer as renderer +import mistletoe.block_token as token +import pprint + +class ContextParser(object): + + def is_command_block(self, block): + if block['type'] == 'BlockCode' and block['language'] == 'shell': + return True + return False + + def is_result_block(self, block): + # This is different than previous SimDem because it didn't require a language for the result. + # I believe this approach is more declarative. + if block['type'] == 'BlockCode' and block['language'] == 'result': + return True + return False + + """ + I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now + Will look like this coming out of AST + {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], + 'level': 1, + 'type': 'Heading'}, + """ + def is_prerequisite_block(self, block): + if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ + and 'prerequisite' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': + return True + return False + + def parse_file(self, file_path): + with open(file_path, 'r') as fin: + ast = renderer.get_ast(token.Document(fin)) + + res = { + 'prerequisites': [], + 'commands': [] + } + + idx = 0 + blocks = ast['children'] + #logging.debug("parse_file():processing " + str(blocks) + while idx < len(blocks): + block = blocks[idx] + logging.debug("parse_file():processing " + str(block)) + + if self.is_prerequisite_block(block): + logging.debug("parse_file():found preqreq block") + res['prerequisites'] = [x['children'][0]['target'] for x in blocks[idx+1]['children']] + # No need to process the next block since that's the prereqs + idx = idx + 1 + + elif self.is_result_block(block): + # Assume that the result block is for the previous command block + logging.debug("parse_file():is_result_block") + content = block['children'][0]['content'].rstrip() + if content: + res['commands'][len(res['commands']) - 1]['expected_result'] = content + + elif self.is_command_block(block): + logging.debug("parse_file():is_command_block") + for line in block['children'][0]['content'].split("\n"): + if line: + res['commands'].append({ 'command': line }) + + else: + logging.info("get_commands():unknown_block. Ignoring") + + idx = idx + 1 + #logging.debug(res) + return res diff --git a/tests/context.py b/tests/context.py index e034c83..80a7a43 100644 --- a/tests/context.py +++ b/tests/context.py @@ -6,4 +6,4 @@ import simdem from simdem.render import demo -from simdem.parser import codeblock \ No newline at end of file +from simdem.parser import codeblock,context \ No newline at end of file diff --git a/tests/test_mistletoe_parser.py b/tests/test_parser_codeblock.py similarity index 87% rename from tests/test_mistletoe_parser.py rename to tests/test_parser_codeblock.py index 1b1c997..691931a 100644 --- a/tests/test_mistletoe_parser.py +++ b/tests/test_parser_codeblock.py @@ -16,10 +16,10 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.renderer = codeblock.CodeBlock() + self.renderer = codeblock.CodeBlockParser() def test_full(self): - file_path = 'content/complete-features/README.md' + file_path = 'content/complete-features/codeblock.md' res = self.renderer.parse_file(file_path) exp_res = { 'prerequisites': ['prereq.md', 'prereq-2.md'], diff --git a/tests/test_parser_context.py b/tests/test_parser_context.py new file mode 100644 index 0000000..d49cdc8 --- /dev/null +++ b/tests/test_parser_context.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from .context import simdem, context + +import unittest +import os.path +import configparser +import mistune + +class MistletoeParserTestSuite(unittest.TestCase): + """Advanced test cases.""" + + parser = None + + def setUp(self): + self.maxDiff = None + config = configparser.ConfigParser() + config.read("content/config/unit_test.ini") + + self.parser = context.ContextParser() + + def test_full(self): + file_path = 'content/complete-features/context.md' + res = self.parser.parse_file(file_path) + exp_res = { + 'prerequisites': ['prereq.md', 'prereq-2.md'], + 'commands': [ + { 'command': 'echo foo' }, + { 'command': 'echo bar' }, + { 'command': 'echo baz', 'expected_result': 'baz' } ] + } + + self.assertEqual(res, exp_res) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_simdem.py b/tests/test_simdem.py index 4e66845..bcfe8eb 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -18,7 +18,7 @@ def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.markdown_parser = simdem.parser.CodeBlock() + self.markdown_parser = simdem.parser.CodeBlockParser() self.simdem = simdem.Core(config, demo.Demo(config), self.markdown_parser) def test_run_cmd(self): diff --git a/tests/test_system.py b/tests/test_system.py index ed07e96..526807a 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -20,7 +20,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), simdem.parser.CodeBlock()) + self.simdem = simdem.Core(config, demo.Demo(config), simdem.parser.CodeBlockParser()) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() From ecbe8e4507464c2512bcfcd64f3d8419095ec544 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 12:40:07 -0600 Subject: [PATCH 055/167] Add documentation Signed-off-by: Tommy Falgout --- README.md | 13 +++ docs/Makefile | 153 ---------------------------- docs/conf.py | 242 -------------------------------------------- docs/development.md | 27 +++++ docs/features.md | 32 ++++++ docs/index.rst | 22 ---- docs/make.bat | 190 ---------------------------------- docs/syntax.md | 24 +++++ 8 files changed, 96 insertions(+), 607 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/conf.py create mode 100644 docs/development.md create mode 100644 docs/features.md delete mode 100644 docs/index.rst delete mode 100644 docs/make.bat create mode 100644 docs/syntax.md diff --git a/README.md b/README.md index 4da1f3d..318454a 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,16 @@ This project provides ways to write tutorials in markdown that then become inter SimDem v2 is a complete rewrite of SimDem v1. The latest commit for v1 can be found at: https://github.com/Azure/simdem/tree/cb1caf17fd684e125789c26817f43eeae0e1c523 +# Features + +SimDem supports the following features: +* Command execution +* Environment variable injection +* Prerequisites +* Command validation + +Details on the complete feature list can be found in the [feature documentation](docs/features.md). + +# Syntax + +SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index f828634..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,153 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/sample.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/sample.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/sample" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/sample" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index ac03bd0..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,242 +0,0 @@ -# -*- coding: utf-8 -*- -# -# sample documentation build configuration file, created by -# sphinx-quickstart on Mon Apr 16 21:22:43 2012. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'sample' -copyright = u'2012, Kenneth Reitz' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = 'v0.0.1' -# The full version, including alpha/beta/rc tags. -release = 'v0.0.1' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'sampledoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'sample.tex', u'sample Documentation', - u'Kenneth Reitz', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'sample', u'sample Documentation', - [u'Kenneth Reitz'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'sample', u'sample Documentation', - u'Kenneth Reitz', 'sample', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..6f624fc --- /dev/null +++ b/docs/development.md @@ -0,0 +1,27 @@ +# Preface + +We would love for you to contribute to SimDem. If there is anything that would make adoption/developing of SimDem easier, please open up an issue with your request. + +# Setup Dev Environment + +## Prerequisites + +* python3 +* pip3 +* Linux shell + +## Setup + +Fetch the necessary packages + +``` +make init +``` + +## Validation + +Verify the test cases pass + +``` +make test +``` \ No newline at end of file diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 0000000..8d796f0 --- /dev/null +++ b/docs/features.md @@ -0,0 +1,32 @@ +# Features + +This document is intended to be a list of all features supported in SimDem + +## Commands +* Execution - Command extracted from the document can be run in a separate shell + * Shell - All command are run in a Linux shell. (PR for Powershell is welcome) + * Comments - If command starts with `#`, it will be considered a comment and will not be run +* Validation - Validate if the previous command ran correctly. See validation section for details + * If validation passes - Continue processing + * If validation fails - Exit program (default) +* Strip ANSI escape sequences + +## Environment Variables + * Allow the ability to inject environment variables via file or CLI (key/value format) + +## Prerequisites + * Allow ability to link to other SimDem documents to run. (e.g. Need Azure CLI setup before creating resource group.) Will be only run once + * Validation - Validate if the prequisite requirements have been met. See validation section for details + * If validation passes - Don't run prerequisite document + * If validation fails - Continue to run prerequsite document + +## Interrupt running + * When running interactively, allow the ability to interrupt running document to run manual commands. Resume processing document when complete + +## Validation + * Allows the output of the previous code block to be evaluated to determine if it meets an expected result. Validation is either passed/failed. + * Validation types + * Expected Result Similarity - e.g. .80 match + * Result Exact Match + * Result Pattern Match + * Exit code \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 46f3fe8..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. sample documentation master file, created by - sphinx-quickstart on Mon Apr 16 21:22:43 2012. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to sample's documentation! -================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index deed45a..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,190 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\sample.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\sample.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -:end diff --git a/docs/syntax.md b/docs/syntax.md new file mode 100644 index 0000000..18164b0 --- /dev/null +++ b/docs/syntax.md @@ -0,0 +1,24 @@ +# Syntax + +Currently, there are two implementation of Markdown syntax supported. + +## Context Based + +This is similar to SimDem v1's syntax. + +[../content/complete-features/context.md](Example Context Based Document) + +## Codeblock Based + +[../content/complete-features/codeblock.md](Example Codeblock Document) + +## Features + +The only sections processed are code blocks and all features are determined by the code block type. + +Feature | Context Based | Codeblock Based +--- | --- | --- +Command | \```shell | \```shell +Prerequisite | `# Prerequisite` followed by list of links | \```prerequisite +Validation | `# Validation` followed by code block | \```validation +Result | `# Result` followed by a code block with the result | \```result From 27d5e755155d48bc40d2f88521b617f79de4cb80 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 14:33:25 -0600 Subject: [PATCH 056/167] More better documentation Signed-off-by: Tommy Falgout --- README.md | 20 ++++++++++++-------- docs/development.md | 42 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 318454a..5f8912e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,6 @@ # SimDem -This project provides ways to write tutorials in markdown that then become interactive demo's and automated tests. - -# History - -SimDem v2 is a complete rewrite of SimDem v1. The latest commit for v1 can be found at: -https://github.com/Azure/simdem/tree/cb1caf17fd684e125789c26817f43eeae0e1c523 +This project provides an easy way to convert tutorials written in markdown into interactive demos and automated tests. # Features @@ -13,10 +8,19 @@ SimDem supports the following features: * Command execution * Environment variable injection * Prerequisites -* Command validation +* Output validation Details on the complete feature list can be found in the [feature documentation](docs/features.md). # Syntax -SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). \ No newline at end of file +Currently, SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). + +# Development + +We would love to have you be a part of the SimDem development team. For details, see the [development documentation](docs/development.md). + +# History + +SimDem v2 is a complete rewrite of SimDem v1. The latest commit for v1 can be found at: +https://github.com/Azure/simdem/tree/cb1caf17fd684e125789c26817f43eeae0e1c523 diff --git a/docs/development.md b/docs/development.md index 6f624fc..a537d22 100644 --- a/docs/development.md +++ b/docs/development.md @@ -10,7 +10,7 @@ We would love for you to contribute to SimDem. If there is anything that would * pip3 * Linux shell -## Setup +## Initialize Fetch the necessary packages @@ -20,8 +20,44 @@ make init ## Validation -Verify the test cases pass +Verify the tests pass ``` make test -``` \ No newline at end of file +``` + +# Code Structure + +SimDem is broken into the following class types + +## Parse + +This class type parses the markdown document into a "SimDem Execution Object". This object has everything SimDem needs to know on how to run the document (e.g. prerequisites, commands, validations, etc.) + +Implementations: +* Context +* CodeBlock + +Example SimDem Execution Object: +``` +{ 'prerequisites': ['prereq.md', 'prereq-2.md'], + 'commands': [ { 'command': 'echo foo' }, + { 'command': 'echo bar' }, + { 'command': 'echo baz', 'expected_result': 'baz' } ] + } +``` + +## Render + +This class type formats the result of running the commands into the desired output + +Implementations: +* Demonstration mode +* Automated mode (coming soon) + +## Execute + +This class type executes the desired commands into the shell + +Implementations: +* Bash \ No newline at end of file From 65335d9b37fb7fb39e63e158e6db7c02aeb4efaa Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 14:40:34 -0600 Subject: [PATCH 057/167] refactor var names --- simdem/core.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/simdem/core.py b/simdem/core.py index 899a060..72281da 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -6,14 +6,14 @@ class Core(object): - rend = None + renderer = None config = None - markdown_parser = None + parser = None - def __init__(self, config, rend, markdown_parser): + def __init__(self, config, renderer, parser): self.config = config - self.rend = rend - self.markdown_parser = markdown_parser + self.renderer = renderer + self.parser = parser def run_code_block(self, cmd_block): # In the future, we'll want to split a code segment into individual lines @@ -26,10 +26,10 @@ def run_code_block(self, cmd_block): return result_latest def run_cmd(self, cmd): - return self.rend.run_cmd(cmd) + return self.renderer.run_cmd(cmd) def process_file(self, file_path): - blocks = self.markdown_parser.parse_file(file_path) + blocks = self.parser.parse_file(file_path) logging.info("process_file():blocks=" + str(blocks)) if 'prerequisites' in blocks: self.process_prereqs(blocks['prerequisites']) From f7a284f063c6689388822094067daa57fdad1900 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 15:03:57 -0600 Subject: [PATCH 058/167] refactor to complete dependency injection --- simdem/core.py | 9 +++++++-- simdem/executor.py | 42 ------------------------------------------ simdem/render/demo.py | 10 ---------- tests/context.py | 3 ++- tests/test_simdem.py | 8 ++++---- tests/test_system.py | 2 +- 6 files changed, 14 insertions(+), 60 deletions(-) delete mode 100644 simdem/executor.py diff --git a/simdem/core.py b/simdem/core.py index 72281da..5e1a145 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -9,11 +9,13 @@ class Core(object): renderer = None config = None parser = None + executor = None - def __init__(self, config, renderer, parser): + def __init__(self, config, renderer, parser, executor): self.config = config self.renderer = renderer self.parser = parser + self.executor = executor def run_code_block(self, cmd_block): # In the future, we'll want to split a code segment into individual lines @@ -26,7 +28,10 @@ def run_code_block(self, cmd_block): return result_latest def run_cmd(self, cmd): - return self.renderer.run_cmd(cmd) + self.renderer.type_command(cmd) + res = self.executor.run_cmd(cmd) + self.renderer.display_result(res) + return res def process_file(self, file_path): blocks = self.parser.parse_file(file_path) diff --git a/simdem/executor.py b/simdem/executor.py deleted file mode 100644 index 0a27853..0000000 --- a/simdem/executor.py +++ /dev/null @@ -1,42 +0,0 @@ -import pexpect -import time -import logging -from pexpect import replwrap - -PEXPECT_PROMPT = u'[PEXPECT_PROMPT>' -PEXPECT_CONTINUATION_PROMPT = u'[PEXPECT_PROMPT+' - -class Executor(object): - _shell = None - _env = None - - def __init__(self): - pass - - - def run_cmd(self, command=None): - command = command.strip() - logging.debug("Execute command: '" + command + "'") - start_time = time.time() - response = self.get_shell().run_command(command) - end_time = time.time() - # https://pexpect.readthedocs.io/en/stable/overview.html#find-the-end-of-line-cr-lf-conventions - # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. This might get tricky if we start supporting windows - # This is because to easily write expected testcase output files, most unix-ish text editors write with \n - response = response.replace("\r\n", "\n") - logging.debug("Response: '" + response + "'") - return response - - def get_shell(self): - """Gets or creates the shell in which to run commands for the - supplied demo - """ - if self._shell == None: - # Should we use spawn or spawnu? - child = pexpect.spawnu('/bin/bash', env=self._env, echo=False, timeout=None) - ps1 = PEXPECT_PROMPT[:5] + u'\[\]' + PEXPECT_PROMPT[5:] - ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + u'\[\]' + PEXPECT_CONTINUATION_PROMPT[5:] - prompt_change = u"PS1='{0}' PS2='{1}' PROMPT_COMMAND=''".format(ps1, ps2) - self._shell = pexpect.replwrap.REPLWrapper(child, u'\$', prompt_change) - return self._shell - diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 3a1ca7b..d141de6 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -1,23 +1,13 @@ import time import random -from .. import executor - class Demo(object): - exe = None config = None def __init__(self, config): self.config = config - self.exe = executor.Executor() pass - def run_cmd(self, cmd): - self.type_command(cmd) - res = self.exe.run_cmd(cmd) - self.display_result(res) - return res - def type_command(self, cmd): """ Displays the command on the screen diff --git a/tests/context.py b/tests/context.py index 80a7a43..68cfa8c 100644 --- a/tests/context.py +++ b/tests/context.py @@ -6,4 +6,5 @@ import simdem from simdem.render import demo -from simdem.parser import codeblock,context \ No newline at end of file +from simdem.parser import codeblock,context +from simdem.executor import bash \ No newline at end of file diff --git a/tests/test_simdem.py b/tests/test_simdem.py index bcfe8eb..29856cd 100644 --- a/tests/test_simdem.py +++ b/tests/test_simdem.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from .context import simdem, demo +from .context import simdem, demo, bash, codeblock import unittest import os.path @@ -17,9 +17,9 @@ class SimDemTestSuite(unittest.TestCase): def setUp(self): os.remove(self.test_file) if os.path.exists(self.test_file) else None config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") - self.markdown_parser = simdem.parser.CodeBlockParser() - self.simdem = simdem.Core(config, demo.Demo(config), self.markdown_parser) + config.read('content/config/unit_test.ini') + self.markdown_parser = codeblock.CodeBlockParser() + self.simdem = simdem.Core(config, demo.Demo(config), self.markdown_parser, bash.BashExecutor()) def test_run_cmd(self): self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) diff --git a/tests/test_system.py b/tests/test_system.py index 526807a..ae08e86 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -20,7 +20,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), simdem.parser.CodeBlockParser()) + self.simdem = simdem.Core(config, demo.Demo(config), simdem.parser.CodeBlockParser(), simdem.executor.BashExecutor()) logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) rootLogger = logging.getLogger() From d8ac944978af7fffe94c1a1c45a3310e59f2752b Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 15:07:43 -0600 Subject: [PATCH 059/167] Add bash executor class Signed-off-by: Tommy Falgout --- simdem/executor/__init__.py | 1 + simdem/executor/bash.py | 42 ++++++++++++++++++++++++++ tests/{test_simdem.py => test_core.py} | 0 3 files changed, 43 insertions(+) create mode 100644 simdem/executor/__init__.py create mode 100644 simdem/executor/bash.py rename tests/{test_simdem.py => test_core.py} (100%) diff --git a/simdem/executor/__init__.py b/simdem/executor/__init__.py new file mode 100644 index 0000000..98e1905 --- /dev/null +++ b/simdem/executor/__init__.py @@ -0,0 +1 @@ +from .bash import BashExecutor \ No newline at end of file diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py new file mode 100644 index 0000000..c08207b --- /dev/null +++ b/simdem/executor/bash.py @@ -0,0 +1,42 @@ +import pexpect +import time +import logging +from pexpect import replwrap + +PEXPECT_PROMPT = u'[PEXPECT_PROMPT>' +PEXPECT_CONTINUATION_PROMPT = u'[PEXPECT_PROMPT+' + +class BashExecutor(object): + _shell = None + _env = None + + def __init__(self): + pass + + + def run_cmd(self, command): + command = command.strip() + logging.debug("Execute command: '" + command + "'") + start_time = time.time() + response = self.get_shell().run_command(command) + end_time = time.time() + # https://pexpect.readthedocs.io/en/stable/overview.html#find-the-end-of-line-cr-lf-conventions + # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. This might get tricky if we start supporting windows + # This is because to easily write expected testcase output files, most unix-ish text editors write with \n + response = response.replace("\r\n", "\n") + logging.debug("Response: '" + response + "'") + return response + + def get_shell(self): + """Gets or creates the shell in which to run commands for the + supplied demo + """ + if self._shell == None: + # Should we use spawn or spawnu? + child = pexpect.spawnu('/bin/bash', env=self._env, echo=False, timeout=None) + ps1 = PEXPECT_PROMPT[:5] + u'\[\]' + PEXPECT_PROMPT[5:] + ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + u'\[\]' + PEXPECT_CONTINUATION_PROMPT[5:] + prompt_change = u"PS1='{0}' PS2='{1}' PROMPT_COMMAND=''".format(ps1, ps2) + self._shell = pexpect.replwrap.REPLWrapper(child, u'\$', prompt_change) + return self._shell + diff --git a/tests/test_simdem.py b/tests/test_core.py similarity index 100% rename from tests/test_simdem.py rename to tests/test_core.py From 296a7e545169e2c55d851d5c38baedbae305b2a4 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 15:23:44 -0600 Subject: [PATCH 060/167] refactor import statements Signed-off-by: Tommy Falgout --- simdem-env/bin/activate | 78 ++++++++++++++++++++++++++++++++ simdem-env/bin/activate.csh | 36 +++++++++++++++ simdem-env/bin/activate.fish | 76 +++++++++++++++++++++++++++++++ simdem-env/bin/activate_this.py | 34 ++++++++++++++ simdem-env/bin/easy_install | 11 +++++ simdem-env/bin/easy_install-3.6 | 11 +++++ simdem-env/bin/flask | 11 +++++ simdem-env/bin/pip | 11 +++++ simdem-env/bin/pip3 | 11 +++++ simdem-env/bin/pip3.6 | 11 +++++ simdem-env/bin/python | 1 + simdem-env/bin/python-config | 78 ++++++++++++++++++++++++++++++++ simdem-env/bin/python3 | 1 + simdem-env/bin/python3.6 | Bin 0 -> 13068 bytes simdem-env/bin/wheel | 11 +++++ simdem-env/include/python3.6m | 1 + simdem-env/pip-selfcheck.json | 1 + simdem/core.py | 6 ++- simdem/executor/bash.py | 6 +-- simdem/parser/codeblock.py | 9 ++-- simdem/parser/context.py | 6 ++- simdem/render/demo.py | 3 +- tests/context.py | 9 ++-- tests/test_core.py | 10 ++-- tests/test_mistletoe.py | 5 +- tests/test_parser_codeblock.py | 9 ++-- tests/test_parser_context.py | 10 ++-- tests/test_system.py | 14 +++--- 28 files changed, 434 insertions(+), 36 deletions(-) create mode 100644 simdem-env/bin/activate create mode 100644 simdem-env/bin/activate.csh create mode 100644 simdem-env/bin/activate.fish create mode 100644 simdem-env/bin/activate_this.py create mode 100755 simdem-env/bin/easy_install create mode 100755 simdem-env/bin/easy_install-3.6 create mode 100755 simdem-env/bin/flask create mode 100755 simdem-env/bin/pip create mode 100755 simdem-env/bin/pip3 create mode 100755 simdem-env/bin/pip3.6 create mode 120000 simdem-env/bin/python create mode 100755 simdem-env/bin/python-config create mode 120000 simdem-env/bin/python3 create mode 100755 simdem-env/bin/python3.6 create mode 100755 simdem-env/bin/wheel create mode 120000 simdem-env/include/python3.6m create mode 100644 simdem-env/pip-selfcheck.json diff --git a/simdem-env/bin/activate b/simdem-env/bin/activate new file mode 100644 index 0000000..b84a1df --- /dev/null +++ b/simdem-env/bin/activate @@ -0,0 +1,78 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + unset -f pydoc >/dev/null 2>&1 + + # reset old environment variables + # ! [ -z ${VAR+_} ] returns true if VAR is declared at all + if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then + PATH="$_OLD_VIRTUAL_PATH" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then + PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then + hash -r 2>/dev/null + fi + + if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then + PS1="$_OLD_VIRTUAL_PS1" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "${1-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/Users/thfalgou/git/simdem/venv" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +if ! [ -z "${PYTHONHOME+_}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then + _OLD_VIRTUAL_PS1="$PS1" + if [ "x" != x ] ; then + PS1="$PS1" + else + PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1" + fi + export PS1 +fi + +# Make sure to unalias pydoc if it's already there +alias pydoc 2>/dev/null >/dev/null && unalias pydoc + +pydoc () { + python -m pydoc "$@" +} + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then + hash -r 2>/dev/null +fi diff --git a/simdem-env/bin/activate.csh b/simdem-env/bin/activate.csh new file mode 100644 index 0000000..9b3e2a2 --- /dev/null +++ b/simdem-env/bin/activate.csh @@ -0,0 +1,36 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/thfalgou/git/simdem/venv" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + + +if ("" != "") then + set env_name = "" +else + set env_name = `basename "$VIRTUAL_ENV"` +endif + +# Could be in a non-interactive environment, +# in which case, $prompt is undefined and we wouldn't +# care about the prompt anyway. +if ( $?prompt ) then + set _OLD_VIRTUAL_PROMPT="$prompt" + set prompt = "[$env_name] $prompt" +endif + +unset env_name + +alias pydoc python -m pydoc + +rehash + diff --git a/simdem-env/bin/activate.fish b/simdem-env/bin/activate.fish new file mode 100644 index 0000000..cb52ddc --- /dev/null +++ b/simdem-env/bin/activate.fish @@ -0,0 +1,76 @@ +# This file must be used using `. bin/activate.fish` *within a running fish ( http://fishshell.com ) session*. +# Do not run it directly. + +function deactivate -d 'Exit virtualenv mode and return to the normal environment.' + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`. + set -l fish_function_path + + # Erase virtualenv's `fish_prompt` and restore the original. + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + end + + set -e VIRTUAL_ENV + + if test "$argv[1]" != 'nondestructive' + # Self-destruct! + functions -e pydoc + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/thfalgou/git/simdem/venv" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# Unset `$PYTHONHOME` if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +function pydoc + python -m pydoc $argv +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # Copy the current `fish_prompt` function as `_old_fish_prompt`. + functions -c fish_prompt _old_fish_prompt + + function fish_prompt + # Save the current $status, for fish_prompts that display it. + set -l old_status $status + + # Prompt override provided? + # If not, just prepend the environment name. + if test -n "" + printf '%s%s' "" (set_color normal) + else + printf '%s(%s) ' (set_color normal) (basename "$VIRTUAL_ENV") + end + + # Restore the original $status + echo "exit $old_status" | source + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/simdem-env/bin/activate_this.py b/simdem-env/bin/activate_this.py new file mode 100644 index 0000000..f18193b --- /dev/null +++ b/simdem-env/bin/activate_this.py @@ -0,0 +1,34 @@ +"""By using execfile(this_file, dict(__file__=this_file)) you will +activate this virtualenv environment. + +This can be used when you must use an existing Python interpreter, not +the virtualenv bin/python +""" + +try: + __file__ +except NameError: + raise AssertionError( + "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") +import sys +import os + +old_os_path = os.environ.get('PATH', '') +os.environ['PATH'] = os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path +base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if sys.platform == 'win32': + site_packages = os.path.join(base, 'Lib', 'site-packages') +else: + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') +prev_sys_path = list(sys.path) +import site +site.addsitedir(site_packages) +sys.real_prefix = sys.prefix +sys.prefix = base +# Move the added items to the front of the path: +new_sys_path = [] +for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) +sys.path[:0] = new_sys_path diff --git a/simdem-env/bin/easy_install b/simdem-env/bin/easy_install new file mode 100755 index 0000000..375bf77 --- /dev/null +++ b/simdem-env/bin/easy_install @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python3.6 + +# -*- coding: utf-8 -*- +import re +import sys + +from setuptools.command.easy_install import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/easy_install-3.6 b/simdem-env/bin/easy_install-3.6 new file mode 100755 index 0000000..375bf77 --- /dev/null +++ b/simdem-env/bin/easy_install-3.6 @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python3.6 + +# -*- coding: utf-8 -*- +import re +import sys + +from setuptools.command.easy_install import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/flask b/simdem-env/bin/flask new file mode 100755 index 0000000..22e6196 --- /dev/null +++ b/simdem-env/bin/flask @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python3.6 + +# -*- coding: utf-8 -*- +import re +import sys + +from flask.cli import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/pip b/simdem-env/bin/pip new file mode 100755 index 0000000..d9963b8 --- /dev/null +++ b/simdem-env/bin/pip @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python3.6 + +# -*- coding: utf-8 -*- +import re +import sys + +from pip import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/pip3 b/simdem-env/bin/pip3 new file mode 100755 index 0000000..d9963b8 --- /dev/null +++ b/simdem-env/bin/pip3 @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python3.6 + +# -*- coding: utf-8 -*- +import re +import sys + +from pip import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/pip3.6 b/simdem-env/bin/pip3.6 new file mode 100755 index 0000000..d9963b8 --- /dev/null +++ b/simdem-env/bin/pip3.6 @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python3.6 + +# -*- coding: utf-8 -*- +import re +import sys + +from pip import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/python b/simdem-env/bin/python new file mode 120000 index 0000000..039b719 --- /dev/null +++ b/simdem-env/bin/python @@ -0,0 +1 @@ +python3.6 \ No newline at end of file diff --git a/simdem-env/bin/python-config b/simdem-env/bin/python-config new file mode 100755 index 0000000..f23f039 --- /dev/null +++ b/simdem-env/bin/python-config @@ -0,0 +1,78 @@ +#!/Users/thfalgou/git/simdem/venv/bin/python + +import sys +import getopt +import sysconfig + +valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', + 'ldflags', 'help'] + +if sys.version_info >= (3, 2): + valid_opts.insert(-1, 'extension-suffix') + valid_opts.append('abiflags') +if sys.version_info >= (3, 3): + valid_opts.append('configdir') + + +def exit_with_usage(code=1): + sys.stderr.write("Usage: {0} [{1}]\n".format( + sys.argv[0], '|'.join('--'+opt for opt in valid_opts))) + sys.exit(code) + +try: + opts, args = getopt.getopt(sys.argv[1:], '', valid_opts) +except getopt.error: + exit_with_usage() + +if not opts: + exit_with_usage() + +pyver = sysconfig.get_config_var('VERSION') +getvar = sysconfig.get_config_var + +opt_flags = [flag for (flag, val) in opts] + +if '--help' in opt_flags: + exit_with_usage(code=0) + +for opt in opt_flags: + if opt == '--prefix': + print(sysconfig.get_config_var('prefix')) + + elif opt == '--exec-prefix': + print(sysconfig.get_config_var('exec_prefix')) + + elif opt in ('--includes', '--cflags'): + flags = ['-I' + sysconfig.get_path('include'), + '-I' + sysconfig.get_path('platinclude')] + if opt == '--cflags': + flags.extend(getvar('CFLAGS').split()) + print(' '.join(flags)) + + elif opt in ('--libs', '--ldflags'): + abiflags = getattr(sys, 'abiflags', '') + libs = ['-lpython' + pyver + abiflags] + libs += getvar('LIBS').split() + libs += getvar('SYSLIBS').split() + # add the prefix/lib/pythonX.Y/config dir, but only if there is no + # shared library in prefix/lib/. + if opt == '--ldflags': + if not getvar('Py_ENABLE_SHARED'): + libs.insert(0, '-L' + getvar('LIBPL')) + if not getvar('PYTHONFRAMEWORK'): + libs.extend(getvar('LINKFORSHARED').split()) + print(' '.join(libs)) + + elif opt == '--extension-suffix': + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + if ext_suffix is None: + ext_suffix = sysconfig.get_config_var('SO') + print(ext_suffix) + + elif opt == '--abiflags': + if not getattr(sys, 'abiflags', None): + exit_with_usage() + print(sys.abiflags) + + elif opt == '--configdir': + print(sysconfig.get_config_var('LIBPL')) diff --git a/simdem-env/bin/python3 b/simdem-env/bin/python3 new file mode 120000 index 0000000..039b719 --- /dev/null +++ b/simdem-env/bin/python3 @@ -0,0 +1 @@ +python3.6 \ No newline at end of file diff --git a/simdem-env/bin/python3.6 b/simdem-env/bin/python3.6 new file mode 100755 index 0000000000000000000000000000000000000000..12874ce8175ef5e27e0834e9290154761ae65587 GIT binary patch literal 13068 zcmeHOUu+ab7@wAc2UZLI2}cy%panGMjw{gAipeeX>Rz~Np#>{Y#=W~OcXD@o+1pFG zh@?qrbaQFK6B3_DjEShx7aj;0tO2Ax7z;6mM4%=_vj_-@1`-olzi)Q8-P;!R&6oMY zH{YN6_M4gC%=KaB2)qIjeg9~mf87Qoe0-e<%&<_lmjk}&-(nuz~=R2-tT zkfWtaKB?wtQP*bV3kdf7-07i>^Kl9DEpQExl;-4r$`=&ud@gQid-LIX8-ZOy-0a^~ zBL2#zSV!FbO{})_G`b&yjpW*x$2r=}zZ{M_3xNi)34mP1dhBL3*vCONLBc<_mqCz1 zgph|ocC-*O4)Q_ZZeS$L?=tQ^_T7(eL!QTH>3kS`0x+&QTrO$hTs9d_S8{2vk)8m? zc?`{~|NeYtjdkRsqr=PJA3ORC_!a__MnE6?lbWnm!kwC&R|ZvWPbs`n)s#-PTu94$RxN}E+@A0@MJs`)6c&9k zkbA~B^I)TZqsm?-Ro3NXPN79v&xAvvaHzYYXVd~|4Yhd3wY{^AwI`pOB0z=2ez_()t9;Ueb8++Z)fdJ*6w2M8>~&THqF`r)@Gnhj4oXQLhP}c=Yu?mZ${+W zFP3Fx@EPmcCDfir?E-2WQ5#2X3u>oO!>8M8$54A0wU1GI549txeTdp2(DHIFr>2Nn z)&+Gy$SZkOtH7I9r>x64!G7ft6irjLcA;FrcOyYpg|w1V)1YM(A*JT?av?3`vIRwu zwcX{sQqYB`o3oI_exG|16(Fy~w=aAfTMH8TB)l&pp*{z=5#Ep%0wbO8n`C4JZjsQo1Bft^h2yg^A0vrL307rl$z!BgGa0LDb z1Qsn1zK+8_8l_meo`xHVBH65wgHbHjWySF{3H9Po%zhB#wBRFA+Fj{V@^p(l*r_QB z_%*@n?6NODz`csOX$QO1NT?|}$D&u;?k+i7z>nL&oKD%JYw2p3? zr#F?IlWEgAA>gPz8|YxCV&-{pJ^qv6Iki*A!956`BQWDIYjRp^4~N&aM%J!d3pMaa Fe* Date: Tue, 2 Jan 2018 15:24:57 -0600 Subject: [PATCH 061/167] refactor import statements Signed-off-by: Tommy Falgout --- tests/context.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/context.py b/tests/context.py index cceacc3..68cfa8c 100644 --- a/tests/context.py +++ b/tests/context.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- -import os import sys +import os +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import simdem -from simdem.executor import bash -from simdem.parser import codeblock, context from simdem.render import demo - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from simdem.parser import codeblock,context +from simdem.executor import bash \ No newline at end of file From 6f6b6db9b8415ff2e1d663fbf079a4c0fdc02fba Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 16:12:28 -0600 Subject: [PATCH 062/167] started fixing pylint issues Signed-off-by: Tommy Falgout --- simdem/executor/bash.py | 10 ++++++++-- simdem/parser/codeblock.py | 21 ++++++++++++++++++++- simdem/parser/context.py | 25 ++++++++++++++++--------- tests/test_core.py | 2 -- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py index 81cd4ea..ea39267 100644 --- a/simdem/executor/bash.py +++ b/simdem/executor/bash.py @@ -8,6 +8,10 @@ PEXPECT_CONTINUATION_PROMPT = u'[PEXPECT_PROMPT+' class BashExecutor(object): + """ This class is used to execute Bash commands + Required functions: run_cmd() + """ + _shell = None _env = None @@ -16,11 +20,12 @@ def __init__(self): def run_cmd(self, command): + """ Runs the command passed in the shell + """ + command = command.strip() logging.debug("Execute command: '" + command + "'") - start_time = time.time() response = self.get_shell().run_command(command) - end_time = time.time() # https://pexpect.readthedocs.io/en/stable/overview.html#find-the-end-of-line-cr-lf-conventions # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. This might get tricky if we start supporting windows # This is because to easily write expected testcase output files, most unix-ish text editors write with \n @@ -32,6 +37,7 @@ def get_shell(self): """Gets or creates the shell in which to run commands for the supplied demo """ + if self._shell == None: # Should we use spawn or spawnu? child = pexpect.spawnu('/bin/bash', env=self._env, echo=False, timeout=None) diff --git a/simdem/parser/codeblock.py b/simdem/parser/codeblock.py index 8f72bde..2763115 100644 --- a/simdem/parser/codeblock.py +++ b/simdem/parser/codeblock.py @@ -1,4 +1,6 @@ -import logging +""" This module hosts the CodeBlockParser +""" + from collections import defaultdict from mistletoe import Document @@ -7,11 +9,20 @@ # Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 class CodeBlockParser(BaseRenderer): + """ This class parses Markdown documents into SimDem Execution Objects + This class expects all SimDem primitives to be parsed from code blocks, + and uses the code block language to know which type of execution it is + """ def __init__(self): super().__init__() self.output = defaultdict(list) def render_block_code(self, token): + """ Implemented as part of Mistletoe interface + Given this object parses only code blocks, this is + the only one with relevant logic. The code block language + is parsed to determine what type of object the code underneath is + """ lines = token.children[0].content.splitlines() if 'prerequisite' in token.language: self.output['prerequisites'].extend(lines) @@ -24,13 +35,21 @@ def render_block_code(self, token): return '' def render_document(self, token): + """ Implemented as part of the Mistletoe interface + """ self.render_inner(token) return dict(self.output) def __getattr__(self, name): + """ Implemented as part of the Mistletoe interface + Added to prevent Mistletoe from throwing warnings for missing render functions + """ return lambda token: '' def parse_file(self, file_path): + """ Implemented as part of SimDem interface + Renders the contents of the file path using this object as a Mistletoe renderer + """ with open(file_path, 'r') as fin: with CodeBlockParser() as renderer: rendered = renderer.render(Document(fin)) diff --git a/simdem/parser/context.py b/simdem/parser/context.py index 59b99c6..171fd58 100644 --- a/simdem/parser/context.py +++ b/simdem/parser/context.py @@ -1,14 +1,21 @@ +""" This module hosts the ContextParser +""" + import logging -import pprint -import re import mistletoe.ast_renderer as renderer import mistletoe.block_token as token class ContextParser(object): + """ This class parses the human readable markdown using a defined syntax to + know how to create the SimDem Execution Object + and uses the code block language to know which type of execution it is + """ def is_command_block(self, block): + """ Expects a code block with a shell command in it + """ if block['type'] == 'BlockCode' and block['language'] == 'shell': return True return False @@ -20,14 +27,14 @@ def is_result_block(self, block): return True return False - """ - I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now - Will look like this coming out of AST - {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], - 'level': 1, - 'type': 'Heading'}, - """ def is_prerequisite_block(self, block): + """ + I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now + Will look like this coming out of AST + {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], + 'level': 1, + 'type': 'Heading'}, + """ if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ and 'prerequisite' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': return True diff --git a/tests/test_core.py b/tests/test_core.py index 5f1fb50..51ec879 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,8 +4,6 @@ import os.path import unittest -import mistune - from .context import bash, codeblock, demo, simdem From 0b4ae9e47a2d77dd9f7b9879c010f6e867401449 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 22:54:18 -0600 Subject: [PATCH 063/167] Update syntax.md --- docs/syntax.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/syntax.md b/docs/syntax.md index 18164b0..6086618 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -6,11 +6,11 @@ Currently, there are two implementation of Markdown syntax supported. This is similar to SimDem v1's syntax. -[../content/complete-features/context.md](Example Context Based Document) +[Example Context Based Document](../content/complete-features/context.md) ## Codeblock Based -[../content/complete-features/codeblock.md](Example Codeblock Document) +[Example Codeblock Document](../content/complete-features/codeblock.md) ## Features From 8973e232c204125ab88d3e5167315298fe75dac5 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 2 Jan 2018 23:24:48 -0600 Subject: [PATCH 064/167] Update syntax.md --- docs/syntax.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/syntax.md b/docs/syntax.md index 6086618..b48504e 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -2,12 +2,20 @@ Currently, there are two implementation of Markdown syntax supported. -## Context Based +## Context Based (default) This is similar to SimDem v1's syntax. [Example Context Based Document](../content/complete-features/context.md) +Feature | Implementation +--- | --- +Command | \```shell +Prerequisite | `# Prerequisite` followed by list of links +Validation | `# Validation` followed by code block +Result | `# Result` followed by a code block with the result + + ## Codeblock Based [Example Codeblock Document](../content/complete-features/codeblock.md) @@ -16,9 +24,9 @@ This is similar to SimDem v1's syntax. The only sections processed are code blocks and all features are determined by the code block type. -Feature | Context Based | Codeblock Based ---- | --- | --- -Command | \```shell | \```shell -Prerequisite | `# Prerequisite` followed by list of links | \```prerequisite -Validation | `# Validation` followed by code block | \```validation -Result | `# Result` followed by a code block with the result | \```result +Feature | Implementation +--- | --- +Command | \```shell +Prerequisite | \```prerequisite +Validation | \```validation +Result | \```result From 93ad4c898d314bf2719cd8aca8a86a54c00cb7a9 Mon Sep 17 00:00:00 2001 From: Ross Gardler Date: Wed, 3 Jan 2018 11:21:15 -0800 Subject: [PATCH 065/167] Add links to existing docs --- docs/syntax.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/syntax.md b/docs/syntax.md index b48504e..26d6834 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -11,12 +11,12 @@ This is similar to SimDem v1's syntax. Feature | Implementation --- | --- Command | \```shell -Prerequisite | `# Prerequisite` followed by list of links -Validation | `# Validation` followed by code block -Result | `# Result` followed by a code block with the result +Prerequisite | `# Prerequisite` followed by natural language text containing links to local or remote SimDem documents taht that should be executed prior to the main body of the current document. See [Prerequisites](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/prerequisites) for more details. +Validation | `# Validation` followed by descriptive natural language and code-blocks to be run as tests prior to running the main content in this document. If all tests pass then there is no need to run the main document. See the [validation](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/prerequisites#validation) section of the prerequisites documentation for more details. +Result | `# Result` followed by a code block with the expected result of the code block immediately before the Results header. See the [testing](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/test) documentation for more details. -## Codeblock Based +## Codeblock Based (Experimental) [Example Codeblock Document](../content/complete-features/codeblock.md) From eb3088078d7a69387f4d7607b4daf710dc44d95b Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 7 Jan 2018 13:05:35 -0600 Subject: [PATCH 066/167] update documentation Signed-off-by: Tommy Falgout --- docs/demo.md | 74 ++++++++++++++++++++++++ docs/feature_prerequisite.md | 108 +++++++++++++++++++++++++++++++++++ docs/feature_result.md | 38 ++++++++++++ docs/feature_validation.md | 71 +++++++++++++++++++++++ docs/features.md | 33 +++++++---- docs/intro.md | 36 ++++++++++++ docs/syntax.md | 32 ++--------- docs/syntax_codeblock.md | 18 ++++++ docs/syntax_context.md | 14 +++++ 9 files changed, 384 insertions(+), 40 deletions(-) create mode 100644 docs/demo.md create mode 100644 docs/feature_prerequisite.md create mode 100644 docs/feature_result.md create mode 100644 docs/feature_validation.md create mode 100644 docs/intro.md create mode 100644 docs/syntax_codeblock.md create mode 100644 docs/syntax_context.md diff --git a/docs/demo.md b/docs/demo.md new file mode 100644 index 0000000..c46c51a --- /dev/null +++ b/docs/demo.md @@ -0,0 +1,74 @@ +# Hello World SimDem Demo + +This script is intended to be used to demonstrate the key features of +Simdem. + +When you hit 'spacebar' a command will be displayed and +executed. Actually you can hit almost any key but we recommend +'spacebar' here because we've not told you about the special keys yet +and spacebar is not one of them. + +``` +echo "Hello World" +``` + +That's cool, lets try again: + +``` +echo "It might look like this was typed into the terminal (even more so if you ran SimDem with the '--style simulate' flag, that simulates a person typing), bit it really comes from a markdown file." +``` + +The date command will show that these commands are being executed in real time. + +``` +date +``` + +Results: + +```expected_similarity=0.3 +Sat Mar 12 08:59:01 UTC 2016 +``` + +You can run almost any shell command this way. + +# Special keys + +Although we said "spacebar" above, in reality you can hit almost any +key. There are a few exceptions though: + +## 'd' for description + +Hitting 'd' will print all the text since the last command, that is it +will print the description of the next command to be executed. + +``` +echo "Hitting 'd' now will display the description for this command." +``` + +## 'b' for break + +Hitting 'b' will "break" from the current script. This allows you to +type in commands that are not part of the script. This is particularly +useful when running in demo mode as it alllows you to respond to +questions by entering an unscripted command. + +``` +echo "Give it a go, why not hit 'b' and type 'ls', or some other command" +``` + +NOTE: at the time of writing it is not possible to use interactive +commands or commands. + +# Next Steps + +It's possible to provide a branching point a the end of a script. The +user can select one of a selection of options or they can enter "quit" +(or just "q") to exit SimDem. + + 1. [Write SimDem documents](../syntax/README.md) + 2. [SimDem Index](../README.md) + 3. [Modes of operation](../modes/README.md) + + + diff --git a/docs/feature_prerequisite.md b/docs/feature_prerequisite.md new file mode 100644 index 0000000..34c3fac --- /dev/null +++ b/docs/feature_prerequisite.md @@ -0,0 +1,108 @@ +# Understanding Prerequisites + +Prerequisite scripts are scripts that must be run in order for another +script to work. When SimDem finds a pre-requisite section it will test +whether the steps have been completed (see validation below). If the +validation tests fail then the code blocks in the pre-requisite script +are executed. + +There aren't really any pre-requisites for this tutorial / +demo. Howeve,r this document is inserted as a pre-requisite so that we +can see how they work. + +# Prerequisites Syntax + +The prerequisites section starts with a heading of `# prerequisites`. + +The body of this section will contain 0 or more links to a script that +should be run ahead of the current one. + +The scripts should appear in the order of required exection in the body. + +# Behavior + +When a prerequisite script is identified SimDem will ask the user if +they have satisfied the requirement. If SimDem is running in test or +auto mode it is assumed that prerequisites have been satisified. + +If the user indicates a prerequisite has been satisfied then execution +moves to the next prerequisite or onto the rest of the script. + +If the user indicates a prereqiusite has not been satisfied then the +required script is executed. + +# Automatically validating Pre-requisites + +Some pre-requisite steps can take a long time to execute. For this +reason it is possible to provide some validation checks to see if the +pre-requisite step has been completed. These are defined in a section +towards the end of the script, before the next steps section (if one +exists). The validation steps will be executed by SimDem *before* +running the pre-requisite steps, if the tests in that section pass +then there is no need to run the pre-requisites. + +It's easier to explain with an example. + +Imagine we have a prerequisite step that takes 5 seconds, we don't +want to wait 5 seconds only to find that we already completed that +pre-requisite (OK, we know 5 seconds is not long, but it's long enough +to serve for this demo). For this example we will merely sleep for 5 +seconds then touch a file. To validate this prequisite has been +satisfied we will test the modified date of the file, if it has been +modified in the last 5 minutes then the pre-requisite has been +satisfied. + +``` +sleep 5 +echo $SIMDEM_TEMP_DIR +mkdir -p $SIMDEM_TEMP_DIR +touch $SIMDEM_TEMP_DIR/this_file_must_be_modfied_every_minute.txt +``` + +Now we have a set of commands that should be executed as part of this +pre-requisite. In order to use them we simply add a reference to this +file in the pre-requisites section of any other script. + +Any code in a section headed with '# Validation' will be used by +SimDem to test whether the pre-requisites have been satisfied. If +validation tests pass the pre-requisite step will be skipped over, +otherwise the other commands in the script will be executed. + +# Validation + +In order to continue with our example we include some validation steps +in this script. If you have not run through the commands above less +than one minute ago this validation stage will fail. If you are +working through this tutorial now you just executed the above +statements and so the tests here will pass, but if you include this +file as pre-requisite again it may well fail and thus automatically +execute this script. + +For this pre-requisite we need to ensure that the test.txt file has +been updated in the last 5 minutes. If not then we need to run the +commands in this document. If you are running through this document in +SimDem itself then it might be worth going back to the page that calls +this as a pre-requisite, as long as you do this in the next five +minutes you won't come back here. You can do this by selecting +"Understanding SimDem Syntax" in the next steps section. + +``` +find $SIMDEM_TEMP_DIR -name "this_file_must_be_modfied_every_minute.txt" -newermt "1 minutes ago" +``` + +Results: + +``` +/home//.simdem/tmp/this_file_must_be_modfied_every_minute.txt +``` + +# Next Steps + + 1. [Understanding SimDem Syntax](../syntax/script.md) + 2. [Configure your scripts through variables](../variables/script.md) + 3. [Build a Hello World script](../tutorial/script.md) + 4. [SimDem Index](../script.md) + 5. [Write multi-part documents](../multipart/script.md) + 6. [Use your documents as interactive tutorials or demos](../running/script.md) + + diff --git a/docs/feature_result.md b/docs/feature_result.md new file mode 100644 index 0000000..c8756e4 --- /dev/null +++ b/docs/feature_result.md @@ -0,0 +1,38 @@ +# Command Testing + +An example of a Result test is: + +``` +echo "This test is expected to fail" +``` + +Results: + +``` +It fails because the results we have in the script are significantly +different to the output of the command. +``` + +By default a 66% or more match indicates a pass. However, in some +cases a much lower similarity is expected, for example, the output of +`date` will vary considerably each time it is run. In these situations +you can provide an expected similarity as part of the three backticks +that start a code block, for example ```Expected_Similarity=0.2 which +is low enough for the test to be recorded as a pass. Note, it is +important that you do not insert any spaces in this notation. + +``` +date +``` + +Results: + +```Expected_Similarity=0.2 +Tue Jun 6 15:23:53 UTC 2017 +``` + +` +preparation/README.md +main/README.md +cleanup/README.md +` diff --git a/docs/feature_validation.md b/docs/feature_validation.md new file mode 100644 index 0000000..c7a17bd --- /dev/null +++ b/docs/feature_validation.md @@ -0,0 +1,71 @@ +# Automated Testing + +When running with the `--test` flag will verify that the output of each command is as expected. It does this by comparing the output of the command with the subsequent `Results:` section in the script. + +``` +cat docs/feature_validation.md +``` + +## Running in test mode + +Running with demo renderer will not check the results against expectations. However, running with the `test` command will do so. + +``` +echo "This test is expected to fail" +``` + +Results: + +``` +It fails because the results we have in the script are significantly +different to the output of the command. +``` + +By default a 66% or more match indicates a pass. However, in some +cases a much lower similarity is expected, for example, the output of +`date` will vary considerably each time it is run. In these situations +you can provide an expected similarity as part of the three backticks +that start a code block, for example ```Expected_Similarity=0.2 which +is low enough for the test to be recorded as a pass. Note, it is +important that you do not insert any spaces in this notation. + +``` +date +``` + +Results: + +```Expected_Similarity=0.2 +Tue Jun 6 15:23:53 UTC 2017 +``` + +# Fast Fail + +The default setting is for SimDem to stop the test run on the first +test failure. This can be overridden by setting the command line flag +`--fastfail` to any value other than `True`. + +# Test Plans + +It is often a good idea to split tests into separate files. SimDem +will allow you to do this by providing a `test_plan.txt` file. Each +line in this file is either a comment (lines starting with '#') or a +filename for a SimDem script to be used in testing. Each of these +files will be concatenated together to create a complete test plan. + +For example, the following example `test_plan.txt` will run all the +code and tests in `preparation/README.md` followed by those in +`main/README.md` and finally those in `cleanup/README.md`. + +` +preparation/README.md +main/README.md +cleanup/README.md +` + +# Next Steps + + 1. [SimDem Index](../README.md) + 2. [Build a Hello World script](../tutorial/README.md) + 3. [Write SimDem documents](../syntax/README.md) + 4. [Configure your scripts through variables](../variables/README.md) diff --git a/docs/features.md b/docs/features.md index 8d796f0..bf00ce1 100644 --- a/docs/features.md +++ b/docs/features.md @@ -1,12 +1,13 @@ # Features -This document is intended to be a list of all features supported in SimDem +This document is intended to be a list of all features supported in SimDem. To see examples on how to write documents that use these features, please see the [syntax documentation](syntad.md). + ## Commands -* Execution - Command extracted from the document can be run in a separate shell +* Execution - Command extracted from the document to be run in a shell * Shell - All command are run in a Linux shell. (PR for Powershell is welcome) * Comments - If command starts with `#`, it will be considered a comment and will not be run -* Validation - Validate if the previous command ran correctly. See validation section for details +* Results - Validate if the previous command ran correctly. See validation section for details * If validation passes - Continue processing * If validation fails - Exit program (default) * Strip ANSI escape sequences @@ -15,18 +16,26 @@ This document is intended to be a list of all features supported in SimDem * Allow the ability to inject environment variables via file or CLI (key/value format) ## Prerequisites - * Allow ability to link to other SimDem documents to run. (e.g. Need Azure CLI setup before creating resource group.) Will be only run once - * Validation - Validate if the prequisite requirements have been met. See validation section for details - * If validation passes - Don't run prerequisite document - * If validation fails - Continue to run prerequsite document +* Allows ability to link to other SimDem documents to run prior to main execution. (e.g. Need Azure CLI setup before creating resource group.) Will be only run once. +* Validation - Validate if the prequisite requirements have been met. See validation section for details + * If validation passes - Don't run prerequisite document + * If validation fails - Continue to run prerequsite document + +For details, see the [prerequisite feature documentation](feature_prerequisite.md) ## Interrupt running * When running interactively, allow the ability to interrupt running document to run manual commands. Resume processing document when complete ## Validation + +Validation is used to verify that the expected result is matched. The two main uses for it are for Command Execution and Prerequisites. + * Allows the output of the previous code block to be evaluated to determine if it meets an expected result. Validation is either passed/failed. - * Validation types - * Expected Result Similarity - e.g. .80 match - * Result Exact Match - * Result Pattern Match - * Exit code \ No newline at end of file + +### Validation types + * Expected Result Similarity - e.g. .80 match + * Result Exact Match + * Result Pattern Match + * Exit code + +For details, see the [validation feature documentation](feature_validation.md) \ No newline at end of file diff --git a/docs/intro.md b/docs/intro.md new file mode 100644 index 0000000..3f50f52 --- /dev/null +++ b/docs/intro.md @@ -0,0 +1,36 @@ +# Welcome to SimDem + +SimDem is: + + * Documentation + * An interactive tutorial + * A live demo + * An automated test script + * A Shell script + +## SimDem overview + +Simdem allows you to wite a tutorial in markdown format and then run the commands as a simulated demo, interactive tutorial or even a test script. You can also generate executable shell scripts. + +SimDem reads a Markdown file and executes the code block commands embedded within. It can even look like you are really typing the commands, which is great if you want to concentrate on explaining what you are doing but still run the demo live. + +It's easier to understand through example. If you are viewing this inside the interactive demo, press a key (other than 'b', we'll look at that shortly) to "type" a command, once the command has been "typed" hit a key to execute the command. + +# Next Steps + +Tutorials can branch too, for example you can choose any of the following paths next: + + 1. [Modes of operation](modes/README.md) + 2. [Hello World Demo](demo/README.md) + 3. [Build a Hello World script](tutorial/README.md) + 4. [Write SimDem documents](syntax/README.md) + 5. [Special Commands](special_commands/README.md) + 6. [Configure your scripts through variables](variables/README.md) + 7. [Write multi-part documents](multipart/README.md) + 8. [Use your documents as interactive tutorials or demos](running/README.md) + 9. [Use your documents as automated tests](test/README.md) + 10. [Build an SimDem container](building/README.md) + + + + diff --git a/docs/syntax.md b/docs/syntax.md index b48504e..dc4c437 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -1,32 +1,8 @@ # Syntax -Currently, there are two implementation of Markdown syntax supported. - -## Context Based (default) - -This is similar to SimDem v1's syntax. - -[Example Context Based Document](../content/complete-features/context.md) - -Feature | Implementation ---- | --- -Command | \```shell -Prerequisite | `# Prerequisite` followed by list of links -Validation | `# Validation` followed by code block -Result | `# Result` followed by a code block with the result +The design of SimDem is to allow multiple implementations of Markdown to support different use cases and documentation patterns. +Currently, there are two implementation of Markdown syntax supported. -## Codeblock Based - -[Example Codeblock Document](../content/complete-features/codeblock.md) - -## Features - -The only sections processed are code blocks and all features are determined by the code block type. - -Feature | Implementation ---- | --- -Command | \```shell -Prerequisite | \```prerequisite -Validation | \```validation -Result | \```result +* [Context (default)](syntax_context.md) +* [Codeblock](syntax_codeblock.md) \ No newline at end of file diff --git a/docs/syntax_codeblock.md b/docs/syntax_codeblock.md new file mode 100644 index 0000000..e32074f --- /dev/null +++ b/docs/syntax_codeblock.md @@ -0,0 +1,18 @@ +# Syntax + +This is the syntax for the alternative codeblock format + +## Codeblock Based + +[Example Codeblock Document](../content/complete-features/codeblock.md) + +## Features + +The only sections processed are code blocks and all features are determined by the code block type. + +Feature | Implementation +--- | --- +Command | \```shell +[Prerequisite](feature_prerequisite.md) | \```prerequisite +[Validation](feature_validation.md) | \```validation +Result | \```result diff --git a/docs/syntax_context.md b/docs/syntax_context.md new file mode 100644 index 0000000..4d56a85 --- /dev/null +++ b/docs/syntax_context.md @@ -0,0 +1,14 @@ +# Syntax + +This is the syntax for the default codeblock format. It's design is to allow more natural, expressive, and readable documentation. It is based off of SimDem v1's syntax + +## Context Based (default) + +[Example Context Based Document](../content/complete-features/context.md) + +Feature | Implementation +--- | --- +Command | \```shell +Prerequisite | `# Prerequisite` followed by natural language text containing links to local or remote SimDem documents taht that should be executed prior to the main body of the current document. See [Prerequisites](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/prerequisites) for more details. +Validation | `# Validation` followed by descriptive natural language and code-blocks to be run as tests prior to running the main content in this document. If all tests pass then there is no need to run the main document. See the [validation](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/prerequisites#validation) section of the prerequisites documentation for more details. +Result | `# Result` followed by a code block with the expected result of the code block immediately before the Results header. See the [testing](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/test) documentation for more details. \ No newline at end of file From f27704556d96d64714213a112447ba8a7768f9a7 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 7 Jan 2018 14:37:17 -0600 Subject: [PATCH 067/167] Implemented context parser for next_steps Signed-off-by: Tommy Falgout --- content/complete-features/context.md | 8 ++++++++ simdem/core.py | 5 ----- simdem/executor/bash.py | 1 - simdem/parser/codeblock.py | 1 + simdem/parser/context.py | 20 ++++++++++++++++++++ tests/test_parser_context.py | 18 +++++++++--------- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/content/complete-features/context.md b/content/complete-features/context.md index ea7c9b7..795e38f 100644 --- a/content/complete-features/context.md +++ b/content/complete-features/context.md @@ -27,3 +27,11 @@ We assume the result is for the last command of the last code block baz ``` +# Next Steps + +The list inside this block are steps that could be followed when performing an interactive tutorial + + 1. [Step #1](step-1.md) + 1. [Step #2](step-2.md) + + diff --git a/simdem/core.py b/simdem/core.py index 073d422..b6d012e 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- import difflib import logging -import re - -from . import executor - class Core(object): @@ -51,7 +47,6 @@ def process_prereqs(self, prereqs): def run_blocks(self, commands): logging.info("run_blocks():blocks=" + str(commands)) - results_latest = None for command in commands: logging.info("run_blocks():processing " + str(command)) result = self.run_code_block(command['command']) diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py index ea39267..28d1b92 100644 --- a/simdem/executor/bash.py +++ b/simdem/executor/bash.py @@ -1,5 +1,4 @@ import logging -import time import pexpect from pexpect import replwrap diff --git a/simdem/parser/codeblock.py b/simdem/parser/codeblock.py index 2763115..77caabe 100644 --- a/simdem/parser/codeblock.py +++ b/simdem/parser/codeblock.py @@ -5,6 +5,7 @@ from mistletoe import Document from mistletoe.base_renderer import BaseRenderer +import logging # Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 diff --git a/simdem/parser/context.py b/simdem/parser/context.py index 171fd58..81085c0 100644 --- a/simdem/parser/context.py +++ b/simdem/parser/context.py @@ -27,6 +27,15 @@ def is_result_block(self, block): return True return False + def is_next_step_block(self, block): + """ This block is similiar to "choose your own adventure" games + This feature allows you to determine which document you want to process after you complete the current one + """ + if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ + and 'next step' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': + return True + return False + def is_prerequisite_block(self, block): """ I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now @@ -75,6 +84,17 @@ def parse_file(self, file_path): if line: res['commands'].append({ 'command': line }) + elif self.is_next_step_block(block): + logging.debug("parse_file():is_next_step_block") + block = blocks[idx] + # Fast-forward to find the list block inside this header block. + # Maybe we should also test to verify we haven't gone past the header block? + while 'List' not in block['type'] and idx < len(blocks): + idx = idx + 1 + block = blocks[idx] + print(block) + res['next_steps'] = [{'target': x['target'], 'title': x['children'][0]['content']} for x in block['children'][0]['children'] if x['type'] == 'Link'] + else: logging.info("get_commands():unknown_block. Ignoring") diff --git a/tests/test_parser_context.py b/tests/test_parser_context.py index c89e7f5..40e63e2 100644 --- a/tests/test_parser_context.py +++ b/tests/test_parser_context.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- +"""SimDem Test Case""" import configparser -import os.path import unittest -import mistune - -from .context import context, simdem - +from .context import context class MistletoeParserTestSuite(unittest.TestCase): """Advanced test cases.""" @@ -15,7 +12,6 @@ class MistletoeParserTestSuite(unittest.TestCase): parser = None def setUp(self): - self.maxDiff = None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") @@ -27,9 +23,13 @@ def test_full(self): exp_res = { 'prerequisites': ['prereq.md', 'prereq-2.md'], 'commands': [ - { 'command': 'echo foo' }, - { 'command': 'echo bar' }, - { 'command': 'echo baz', 'expected_result': 'baz' } ] + {'command': 'echo foo'}, + {'command': 'echo bar'}, + {'command': 'echo baz', 'expected_result': 'baz'}], + 'next_steps': [ + {'title': 'Step #1', 'target': 'step-1.md'}, + {'title': 'Step #2', 'target': 'step-2.md'} + ] } self.assertEqual(res, exp_res) From 729ecdd483006616d69480d45983f2edf8eb5340 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 7 Jan 2018 23:43:30 -0600 Subject: [PATCH 068/167] fix pylint issues Signed-off-by: Tommy Falgout --- simdem/core.py | 31 +++++++++++++++++--------- simdem/executor/__init__.py | 2 +- simdem/executor/bash.py | 14 +++++++----- simdem/parser/codeblock.py | 6 ++--- simdem/parser/context.py | 43 ++++++++++++++++++++++-------------- simdem/render/demo.py | 4 +++- tests/test_mistletoe.py | 25 +++++++++++---------- tests/test_parser_context.py | 1 + 8 files changed, 77 insertions(+), 49 deletions(-) diff --git a/simdem/core.py b/simdem/core.py index b6d012e..8b60ca9 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -1,8 +1,13 @@ # -*- coding: utf-8 -*- +"""Contains only core SimDem object""" + import difflib import logging class Core(object): + """The core glue for SimDem is here. It uses dependency injection so that the + implementation of the renderer, parser and executor are left to the classes passed in. + """ renderer = None config = None @@ -16,10 +21,12 @@ def __init__(self, config, renderer, parser, executor): self.executor = executor def run_code_block(self, cmd_block): - # In the future, we'll want to split a code segment into individual lines - # For now, assume just one command in a block - # Returning the latest result so we can validate the result. - # We might want to validate the result of the entire block, but for now, validate just the last run command + """In the future, we'll want to split a code segment into individual lines + For now, assume just one command in a block + Returning the latest result so we can validate the result. + We might want to validate the result of the entire block, but for now, + validate just the last run command + """ result_latest = None for cmd in cmd_block.split("\n"): result_latest = self.run_cmd(cmd) @@ -57,8 +64,8 @@ def run_blocks(self, commands): logging.error("Result did not pass") return - - def is_result_valid(self, expected_results, actual_results, expected_similarity = 1.0): + @staticmethod + def is_result_valid(expected_results, actual_results, expected_similarity=1.0): """Checks to see if a command execution passes. If actual results compared to expected results is within the expected similarity level then it's considered a pass. @@ -72,12 +79,16 @@ def is_result_valid(self, expected_results, actual_results, expected_similarity logging.error("is_result_valid(): actual_results is empty.") return False - logging.debug("is_result_valid(" + expected_results + "," + actual_results + "," + str(expected_similarity) + ")") + logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ + "," + str(expected_similarity) + ")") expected_results_str = expected_results.rstrip() actual_results_str = actual_results.rstrip() - logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + "," + str(expected_similarity) + ")") - seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", actual_results_str, expected_results_str) + logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ + "," + str(expected_similarity) + ")") + seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", + actual_results_str, + expected_results_str) is_pass = seq.ratio() >= expected_similarity @@ -88,4 +99,4 @@ def is_result_valid(self, expected_results, actual_results, expected_similarity logging.error("actual_results = " + actual_results) logging.error("expected_results = " + expected_results) - return is_pass + return is_pass diff --git a/simdem/executor/__init__.py b/simdem/executor/__init__.py index 98e1905..e988b77 100644 --- a/simdem/executor/__init__.py +++ b/simdem/executor/__init__.py @@ -1 +1 @@ -from .bash import BashExecutor \ No newline at end of file +from .bash import BashExecutor diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py index 28d1b92..cb9ade1 100644 --- a/simdem/executor/bash.py +++ b/simdem/executor/bash.py @@ -10,7 +10,7 @@ class BashExecutor(object): """ This class is used to execute Bash commands Required functions: run_cmd() """ - + _shell = None _env = None @@ -21,13 +21,15 @@ def __init__(self): def run_cmd(self, command): """ Runs the command passed in the shell """ - + command = command.strip() logging.debug("Execute command: '" + command + "'") response = self.get_shell().run_command(command) # https://pexpect.readthedocs.io/en/stable/overview.html#find-the-end-of-line-cr-lf-conventions - # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. This might get tricky if we start supporting windows - # This is because to easily write expected testcase output files, most unix-ish text editors write with \n + # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. + # This might get tricky if we start supporting windows + # This is because to easily write expected testcase output files, + # most unix-ish text editors write with \n response = response.replace("\r\n", "\n") logging.debug("Response: '" + response + "'") return response @@ -36,8 +38,8 @@ def get_shell(self): """Gets or creates the shell in which to run commands for the supplied demo """ - - if self._shell == None: + + if self._shell is None: # Should we use spawn or spawnu? child = pexpect.spawnu('/bin/bash', env=self._env, echo=False, timeout=None) ps1 = PEXPECT_PROMPT[:5] + u'\[\]' + PEXPECT_PROMPT[5:] diff --git a/simdem/parser/codeblock.py b/simdem/parser/codeblock.py index 77caabe..0c17da3 100644 --- a/simdem/parser/codeblock.py +++ b/simdem/parser/codeblock.py @@ -1,11 +1,10 @@ """ This module hosts the CodeBlockParser """ +import logging from collections import defaultdict - from mistletoe import Document from mistletoe.base_renderer import BaseRenderer -import logging # Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 @@ -47,7 +46,8 @@ def __getattr__(self, name): """ return lambda token: '' - def parse_file(self, file_path): + @staticmethod + def parse_file(file_path): """ Implemented as part of SimDem interface Renders the contents of the file path using this object as a Mistletoe renderer """ diff --git a/simdem/parser/context.py b/simdem/parser/context.py index 81085c0..bbb8519 100644 --- a/simdem/parser/context.py +++ b/simdem/parser/context.py @@ -19,40 +19,48 @@ def is_command_block(self, block): if block['type'] == 'BlockCode' and block['language'] == 'shell': return True return False - + def is_result_block(self, block): - # This is different than previous SimDem because it didn't require a language for the result. - # I believe this approach is more declarative. + """ This is different than previous SimDem because it didn't require + a language for the result. + I believe this approach is more declarative. + """ if block['type'] == 'BlockCode' and block['language'] == 'result': return True return False def is_next_step_block(self, block): - """ This block is similiar to "choose your own adventure" games - This feature allows you to determine which document you want to process after you complete the current one + """ This block is similiar to "choose your own adventure" games + This feature allows you to determine which document you want to process + after you complete the current one """ - if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ - and 'next step' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': + if 'children' in block and len(block['children']) \ + and 'content' in block['children'][0] \ + and 'next step' in block['children'][0]['content'].lower() \ + and block['type'].lower() == 'heading': return True return False def is_prerequisite_block(self, block): """ - I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now + I'm not a fan of denoting prerequisites by using a header title, + but that will suffice for now Will look like this coming out of AST {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], 'level': 1, 'type': 'Heading'}, """ - if 'children' in block and len(block['children']) and 'content' in block['children'][0] \ - and 'prerequisite' in block['children'][0]['content'].lower() and block['type'].lower() == 'heading': + if 'children' in block and len(block['children']) \ + and 'content' in block['children'][0] \ + and 'prerequisite' in block['children'][0]['content'].lower() \ + and block['type'].lower() == 'heading': return True return False def parse_file(self, file_path): with open(file_path, 'r') as fin: ast = renderer.get_ast(token.Document(fin)) - + res = { 'prerequisites': [], 'commands': [] @@ -64,10 +72,11 @@ def parse_file(self, file_path): while idx < len(blocks): block = blocks[idx] logging.debug("parse_file():processing " + str(block)) - + if self.is_prerequisite_block(block): logging.debug("parse_file():found preqreq block") - res['prerequisites'] = [x['children'][0]['target'] for x in blocks[idx+1]['children']] + res['prerequisites'] = \ + [x['children'][0]['target'] for x in blocks[idx+1]['children']] # No need to process the next block since that's the prereqs idx = idx + 1 @@ -81,8 +90,8 @@ def parse_file(self, file_path): elif self.is_command_block(block): logging.debug("parse_file():is_command_block") for line in block['children'][0]['content'].split("\n"): - if line: - res['commands'].append({ 'command': line }) + if line: + res['commands'].append({'command': line}) elif self.is_next_step_block(block): logging.debug("parse_file():is_next_step_block") @@ -93,7 +102,9 @@ def parse_file(self, file_path): idx = idx + 1 block = blocks[idx] print(block) - res['next_steps'] = [{'target': x['target'], 'title': x['children'][0]['content']} for x in block['children'][0]['children'] if x['type'] == 'Link'] + res['next_steps'] = [{'target': x['target'], 'title': x['children'][0]['content']} + for x in block['children'][0]['children'] + if x['type'] == 'Link'] else: logging.info("get_commands():unknown_block. Ignoring") diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 44fcbfa..8982762 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -26,5 +26,7 @@ def type_command(self, cmd): print(char, end="", flush=True) print("", flush=True) - def display_result(self, res): + @staticmethod + def display_result(res): + """Demo specific implementation of displaying to the screen""" print(res, end="", flush=True) diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index b10194f..0c81b90 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +"""SimDem Test Case""" import unittest -import mistletoe import mistletoe.ast_renderer as renderer import mistletoe.block_token as token @@ -11,25 +11,26 @@ class SimDemMistletoeTestSuite(unittest.TestCase): """Lexer test cases.""" def test_ast(self): - self.maxDiff = None + """Verify we understand how the Mistletoe AST parsers work""" + file_path = 'content/simple/README.md' with open(file_path, 'r') as fin: res = renderer.get_ast(token.Document(fin)) exp_res = {'children': [{'children': [{'content': 'this is text', 'type': 'RawText'}], - 'type': 'Paragraph'}, + 'type': 'Paragraph'}, {'children': [{'content': 'echo foo\necho bar\n', - 'type': 'RawText'}], - 'language': 'shell', - 'type': 'BlockCode'}, + 'type': 'RawText'}], + 'language': 'shell', + 'type': 'BlockCode'}, {'children': [{'content': 'more text', 'type': 'RawText'}], - 'type': 'Paragraph'}, + 'type': 'Paragraph'}, {'children': [{'content': 'echo baz\n', 'type': 'RawText'}], - 'language': 'shell', - 'type': 'BlockCode'}, + 'language': 'shell', + 'type': 'BlockCode'}, {'children': [{'content': 'even more text', 'type': 'RawText'}], - 'type': 'Paragraph'}], - 'footnotes': {}, - 'type': 'Document'} + 'type': 'Paragraph'}], + 'footnotes': {}, + 'type': 'Document'} self.assertEqual(res, exp_res) diff --git a/tests/test_parser_context.py b/tests/test_parser_context.py index 40e63e2..ea345d3 100644 --- a/tests/test_parser_context.py +++ b/tests/test_parser_context.py @@ -18,6 +18,7 @@ def setUp(self): self.parser = context.ContextParser() def test_full(self): + """Test parsing a document with all features in it""" file_path = 'content/complete-features/context.md' res = self.parser.parse_file(file_path) exp_res = { From 67000f4e29eb91c0b2271c54ba6ece21fa55f7ba Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 8 Jan 2018 11:02:43 -0600 Subject: [PATCH 069/167] Fix pylint issues Signed-off-by: Tommy Falgout --- simdem/core.py | 4 ++++ simdem/executor/__init__.py | 1 + simdem/executor/bash.py | 12 ++++++++---- simdem/parser/__init__.py | 3 ++- simdem/parser/context.py | 22 +++++++++++++++------- simdem/render/__init__.py | 3 ++- simdem/render/demo.py | 8 ++++++-- tests/test_core.py | 13 +++++++++---- tests/test_system.py | 34 ++++++++++++++++++++-------------- 9 files changed, 67 insertions(+), 33 deletions(-) diff --git a/simdem/core.py b/simdem/core.py index 8b60ca9..875ba5a 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -33,12 +33,14 @@ def run_code_block(self, cmd_block): return result_latest def run_cmd(self, cmd): + """ prints out the command, runs it and then displays the results """ self.renderer.type_command(cmd) res = self.executor.run_cmd(cmd) self.renderer.display_result(res) return res def process_file(self, file_path): + """ Parses the file and then runs any prerequisites """ blocks = self.parser.parse_file(file_path) logging.info("process_file():blocks=" + str(blocks)) if 'prerequisites' in blocks: @@ -48,11 +50,13 @@ def process_file(self, file_path): return result def process_prereqs(self, prereqs): + """ Loops through each of the prerequisites """ logging.info("process_preqreqs():" + str(prereqs)) for prereq_file in prereqs: self.process_file(prereq_file) def run_blocks(self, commands): + """ For each block, run and validate the output """ logging.info("run_blocks():blocks=" + str(commands)) for command in commands: logging.info("run_blocks():processing " + str(command)) diff --git a/simdem/executor/__init__.py b/simdem/executor/__init__.py index e988b77..767b2f0 100644 --- a/simdem/executor/__init__.py +++ b/simdem/executor/__init__.py @@ -1 +1,2 @@ +""" Import each of the executors """ from .bash import BashExecutor diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py index cb9ade1..b7af93f 100644 --- a/simdem/executor/bash.py +++ b/simdem/executor/bash.py @@ -1,3 +1,5 @@ +""" This file contains the BashExecutor object """ + import logging import pexpect @@ -40,10 +42,12 @@ def get_shell(self): """ if self._shell is None: - # Should we use spawn or spawnu? + # Should we use spawn or spawnu? + # The prompts used to be u'\[\]', but pylint prefers r'\[\]'. + # Noting just in case this bites us later child = pexpect.spawnu('/bin/bash', env=self._env, echo=False, timeout=None) - ps1 = PEXPECT_PROMPT[:5] + u'\[\]' + PEXPECT_PROMPT[5:] - ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + u'\[\]' + PEXPECT_CONTINUATION_PROMPT[5:] + ps1 = PEXPECT_PROMPT[:5] + r'\[\]' + PEXPECT_PROMPT[5:] + ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + r'\[\]' + PEXPECT_CONTINUATION_PROMPT[5:] prompt_change = u"PS1='{0}' PS2='{1}' PROMPT_COMMAND=''".format(ps1, ps2) - self._shell = pexpect.replwrap.REPLWrapper(child, u'\$', prompt_change) + self._shell = replwrap.REPLWrapper(child, r'\$', prompt_change) return self._shell diff --git a/simdem/parser/__init__.py b/simdem/parser/__init__.py index b03f17d..993f899 100644 --- a/simdem/parser/__init__.py +++ b/simdem/parser/__init__.py @@ -1,2 +1,3 @@ +""" Import each of the parsers. """ from .codeblock import CodeBlockParser -from .context import ContextParser \ No newline at end of file +from .context import ContextParser diff --git a/simdem/parser/context.py b/simdem/parser/context.py index bbb8519..a0aea1b 100644 --- a/simdem/parser/context.py +++ b/simdem/parser/context.py @@ -8,19 +8,21 @@ class ContextParser(object): - """ This class parses the human readable markdown using a defined syntax to + """ This class parses the human readable markdown using a defined syntax to know how to create the SimDem Execution Object and uses the code block language to know which type of execution it is """ - def is_command_block(self, block): + @staticmethod + def is_command_block(block): """ Expects a code block with a shell command in it """ if block['type'] == 'BlockCode' and block['language'] == 'shell': return True return False - def is_result_block(self, block): + @staticmethod + def is_result_block(block): """ This is different than previous SimDem because it didn't require a language for the result. I believe this approach is more declarative. @@ -29,19 +31,21 @@ def is_result_block(self, block): return True return False - def is_next_step_block(self, block): + @staticmethod + def is_next_step_block(block): """ This block is similiar to "choose your own adventure" games This feature allows you to determine which document you want to process after you complete the current one """ - if 'children' in block and len(block['children']) \ + if 'children' in block and block['children'] \ and 'content' in block['children'][0] \ and 'next step' in block['children'][0]['content'].lower() \ and block['type'].lower() == 'heading': return True return False - def is_prerequisite_block(self, block): + @staticmethod + def is_prerequisite_block(block): """ I'm not a fan of denoting prerequisites by using a header title, but that will suffice for now @@ -50,7 +54,7 @@ def is_prerequisite_block(self, block): 'level': 1, 'type': 'Heading'}, """ - if 'children' in block and len(block['children']) \ + if 'children' in block and block['children'] \ and 'content' in block['children'][0] \ and 'prerequisite' in block['children'][0]['content'].lower() \ and block['type'].lower() == 'heading': @@ -58,6 +62,10 @@ def is_prerequisite_block(self, block): return False def parse_file(self, file_path): + """ The main meat for parsing the file. Uses mistletoe's AST parser + to create a tokenized object and then parses that tokenized object + into SimDem Execution Object + """ with open(file_path, 'r') as fin: ast = renderer.get_ast(token.Document(fin)) diff --git a/simdem/render/__init__.py b/simdem/render/__init__.py index 3c2343c..97224c7 100644 --- a/simdem/render/__init__.py +++ b/simdem/render/__init__.py @@ -1 +1,2 @@ -from .demo import Demo \ No newline at end of file +""" Import each of the renderers """ +from .demo import Demo diff --git a/simdem/render/demo.py b/simdem/render/demo.py index 8982762..d3b699a 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -1,13 +1,17 @@ +"""Demo (default) renderer for SimDem""" + import random import time class Demo(object): + """ This class is used to render the output of the commands into something + that looks like you're typing + """ config = None def __init__(self, config): self.config = config - pass def type_command(self, cmd): """ @@ -17,7 +21,7 @@ def type_command(self, cmd): # Must add ' ' when typing command because whitespaces are removed from configparser # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ', end="", flush=True) - for idx, char in enumerate(cmd): + for _, char in enumerate(cmd): if char != "\n": typing_delay = float(self.config.get('RENDER', 'TYPING_DELAY')) if typing_delay: diff --git a/tests/test_core.py b/tests/test_core.py index 51ec879..0123593 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +""" Advanced test cases for SimDem """ import configparser import os.path @@ -15,16 +16,20 @@ class SimDemTestSuite(unittest.TestCase): markdown_parser = None def setUp(self): - os.remove(self.test_file) if os.path.exists(self.test_file) else None + if os.path.exists(self.test_file): + os.remove(self.test_file) config = configparser.ConfigParser() config.read('content/config/unit_test.ini') self.markdown_parser = codeblock.CodeBlockParser() - self.simdem = simdem.Core(config, demo.Demo(config), self.markdown_parser, bash.BashExecutor()) + self.simdem = simdem.Core(config, demo.Demo(config), + self.markdown_parser, bash.BashExecutor()) def test_run_cmd(self): - self.assertEquals("foobar\n", self.simdem.run_cmd('echo foobar')) - + """ Validate running a command only prints out the result """ + self.assertEqual("foobar\n", self.simdem.run_cmd('echo foobar')) + def test_run_blocks(self): + """ Validate running a document that creates a file actually creates it """ self.assertFalse(os.path.exists(self.test_file)) blocks = self.markdown_parser.parse_file('content/create-file/README.md') self.simdem.run_blocks(blocks['commands']) diff --git a/tests/test_system.py b/tests/test_system.py index a0cb021..5d36ec0 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- +""" system level test class """ import configparser import logging -import os import sys import unittest -import mistune from ddt import data, ddt from .context import demo, simdem @@ -22,23 +21,30 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), simdem.parser.CodeBlockParser(), simdem.executor.BashExecutor()) + self.simdem = simdem.Core(config, demo.Demo(config), + simdem.parser.CodeBlockParser(), simdem.executor.BashExecutor()) - logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) - rootLogger = logging.getLogger() - rootLogger.setLevel(config.get('LOG', 'LEVEL')) - fileHandler = logging.FileHandler(config.get('LOG', 'FILE')) - fileHandler.setFormatter(logFormatter) - rootLogger.addHandler(fileHandler) + log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + root_logger = logging.getLogger() + root_logger.setLevel(config.get('LOG', 'LEVEL')) + file_handler = logging.FileHandler(config.get('LOG', 'FILE')) + file_handler.setFormatter(log_formatter) + root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer # @data('prerequisite-run') - @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'create-file', 'prerequisite-run') - def test_process(self, dir): - self.simdem.process_file('./content/' + dir + '/README.md') + @data('simple', 'simple-variable', 'results-block', + 'results-block-fail', 'create-file', 'prerequisite-run') + def test_process(self, directory): + """ Each content directory is expected to have a README.md and an expected_result.out + this allows us to test each of them easily + """ + self.simdem.process_file('./content/' + directory + '/README.md') + # TODO: Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. + # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. res = sys.stdout.getvalue() - exp_res = open('./content/' + dir + '/expected_result.out', 'r').read() - self.assertEquals(exp_res, res) + exp_res = open('./content/' + directory + '/expected_result.out', 'r').read() + self.assertEqual(exp_res, res) if __name__ == '__main__': From 65a958ab017e382e3a2825e81f4324019161133c Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 8 Jan 2018 11:04:29 -0600 Subject: [PATCH 070/167] Disable pylint error. Signed-off-by: Tommy Falgout --- tests/test_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_system.py b/tests/test_system.py index 5d36ec0..83782e2 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -40,9 +40,9 @@ def test_process(self, directory): this allows us to test each of them easily """ self.simdem.process_file('./content/' + directory + '/README.md') - # TODO: Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. + # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. - res = sys.stdout.getvalue() + res = sys.stdout.getvalue() # pylint: disable=E1101 exp_res = open('./content/' + directory + '/expected_result.out', 'r').read() self.assertEqual(exp_res, res) From 0002b678b3af69d2a51a2786db5d77264fae9190 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 8 Jan 2018 11:11:24 -0600 Subject: [PATCH 071/167] Add next_steps to codeblock parser Signed-off-by: Tommy Falgout --- content/complete-features/codeblock.md | 8 ++++++++ simdem/parser/codeblock.py | 4 ++++ tests/test_parser_codeblock.py | 16 ++++++++++------ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/content/complete-features/codeblock.md b/content/complete-features/codeblock.md index bf941ae..80ff378 100644 --- a/content/complete-features/codeblock.md +++ b/content/complete-features/codeblock.md @@ -28,3 +28,11 @@ We assume the result is for the last command of the last code block baz ``` +# Process these steps nexts + +The list inside this block are steps that could be followed when performing an interactive tutorial + +```next_steps +step-1.md +step-2.md +``` \ No newline at end of file diff --git a/simdem/parser/codeblock.py b/simdem/parser/codeblock.py index 0c17da3..767a46f 100644 --- a/simdem/parser/codeblock.py +++ b/simdem/parser/codeblock.py @@ -30,6 +30,10 @@ def render_block_code(self, token): self.output['commands'][-1]['expected_result'] = lines[0] elif 'shell' in token.language: self.output['commands'].extend({'command': line} for line in lines) + elif 'next_steps' in token.language: + self.output['next_steps'].extend({'title': line, 'target': line} + for line in lines + if line != '```') else: logging.info("get_commands():unknown_block. Ignoring") return '' diff --git a/tests/test_parser_codeblock.py b/tests/test_parser_codeblock.py index 2d06679..658c0a8 100644 --- a/tests/test_parser_codeblock.py +++ b/tests/test_parser_codeblock.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- +""" Test Codeblock Parser """ import configparser -import os.path import unittest -from .context import codeblock, simdem +from .context import codeblock class MistletoeParserTestSuite(unittest.TestCase): @@ -13,21 +13,25 @@ class MistletoeParserTestSuite(unittest.TestCase): renderer = None def setUp(self): - self.maxDiff = None config = configparser.ConfigParser() config.read("content/config/unit_test.ini") self.renderer = codeblock.CodeBlockParser() def test_full(self): + """ Test the full featureset of codeblock parser """ file_path = 'content/complete-features/codeblock.md' res = self.renderer.parse_file(file_path) exp_res = { 'prerequisites': ['prereq.md', 'prereq-2.md'], 'commands': [ - { 'command': 'echo foo' }, - { 'command': 'echo bar' }, - { 'command': 'echo baz', 'expected_result': 'baz' } ] + {'command': 'echo foo'}, + {'command': 'echo bar'}, + {'command': 'echo baz', 'expected_result': 'baz'}], + 'next_steps': [ + {'title': 'step-1.md', 'target': 'step-1.md'}, + {'title': 'step-2.md', 'target': 'step-2.md'} + ] } self.assertEqual(res, exp_res) From 80c6249414e05a1f45139f32b8cb14140b8c6e60 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 8 Jan 2018 13:00:30 -0600 Subject: [PATCH 072/167] Fix CLI after major core refactoring Signed-off-by: Tommy Falgout --- content/next-steps/README.md | 13 ++++ content/next-steps/expected_result.out | 5 ++ content/next-steps/step-1.md | 4 + content/next-steps/step-2.md | 4 + main.py | 100 ++++++++++++++----------- simdem/core.py | 7 ++ simdem/render/demo.py | 15 ++++ 7 files changed, 103 insertions(+), 45 deletions(-) create mode 100644 content/next-steps/README.md create mode 100644 content/next-steps/expected_result.out create mode 100644 content/next-steps/step-1.md create mode 100644 content/next-steps/step-2.md diff --git a/content/next-steps/README.md b/content/next-steps/README.md new file mode 100644 index 0000000..7409350 --- /dev/null +++ b/content/next-steps/README.md @@ -0,0 +1,13 @@ +this is text + +```shell +echo main +``` + +# Next steps + +The list inside this block are steps that could be followed when performing an interactive tutorial + + 1. [Step #1](step-1.md) + 1. [Step #2](step-2.md) + diff --git a/content/next-steps/expected_result.out b/content/next-steps/expected_result.out new file mode 100644 index 0000000..f538837 --- /dev/null +++ b/content/next-steps/expected_result.out @@ -0,0 +1,5 @@ +main + +Which step do you want to take next? +1.) Step #1 +2.) Step #2 \ No newline at end of file diff --git a/content/next-steps/step-1.md b/content/next-steps/step-1.md new file mode 100644 index 0000000..b07c629 --- /dev/null +++ b/content/next-steps/step-1.md @@ -0,0 +1,4 @@ + +```shell +echo step-1 +``` \ No newline at end of file diff --git a/content/next-steps/step-2.md b/content/next-steps/step-2.md new file mode 100644 index 0000000..9e5fc4d --- /dev/null +++ b/content/next-steps/step-2.md @@ -0,0 +1,4 @@ + +```shell +echo step-2 +``` \ No newline at end of file diff --git a/main.py b/main.py index 54eb18f..517d58d 100755 --- a/main.py +++ b/main.py @@ -1,26 +1,35 @@ #!/usr/local/bin/python3 +""" Entrypoint to Simdem """ +import configparser +import logging +import argparse +import os + from simdem import core +from simdem.executor import bash +from simdem.parser import codeblock, context from simdem.render import demo -import os -import sys -import mistune -import logging -import optparse -import configparser + def main(): - p = optparse.OptionParser("%prog file", version="%prog 1.0") - p.add_option('--debug', '-d', action="store_true", - help="Turn on logging to console") - p.add_option('--config-file', '-f', default="content/config/demo.ini", - help="Config file to use") - p.add_option('--render', '-r', default="demo", - help="Render class to use") - p.add_option('--lexer', '-l', default="mistune.BlockLexer", - help="Lexer class to use") - options, arguments = p.parse_args() - - validate_error = validate(options, arguments) + """ Main execution function """ + argp = argparse.ArgumentParser() + argp.add_argument('file', metavar='file', + help='file to process') + argp.add_argument('--debug', '-d', action="store_true", + help="Turn on logging to console") + argp.add_argument('--config-file', '-c', default="content/config/demo.ini", + help="Config file to use") + argp.add_argument('--renderer', '-r', default="demo", + help="Render class to use", choices=['demo']) + argp.add_argument('--parser', '-p', default="context", + help="Parser class to use", choices=['context', 'codeblock']) + argp.add_argument('--executor', '-e', default="bash", + help="Executor class to use", choices=['bash']) + options = argp.parse_args() + print(options) + file_path = options.file + validate_error = validate(options, file_path) if validate_error: print(validate_error) exit(1) @@ -30,48 +39,49 @@ def main(): setup_logging(config, options) - simdem = core.Core(config, get_render(options, config), get_lexer(options)) + simdem = core.Core(config, get_render(options, config), + get_parser(options), get_executor(options)) - file_path = arguments[0] simdem.process_file(file_path) -def validate(options, arguments): - if len(arguments) != 1: - return "Must provide one and only one argument: " + str(arguments) - - file_path = arguments[0] +def validate(options, file_path): + """ validate all passed in arguments """ if not os.path.isfile(file_path): return "Unable to find file: " + file_path if not os.path.isfile(options.config_file): return "Unable to find config file: " + options.config_file - if options.render not in ['demo']: - return "Unknown Render: " + options.render - - if options.lexer not in ['mistune.BlockLexer']: - return "Unknown Lexer: " + options.lexer - def get_render(options, config): - if options.render == 'demo': + """ Returns correct renderer object """ + if options.renderer == 'demo': return demo.Demo(config) -def get_lexer(options): - if options.lexer == 'mistune.BlockLexer': - return mistune.BlockLexer() +def get_parser(options): + """ Returns correct parser object """ + if options.parser == 'codeblock': + return codeblock.CodeBlockParser() + elif options.parser == 'context': + return context.ContextParser() + +def get_executor(options): + """ Returns correct executor object """ + if options.executor == 'bash': + return bash.BashExecutor() def setup_logging(config, options): - logFormatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) - rootLogger = logging.getLogger() - rootLogger.setLevel(config.get('LOG', 'LEVEL')) + """ Establishes logging level and format """ + log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + root_logger = logging.getLogger() + root_logger.setLevel(config.get('LOG', 'LEVEL')) - fileHandler = logging.FileHandler(config.get('LOG', 'FILE')) - fileHandler.setFormatter(logFormatter) - rootLogger.addHandler(fileHandler) + file_handler = logging.FileHandler(config.get('LOG', 'FILE')) + file_handler.setFormatter(log_formatter) + root_logger.addHandler(file_handler) if options.debug: - consoleHandler = logging.StreamHandler() - consoleHandler.setFormatter(logFormatter) - rootLogger.addHandler(consoleHandler) + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_formatter) + root_logger.addHandler(console_handler) -main() \ No newline at end of file +main() diff --git a/simdem/core.py b/simdem/core.py index 875ba5a..66444fd 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -68,6 +68,13 @@ def run_blocks(self, commands): logging.error("Result did not pass") return + def show_next_steps(self, steps): + """ If there's any next steps to take, let the renderer process them """ + logging.info("run_blocks():steps=" + str(steps)) + step_request_target = self.renderer.get_next_step(steps) + if step_request_target: + self.process_file(step_request_target) + @staticmethod def is_result_valid(expected_results, actual_results, expected_similarity=1.0): """Checks to see if a command execution passes. diff --git a/simdem/render/demo.py b/simdem/render/demo.py index d3b699a..a12c55e 100644 --- a/simdem/render/demo.py +++ b/simdem/render/demo.py @@ -34,3 +34,18 @@ def type_command(self, cmd): def display_result(res): """Demo specific implementation of displaying to the screen""" print(res, end="", flush=True) + + @staticmethod + def show_next_steps(steps): + """ Is there a good way to test this that doesn't involve lots of test code + expect? + """ + idx = 1 + if steps: + print("Next steps available:") + for step in steps: + print(idx + ".) " + step['title']) + idx += 1 + step_request = input("Which step do you want to take next?") + if step_request: + return steps[step_request+1]['target'] + return From b37ee950d07faf0a0d31d7d3ae3399a7df05345b Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 10 Jan 2018 21:09:20 -0600 Subject: [PATCH 073/167] rename run_blocks Signed-off-by: Tommy Falgout --- simdem/core.py | 8 ++++---- tests/test_core.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/simdem/core.py b/simdem/core.py index 66444fd..30491b9 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -46,7 +46,7 @@ def process_file(self, file_path): if 'prerequisites' in blocks: self.process_prereqs(blocks['prerequisites']) logging.info("process_file():completed process_prereqs()") - result = self.run_blocks(blocks['commands']) + result = self.run_command_blocks(blocks['commands']) return result def process_prereqs(self, prereqs): @@ -55,11 +55,11 @@ def process_prereqs(self, prereqs): for prereq_file in prereqs: self.process_file(prereq_file) - def run_blocks(self, commands): + def run_command_blocks(self, commands): """ For each block, run and validate the output """ - logging.info("run_blocks():blocks=" + str(commands)) + logging.info("run_command_blocks():blocks=" + str(commands)) for command in commands: - logging.info("run_blocks():processing " + str(command)) + logging.info("run_command_blocks():processing " + str(command)) result = self.run_code_block(command['command']) if 'expected_result' in command: if self.is_result_valid(command['expected_result'], result): diff --git a/tests/test_core.py b/tests/test_core.py index 0123593..fd6bc5a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -32,7 +32,7 @@ def test_run_blocks(self): """ Validate running a document that creates a file actually creates it """ self.assertFalse(os.path.exists(self.test_file)) blocks = self.markdown_parser.parse_file('content/create-file/README.md') - self.simdem.run_blocks(blocks['commands']) + self.simdem.run_command_blocks(blocks['commands']) self.assertTrue(os.path.exists(self.test_file)) if __name__ == '__main__': From 0699494bd92d74764ce176e16aa92e61c85fe80f Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 11 Jan 2018 22:02:41 -0600 Subject: [PATCH 074/167] Major refactor. Created modes and default demo mode. Signed-off-by: Tommy Falgout --- content/simdem1/README.md | 53 ++++++++++++ content/simdem1/prereq-ignored.md | 31 +++++++ content/simdem1/prereq-processed.md | 31 +++++++ main.py | 38 +++++--- simdem-env/bin/autopep8 | 11 +++ simdem-env/bin/pycodestyle | 11 +++ simdem/core.py | 10 +-- simdem/mode/debug.py | 18 ++++ simdem/mode/demo.py | 129 ++++++++++++++++++++++++++++ simdem/parser/ast.py | 21 +++++ simdem/parser/simdem1.py | 119 +++++++++++++++++++++++++ tests/context.py | 2 +- tests/test_parser_simdem1.py | 64 ++++++++++++++ 13 files changed, 518 insertions(+), 20 deletions(-) create mode 100644 content/simdem1/README.md create mode 100644 content/simdem1/prereq-ignored.md create mode 100644 content/simdem1/prereq-processed.md create mode 100755 simdem-env/bin/autopep8 create mode 100755 simdem-env/bin/pycodestyle create mode 100644 simdem/mode/debug.py create mode 100644 simdem/mode/demo.py create mode 100644 simdem/parser/ast.py create mode 100644 simdem/parser/simdem1.py create mode 100644 tests/test_parser_simdem1.py diff --git a/content/simdem1/README.md b/content/simdem1/README.md new file mode 100644 index 0000000..fdebd58 --- /dev/null +++ b/content/simdem1/README.md @@ -0,0 +1,53 @@ +# Prerequisites + +This is the prerequisite section. SimDem looks for a set of links to extract and run through first + +* [prereq-ignored](content/simdem1/prereq-ignored.md) + +They don't even need to be in the same list + +* [prereq-processed](content/simdem1/prereq-processed.md) + +By this point, the prerequisites have either run or have passed their validation + +# Did our prerequisites run? + +```shell +echo prereq_ignored = $prereq_ignored +echo prereq_processed = $prereq_processed +``` + +# Do stuff here + +We want to execute this because the code type is shell + +```shell +echo foo +echo bar +var=foo +``` + + +# Do more stuff here + +```shell +echo baz +``` + +# Results + +The only thing that makes it a result is the code type is result. +We assume the result is for the last command of the last code block + +```result +baz +``` + +# Next Steps + +The list inside this block are steps that could be followed when performing an interactive tutorial + + 1. [Step #1](step-1.md) + 1. [Step #2](step-2.md) + + diff --git a/content/simdem1/prereq-ignored.md b/content/simdem1/prereq-ignored.md new file mode 100644 index 0000000..0566eab --- /dev/null +++ b/content/simdem1/prereq-ignored.md @@ -0,0 +1,31 @@ +# WE ARE IN prereq-ignored.md + +This file is designed to have the validation pass. This means that we should stop executing this document after the results section + +# Validation + +This is a validation section. If this validation section passes, we stop processing this file. The command below should be the last text displayed + +``` +echo prereq_validation_pass +``` + +Results: + +``` +prereq_validation_pass +``` + +# Main area + +This should never be run since the validation for the prereq has been met + +``` +echo YOU SHOULD NOT SEE THIS +``` + +# Set a variable that passes through + +``` +prereq_ignored=true +``` diff --git a/content/simdem1/prereq-processed.md b/content/simdem1/prereq-processed.md new file mode 100644 index 0000000..019053e --- /dev/null +++ b/content/simdem1/prereq-processed.md @@ -0,0 +1,31 @@ +# WE ARE IN prereq-processed.md + +This file is designed to have the validation fail. This means that we should completely execute this file + +# Validation + +This is a validation section. If this validation section passes, we stop processing this file + +``` +echo prereq_validation_fail +``` + +Results: + +``` +blah +``` + +# Main area + +This should be run since the validation for the prereq has failed + +``` +echo YOU SHOULD SEE THIS +``` + +# Set a variable that passes through + +``` +prereq_processed=true +``` diff --git a/main.py b/main.py index 517d58d..7c6d245 100755 --- a/main.py +++ b/main.py @@ -7,8 +7,10 @@ from simdem import core from simdem.executor import bash -from simdem.parser import codeblock, context -from simdem.render import demo +from simdem.parser import ast, simdem1 +from simdem.render import demo as demo_r +from simdem.mode import demo as demo_m +from simdem.mode import debug as debug_m def main(): @@ -22,12 +24,14 @@ def main(): help="Config file to use") argp.add_argument('--renderer', '-r', default="demo", help="Render class to use", choices=['demo']) - argp.add_argument('--parser', '-p', default="context", - help="Parser class to use", choices=['context', 'codeblock']) + argp.add_argument('--mode', '-m', default="demo", + help="Mode to use", choices=['demo', 'debug']) + argp.add_argument('--parser', '-p', default="simdem1", + help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", help="Executor class to use", choices=['bash']) options = argp.parse_args() - print(options) + file_path = options.file validate_error = validate(options, file_path) if validate_error: @@ -39,9 +43,9 @@ def main(): setup_logging(config, options) - simdem = core.Core(config, get_render(options, config), - get_parser(options), get_executor(options)) - +# simdem = core.Core(config, get_render(options, config), +# get_parser(options), get_executor(options), get_mode(options, config)) + simdem = get_mode(options, config) simdem.process_file(file_path) def validate(options, file_path): @@ -55,14 +59,22 @@ def validate(options, file_path): def get_render(options, config): """ Returns correct renderer object """ if options.renderer == 'demo': - return demo.Demo(config) + return demo_r.Demo(config) + +def get_mode(options, config): + """ Returns correct renderer object """ + if options.mode == 'demo': + return demo_m.DemoMode(config, get_executor(options), get_parser(options)) + + if options.mode == 'debug': + return debug_m.DebugMode(config, get_parser(options)) def get_parser(options): """ Returns correct parser object """ - if options.parser == 'codeblock': - return codeblock.CodeBlockParser() - elif options.parser == 'context': - return context.ContextParser() + if options.parser == 'ast': + return ast.AstParser() + elif options.parser == 'simdem1': + return simdem1.SimDem1Parser() def get_executor(options): """ Returns correct executor object """ diff --git a/simdem-env/bin/autopep8 b/simdem-env/bin/autopep8 new file mode 100755 index 0000000..9af5c95 --- /dev/null +++ b/simdem-env/bin/autopep8 @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/simdem-env/bin/python + +# -*- coding: utf-8 -*- +import re +import sys + +from autopep8 import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/simdem-env/bin/pycodestyle b/simdem-env/bin/pycodestyle new file mode 100755 index 0000000..a532603 --- /dev/null +++ b/simdem-env/bin/pycodestyle @@ -0,0 +1,11 @@ +#!/Users/thfalgou/git/simdem/simdem-env/bin/python + +# -*- coding: utf-8 -*- +import re +import sys + +from pycodestyle import _main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(_main()) diff --git a/simdem/core.py b/simdem/core.py index 30491b9..4975643 100644 --- a/simdem/core.py +++ b/simdem/core.py @@ -13,12 +13,14 @@ class Core(object): config = None parser = None executor = None + processor = None - def __init__(self, config, renderer, parser, executor): + def __init__(self, config, renderer, parser, executor, processor): self.config = config self.renderer = renderer self.parser = parser self.executor = executor + self.processor = processor def run_code_block(self, cmd_block): """In the future, we'll want to split a code segment into individual lines @@ -43,11 +45,7 @@ def process_file(self, file_path): """ Parses the file and then runs any prerequisites """ blocks = self.parser.parse_file(file_path) logging.info("process_file():blocks=" + str(blocks)) - if 'prerequisites' in blocks: - self.process_prereqs(blocks['prerequisites']) - logging.info("process_file():completed process_prereqs()") - result = self.run_command_blocks(blocks['commands']) - return result + return self.processor.process(blocks) def process_prereqs(self, prereqs): """ Loops through each of the prerequisites """ diff --git a/simdem/mode/debug.py b/simdem/mode/debug.py new file mode 100644 index 0000000..c698f45 --- /dev/null +++ b/simdem/mode/debug.py @@ -0,0 +1,18 @@ +"""Demo (default) renderer for SimDem""" + +import pprint + +class DebugMode(object): + """ This class is used to render the output of the commands into something + that looks like you're typing + """ + config = None + parser = None + + def __init__(self, config, parser): + self.config = config + self.parser = parser + + def process_file(self, file_path): + steps = self.parser.parse_file(file_path) + pprint.pprint(steps) diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py new file mode 100644 index 0000000..09c4073 --- /dev/null +++ b/simdem/mode/demo.py @@ -0,0 +1,129 @@ +"""Demo (default) renderer for SimDem""" + +import random +import time +import logging +import difflib + +class DemoMode(object): + """ This class is used to render the output of the commands into something + that looks like you're typing + """ + config = None + executor = None + parser = None + + def __init__(self, config, executor, parser): + self.config = config + self.executor = executor + self.parser = parser + + def process_file(self, file_path): + print("*** Processing " + file_path + " ***") + steps = self.parser.parse_file(file_path) + self.process(steps) + + def process(self, steps): + last_command_result = None + + for step in steps: + if step['type'] == 'heading': + self.process_heading(step) + elif step['type'] == 'text': + self.process_text(step) + elif step['type'] == 'commands': + self.process_commands(step) + elif step['type'] == 'result': + self.process_result(step) + elif step['type'] == 'prerequisites': + for prereq_file in step['content']: + self.process_file(prereq_file) + elif step['type'] == 'validation_command': + last_command_result = self.process_commands(step) + elif step['type'] == 'validation_result': + if self.is_result_valid(step['content'], last_command_result): + logging.info("Result passed") + print('***VALIDATION PASSED***') + return + else: + logging.error("Result did not pass") + print('***VALIDATION FAILED***') + + + def process_heading(self, step): + print(step['level'] * '#' + ' ' + step['content']) + print() + + def process_text(self, step): + print(step['content']) + print() + + def process_commands(self, step): + for cmd in step['content']: + self.type_command(cmd) + results = self.executor.run_cmd(cmd) + self.display_result(results) + print() + return results + + def process_result(self, step): + print('CHECKING RESULT') + print('RES= ' + step['content']) + print() + + def type_command(self, cmd): + """ Displays the command on the screen """ + + # Must add ' ' when typing command because whitespaces are removed from configparser + # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure + print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ', end="", flush=True) + for _, char in enumerate(cmd): + if char != "\n": + typing_delay = None #float(self.config.get('RENDER', 'TYPING_DELAY')) + if typing_delay: + delay = random.uniform(0.02, typing_delay) + time.sleep(delay) + print(char, end="", flush=True) + print("", flush=True) + + @staticmethod + def display_result(res): + """Demo specific implementation of displaying to the screen""" + print(res, end="", flush=True) + + @staticmethod + def is_result_valid(expected_results, actual_results, expected_similarity=1.0): + """Checks to see if a command execution passes. + If actual results compared to expected results is within + the expected similarity level then it's considered a pass. + + expected_similarity = 1.0 could be a breaking change for older SimDem scripts. + explicit fails > implicit passes + Ross may disagree with me. Let's see how this story unfolds. + """ + + if not actual_results: + logging.error("is_result_valid(): actual_results is empty.") + return False + + logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ + "," + str(expected_similarity) + ")") + + expected_results_str = expected_results.rstrip() + actual_results_str = actual_results.rstrip() + logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ + "," + str(expected_similarity) + ")") + seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", + actual_results_str, + expected_results_str) + + is_pass = seq.ratio() >= expected_similarity + + if is_pass: + logging.info("is_result_valid passed") + else: + logging.error("is_result_valid failed") + logging.error("actual_results = " + actual_results) + logging.error("expected_results = " + expected_results) + + return is_pass \ No newline at end of file diff --git a/simdem/parser/ast.py b/simdem/parser/ast.py new file mode 100644 index 0000000..96fd0f3 --- /dev/null +++ b/simdem/parser/ast.py @@ -0,0 +1,21 @@ +""" This module hosts the ContextParser +""" + +import mistletoe.ast_renderer as renderer +import mistletoe.block_token + + +class AstParser(object): + """ This class parses the human readable markdown using a defined syntax to + know how to create the SimDem Execution Object + and uses the code block language to know which type of execution it is + """ + + def parse_file(self, file_path): + """ The main meat for parsing the file. Uses mistletoe's AST parser + to create a tokenized object and then parses that tokenized object + into SimDem Execution Object + """ + with open(file_path, 'r') as fin: + ast = renderer.get_ast(mistletoe.block_token.Document(fin)) + return ast diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py new file mode 100644 index 0000000..6103d25 --- /dev/null +++ b/simdem/parser/simdem1.py @@ -0,0 +1,119 @@ +""" This module hosts the ContextParser +""" + +import logging + +import mistletoe.ast_renderer as renderer +import mistletoe.block_token + + +class SimDem1Parser(object): + """ This class parses the human readable markdown using a defined syntax to + know how to create the SimDem Execution Object + and uses the code block language to know which type of execution it is + """ + + mode = None + + def set_mode(self, mode): + logging.debug('set_mode(' + str(mode) + ')') + self.mode = mode + + def parse_file(self, file_path): + """ The main meat for parsing the file. Uses mistletoe's AST parser + to create a tokenized object and then parses that tokenized object + into SimDem Execution Object + """ + with open(file_path, 'r') as fin: + ast = renderer.get_ast(mistletoe.block_token.Document(fin)) + res = [] + + for token in ast['children']: + logging.debug("parse_file():processing " + str(token)) + + # Heading + if token['type'] == 'Heading': + res.append({'type': 'heading', 'level': token['level'], + 'content': token['children'][0]['content']}) + logging.debug("parse_file():found heading") + + # Prerequisite Heading + if token['children'][0]['content'].lower() == 'prerequisites': + self.set_mode('prerequisites') + + # Next Steps Heading + elif token['children'][0]['content'].lower() == 'next steps': + self.set_mode('next_steps') + + # Results Heading + elif token['children'][0]['content'].lower() == 'result': + self.set_mode('result') + + # Validation Heading + elif token['children'][0]['content'].lower() == 'validation': + self.set_mode('validation') + + else: + logging.debug("parse_file():unable to determing header type.") + self.set_mode(None) + + # Prerequisite List + elif token['type'] == 'List' and self.mode == 'prerequisites': + logging.debug("parse_file():found prerequisites list") + titles = [x['children'][0]['children'][0]['content'] for x in token['children']] + content_t = "\n".join(' * ' + x for x in titles) + # Make sure to print out the page contents too + res.append({'type': 'text', 'content': content_t}) + + content_p = [x['children'][0]['target'] for x in token['children']] + res.append({'type': 'prerequisites', 'content': content_p}) + + # Paragraph + elif token['type'] == 'Paragraph': + logging.debug("parse_file():found paragraph") + if token['children']: + content = token['children'][0]['content'] + # Result Paragraph + if content == 'Results:': + if self.mode == 'validation': + self.set_mode('validation_result') + else: + self.set_mode('result') + #res.append({'type': 'result', 'content': content}) + # Regular Paragraph + else: + res.append({'type': 'text', 'content': content}) + + elif token['type'] == 'BlockCode': + # Result Block Code + if self.mode == 'validation': + logging.debug("parse_file():found validation block code") + content = [line for line in token['children'][0]['content'].split("\n") if line] + res.append({'type': 'validation_command', 'content': content}) + + # Validation Result Block Code + elif self.mode == 'validation_result': + logging.debug("parse_file():found validation result block code") + content = token['children'][0]['content'].rstrip() + res.append({'type': 'validation_result', 'content': content}) + + # Result Block Code + elif self.mode == 'result': + logging.debug("parse_file():found result block code") + content = token['children'][0]['content'].rstrip() + res.append({'type': 'result', 'content': content}) + + # Block Code (regular) + elif self.mode is None: + logging.debug("parse_file():found block code") + content = [line for line in token['children'][0]['content'].split("\n") if line] + res.append({'type': 'commands', 'content': content}) + + else: + logging.debug('parse_file():unexpected Block Code mode (' + self.mode + ')') + + else: + logging.debug('parse_file():unknown token type (type=' + token['type'] + + ';mode=' + self.mode + '). Igorning') + + return res diff --git a/tests/context.py b/tests/context.py index 68cfa8c..f42c5f5 100644 --- a/tests/context.py +++ b/tests/context.py @@ -6,5 +6,5 @@ import simdem from simdem.render import demo -from simdem.parser import codeblock,context +from simdem.parser import codeblock,context,simdem1 from simdem.executor import bash \ No newline at end of file diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py new file mode 100644 index 0000000..b1feb48 --- /dev/null +++ b/tests/test_parser_simdem1.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +"""SimDem Test Case""" + +import configparser +import unittest +import pprint +from .context import simdem1 + +class SimDem1ParserTestSuite(unittest.TestCase): + """Advanced test cases.""" + + parser = None + + def setUp(self): + config = configparser.ConfigParser() + config.read("content/config/unit_test.ini") + + self.parser = simdem1.SimDem1Parser() + + def test_full(self): + """Test parsing a document with all features in it""" + self.maxDiff = None + file_path = 'content/simdem1/README.md' + res = self.parser.parse_file(file_path) + + exp_res = [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, + {'content': 'This is the prerequisite section. SimDem looks for a set of ' + 'links to extract and run through first', + 'type': 'text'}, + {'content': ' * prereq-validation-pass', 'type': 'text'}, + {'content': ['prereq-validaiton-pass.md'], 'type': 'prerequisite'}, + {'content': "They don't even need to be in the same list", 'type': 'text'}, + {'content': ' * prereq-validation-fail', 'type': 'text'}, + {'content': ['prereq-validaiton-fail.md'], 'type': 'prerequisite'}, + {'content': 'By this point, the prerequisites have either run or have passed ' + 'their validation', + 'type': 'text'}, + {'content': 'Do stuff here', 'level': 1, 'type': 'heading'}, + {'content': 'We want to execute this because the code type is shell', + 'type': 'text'}, + {'content': ['echo foo', 'echo bar', 'var=foo'], 'type': 'commands'}, + {'content': 'Validation', 'level': 1, 'type': 'heading'}, + {'content': 'This is a validation section. If this validation section ' + 'passes, we stop processing this file', + 'type': 'text'}, + {'content': 'Results:', 'type': 'text'}, + {'content': 'Do more stuff here', 'level': 1, 'type': 'heading'}, + {'content': ['echo baz'], 'type': 'commands'}, + {'content': 'Results', 'level': 1, 'type': 'heading'}, + {'content': 'The only thing that makes it a result is the code type is ' + 'result. We assume the result is for the last command of the last ' + 'code block', + 'type': 'text'}, + {'content': 'baz', 'type': 'result'}, + {'content': 'Next Steps', 'level': 1, 'type': 'heading'}, + {'content': 'The list inside this block are steps that could be followed when ' + 'performing an interactive tutorial', + 'type': 'text'}] + pprint.pprint(res) + self.assertEqual(res, exp_res) + + +if __name__ == '__main__': + unittest.main() From aa12f041954a1fc6ff17e46792a2010cbd35db54 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 11 Jan 2018 22:18:06 -0600 Subject: [PATCH 075/167] Updated for clearer debugging Signed-off-by: Tommy Falgout --- content/simdem1/README.md | 13 ++++++------- simdem/mode/demo.py | 16 +++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/content/simdem1/README.md b/content/simdem1/README.md index fdebd58..4192c87 100644 --- a/content/simdem1/README.md +++ b/content/simdem1/README.md @@ -23,24 +23,23 @@ We want to execute this because the code type is shell ```shell echo foo -echo bar -var=foo +var=bar ``` # Do more stuff here +We assume the result is for the last command of the last code block + ```shell echo baz +echo $var ``` -# Results - -The only thing that makes it a result is the code type is result. -We assume the result is for the last command of the last code block +Results: ```result -baz +bar ``` # Next Steps diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index 09c4073..c4a699e 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -22,6 +22,7 @@ def process_file(self, file_path): print("*** Processing " + file_path + " ***") steps = self.parser.parse_file(file_path) self.process(steps) + print("*** Completed Processing " + file_path + " ***") def process(self, steps): last_command_result = None @@ -32,9 +33,12 @@ def process(self, steps): elif step['type'] == 'text': self.process_text(step) elif step['type'] == 'commands': - self.process_commands(step) + last_command_result = self.process_commands(step) elif step['type'] == 'result': - self.process_result(step) + if self.is_result_valid(step['content'], last_command_result): + print('***VALIDATION PASSED***') + else: + print('***VALIDATION FAILED***') elif step['type'] == 'prerequisites': for prereq_file in step['content']: self.process_file(prereq_file) @@ -42,11 +46,9 @@ def process(self, steps): last_command_result = self.process_commands(step) elif step['type'] == 'validation_result': if self.is_result_valid(step['content'], last_command_result): - logging.info("Result passed") print('***VALIDATION PASSED***') return else: - logging.error("Result did not pass") print('***VALIDATION FAILED***') @@ -66,8 +68,8 @@ def process_commands(self, step): print() return results - def process_result(self, step): - print('CHECKING RESULT') + def process_result(self, ): + print('***CHECKING RESULT***') print('RES= ' + step['content']) print() @@ -126,4 +128,4 @@ def is_result_valid(expected_results, actual_results, expected_similarity=1.0): logging.error("actual_results = " + actual_results) logging.error("expected_results = " + expected_results) - return is_pass \ No newline at end of file + return is_pass From 39317d3c33545bf3302ee9211512311e737604bf Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 12 Jan 2018 16:31:20 -0600 Subject: [PATCH 076/167] Fix testcases after mode refactor Signed-off-by: Tommy Falgout --- ...pected_result.out => expected_result.demo} | 1 + content/prerequisite-run/README.md | 6 +- content/prerequisite-run/expected_result.demo | 16 ++ content/prerequisite-run/expected_result.out | 4 - content/results-block-fail/README.md | 5 +- .../results-block-fail/expected_result.demo | 14 ++ .../results-block-fail/expected_result.out | 4 - content/results-block/expected_result.demo | 9 + content/results-block/expected_result.out | 4 - content/simple-variable/expected_result.demo | 9 + content/simple-variable/expected_result.out | 3 - ...pected_result.out => expected_result.demo} | 8 + main.py | 24 +-- simdem/__init__.py | 1 - simdem/core.py | 111 ------------ simdem/mode/demo.py | 64 ++++--- simdem/mode/{debug.py => dump.py} | 10 +- simdem/parser/ast.py | 5 +- simdem/parser/simdem1.py | 167 ++++++++++-------- simdem/render/__init__.py | 2 - simdem/render/demo.py | 51 ------ tests/context.py | 10 -- tests/test_bash.py | 22 +++ tests/test_core.py | 39 ---- tests/{test_system.py => test_demo_mode.py} | 12 +- tests/test_parser_codeblock.py | 41 ----- tests/test_parser_context.py | 40 ----- tests/test_parser_simdem1.py | 48 +++-- 28 files changed, 263 insertions(+), 467 deletions(-) rename content/create-file/{expected_result.out => expected_result.demo} (98%) create mode 100644 content/prerequisite-run/expected_result.demo delete mode 100644 content/prerequisite-run/expected_result.out create mode 100644 content/results-block-fail/expected_result.demo delete mode 100644 content/results-block-fail/expected_result.out create mode 100644 content/results-block/expected_result.demo delete mode 100644 content/results-block/expected_result.out create mode 100644 content/simple-variable/expected_result.demo delete mode 100644 content/simple-variable/expected_result.out rename content/simple/{expected_result.out => expected_result.demo} (51%) delete mode 100644 simdem/core.py rename simdem/mode/{debug.py => dump.py} (56%) delete mode 100644 simdem/render/__init__.py delete mode 100644 simdem/render/demo.py delete mode 100644 tests/context.py create mode 100644 tests/test_bash.py delete mode 100644 tests/test_core.py rename tests/{test_system.py => test_demo_mode.py} (82%) delete mode 100644 tests/test_parser_codeblock.py delete mode 100644 tests/test_parser_context.py diff --git a/content/create-file/expected_result.out b/content/create-file/expected_result.demo similarity index 98% rename from content/create-file/expected_result.out rename to content/create-file/expected_result.demo index ae4b462..dd8004e 100644 --- a/content/create-file/expected_result.out +++ b/content/create-file/expected_result.demo @@ -1,3 +1,4 @@ $ echo 'abc' > scratch/foo $ cat scratch/foo abc + diff --git a/content/prerequisite-run/README.md b/content/prerequisite-run/README.md index 5b1088d..7f84242 100644 --- a/content/prerequisite-run/README.md +++ b/content/prerequisite-run/README.md @@ -1,8 +1,6 @@ -# Prerequisite +# Prerequisites -```prerequisites -content/prerequisite-run/prereq.md -``` +* [prereq](content/prerequisite-run/prereq.md) # Main diff --git a/content/prerequisite-run/expected_result.demo b/content/prerequisite-run/expected_result.demo new file mode 100644 index 0000000..d045721 --- /dev/null +++ b/content/prerequisite-run/expected_result.demo @@ -0,0 +1,16 @@ +# Prerequisites + + * prereq + +This text should be run + +$ echo running-prereq +running-prereq + +End running text + +# Main + +$ echo post-prereq-run +post-prereq-run + diff --git a/content/prerequisite-run/expected_result.out b/content/prerequisite-run/expected_result.out deleted file mode 100644 index d5991b2..0000000 --- a/content/prerequisite-run/expected_result.out +++ /dev/null @@ -1,4 +0,0 @@ -$ echo running-prereq -running-prereq -$ echo post-prereq-run -post-prereq-run diff --git a/content/results-block-fail/README.md b/content/results-block-fail/README.md index 0025574..5c8918a 100644 --- a/content/results-block-fail/README.md +++ b/content/results-block-fail/README.md @@ -11,9 +11,10 @@ Results: barr ``` -After this, we should not pass +In demo mode, we will continue processing. + ```shell -echo YOU SHALL NOT PASS! +echo post_result_block_fail ``` even more text diff --git a/content/results-block-fail/expected_result.demo b/content/results-block-fail/expected_result.demo new file mode 100644 index 0000000..6e279c2 --- /dev/null +++ b/content/results-block-fail/expected_result.demo @@ -0,0 +1,14 @@ +this is text + +$ echo foo +foo +$ echo bar +bar + +In demo mode, we will continue processing. + +$ echo post_result_block_fail +post_result_block_fail + +even more text + diff --git a/content/results-block-fail/expected_result.out b/content/results-block-fail/expected_result.out deleted file mode 100644 index ac40788..0000000 --- a/content/results-block-fail/expected_result.out +++ /dev/null @@ -1,4 +0,0 @@ -$ echo foo -foo -$ echo bar -bar diff --git a/content/results-block/expected_result.demo b/content/results-block/expected_result.demo new file mode 100644 index 0000000..635d3c6 --- /dev/null +++ b/content/results-block/expected_result.demo @@ -0,0 +1,9 @@ +this is text + +$ echo foo +foo +$ echo bar +bar + +even more text + diff --git a/content/results-block/expected_result.out b/content/results-block/expected_result.out deleted file mode 100644 index ac40788..0000000 --- a/content/results-block/expected_result.out +++ /dev/null @@ -1,4 +0,0 @@ -$ echo foo -foo -$ echo bar -bar diff --git a/content/simple-variable/expected_result.demo b/content/simple-variable/expected_result.demo new file mode 100644 index 0000000..7dbae61 --- /dev/null +++ b/content/simple-variable/expected_result.demo @@ -0,0 +1,9 @@ +this is text + +$ FOO=bar + +$ echo $FOO +bar + +more text + diff --git a/content/simple-variable/expected_result.out b/content/simple-variable/expected_result.out deleted file mode 100644 index 2b21e48..0000000 --- a/content/simple-variable/expected_result.out +++ /dev/null @@ -1,3 +0,0 @@ -$ FOO=bar -$ echo $FOO -bar diff --git a/content/simple/expected_result.out b/content/simple/expected_result.demo similarity index 51% rename from content/simple/expected_result.out rename to content/simple/expected_result.demo index 166ed98..39cb179 100644 --- a/content/simple/expected_result.out +++ b/content/simple/expected_result.demo @@ -1,6 +1,14 @@ +this is text + $ echo foo foo $ echo bar bar + +more text + $ echo baz baz + +even more text + diff --git a/main.py b/main.py index 7c6d245..20f7310 100755 --- a/main.py +++ b/main.py @@ -5,12 +5,9 @@ import argparse import os -from simdem import core from simdem.executor import bash from simdem.parser import ast, simdem1 -from simdem.render import demo as demo_r -from simdem.mode import demo as demo_m -from simdem.mode import debug as debug_m +from simdem.mode import demo, dump def main(): @@ -25,7 +22,7 @@ def main(): argp.add_argument('--renderer', '-r', default="demo", help="Render class to use", choices=['demo']) argp.add_argument('--mode', '-m', default="demo", - help="Mode to use", choices=['demo', 'debug']) + help="Mode to use", choices=['demo', 'dump']) argp.add_argument('--parser', '-p', default="simdem1", help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", @@ -43,10 +40,8 @@ def main(): setup_logging(config, options) -# simdem = core.Core(config, get_render(options, config), -# get_parser(options), get_executor(options), get_mode(options, config)) - simdem = get_mode(options, config) - simdem.process_file(file_path) + mode = get_mode(options, config) + mode.process_file(file_path) def validate(options, file_path): """ validate all passed in arguments """ @@ -56,18 +51,13 @@ def validate(options, file_path): if not os.path.isfile(options.config_file): return "Unable to find config file: " + options.config_file -def get_render(options, config): - """ Returns correct renderer object """ - if options.renderer == 'demo': - return demo_r.Demo(config) - def get_mode(options, config): """ Returns correct renderer object """ if options.mode == 'demo': - return demo_m.DemoMode(config, get_executor(options), get_parser(options)) + return demo.DemoMode(config, get_parser(options), get_executor(options)) - if options.mode == 'debug': - return debug_m.DebugMode(config, get_parser(options)) + if options.mode == 'dump': + return dump.DumpMode(config, get_parser(options)) def get_parser(options): """ Returns correct parser object """ diff --git a/simdem/__init__.py b/simdem/__init__.py index a8d584a..e69de29 100644 --- a/simdem/__init__.py +++ b/simdem/__init__.py @@ -1 +0,0 @@ -from .core import Core \ No newline at end of file diff --git a/simdem/core.py b/simdem/core.py deleted file mode 100644 index 4975643..0000000 --- a/simdem/core.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -"""Contains only core SimDem object""" - -import difflib -import logging - -class Core(object): - """The core glue for SimDem is here. It uses dependency injection so that the - implementation of the renderer, parser and executor are left to the classes passed in. - """ - - renderer = None - config = None - parser = None - executor = None - processor = None - - def __init__(self, config, renderer, parser, executor, processor): - self.config = config - self.renderer = renderer - self.parser = parser - self.executor = executor - self.processor = processor - - def run_code_block(self, cmd_block): - """In the future, we'll want to split a code segment into individual lines - For now, assume just one command in a block - Returning the latest result so we can validate the result. - We might want to validate the result of the entire block, but for now, - validate just the last run command - """ - result_latest = None - for cmd in cmd_block.split("\n"): - result_latest = self.run_cmd(cmd) - return result_latest - - def run_cmd(self, cmd): - """ prints out the command, runs it and then displays the results """ - self.renderer.type_command(cmd) - res = self.executor.run_cmd(cmd) - self.renderer.display_result(res) - return res - - def process_file(self, file_path): - """ Parses the file and then runs any prerequisites """ - blocks = self.parser.parse_file(file_path) - logging.info("process_file():blocks=" + str(blocks)) - return self.processor.process(blocks) - - def process_prereqs(self, prereqs): - """ Loops through each of the prerequisites """ - logging.info("process_preqreqs():" + str(prereqs)) - for prereq_file in prereqs: - self.process_file(prereq_file) - - def run_command_blocks(self, commands): - """ For each block, run and validate the output """ - logging.info("run_command_blocks():blocks=" + str(commands)) - for command in commands: - logging.info("run_command_blocks():processing " + str(command)) - result = self.run_code_block(command['command']) - if 'expected_result' in command: - if self.is_result_valid(command['expected_result'], result): - logging.info("Result passed") - else: - logging.error("Result did not pass") - return - - def show_next_steps(self, steps): - """ If there's any next steps to take, let the renderer process them """ - logging.info("run_blocks():steps=" + str(steps)) - step_request_target = self.renderer.get_next_step(steps) - if step_request_target: - self.process_file(step_request_target) - - @staticmethod - def is_result_valid(expected_results, actual_results, expected_similarity=1.0): - """Checks to see if a command execution passes. - If actual results compared to expected results is within - the expected similarity level then it's considered a pass. - - expected_similarity = 1.0 could be a breaking change for older SimDem scripts. - explicit fails > implicit passes - Ross may disagree with me. Let's see how this story unfolds. - """ - - if not actual_results: - logging.error("is_result_valid(): actual_results is empty.") - return False - - logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ - "," + str(expected_similarity) + ")") - - expected_results_str = expected_results.rstrip() - actual_results_str = actual_results.rstrip() - logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ - "," + str(expected_similarity) + ")") - seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", - actual_results_str, - expected_results_str) - - is_pass = seq.ratio() >= expected_similarity - - if is_pass: - logging.info("is_result_valid passed") - else: - logging.error("is_result_valid failed") - logging.error("actual_results = " + actual_results) - logging.error("expected_results = " + expected_results) - - return is_pass diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index c4a699e..ff29116 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -1,4 +1,4 @@ -"""Demo (default) renderer for SimDem""" +"""Demo (default) mode for SimDem""" import random import time @@ -6,27 +6,31 @@ import difflib class DemoMode(object): - """ This class is used to render the output of the commands into something - that looks like you're typing + """ This class is the default SimDem file processor. + It's designed for running files in a demo-able mode that looks like a human is typing it """ config = None executor = None parser = None - def __init__(self, config, executor, parser): + def __init__(self, config, parser, executor): self.config = config - self.executor = executor self.parser = parser + self.executor = executor def process_file(self, file_path): - print("*** Processing " + file_path + " ***") + """ Parses the file and starts processing it """ + #print("*** Processing " + file_path + " ***") steps = self.parser.parse_file(file_path) self.process(steps) - print("*** Completed Processing " + file_path + " ***") + #print("*** Completed Processing " + file_path + " ***") def process(self, steps): + """ Parses the file and starts processing it """ last_command_result = None + """ I'd like to use a dispatcher for this; however, we need to exit processing + if the validation fails. """ for step in steps: if step['type'] == 'heading': self.process_heading(step) @@ -34,11 +38,11 @@ def process(self, steps): self.process_text(step) elif step['type'] == 'commands': last_command_result = self.process_commands(step) - elif step['type'] == 'result': - if self.is_result_valid(step['content'], last_command_result): - print('***VALIDATION PASSED***') - else: - print('***VALIDATION FAILED***') +# elif step['type'] == 'result': +# if self.is_result_valid(step['content'], last_command_result): +# print('***VALIDATION PASSED***') +# else: +# print('***VALIDATION FAILED***') elif step['type'] == 'prerequisites': for prereq_file in step['content']: self.process_file(prereq_file) @@ -46,21 +50,25 @@ def process(self, steps): last_command_result = self.process_commands(step) elif step['type'] == 'validation_result': if self.is_result_valid(step['content'], last_command_result): - print('***VALIDATION PASSED***') +# print('***VALIDATION PASSED***') return - else: - print('***VALIDATION FAILED***') +# else: +# print('***VALIDATION FAILED***') - - def process_heading(self, step): + @staticmethod + def process_heading(step): + """ Print out the heading exactly as we found it """ print(step['level'] * '#' + ' ' + step['content']) print() - def process_text(self, step): + @staticmethod + def process_text(step): + """ Print out the text exactly as we found it """ print(step['content']) print() def process_commands(self, step): + """ Pretend to type the command, run it and then display the output """ for cmd in step['content']: self.type_command(cmd) results = self.executor.run_cmd(cmd) @@ -68,11 +76,6 @@ def process_commands(self, step): print() return results - def process_result(self, ): - print('***CHECKING RESULT***') - print('RES= ' + step['content']) - print() - def type_command(self, cmd): """ Displays the command on the screen """ @@ -129,3 +132,18 @@ def is_result_valid(expected_results, actual_results, expected_similarity=1.0): logging.error("expected_results = " + expected_results) return is_pass + + @staticmethod + def process_next_steps(steps): + """ Is there a good way to test this that doesn't involve lots of test code + expect? + """ + idx = 1 + if steps: + print("Next steps available:") + for step in steps: + print(idx + ".) " + step['title']) + idx += 1 + step_request = input("Which step do you want to take next?") + if step_request: + return steps[step_request+1]['target'] + return diff --git a/simdem/mode/debug.py b/simdem/mode/dump.py similarity index 56% rename from simdem/mode/debug.py rename to simdem/mode/dump.py index c698f45..817ada0 100644 --- a/simdem/mode/debug.py +++ b/simdem/mode/dump.py @@ -1,11 +1,10 @@ -"""Demo (default) renderer for SimDem""" +""" Debug renderer for SimDem""" import pprint -class DebugMode(object): - """ This class is used to render the output of the commands into something - that looks like you're typing - """ +class DumpMode(object): # pylint: disable=R0903 + """ This class is used to pretty print a parsed file """ + config = None parser = None @@ -14,5 +13,6 @@ def __init__(self, config, parser): self.parser = parser def process_file(self, file_path): + """ Parse the file and print it. Not very exciting. """ steps = self.parser.parse_file(file_path) pprint.pprint(steps) diff --git a/simdem/parser/ast.py b/simdem/parser/ast.py index 96fd0f3..4f7f530 100644 --- a/simdem/parser/ast.py +++ b/simdem/parser/ast.py @@ -5,13 +5,14 @@ import mistletoe.block_token -class AstParser(object): +class AstParser(object): # pylint: disable=R0903 """ This class parses the human readable markdown using a defined syntax to know how to create the SimDem Execution Object and uses the code block language to know which type of execution it is """ - def parse_file(self, file_path): + @staticmethod + def parse_file(file_path): """ The main meat for parsing the file. Uses mistletoe's AST parser to create a tokenized object and then parses that tokenized object into SimDem Execution Object diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 6103d25..24ff3bc 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -16,6 +16,7 @@ class SimDem1Parser(object): mode = None def set_mode(self, mode): + """ Used mainly for debugging. No real need to have as a separate function """ logging.debug('set_mode(' + str(mode) + ')') self.mode = mode @@ -33,87 +34,107 @@ def parse_file(self, file_path): # Heading if token['type'] == 'Heading': - res.append({'type': 'heading', 'level': token['level'], - 'content': token['children'][0]['content']}) - logging.debug("parse_file():found heading") + res = self.parse_heading(token, res) + elif token['type'] == 'List': + res = self.parse_list(token, res) + elif token['type'] == 'Paragraph': + res = self.parse_paragraph(token, res) + elif token['type'] == 'BlockCode': + res = self.parse_block_code(token, res) + else: + logging.debug('parse_file():unknown token type (type=' + token['type'] + + ';mode=' + self.mode + '). Igorning') - # Prerequisite Heading - if token['children'][0]['content'].lower() == 'prerequisites': - self.set_mode('prerequisites') + return res - # Next Steps Heading - elif token['children'][0]['content'].lower() == 'next steps': - self.set_mode('next_steps') + def parse_heading(self, token, res): + """ Parse heading """ + res.append({'type': 'heading', 'level': token['level'], + 'content': token['children'][0]['content']}) + logging.debug("parse_file():found heading") - # Results Heading - elif token['children'][0]['content'].lower() == 'result': - self.set_mode('result') + # Prerequisite Heading + if token['children'][0]['content'].lower() == 'prerequisites': + self.set_mode('prerequisites') - # Validation Heading - elif token['children'][0]['content'].lower() == 'validation': - self.set_mode('validation') + # Next Steps Heading + elif token['children'][0]['content'].lower() == 'next steps': + self.set_mode('next_steps') - else: - logging.debug("parse_file():unable to determing header type.") - self.set_mode(None) - - # Prerequisite List - elif token['type'] == 'List' and self.mode == 'prerequisites': - logging.debug("parse_file():found prerequisites list") - titles = [x['children'][0]['children'][0]['content'] for x in token['children']] - content_t = "\n".join(' * ' + x for x in titles) - # Make sure to print out the page contents too - res.append({'type': 'text', 'content': content_t}) - - content_p = [x['children'][0]['target'] for x in token['children']] - res.append({'type': 'prerequisites', 'content': content_p}) - - # Paragraph - elif token['type'] == 'Paragraph': - logging.debug("parse_file():found paragraph") - if token['children']: - content = token['children'][0]['content'] - # Result Paragraph - if content == 'Results:': - if self.mode == 'validation': - self.set_mode('validation_result') - else: - self.set_mode('result') - #res.append({'type': 'result', 'content': content}) - # Regular Paragraph - else: - res.append({'type': 'text', 'content': content}) + # Results Heading + elif token['children'][0]['content'].lower() == 'result': + self.set_mode('result') - elif token['type'] == 'BlockCode': - # Result Block Code - if self.mode == 'validation': - logging.debug("parse_file():found validation block code") - content = [line for line in token['children'][0]['content'].split("\n") if line] - res.append({'type': 'validation_command', 'content': content}) - - # Validation Result Block Code - elif self.mode == 'validation_result': - logging.debug("parse_file():found validation result block code") - content = token['children'][0]['content'].rstrip() - res.append({'type': 'validation_result', 'content': content}) - - # Result Block Code - elif self.mode == 'result': - logging.debug("parse_file():found result block code") - content = token['children'][0]['content'].rstrip() - res.append({'type': 'result', 'content': content}) - - # Block Code (regular) - elif self.mode is None: - logging.debug("parse_file():found block code") - content = [line for line in token['children'][0]['content'].split("\n") if line] - res.append({'type': 'commands', 'content': content}) + # Validation Heading + elif token['children'][0]['content'].lower() == 'validation': + self.set_mode('validation') - else: - logging.debug('parse_file():unexpected Block Code mode (' + self.mode + ')') + else: + logging.debug("parse_file():unable to determing header type.") + self.set_mode(None) + return res + def parse_list(self, token, res): + """ Parse list """ + logging.debug("parse_list()") + if self.mode == 'prerequisites': + logging.debug("parse_file():found prerequisites list") + titles = [x['children'][0]['children'][0]['content'] for x in token['children']] + content_t = "\n".join(' * ' + x for x in titles) + # Make sure to print out the page contents too + res.append({'type': 'text', 'content': content_t}) + + content_p = [x['children'][0]['target'] for x in token['children']] + res.append({'type': 'prerequisites', 'content': content_p}) + return res + + def parse_paragraph(self, token, res): + """ Parse Paragraph """ + logging.debug("parse_paragraph()") + if token['children']: + content = token['children'][0]['content'] + # Result Paragraph + if content == 'Results:': + if self.mode == 'validation': + self.set_mode('validation_result') + else: + self.set_mode('result') + #res.append({'type': 'result', 'content': content}) + # Regular Paragraph else: - logging.debug('parse_file():unknown token type (type=' + token['type'] + - ';mode=' + self.mode + '). Igorning') + res.append({'type': 'text', 'content': content}) + return res + + def parse_block_code(self, token, res): + """ Parse block code """ + logging.debug("parse_block_code()") + + # Result Block Code + if self.mode == 'validation': + logging.debug("parse_block_code():found validation block code") + content = [line for line in token['children'][0]['content'].split("\n") if line] + res.append({'type': 'validation_command', 'content': content}) + + # Validation Result Block Code + elif self.mode == 'validation_result': + logging.debug("parse_block_code():found validation result block code") + content = token['children'][0]['content'].rstrip() + res.append({'type': 'validation_result', 'content': content}) + + # Result Block Code + elif self.mode == 'result': + logging.debug("parse_block_code():found result block code") + content = token['children'][0]['content'].rstrip() + res.append({'type': 'result', 'content': content}) + self.set_mode(None) + + # Block Code (regular) + elif self.mode is None: + logging.debug("parse_block_code():found default block code") + content = [line for line in token['children'][0]['content'].split("\n") if line] + res.append({'type': 'commands', 'content': content}) + + else: + logging.debug('parse_file():unexpected Block Code mode (' + self.mode + ')') return res diff --git a/simdem/render/__init__.py b/simdem/render/__init__.py deleted file mode 100644 index 97224c7..0000000 --- a/simdem/render/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -""" Import each of the renderers """ -from .demo import Demo diff --git a/simdem/render/demo.py b/simdem/render/demo.py deleted file mode 100644 index a12c55e..0000000 --- a/simdem/render/demo.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Demo (default) renderer for SimDem""" - -import random -import time - - -class Demo(object): - """ This class is used to render the output of the commands into something - that looks like you're typing - """ - config = None - - def __init__(self, config): - self.config = config - - def type_command(self, cmd): - """ - Displays the command on the screen - """ - - # Must add ' ' when typing command because whitespaces are removed from configparser - # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure - print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ', end="", flush=True) - for _, char in enumerate(cmd): - if char != "\n": - typing_delay = float(self.config.get('RENDER', 'TYPING_DELAY')) - if typing_delay: - delay = random.uniform(0.02, typing_delay) - time.sleep(delay) - print(char, end="", flush=True) - print("", flush=True) - - @staticmethod - def display_result(res): - """Demo specific implementation of displaying to the screen""" - print(res, end="", flush=True) - - @staticmethod - def show_next_steps(steps): - """ Is there a good way to test this that doesn't involve lots of test code + expect? - """ - idx = 1 - if steps: - print("Next steps available:") - for step in steps: - print(idx + ".) " + step['title']) - idx += 1 - step_request = input("Which step do you want to take next?") - if step_request: - return steps[step_request+1]['target'] - return diff --git a/tests/context.py b/tests/context.py deleted file mode 100644 index f42c5f5..0000000 --- a/tests/context.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -import os -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -import simdem -from simdem.render import demo -from simdem.parser import codeblock,context,simdem1 -from simdem.executor import bash \ No newline at end of file diff --git a/tests/test_bash.py b/tests/test_bash.py new file mode 100644 index 0000000..0507c71 --- /dev/null +++ b/tests/test_bash.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" Advanced test cases for SimDem """ + +import unittest + +from simdem.executor import bash + +class SimDemTestSuite(unittest.TestCase): + """Advanced test cases.""" + + bash = None + + def setUp(self): + self.bash = bash.BashExecutor() + + def test_run_cmd(self): + """ Validate running a command only prints out the result """ + self.assertEqual("foobar\n", self.bash.run_cmd('echo foobar')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_core.py b/tests/test_core.py deleted file mode 100644 index fd6bc5a..0000000 --- a/tests/test_core.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -""" Advanced test cases for SimDem """ - -import configparser -import os.path -import unittest - -from .context import bash, codeblock, demo, simdem - - -class SimDemTestSuite(unittest.TestCase): - """Advanced test cases.""" - - test_file = 'scratch/foo' - simdem = None - markdown_parser = None - - def setUp(self): - if os.path.exists(self.test_file): - os.remove(self.test_file) - config = configparser.ConfigParser() - config.read('content/config/unit_test.ini') - self.markdown_parser = codeblock.CodeBlockParser() - self.simdem = simdem.Core(config, demo.Demo(config), - self.markdown_parser, bash.BashExecutor()) - - def test_run_cmd(self): - """ Validate running a command only prints out the result """ - self.assertEqual("foobar\n", self.simdem.run_cmd('echo foobar')) - - def test_run_blocks(self): - """ Validate running a document that creates a file actually creates it """ - self.assertFalse(os.path.exists(self.test_file)) - blocks = self.markdown_parser.parse_file('content/create-file/README.md') - self.simdem.run_command_blocks(blocks['commands']) - self.assertTrue(os.path.exists(self.test_file)) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_system.py b/tests/test_demo_mode.py similarity index 82% rename from tests/test_system.py rename to tests/test_demo_mode.py index 83782e2..d89338c 100644 --- a/tests/test_system.py +++ b/tests/test_demo_mode.py @@ -8,7 +8,9 @@ from ddt import data, ddt -from .context import demo, simdem +from simdem.parser import simdem1 +from simdem.executor import bash +from simdem.mode import demo @ddt @@ -21,8 +23,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("content/config/unit_test.ini") - self.simdem = simdem.Core(config, demo.Demo(config), - simdem.parser.CodeBlockParser(), simdem.executor.BashExecutor()) + self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) root_logger = logging.getLogger() @@ -32,18 +33,17 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer -# @data('prerequisite-run') @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'create-file', 'prerequisite-run') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.out this allows us to test each of them easily """ - self.simdem.process_file('./content/' + directory + '/README.md') + self.demo.process_file('./content/' + directory + '/README.md') # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. res = sys.stdout.getvalue() # pylint: disable=E1101 - exp_res = open('./content/' + directory + '/expected_result.out', 'r').read() + exp_res = open('./content/' + directory + '/expected_result.demo', 'r').read() self.assertEqual(exp_res, res) diff --git a/tests/test_parser_codeblock.py b/tests/test_parser_codeblock.py deleted file mode 100644 index 658c0a8..0000000 --- a/tests/test_parser_codeblock.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -""" Test Codeblock Parser """ - -import configparser -import unittest - -from .context import codeblock - - -class MistletoeParserTestSuite(unittest.TestCase): - """Advanced test cases.""" - - renderer = None - - def setUp(self): - config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") - - self.renderer = codeblock.CodeBlockParser() - - def test_full(self): - """ Test the full featureset of codeblock parser """ - file_path = 'content/complete-features/codeblock.md' - res = self.renderer.parse_file(file_path) - exp_res = { - 'prerequisites': ['prereq.md', 'prereq-2.md'], - 'commands': [ - {'command': 'echo foo'}, - {'command': 'echo bar'}, - {'command': 'echo baz', 'expected_result': 'baz'}], - 'next_steps': [ - {'title': 'step-1.md', 'target': 'step-1.md'}, - {'title': 'step-2.md', 'target': 'step-2.md'} - ] - } - - self.assertEqual(res, exp_res) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_parser_context.py b/tests/test_parser_context.py deleted file mode 100644 index ea345d3..0000000 --- a/tests/test_parser_context.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -"""SimDem Test Case""" - -import configparser -import unittest - -from .context import context - -class MistletoeParserTestSuite(unittest.TestCase): - """Advanced test cases.""" - - parser = None - - def setUp(self): - config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") - - self.parser = context.ContextParser() - - def test_full(self): - """Test parsing a document with all features in it""" - file_path = 'content/complete-features/context.md' - res = self.parser.parse_file(file_path) - exp_res = { - 'prerequisites': ['prereq.md', 'prereq-2.md'], - 'commands': [ - {'command': 'echo foo'}, - {'command': 'echo bar'}, - {'command': 'echo baz', 'expected_result': 'baz'}], - 'next_steps': [ - {'title': 'Step #1', 'target': 'step-1.md'}, - {'title': 'Step #2', 'target': 'step-2.md'} - ] - } - - self.assertEqual(res, exp_res) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index b1feb48..9225202 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -4,7 +4,7 @@ import configparser import unittest import pprint -from .context import simdem1 +from simdem.parser import simdem1 class SimDem1ParserTestSuite(unittest.TestCase): """Advanced test cases.""" @@ -19,45 +19,43 @@ def setUp(self): def test_full(self): """Test parsing a document with all features in it""" - self.maxDiff = None + #self.maxDiff = None file_path = 'content/simdem1/README.md' res = self.parser.parse_file(file_path) - exp_res = [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, + # This is pretty brittle. It might be valuable to have a test document with less content + exp_resl = [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, {'content': 'This is the prerequisite section. SimDem looks for a set of ' 'links to extract and run through first', - 'type': 'text'}, - {'content': ' * prereq-validation-pass', 'type': 'text'}, - {'content': ['prereq-validaiton-pass.md'], 'type': 'prerequisite'}, + 'type': 'text'}, + {'content': ' * prereq-ignored', 'type': 'text'}, + {'content': ['content/simdem1/prereq-ignored.md'], 'type': 'prerequisites'}, {'content': "They don't even need to be in the same list", 'type': 'text'}, - {'content': ' * prereq-validation-fail', 'type': 'text'}, - {'content': ['prereq-validaiton-fail.md'], 'type': 'prerequisite'}, + {'content': ' * prereq-processed', 'type': 'text'}, + {'content': ['content/simdem1/prereq-processed.md'], 'type': 'prerequisites'}, {'content': 'By this point, the prerequisites have either run or have passed ' 'their validation', - 'type': 'text'}, + 'type': 'text'}, + {'content': 'Did our prerequisites run?', 'level': 1, 'type': 'heading'}, + {'content': ['echo prereq_ignored = $prereq_ignored', + 'echo prereq_processed = $prereq_processed'], + 'type': 'commands'}, {'content': 'Do stuff here', 'level': 1, 'type': 'heading'}, {'content': 'We want to execute this because the code type is shell', - 'type': 'text'}, - {'content': ['echo foo', 'echo bar', 'var=foo'], 'type': 'commands'}, - {'content': 'Validation', 'level': 1, 'type': 'heading'}, - {'content': 'This is a validation section. If this validation section ' - 'passes, we stop processing this file', - 'type': 'text'}, - {'content': 'Results:', 'type': 'text'}, + 'type': 'text'}, + {'content': ['echo foo', 'var=bar'], 'type': 'commands'}, {'content': 'Do more stuff here', 'level': 1, 'type': 'heading'}, - {'content': ['echo baz'], 'type': 'commands'}, - {'content': 'Results', 'level': 1, 'type': 'heading'}, - {'content': 'The only thing that makes it a result is the code type is ' - 'result. We assume the result is for the last command of the last ' - 'code block', - 'type': 'text'}, - {'content': 'baz', 'type': 'result'}, + {'content': 'We assume the result is for the last command of the last code ' + 'block', + 'type': 'text'}, + {'content': ['echo baz', 'echo $var'], 'type': 'commands'}, + {'content': 'bar', 'type': 'result'}, {'content': 'Next Steps', 'level': 1, 'type': 'heading'}, {'content': 'The list inside this block are steps that could be followed when ' 'performing an interactive tutorial', - 'type': 'text'}] + 'type': 'text'}] pprint.pprint(res) - self.assertEqual(res, exp_res) + self.assertEqual(res, exp_resl) if __name__ == '__main__': From cc4b64f21d15973d4b901e423d2b6ffc7aa582e0 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 12 Jan 2018 16:57:54 -0600 Subject: [PATCH 077/167] Fix setup.py Signed-off-by: Tommy Falgout --- setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2a0fb08..1f54e5f 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ license = f.read() setup( - name='sample', + name='simdem', version='0.1.0', description='SimDem', long_description=readme, @@ -20,6 +20,11 @@ author_email='thfalgou@microsoft.com', url='https://github.com/Azure/simdem', license=license, - packages=find_packages(exclude=('tests', 'docs')) + packages=find_packages(exclude=('tests', 'docs')), + entry_points={ + 'console_scripts': [ + 'simdem=main:main' + ] + } ) From a85adf7d3791231246d1ea14738ca740c7efdf55 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 12 Jan 2018 17:05:22 -0600 Subject: [PATCH 078/167] Move CLI code to standard location Signed-off-by: Tommy Falgout --- setup.py | 2 +- main.py => simdem/cli.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename main.py => simdem/cli.py (100%) diff --git a/setup.py b/setup.py index 1f54e5f..de5c16f 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ packages=find_packages(exclude=('tests', 'docs')), entry_points={ 'console_scripts': [ - 'simdem=main:main' + 'simdem=simdem.cli:main' ] } ) diff --git a/main.py b/simdem/cli.py similarity index 100% rename from main.py rename to simdem/cli.py From 2c51a71a6db28e23d0750aab50dccfa70a15b205 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 10:36:26 -0600 Subject: [PATCH 079/167] Delete dead code Signed-off-by: Tommy Falgout --- simdem/parser/codeblock.py | 61 ------------------- simdem/parser/context.py | 122 ------------------------------------- 2 files changed, 183 deletions(-) delete mode 100644 simdem/parser/codeblock.py delete mode 100644 simdem/parser/context.py diff --git a/simdem/parser/codeblock.py b/simdem/parser/codeblock.py deleted file mode 100644 index 767a46f..0000000 --- a/simdem/parser/codeblock.py +++ /dev/null @@ -1,61 +0,0 @@ -""" This module hosts the CodeBlockParser -""" - -import logging -from collections import defaultdict -from mistletoe import Document -from mistletoe.base_renderer import BaseRenderer - -# Inspired by: https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 - -class CodeBlockParser(BaseRenderer): - """ This class parses Markdown documents into SimDem Execution Objects - This class expects all SimDem primitives to be parsed from code blocks, - and uses the code block language to know which type of execution it is - """ - def __init__(self): - super().__init__() - self.output = defaultdict(list) - - def render_block_code(self, token): - """ Implemented as part of Mistletoe interface - Given this object parses only code blocks, this is - the only one with relevant logic. The code block language - is parsed to determine what type of object the code underneath is - """ - lines = token.children[0].content.splitlines() - if 'prerequisite' in token.language: - self.output['prerequisites'].extend(lines) - elif 'result' in token.language: - self.output['commands'][-1]['expected_result'] = lines[0] - elif 'shell' in token.language: - self.output['commands'].extend({'command': line} for line in lines) - elif 'next_steps' in token.language: - self.output['next_steps'].extend({'title': line, 'target': line} - for line in lines - if line != '```') - else: - logging.info("get_commands():unknown_block. Ignoring") - return '' - - def render_document(self, token): - """ Implemented as part of the Mistletoe interface - """ - self.render_inner(token) - return dict(self.output) - - def __getattr__(self, name): - """ Implemented as part of the Mistletoe interface - Added to prevent Mistletoe from throwing warnings for missing render functions - """ - return lambda token: '' - - @staticmethod - def parse_file(file_path): - """ Implemented as part of SimDem interface - Renders the contents of the file path using this object as a Mistletoe renderer - """ - with open(file_path, 'r') as fin: - with CodeBlockParser() as renderer: - rendered = renderer.render(Document(fin)) - return rendered diff --git a/simdem/parser/context.py b/simdem/parser/context.py deleted file mode 100644 index a0aea1b..0000000 --- a/simdem/parser/context.py +++ /dev/null @@ -1,122 +0,0 @@ -""" This module hosts the ContextParser -""" - -import logging - -import mistletoe.ast_renderer as renderer -import mistletoe.block_token as token - - -class ContextParser(object): - """ This class parses the human readable markdown using a defined syntax to - know how to create the SimDem Execution Object - and uses the code block language to know which type of execution it is - """ - - @staticmethod - def is_command_block(block): - """ Expects a code block with a shell command in it - """ - if block['type'] == 'BlockCode' and block['language'] == 'shell': - return True - return False - - @staticmethod - def is_result_block(block): - """ This is different than previous SimDem because it didn't require - a language for the result. - I believe this approach is more declarative. - """ - if block['type'] == 'BlockCode' and block['language'] == 'result': - return True - return False - - @staticmethod - def is_next_step_block(block): - """ This block is similiar to "choose your own adventure" games - This feature allows you to determine which document you want to process - after you complete the current one - """ - if 'children' in block and block['children'] \ - and 'content' in block['children'][0] \ - and 'next step' in block['children'][0]['content'].lower() \ - and block['type'].lower() == 'heading': - return True - return False - - @staticmethod - def is_prerequisite_block(block): - """ - I'm not a fan of denoting prerequisites by using a header title, - but that will suffice for now - Will look like this coming out of AST - {'children': [{'children': [{'content': 'Prerequisites', 'type': 'RawText'}], - 'level': 1, - 'type': 'Heading'}, - """ - if 'children' in block and block['children'] \ - and 'content' in block['children'][0] \ - and 'prerequisite' in block['children'][0]['content'].lower() \ - and block['type'].lower() == 'heading': - return True - return False - - def parse_file(self, file_path): - """ The main meat for parsing the file. Uses mistletoe's AST parser - to create a tokenized object and then parses that tokenized object - into SimDem Execution Object - """ - with open(file_path, 'r') as fin: - ast = renderer.get_ast(token.Document(fin)) - - res = { - 'prerequisites': [], - 'commands': [] - } - - idx = 0 - blocks = ast['children'] - #logging.debug("parse_file():processing " + str(blocks) - while idx < len(blocks): - block = blocks[idx] - logging.debug("parse_file():processing " + str(block)) - - if self.is_prerequisite_block(block): - logging.debug("parse_file():found preqreq block") - res['prerequisites'] = \ - [x['children'][0]['target'] for x in blocks[idx+1]['children']] - # No need to process the next block since that's the prereqs - idx = idx + 1 - - elif self.is_result_block(block): - # Assume that the result block is for the previous command block - logging.debug("parse_file():is_result_block") - content = block['children'][0]['content'].rstrip() - if content: - res['commands'][len(res['commands']) - 1]['expected_result'] = content - - elif self.is_command_block(block): - logging.debug("parse_file():is_command_block") - for line in block['children'][0]['content'].split("\n"): - if line: - res['commands'].append({'command': line}) - - elif self.is_next_step_block(block): - logging.debug("parse_file():is_next_step_block") - block = blocks[idx] - # Fast-forward to find the list block inside this header block. - # Maybe we should also test to verify we haven't gone past the header block? - while 'List' not in block['type'] and idx < len(blocks): - idx = idx + 1 - block = blocks[idx] - print(block) - res['next_steps'] = [{'target': x['target'], 'title': x['children'][0]['content']} - for x in block['children'][0]['children'] - if x['type'] == 'Link'] - - else: - logging.info("get_commands():unknown_block. Ignoring") - - idx = idx + 1 - #logging.debug(res) - return res From 40958141479e37f1a50b594a5ab6b303f7c834e0 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 10:45:05 -0600 Subject: [PATCH 080/167] Fix unexpected content Signed-off-by: Tommy Falgout --- simdem/parser/__init__.py | 3 --- simdem/parser/simdem1.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/simdem/parser/__init__.py b/simdem/parser/__init__.py index 993f899..e69de29 100644 --- a/simdem/parser/__init__.py +++ b/simdem/parser/__init__.py @@ -1,3 +0,0 @@ -""" Import each of the parsers. """ -from .codeblock import CodeBlockParser -from .context import ContextParser diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 24ff3bc..c8851d7 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -91,7 +91,7 @@ def parse_list(self, token, res): def parse_paragraph(self, token, res): """ Parse Paragraph """ logging.debug("parse_paragraph()") - if token['children']: + if 'content' in token['children'][0]: content = token['children'][0]['content'] # Result Paragraph if content == 'Results:': From 2d10f7f9a7625d79ea4a5af93e311b3529722f30 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 11:26:15 -0600 Subject: [PATCH 081/167] Implement Mode Common for common mode code Signed-off-by: Tommy Falgout --- hello_world/README.md | 1 + simdem/mode/__init__.py | 0 simdem/mode/automated.py | 66 ++++++++++++++++++++++++++++++++++++ simdem/mode/common.py | 53 +++++++++++++++++++++++++++++ simdem/mode/demo.py | 73 +++------------------------------------- simdem/mode/dump.py | 10 ++---- 6 files changed, 126 insertions(+), 77 deletions(-) create mode 100644 hello_world/README.md create mode 100644 simdem/mode/__init__.py create mode 100644 simdem/mode/automated.py create mode 100644 simdem/mode/common.py diff --git a/hello_world/README.md b/hello_world/README.md new file mode 100644 index 0000000..1934e6f --- /dev/null +++ b/hello_world/README.md @@ -0,0 +1 @@ +# Hello World Script diff --git a/simdem/mode/__init__.py b/simdem/mode/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/simdem/mode/automated.py b/simdem/mode/automated.py new file mode 100644 index 0000000..178f5ba --- /dev/null +++ b/simdem/mode/automated.py @@ -0,0 +1,66 @@ +"""Demo (default) mode for SimDem""" + +from simdem.mode.common import ModeCommon + +class AutomatedMode(ModeCommon): + """ This class is the default SimDem file processor. + It's designed for running files in a demo-able mode that looks like a human is typing it + """ + + def process_file(self, file_path): + """ Parses the file and starts processing it """ + #print("*** Processing " + file_path + " ***") + steps = self.parser.parse_file(file_path) + self.process(steps) + #print("*** Completed Processing " + file_path + " ***") + + def process(self, steps): + """ Parses the file and starts processing it """ + last_command_result = None + + """ I'd like to use a dispatcher for this; however, we need to exit processing + if the validation fails. """ + for step in steps: + if step['type'] == 'heading': + self.process_heading(step) + elif step['type'] == 'text': + self.process_text(step) + elif step['type'] == 'commands': + last_command_result = self.process_commands(step) +# elif step['type'] == 'result': +# if self.is_result_valid(step['content'], last_command_result): +# print('***VALIDATION PASSED***') +# else: +# print('***VALIDATION FAILED***') + elif step['type'] == 'prerequisites': + for prereq_file in step['content']: + self.process_file(prereq_file) + elif step['type'] == 'validation_command': + last_command_result = self.process_commands(step) + elif step['type'] == 'validation_result': + if self.is_result_valid(step['content'], last_command_result): +# print('***VALIDATION PASSED***') + return +# else: +# print('***VALIDATION FAILED***') + + @staticmethod + def process_heading(step): + """ Print out the heading exactly as we found it """ + print(step['level'] * '#' + ' ' + step['content']) + print() + + @staticmethod + def process_text(step): + """ Print out the text exactly as we found it """ + print(step['content']) + print() + + def process_commands(self, step): + """ Pretend to type the command, run it and then display the output """ + for cmd in step['content']: + print(cmd, end="", flush=True) + results = self.executor.run_cmd(cmd) + print(results, end="", flush=True) + print() + return results diff --git a/simdem/mode/common.py b/simdem/mode/common.py new file mode 100644 index 0000000..0d25c96 --- /dev/null +++ b/simdem/mode/common.py @@ -0,0 +1,53 @@ +""" Common mode for SimDem mode """ + +import logging +import difflib + +class ModeCommon(object): + """ This class is designed to hold any shared code across modes + """ + config = None + executor = None + parser = None + + def __init__(self, config, parser, executor): + self.config = config + self.parser = parser + self.executor = executor + + @staticmethod + def is_result_valid(expected_results, actual_results, expected_similarity=1.0): + """Checks to see if a command execution passes. + If actual results compared to expected results is within + the expected similarity level then it's considered a pass. + + expected_similarity = 1.0 could be a breaking change for older SimDem scripts. + explicit fails > implicit passes + Ross may disagree with me. Let's see how this story unfolds. + """ + + if not actual_results: + logging.error("is_result_valid(): actual_results is empty.") + return False + + logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ + "," + str(expected_similarity) + ")") + + expected_results_str = expected_results.rstrip() + actual_results_str = actual_results.rstrip() + logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ + "," + str(expected_similarity) + ")") + seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", + actual_results_str, + expected_results_str) + + is_pass = seq.ratio() >= expected_similarity + + if is_pass: + logging.info("is_result_valid passed") + else: + logging.error("is_result_valid failed") + logging.error("actual_results = " + actual_results) + logging.error("expected_results = " + expected_results) + + return is_pass diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index ff29116..5e33911 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -2,21 +2,12 @@ import random import time -import logging -import difflib +from simdem.mode.common import ModeCommon -class DemoMode(object): +class DemoMode(ModeCommon): """ This class is the default SimDem file processor. It's designed for running files in a demo-able mode that looks like a human is typing it """ - config = None - executor = None - parser = None - - def __init__(self, config, parser, executor): - self.config = config - self.parser = parser - self.executor = executor def process_file(self, file_path): """ Parses the file and starts processing it """ @@ -27,7 +18,6 @@ def process_file(self, file_path): def process(self, steps): """ Parses the file and starts processing it """ - last_command_result = None """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ @@ -37,23 +27,10 @@ def process(self, steps): elif step['type'] == 'text': self.process_text(step) elif step['type'] == 'commands': - last_command_result = self.process_commands(step) -# elif step['type'] == 'result': -# if self.is_result_valid(step['content'], last_command_result): -# print('***VALIDATION PASSED***') -# else: -# print('***VALIDATION FAILED***') + self.process_commands(step) elif step['type'] == 'prerequisites': for prereq_file in step['content']: self.process_file(prereq_file) - elif step['type'] == 'validation_command': - last_command_result = self.process_commands(step) - elif step['type'] == 'validation_result': - if self.is_result_valid(step['content'], last_command_result): -# print('***VALIDATION PASSED***') - return -# else: -# print('***VALIDATION FAILED***') @staticmethod def process_heading(step): @@ -72,7 +49,7 @@ def process_commands(self, step): for cmd in step['content']: self.type_command(cmd) results = self.executor.run_cmd(cmd) - self.display_result(results) + print(results, end="", flush=True) print() return results @@ -91,48 +68,6 @@ def type_command(self, cmd): print(char, end="", flush=True) print("", flush=True) - @staticmethod - def display_result(res): - """Demo specific implementation of displaying to the screen""" - print(res, end="", flush=True) - - @staticmethod - def is_result_valid(expected_results, actual_results, expected_similarity=1.0): - """Checks to see if a command execution passes. - If actual results compared to expected results is within - the expected similarity level then it's considered a pass. - - expected_similarity = 1.0 could be a breaking change for older SimDem scripts. - explicit fails > implicit passes - Ross may disagree with me. Let's see how this story unfolds. - """ - - if not actual_results: - logging.error("is_result_valid(): actual_results is empty.") - return False - - logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ - "," + str(expected_similarity) + ")") - - expected_results_str = expected_results.rstrip() - actual_results_str = actual_results.rstrip() - logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ - "," + str(expected_similarity) + ")") - seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", - actual_results_str, - expected_results_str) - - is_pass = seq.ratio() >= expected_similarity - - if is_pass: - logging.info("is_result_valid passed") - else: - logging.error("is_result_valid failed") - logging.error("actual_results = " + actual_results) - logging.error("expected_results = " + expected_results) - - return is_pass - @staticmethod def process_next_steps(steps): """ Is there a good way to test this that doesn't involve lots of test code + expect? diff --git a/simdem/mode/dump.py b/simdem/mode/dump.py index 817ada0..d4c2f5e 100644 --- a/simdem/mode/dump.py +++ b/simdem/mode/dump.py @@ -1,17 +1,11 @@ """ Debug renderer for SimDem""" import pprint +from simdem.mode.common import ModeCommon -class DumpMode(object): # pylint: disable=R0903 +class DumpMode(ModeCommon): # pylint: disable=R0903 """ This class is used to pretty print a parsed file """ - config = None - parser = None - - def __init__(self, config, parser): - self.config = config - self.parser = parser - def process_file(self, file_path): """ Parse the file and print it. Not very exciting. """ steps = self.parser.parse_file(file_path) From 6292e0d163f01c9c076618087bec08afb97fc348 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 13:34:55 -0600 Subject: [PATCH 082/167] Commit to debug double-print --- content/simple-variable/expected_result.demo | 7 +- content/simple/README.md | 11 +-- content/simple/expected_result.demo | 6 -- simdem/cli.py | 20 ++++- simdem/mode/automated.py | 79 +++++++++++++------- simdem/mode/common.py | 40 ---------- simdem/mode/demo.py | 18 +---- tests/test_demo_mode.py | 5 +- 8 files changed, 76 insertions(+), 110 deletions(-) diff --git a/content/simple-variable/expected_result.demo b/content/simple-variable/expected_result.demo index 7dbae61..f17fa34 100644 --- a/content/simple-variable/expected_result.demo +++ b/content/simple-variable/expected_result.demo @@ -1,9 +1,10 @@ -this is text - $ FOO=bar $ echo $FOO bar -more text +$ FOO=bar + +$ echo $FOO +bar diff --git a/content/simple/README.md b/content/simple/README.md index 0f2a643..d429891 100644 --- a/content/simple/README.md +++ b/content/simple/README.md @@ -1,14 +1,5 @@ -this is text +# Simple ```shell echo foo -echo bar ``` - -more text - -```shell -echo baz -``` - -even more text diff --git a/content/simple/expected_result.demo b/content/simple/expected_result.demo index 39cb179..1eaa372 100644 --- a/content/simple/expected_result.demo +++ b/content/simple/expected_result.demo @@ -1,14 +1,8 @@ -this is text - $ echo foo foo $ echo bar bar -more text - $ echo baz baz -even more text - diff --git a/simdem/cli.py b/simdem/cli.py index 20f7310..6f93b42 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -7,7 +7,7 @@ from simdem.executor import bash from simdem.parser import ast, simdem1 -from simdem.mode import demo, dump +from simdem.mode import demo, dump, automated, tutorial def main(): @@ -22,7 +22,7 @@ def main(): argp.add_argument('--renderer', '-r', default="demo", help="Render class to use", choices=['demo']) argp.add_argument('--mode', '-m', default="demo", - help="Mode to use", choices=['demo', 'dump']) + help="Mode to use", choices=['demo', 'dump', 'automated', 'tutorial']) argp.add_argument('--parser', '-p', default="simdem1", help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", @@ -53,11 +53,21 @@ def validate(options, file_path): def get_mode(options, config): """ Returns correct renderer object """ + + parser = get_parser(options) + executor = get_executor(options) + if options.mode == 'demo': - return demo.DemoMode(config, get_parser(options), get_executor(options)) + return demo.DemoMode(config, parser, executor) if options.mode == 'dump': - return dump.DumpMode(config, get_parser(options)) + return dump.DumpMode(config, parser, executor) + + if options.mode == 'automated': + return automated.AutomatedMode(config, parser, executor) + + if options.mode == 'tutorial': + return tutorial.TutorialMode(config, parser, executor) def get_parser(options): """ Returns correct parser object """ @@ -86,4 +96,6 @@ def setup_logging(config, options): console_handler.setFormatter(log_formatter) root_logger.addHandler(console_handler) +print("***MAIN") main() +print("***END MAIN") diff --git a/simdem/mode/automated.py b/simdem/mode/automated.py index 178f5ba..8f4fecc 100644 --- a/simdem/mode/automated.py +++ b/simdem/mode/automated.py @@ -1,10 +1,14 @@ -"""Demo (default) mode for SimDem""" +""" Automated mode for SimDem """ +import logging +import difflib from simdem.mode.common import ModeCommon class AutomatedMode(ModeCommon): - """ This class is the default SimDem file processor. - It's designed for running files in a demo-able mode that looks like a human is typing it + """ This class is the automated SimDem mode + Does not display the descriptive text, but pauses at each + code block. When the user hits a key the command is "typed", a + second keypress executes the command. """ def process_file(self, file_path): @@ -21,40 +25,19 @@ def process(self, steps): """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ for step in steps: - if step['type'] == 'heading': - self.process_heading(step) - elif step['type'] == 'text': - self.process_text(step) - elif step['type'] == 'commands': + if step['type'] == 'commands': last_command_result = self.process_commands(step) -# elif step['type'] == 'result': -# if self.is_result_valid(step['content'], last_command_result): -# print('***VALIDATION PASSED***') -# else: -# print('***VALIDATION FAILED***') + elif step['type'] == 'result': + self.is_result_valid(step['content'], last_command_result) elif step['type'] == 'prerequisites': for prereq_file in step['content']: self.process_file(prereq_file) elif step['type'] == 'validation_command': last_command_result = self.process_commands(step) elif step['type'] == 'validation_result': - if self.is_result_valid(step['content'], last_command_result): -# print('***VALIDATION PASSED***') + if not self.is_result_valid(step['content'], last_command_result): return -# else: -# print('***VALIDATION FAILED***') - @staticmethod - def process_heading(step): - """ Print out the heading exactly as we found it """ - print(step['level'] * '#' + ' ' + step['content']) - print() - - @staticmethod - def process_text(step): - """ Print out the text exactly as we found it """ - print(step['content']) - print() def process_commands(self, step): """ Pretend to type the command, run it and then display the output """ @@ -64,3 +47,43 @@ def process_commands(self, step): print(results, end="", flush=True) print() return results + + @staticmethod + def is_result_valid(expected_results, actual_results, expected_similarity=1.0): + """Checks to see if a command execution passes. + If actual results compared to expected results is within + the expected similarity level then it's considered a pass. + + expected_similarity = 1.0 could be a breaking change for older SimDem scripts. + explicit fails > implicit passes + Ross may disagree with me. Let's see how this story unfolds. + """ + + if not actual_results: + logging.error("is_result_valid(): actual_results is empty.") + return False + + logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ + "," + str(expected_similarity) + ")") + + expected_results_str = expected_results.rstrip() + actual_results_str = actual_results.rstrip() + logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ + "," + str(expected_similarity) + ")") + seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", + actual_results_str, + expected_results_str) + + is_pass = seq.ratio() >= expected_similarity + + if is_pass: + logging.info("is_result_valid passed") + print('***VALIDATION PASSED***') + + else: + logging.error("is_result_valid failed") + logging.error("actual_results = " + actual_results) + logging.error("expected_results = " + expected_results) + print('***VALIDATION FAILED***') + + return is_pass diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 0d25c96..8bb96b6 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -1,8 +1,5 @@ """ Common mode for SimDem mode """ -import logging -import difflib - class ModeCommon(object): """ This class is designed to hold any shared code across modes """ @@ -14,40 +11,3 @@ def __init__(self, config, parser, executor): self.config = config self.parser = parser self.executor = executor - - @staticmethod - def is_result_valid(expected_results, actual_results, expected_similarity=1.0): - """Checks to see if a command execution passes. - If actual results compared to expected results is within - the expected similarity level then it's considered a pass. - - expected_similarity = 1.0 could be a breaking change for older SimDem scripts. - explicit fails > implicit passes - Ross may disagree with me. Let's see how this story unfolds. - """ - - if not actual_results: - logging.error("is_result_valid(): actual_results is empty.") - return False - - logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ - "," + str(expected_similarity) + ")") - - expected_results_str = expected_results.rstrip() - actual_results_str = actual_results.rstrip() - logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ - "," + str(expected_similarity) + ")") - seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", - actual_results_str, - expected_results_str) - - is_pass = seq.ratio() >= expected_similarity - - if is_pass: - logging.info("is_result_valid passed") - else: - logging.error("is_result_valid failed") - logging.error("actual_results = " + actual_results) - logging.error("expected_results = " + expected_results) - - return is_pass diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index 5e33911..d236755 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -22,28 +22,12 @@ def process(self, steps): """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ for step in steps: - if step['type'] == 'heading': - self.process_heading(step) - elif step['type'] == 'text': - self.process_text(step) - elif step['type'] == 'commands': + if step['type'] == 'commands': self.process_commands(step) elif step['type'] == 'prerequisites': for prereq_file in step['content']: self.process_file(prereq_file) - @staticmethod - def process_heading(step): - """ Print out the heading exactly as we found it """ - print(step['level'] * '#' + ' ' + step['content']) - print() - - @staticmethod - def process_text(step): - """ Print out the text exactly as we found it """ - print(step['content']) - print() - def process_commands(self, step): """ Pretend to type the command, run it and then display the output """ for cmd in step['content']: diff --git a/tests/test_demo_mode.py b/tests/test_demo_mode.py index d89338c..00c75ec 100644 --- a/tests/test_demo_mode.py +++ b/tests/test_demo_mode.py @@ -33,8 +33,9 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable', 'results-block', - 'results-block-fail', 'create-file', 'prerequisite-run') + @data('simple-variable') + #@data('simple', 'simple-variable', 'results-block', + # 'results-block-fail', 'create-file', 'prerequisite-run') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.out this allows us to test each of them easily From a211750a28fa3718a5251ba2ba99e94581494515 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 13:41:04 -0600 Subject: [PATCH 083/167] Remove double call of main() --- simdem/cli.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/simdem/cli.py b/simdem/cli.py index 6f93b42..35b33ff 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -95,7 +95,3 @@ def setup_logging(config, options): console_handler = logging.StreamHandler() console_handler.setFormatter(log_formatter) root_logger.addHandler(console_handler) - -print("***MAIN") -main() -print("***END MAIN") From 418efcb4c52aea47929d814ca702090bdc7c96cf Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 13:55:56 -0600 Subject: [PATCH 084/167] Split demo + tutorial expected_outputs Signed-off-by: Tommy Falgout --- content/create-file/expected_result.tutorial | 4 ++ content/prerequisite-run/expected_result.demo | 10 --- .../prerequisite-run/expected_result.tutorial | 16 +++++ .../results-block-fail/expected_result.demo | 6 -- .../expected_result.tutorial | 14 ++++ content/results-block/expected_result.demo | 4 -- .../results-block/expected_result.tutorial | 9 +++ content/simple-variable/expected_result.demo | 5 -- .../simple-variable/expected_result.tutorial | 9 +++ content/simple/expected_result.demo | 5 -- content/simple/expected_result.tutorial | 5 ++ simdem/mode/tutorial.py | 68 +++++++++++++++++++ tests/test_demo_mode.py | 2 +- tests/test_mistletoe.py | 17 ++--- tests/test_tutorial_mode.py | 52 ++++++++++++++ 15 files changed, 184 insertions(+), 42 deletions(-) create mode 100644 content/create-file/expected_result.tutorial create mode 100644 content/prerequisite-run/expected_result.tutorial create mode 100644 content/results-block-fail/expected_result.tutorial create mode 100644 content/results-block/expected_result.tutorial create mode 100644 content/simple-variable/expected_result.tutorial create mode 100644 content/simple/expected_result.tutorial create mode 100644 simdem/mode/tutorial.py create mode 100644 tests/test_tutorial_mode.py diff --git a/content/create-file/expected_result.tutorial b/content/create-file/expected_result.tutorial new file mode 100644 index 0000000..dd8004e --- /dev/null +++ b/content/create-file/expected_result.tutorial @@ -0,0 +1,4 @@ +$ echo 'abc' > scratch/foo +$ cat scratch/foo +abc + diff --git a/content/prerequisite-run/expected_result.demo b/content/prerequisite-run/expected_result.demo index d045721..9171799 100644 --- a/content/prerequisite-run/expected_result.demo +++ b/content/prerequisite-run/expected_result.demo @@ -1,16 +1,6 @@ -# Prerequisites - - * prereq - -This text should be run - $ echo running-prereq running-prereq -End running text - -# Main - $ echo post-prereq-run post-prereq-run diff --git a/content/prerequisite-run/expected_result.tutorial b/content/prerequisite-run/expected_result.tutorial new file mode 100644 index 0000000..d045721 --- /dev/null +++ b/content/prerequisite-run/expected_result.tutorial @@ -0,0 +1,16 @@ +# Prerequisites + + * prereq + +This text should be run + +$ echo running-prereq +running-prereq + +End running text + +# Main + +$ echo post-prereq-run +post-prereq-run + diff --git a/content/results-block-fail/expected_result.demo b/content/results-block-fail/expected_result.demo index 6e279c2..5566f95 100644 --- a/content/results-block-fail/expected_result.demo +++ b/content/results-block-fail/expected_result.demo @@ -1,14 +1,8 @@ -this is text - $ echo foo foo $ echo bar bar -In demo mode, we will continue processing. - $ echo post_result_block_fail post_result_block_fail -even more text - diff --git a/content/results-block-fail/expected_result.tutorial b/content/results-block-fail/expected_result.tutorial new file mode 100644 index 0000000..6e279c2 --- /dev/null +++ b/content/results-block-fail/expected_result.tutorial @@ -0,0 +1,14 @@ +this is text + +$ echo foo +foo +$ echo bar +bar + +In demo mode, we will continue processing. + +$ echo post_result_block_fail +post_result_block_fail + +even more text + diff --git a/content/results-block/expected_result.demo b/content/results-block/expected_result.demo index 635d3c6..8a8f518 100644 --- a/content/results-block/expected_result.demo +++ b/content/results-block/expected_result.demo @@ -1,9 +1,5 @@ -this is text - $ echo foo foo $ echo bar bar -even more text - diff --git a/content/results-block/expected_result.tutorial b/content/results-block/expected_result.tutorial new file mode 100644 index 0000000..635d3c6 --- /dev/null +++ b/content/results-block/expected_result.tutorial @@ -0,0 +1,9 @@ +this is text + +$ echo foo +foo +$ echo bar +bar + +even more text + diff --git a/content/simple-variable/expected_result.demo b/content/simple-variable/expected_result.demo index f17fa34..0ca9529 100644 --- a/content/simple-variable/expected_result.demo +++ b/content/simple-variable/expected_result.demo @@ -3,8 +3,3 @@ $ FOO=bar $ echo $FOO bar -$ FOO=bar - -$ echo $FOO -bar - diff --git a/content/simple-variable/expected_result.tutorial b/content/simple-variable/expected_result.tutorial new file mode 100644 index 0000000..7dbae61 --- /dev/null +++ b/content/simple-variable/expected_result.tutorial @@ -0,0 +1,9 @@ +this is text + +$ FOO=bar + +$ echo $FOO +bar + +more text + diff --git a/content/simple/expected_result.demo b/content/simple/expected_result.demo index 1eaa372..3e08e67 100644 --- a/content/simple/expected_result.demo +++ b/content/simple/expected_result.demo @@ -1,8 +1,3 @@ $ echo foo foo -$ echo bar -bar - -$ echo baz -baz diff --git a/content/simple/expected_result.tutorial b/content/simple/expected_result.tutorial new file mode 100644 index 0000000..d54007a --- /dev/null +++ b/content/simple/expected_result.tutorial @@ -0,0 +1,5 @@ +# Simple + +$ echo foo +foo + diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py new file mode 100644 index 0000000..82c8f23 --- /dev/null +++ b/simdem/mode/tutorial.py @@ -0,0 +1,68 @@ +""" Tutorial mode for SimDem""" + +from simdem.mode.common import ModeCommon + +class TutorialMode(ModeCommon): + """ This class is the tutorial mode class for SimDem. + It's designed for running files in a tutorial mode which Displays the descriptive text + of the tutorial and pauses at code blocks to allow user interaction. + """ + + def process_file(self, file_path): + """ Parses the file and starts processing it """ + #print("*** Processing " + file_path + " ***") + steps = self.parser.parse_file(file_path) + self.process(steps) + #print("*** Completed Processing " + file_path + " ***") + + def process(self, steps): + """ Parses the file and starts processing it """ + + """ I'd like to use a dispatcher for this; however, we need to exit processing + if the validation fails. """ + for step in steps: + if step['type'] == 'heading': + self.process_heading(step) + elif step['type'] == 'text': + self.process_text(step) + elif step['type'] == 'commands': + self.process_commands(step) + elif step['type'] == 'prerequisites': + for prereq_file in step['content']: + self.process_file(prereq_file) + + @staticmethod + def process_heading(step): + """ Print out the heading exactly as we found it """ + print(step['level'] * '#' + ' ' + step['content']) + print() + + @staticmethod + def process_text(step): + """ Print out the text exactly as we found it """ + print(step['content']) + print() + + def process_commands(self, step): + """ Pretend to type the command, run it and then display the output """ + for cmd in step['content']: + print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ' + cmd) + results = self.executor.run_cmd(cmd) + print(results, end="", flush=True) + print() + return results + + @staticmethod + def process_next_steps(steps): + """ Is there a good way to test this that doesn't involve lots of test code + expect? + """ + idx = 1 + if steps: + print("Next steps available:") + for step in steps: + print(idx + ".) " + step['title']) + idx += 1 + step_request = input("Which step do you want to take next?") + if step_request: + return steps[step_request+1]['target'] + return diff --git a/tests/test_demo_mode.py b/tests/test_demo_mode.py index 00c75ec..44ed51d 100644 --- a/tests/test_demo_mode.py +++ b/tests/test_demo_mode.py @@ -33,7 +33,7 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple-variable') + @data('simple', 'simple-variable', 'results-block') #@data('simple', 'simple-variable', 'results-block', # 'results-block-fail', 'create-file', 'prerequisite-run') def test_process(self, directory): diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index 0c81b90..8f04ec4 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -12,23 +12,18 @@ class SimDemMistletoeTestSuite(unittest.TestCase): def test_ast(self): """Verify we understand how the Mistletoe AST parsers work""" + self.maxDiff = None file_path = 'content/simple/README.md' with open(file_path, 'r') as fin: res = renderer.get_ast(token.Document(fin)) - exp_res = {'children': [{'children': [{'content': 'this is text', 'type': 'RawText'}], - 'type': 'Paragraph'}, - {'children': [{'content': 'echo foo\necho bar\n', + exp_res = {'children': [{'children': [{'content': 'Simple', 'type': 'RawText'}], + 'level': 1, + 'type': 'Heading'}, + {'children': [{'content': 'echo foo\n', 'type': 'RawText'}], 'language': 'shell', - 'type': 'BlockCode'}, - {'children': [{'content': 'more text', 'type': 'RawText'}], - 'type': 'Paragraph'}, - {'children': [{'content': 'echo baz\n', 'type': 'RawText'}], - 'language': 'shell', - 'type': 'BlockCode'}, - {'children': [{'content': 'even more text', 'type': 'RawText'}], - 'type': 'Paragraph'}], + 'type': 'BlockCode'}], 'footnotes': {}, 'type': 'Document'} diff --git a/tests/test_tutorial_mode.py b/tests/test_tutorial_mode.py new file mode 100644 index 0000000..02c12b5 --- /dev/null +++ b/tests/test_tutorial_mode.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +""" system level test class """ + +import configparser +import logging +import sys +import unittest + +from ddt import data, ddt + +from simdem.parser import simdem1 +from simdem.executor import bash +from simdem.mode import tutorial + + +@ddt +class SimDemSystemTestSuite(unittest.TestCase): + """Advanced test cases.""" + + simdem = None + + def setUp(self): + + config = configparser.ConfigParser() + config.read("content/config/unit_test.ini") + self.simdem = tutorial.TutorialMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) + + log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + root_logger = logging.getLogger() + root_logger.setLevel(config.get('LOG', 'LEVEL')) + file_handler = logging.FileHandler(config.get('LOG', 'FILE')) + file_handler.setFormatter(log_formatter) + root_logger.addHandler(file_handler) + + # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer + @data('simple', 'simple-variable') + #@data('simple', 'simple-variable', 'results-block', + # 'results-block-fail', 'create-file', 'prerequisite-run') + def test_process(self, directory): + """ Each content directory is expected to have a README.md and an expected_result.tutorial + this allows us to test each of them easily + """ + self.simdem.process_file('./content/' + directory + '/README.md') + # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. + # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. + res = sys.stdout.getvalue() # pylint: disable=E1101 + exp_res = open('./content/' + directory + '/expected_result.tutorial', 'r').read() + self.assertEqual(exp_res, res) + + +if __name__ == '__main__': + unittest.main() From 7f475c26f57c60f008452a09329354d6f1db45a7 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 16 Jan 2018 19:09:57 -0600 Subject: [PATCH 085/167] fix bug in simdem1 parser Signed-off-by: Tommy Falgout --- README.md | 9 +++++++++ content/simple-variable/README.md | 2 +- content/simple-variable/expected_result.tutorial | 2 +- docs/{intro.md => README.md} | 0 simdem/cli.py | 4 +--- simdem/parser/simdem1.py | 2 +- tests/test_mistletoe.py | 2 +- tests/test_parser_simdem1.py | 2 +- 8 files changed, 15 insertions(+), 8 deletions(-) rename docs/{intro.md => README.md} (100%) diff --git a/README.md b/README.md index 5f8912e..a034b21 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,15 @@ Details on the complete feature list can be found in the [feature documentation] Currently, SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). +# Installation + +Currently, it's only available for installation in development mode: + +``` +git clone git@github.com:Azure/simdem.git +pip3 install -v -e . +``` + # Development We would love to have you be a part of the SimDem development team. For details, see the [development documentation](docs/development.md). diff --git a/content/simple-variable/README.md b/content/simple-variable/README.md index 4d74318..d420346 100644 --- a/content/simple-variable/README.md +++ b/content/simple-variable/README.md @@ -1,4 +1,4 @@ -this is text +# this is text ```shell FOO=bar diff --git a/content/simple-variable/expected_result.tutorial b/content/simple-variable/expected_result.tutorial index 7dbae61..43278d6 100644 --- a/content/simple-variable/expected_result.tutorial +++ b/content/simple-variable/expected_result.tutorial @@ -1,4 +1,4 @@ -this is text +# this is text $ FOO=bar diff --git a/docs/intro.md b/docs/README.md similarity index 100% rename from docs/intro.md rename to docs/README.md diff --git a/simdem/cli.py b/simdem/cli.py index 35b33ff..c789289 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -19,9 +19,7 @@ def main(): help="Turn on logging to console") argp.add_argument('--config-file', '-c', default="content/config/demo.ini", help="Config file to use") - argp.add_argument('--renderer', '-r', default="demo", - help="Render class to use", choices=['demo']) - argp.add_argument('--mode', '-m', default="demo", + argp.add_argument('--mode', '-m', default="tutorial", help="Mode to use", choices=['demo', 'dump', 'automated', 'tutorial']) argp.add_argument('--parser', '-p', default="simdem1", help="Parser class to use", choices=['simdem1', 'ast']) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index c8851d7..895e757 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -91,7 +91,7 @@ def parse_list(self, token, res): def parse_paragraph(self, token, res): """ Parse Paragraph """ logging.debug("parse_paragraph()") - if 'content' in token['children'][0]: + if token['children'] and 'content' in token['children'][0]: content = token['children'][0]['content'] # Result Paragraph if content == 'Results:': diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index 8f04ec4..a4f9228 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -12,7 +12,7 @@ class SimDemMistletoeTestSuite(unittest.TestCase): def test_ast(self): """Verify we understand how the Mistletoe AST parsers work""" - self.maxDiff = None + #self.maxDiff = None file_path = 'content/simple/README.md' with open(file_path, 'r') as fin: diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 9225202..72bc559 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -19,7 +19,7 @@ def setUp(self): def test_full(self): """Test parsing a document with all features in it""" - #self.maxDiff = None + self.maxDiff = None file_path = 'content/simdem1/README.md' res = self.parser.parse_file(file_path) From 1f1c4007f74712b3b11ac97e5830489ee57d9a58 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 17 Jan 2018 10:38:04 -0600 Subject: [PATCH 086/167] Refactored simdem parser to not use AST Signed-off-by: Tommy Falgout --- README.md | 18 +- content/prerequisite-run/README.md | 4 +- .../prerequisite-run/expected_result.tutorial | 17 +- content/prerequisite-run/prereq.md | 7 +- content/simple/README.md | 1 + simdem-env/pip-selfcheck.json | 2 +- simdem/mode/automated.py | 23 +- simdem/mode/demo.py | 29 +- simdem/mode/tutorial.py | 14 +- simdem/parser/simdem1.py | 256 ++++++++++-------- simdem/parser/simdem1old.py | 140 ++++++++++ 11 files changed, 327 insertions(+), 184 deletions(-) create mode 100644 simdem/parser/simdem1old.py diff --git a/README.md b/README.md index a034b21..b6b08af 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,9 @@ SimDem supports the following features: Details on the complete feature list can be found in the [feature documentation](docs/features.md). -# Syntax - -Currently, SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). +# Getting Started -# Installation +## Installation Currently, it's only available for installation in development mode: @@ -25,6 +23,18 @@ git clone git@github.com:Azure/simdem.git pip3 install -v -e . ``` +## Running + +A great place to start working on SimDem is to run SimDem on its own documentation + +``` +simdem docs/README.md +``` + +# Syntax + +Currently, SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). + # Development We would love to have you be a part of the SimDem development team. For details, see the [development documentation](docs/development.md). diff --git a/content/prerequisite-run/README.md b/content/prerequisite-run/README.md index 7f84242..86b1888 100644 --- a/content/prerequisite-run/README.md +++ b/content/prerequisite-run/README.md @@ -1,9 +1,9 @@ # Prerequisites -* [prereq](content/prerequisite-run/prereq.md) +This is a [prereq](content/prerequisite-run/prereq.md) # Main ```shell -echo post-prereq-run +echo $prereq_run ``` diff --git a/content/prerequisite-run/expected_result.tutorial b/content/prerequisite-run/expected_result.tutorial index d045721..7625dbf 100644 --- a/content/prerequisite-run/expected_result.tutorial +++ b/content/prerequisite-run/expected_result.tutorial @@ -1,16 +1,17 @@ -# Prerequisites +# Inside prereq.md - * prereq +$ prereq_run=true +$ echo $prereq_run +true -This text should be run +Completed running prereq.md -$ echo running-prereq -running-prereq +# Prerequisites -End running text +This is a prereq # Main -$ echo post-prereq-run -post-prereq-run +$ echo $prereq_run +true diff --git a/content/prerequisite-run/prereq.md b/content/prerequisite-run/prereq.md index c6c9648..db47629 100644 --- a/content/prerequisite-run/prereq.md +++ b/content/prerequisite-run/prereq.md @@ -1,7 +1,8 @@ -This text should be run +# Inside prereq.md ```shell -echo running-prereq +prereq_run=true +echo $prereq_run ``` -End running text +Completed running prereq.md diff --git a/content/simple/README.md b/content/simple/README.md index d429891..d7e193c 100644 --- a/content/simple/README.md +++ b/content/simple/README.md @@ -2,4 +2,5 @@ ```shell echo foo +echo bar ``` diff --git a/simdem-env/pip-selfcheck.json b/simdem-env/pip-selfcheck.json index 5e755af..d251d3d 100644 --- a/simdem-env/pip-selfcheck.json +++ b/simdem-env/pip-selfcheck.json @@ -1 +1 @@ -{"last_check":"2017-08-16T22:13:50Z","pypi_version":"9.0.1"} \ No newline at end of file +{"last_check":"2018-01-17T15:25:04Z","pypi_version":"9.0.1"} \ No newline at end of file diff --git a/simdem/mode/automated.py b/simdem/mode/automated.py index 8f4fecc..fdef4e0 100644 --- a/simdem/mode/automated.py +++ b/simdem/mode/automated.py @@ -15,28 +15,25 @@ def process_file(self, file_path): """ Parses the file and starts processing it """ #print("*** Processing " + file_path + " ***") steps = self.parser.parse_file(file_path) - self.process(steps) - #print("*** Completed Processing " + file_path + " ***") - def process(self, steps): - """ Parses the file and starts processing it """ last_command_result = None + if 'prerequisites' in steps: + for prereq_file in steps['prerequisites']: + self.process_file(prereq_file) + if 'validation_command' in steps: + last_command_result = self.process_commands(steps['validation_command']) + if 'validation_result' in steps: + if not self.is_result_valid(steps['validation_result'], last_command_result): + return + """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ - for step in steps: + for step in steps['body']: if step['type'] == 'commands': last_command_result = self.process_commands(step) elif step['type'] == 'result': self.is_result_valid(step['content'], last_command_result) - elif step['type'] == 'prerequisites': - for prereq_file in step['content']: - self.process_file(prereq_file) - elif step['type'] == 'validation_command': - last_command_result = self.process_commands(step) - elif step['type'] == 'validation_result': - if not self.is_result_valid(step['content'], last_command_result): - return def process_commands(self, step): diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index d236755..e394804 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -13,20 +13,16 @@ def process_file(self, file_path): """ Parses the file and starts processing it """ #print("*** Processing " + file_path + " ***") steps = self.parser.parse_file(file_path) - self.process(steps) - #print("*** Completed Processing " + file_path + " ***") - - def process(self, steps): - """ Parses the file and starts processing it """ """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ - for step in steps: + if 'prerequisites' in steps: + for prereq_file in steps['prerequisites']: + self.process_file(prereq_file) + + for step in steps['body']: if step['type'] == 'commands': self.process_commands(step) - elif step['type'] == 'prerequisites': - for prereq_file in step['content']: - self.process_file(prereq_file) def process_commands(self, step): """ Pretend to type the command, run it and then display the output """ @@ -51,18 +47,3 @@ def type_command(self, cmd): time.sleep(delay) print(char, end="", flush=True) print("", flush=True) - - @staticmethod - def process_next_steps(steps): - """ Is there a good way to test this that doesn't involve lots of test code + expect? - """ - idx = 1 - if steps: - print("Next steps available:") - for step in steps: - print(idx + ".) " + step['title']) - idx += 1 - step_request = input("Which step do you want to take next?") - if step_request: - return steps[step_request+1]['target'] - return diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 82c8f23..a22b2c1 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -12,24 +12,20 @@ def process_file(self, file_path): """ Parses the file and starts processing it """ #print("*** Processing " + file_path + " ***") steps = self.parser.parse_file(file_path) - self.process(steps) - #print("*** Completed Processing " + file_path + " ***") - - def process(self, steps): - """ Parses the file and starts processing it """ """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ - for step in steps: + if 'prerequisites' in steps: + for prereq_file in steps['prerequisites']: + self.process_file(prereq_file) + + for step in steps['body']: if step['type'] == 'heading': self.process_heading(step) elif step['type'] == 'text': self.process_text(step) elif step['type'] == 'commands': self.process_commands(step) - elif step['type'] == 'prerequisites': - for prereq_file in step['content']: - self.process_file(prereq_file) @staticmethod def process_heading(step): diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 895e757..52d285f 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -2,139 +2,155 @@ """ import logging +from collections import defaultdict +from mistletoe.base_renderer import BaseRenderer +from mistletoe import Document -import mistletoe.ast_renderer as renderer -import mistletoe.block_token +class SimDem1Parser(object): # pylint: disable=R0903 + """ Parses markdown based off of Mistletoe renderer """ -class SimDem1Parser(object): - """ This class parses the human readable markdown using a defined syntax to - know how to create the SimDem Execution Object - and uses the code block language to know which type of execution it is - """ - - mode = None - - def set_mode(self, mode): - """ Used mainly for debugging. No real need to have as a separate function """ - logging.debug('set_mode(' + str(mode) + ')') - self.mode = mode - - def parse_file(self, file_path): - """ The main meat for parsing the file. Uses mistletoe's AST parser - to create a tokenized object and then parses that tokenized object - into SimDem Execution Object - """ + @staticmethod + def parse_file(file_path): + """ The main meat for parsing the file. """ with open(file_path, 'r') as fin: - ast = renderer.get_ast(mistletoe.block_token.Document(fin)) - res = [] - - for token in ast['children']: - logging.debug("parse_file():processing " + str(token)) - - # Heading - if token['type'] == 'Heading': - res = self.parse_heading(token, res) - elif token['type'] == 'List': - res = self.parse_list(token, res) - elif token['type'] == 'Paragraph': - res = self.parse_paragraph(token, res) - elif token['type'] == 'BlockCode': - res = self.parse_block_code(token, res) - else: - logging.debug('parse_file():unknown token type (type=' + token['type'] + - ';mode=' + self.mode + '). Igorning') - - return res - - def parse_heading(self, token, res): - """ Parse heading """ - res.append({'type': 'heading', 'level': token['level'], - 'content': token['children'][0]['content']}) - logging.debug("parse_file():found heading") + with SimDemMistletoeRenderer() as renderer: + rendered = renderer.render(Document(fin)) + + # Do stuff here + return rendered + + +class SimDemMistletoeRenderer(BaseRenderer): + """ Based off of https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 """ + links = [] + section = None + block = None + + def __init__(self): + super().__init__() + self.output = defaultdict(list) + self.reset_section() + + def reset_section(self): + """ If we encounter a new section, reset everything we know """ + self.links = [] + self.section = None + self.block = None + + def set_section(self, section): + """ Set the section to the section name. Duh """ + logging.debug('set_section(' + str(section) + ')') + self.section = section + + def set_block(self, block): + """ Set the block to the block name. Duh """ + logging.debug('set_block(' + str(block) + ')') + self.block = block + + def append_validation_command(self, cmd): + """ Assuming validation commands should be a list """ + logging.debug('append_validation(' + str(cmd) + ')') + self.output['validation_command'].append(cmd) + + def append_validation_result(self, result): + """ Assuming validation results should be a list """ + logging.debug('append_validation(' + str(result) + ')') + self.output['validation_expected_result'].append(result) + + def append_body(self, body): + """ Adding the meat of the work to the dict """ + logging.debug('append_body(' + str(body) + ')') + self.output['body'].append(body) + + def append_prereq(self, target): + """ Set Prereqs """ + logging.debug('append_prereq(' + target + ')') + self.output['prerequisites'].append(target) + + def render_heading(self, token): + """ Render for Heading: # """ + inner = self.render_inner(token) + content = {'type': 'heading', 'level': token.level, 'content': inner} + self.append_body(content) # Prerequisite Heading - if token['children'][0]['content'].lower() == 'prerequisites': - self.set_mode('prerequisites') + if inner.lower() == 'prerequisites': + self.set_section('prerequisites') # Next Steps Heading - elif token['children'][0]['content'].lower() == 'next steps': - self.set_mode('next_steps') - - # Results Heading - elif token['children'][0]['content'].lower() == 'result': - self.set_mode('result') + elif inner.lower() == 'next steps': + self.set_section('next_steps') # Validation Heading - elif token['children'][0]['content'].lower() == 'validation': - self.set_mode('validation') + elif inner.lower() == 'validation': + self.set_section('validation') else: logging.debug("parse_file():unable to determing header type.") - self.set_mode(None) - return res - - def parse_list(self, token, res): - """ Parse list """ - logging.debug("parse_list()") - if self.mode == 'prerequisites': - logging.debug("parse_file():found prerequisites list") - titles = [x['children'][0]['children'][0]['content'] for x in token['children']] - content_t = "\n".join(' * ' + x for x in titles) - # Make sure to print out the page contents too - res.append({'type': 'text', 'content': content_t}) - - content_p = [x['children'][0]['target'] for x in token['children']] - res.append({'type': 'prerequisites', 'content': content_p}) - return res - - def parse_paragraph(self, token, res): - """ Parse Paragraph """ - logging.debug("parse_paragraph()") - if token['children'] and 'content' in token['children'][0]: - content = token['children'][0]['content'] - # Result Paragraph - if content == 'Results:': - if self.mode == 'validation': - self.set_mode('validation_result') - else: - self.set_mode('result') - #res.append({'type': 'result', 'content': content}) - # Regular Paragraph + self.reset_section() + return inner + + def render_list(self, token): + """ Render a markdown list """ + inner = self.render_inner(token) + return inner + + def render_list_item(self, token): + """ Render a markdown list item """ + inner = self.render_inner(token) + return inner + + def render_link(self, token): + """ Due to the way the parser works, we can only return strings + We need to find another way to store link targets. + Unfortunately, I can only think of storing them in an array right now + """ + if self.section and 'prerequisites' in self.section: + self.append_prereq(token.target) + inner = self.render_inner(token) + return inner + + def render_raw_text(self, token): + """ Render raw text. The only thing to look for is the result text indicator """ + if token.content == 'Results:': + self.set_block('results') + return token.content + + def render_paragraph(self, token): + """ Render for Paragraph """ + inner = self.render_inner(token) + body = {'type': 'text', 'content': inner} + self.append_body(body) + return '' + + def render_block_code(self, token): + """ Render a markdown block code """ + #lines = token.children[0].content.splitlines() + content = token.children[0].content + + if self.section == 'validation': + # Validation blocks aren't run like normal blocks + if self.block == 'results': + # Validation result blocks are expecially not checked like normal blocks + self.append_validation_result(content) else: - res.append({'type': 'text', 'content': content}) - return res - - def parse_block_code(self, token, res): - """ Parse block code """ - logging.debug("parse_block_code()") - - # Result Block Code - if self.mode == 'validation': - logging.debug("parse_block_code():found validation block code") - content = [line for line in token['children'][0]['content'].split("\n") if line] - res.append({'type': 'validation_command', 'content': content}) - - # Validation Result Block Code - elif self.mode == 'validation_result': - logging.debug("parse_block_code():found validation result block code") - content = token['children'][0]['content'].rstrip() - res.append({'type': 'validation_result', 'content': content}) - - # Result Block Code - elif self.mode == 'result': - logging.debug("parse_block_code():found result block code") - content = token['children'][0]['content'].rstrip() - res.append({'type': 'result', 'content': content}) - self.set_mode(None) - - # Block Code (regular) - elif self.mode is None: - logging.debug("parse_block_code():found default block code") - content = [line for line in token['children'][0]['content'].split("\n") if line] - res.append({'type': 'commands', 'content': content}) - + self.append_validation_command(content) else: - logging.debug('parse_file():unexpected Block Code mode (' + self.mode + ')') + # Assume this is a normal code block to run block + content = {'type': 'commands', 'content': content.splitlines()} + self.append_body(content) + + inner = self.render_inner(token) + # After this code block, reset the block type (e.g. No longer a "Result block") + self.set_block(None) + + return inner + + def render_document(self, token): + """ Render the entire markdown document """ + self.render_inner(token) + return dict(self.output) - return res + def __getattr__(self, name): + return lambda token: '' diff --git a/simdem/parser/simdem1old.py b/simdem/parser/simdem1old.py new file mode 100644 index 0000000..895e757 --- /dev/null +++ b/simdem/parser/simdem1old.py @@ -0,0 +1,140 @@ +""" This module hosts the ContextParser +""" + +import logging + +import mistletoe.ast_renderer as renderer +import mistletoe.block_token + + +class SimDem1Parser(object): + """ This class parses the human readable markdown using a defined syntax to + know how to create the SimDem Execution Object + and uses the code block language to know which type of execution it is + """ + + mode = None + + def set_mode(self, mode): + """ Used mainly for debugging. No real need to have as a separate function """ + logging.debug('set_mode(' + str(mode) + ')') + self.mode = mode + + def parse_file(self, file_path): + """ The main meat for parsing the file. Uses mistletoe's AST parser + to create a tokenized object and then parses that tokenized object + into SimDem Execution Object + """ + with open(file_path, 'r') as fin: + ast = renderer.get_ast(mistletoe.block_token.Document(fin)) + res = [] + + for token in ast['children']: + logging.debug("parse_file():processing " + str(token)) + + # Heading + if token['type'] == 'Heading': + res = self.parse_heading(token, res) + elif token['type'] == 'List': + res = self.parse_list(token, res) + elif token['type'] == 'Paragraph': + res = self.parse_paragraph(token, res) + elif token['type'] == 'BlockCode': + res = self.parse_block_code(token, res) + else: + logging.debug('parse_file():unknown token type (type=' + token['type'] + + ';mode=' + self.mode + '). Igorning') + + return res + + def parse_heading(self, token, res): + """ Parse heading """ + res.append({'type': 'heading', 'level': token['level'], + 'content': token['children'][0]['content']}) + logging.debug("parse_file():found heading") + + # Prerequisite Heading + if token['children'][0]['content'].lower() == 'prerequisites': + self.set_mode('prerequisites') + + # Next Steps Heading + elif token['children'][0]['content'].lower() == 'next steps': + self.set_mode('next_steps') + + # Results Heading + elif token['children'][0]['content'].lower() == 'result': + self.set_mode('result') + + # Validation Heading + elif token['children'][0]['content'].lower() == 'validation': + self.set_mode('validation') + + else: + logging.debug("parse_file():unable to determing header type.") + self.set_mode(None) + return res + + def parse_list(self, token, res): + """ Parse list """ + logging.debug("parse_list()") + if self.mode == 'prerequisites': + logging.debug("parse_file():found prerequisites list") + titles = [x['children'][0]['children'][0]['content'] for x in token['children']] + content_t = "\n".join(' * ' + x for x in titles) + # Make sure to print out the page contents too + res.append({'type': 'text', 'content': content_t}) + + content_p = [x['children'][0]['target'] for x in token['children']] + res.append({'type': 'prerequisites', 'content': content_p}) + return res + + def parse_paragraph(self, token, res): + """ Parse Paragraph """ + logging.debug("parse_paragraph()") + if token['children'] and 'content' in token['children'][0]: + content = token['children'][0]['content'] + # Result Paragraph + if content == 'Results:': + if self.mode == 'validation': + self.set_mode('validation_result') + else: + self.set_mode('result') + #res.append({'type': 'result', 'content': content}) + # Regular Paragraph + else: + res.append({'type': 'text', 'content': content}) + return res + + def parse_block_code(self, token, res): + """ Parse block code """ + logging.debug("parse_block_code()") + + # Result Block Code + if self.mode == 'validation': + logging.debug("parse_block_code():found validation block code") + content = [line for line in token['children'][0]['content'].split("\n") if line] + res.append({'type': 'validation_command', 'content': content}) + + # Validation Result Block Code + elif self.mode == 'validation_result': + logging.debug("parse_block_code():found validation result block code") + content = token['children'][0]['content'].rstrip() + res.append({'type': 'validation_result', 'content': content}) + + # Result Block Code + elif self.mode == 'result': + logging.debug("parse_block_code():found result block code") + content = token['children'][0]['content'].rstrip() + res.append({'type': 'result', 'content': content}) + self.set_mode(None) + + # Block Code (regular) + elif self.mode is None: + logging.debug("parse_block_code():found default block code") + content = [line for line in token['children'][0]['content'].split("\n") if line] + res.append({'type': 'commands', 'content': content}) + + else: + logging.debug('parse_file():unexpected Block Code mode (' + self.mode + ')') + + return res From 6171c4e00de3cf5de53d59b253e33dd5ff835797 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 17 Jan 2018 10:38:41 -0600 Subject: [PATCH 087/167] Refactor simdem1 parser (remove ast) Signed-off-by: Tommy Falgout --- simdem/parser/simdem1.py | 74 +++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 52d285f..da8ef42 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -32,42 +32,6 @@ def __init__(self): self.output = defaultdict(list) self.reset_section() - def reset_section(self): - """ If we encounter a new section, reset everything we know """ - self.links = [] - self.section = None - self.block = None - - def set_section(self, section): - """ Set the section to the section name. Duh """ - logging.debug('set_section(' + str(section) + ')') - self.section = section - - def set_block(self, block): - """ Set the block to the block name. Duh """ - logging.debug('set_block(' + str(block) + ')') - self.block = block - - def append_validation_command(self, cmd): - """ Assuming validation commands should be a list """ - logging.debug('append_validation(' + str(cmd) + ')') - self.output['validation_command'].append(cmd) - - def append_validation_result(self, result): - """ Assuming validation results should be a list """ - logging.debug('append_validation(' + str(result) + ')') - self.output['validation_expected_result'].append(result) - - def append_body(self, body): - """ Adding the meat of the work to the dict """ - logging.debug('append_body(' + str(body) + ')') - self.output['body'].append(body) - - def append_prereq(self, target): - """ Set Prereqs """ - logging.debug('append_prereq(' + target + ')') - self.output['prerequisites'].append(target) - def render_heading(self, token): """ Render for Heading: # """ inner = self.render_inner(token) @@ -154,3 +118,41 @@ def render_document(self, token): def __getattr__(self, name): return lambda token: '' + + # Everything below here is boring setters + + def reset_section(self): + """ If we encounter a new section, reset everything we know """ + self.links = [] + self.section = None + self.block = None + + def set_section(self, section): + """ Set the section to the section name. Duh """ + logging.debug('set_section(' + str(section) + ')') + self.section = section + + def set_block(self, block): + """ Set the block to the block name. Duh """ + logging.debug('set_block(' + str(block) + ')') + self.block = block + + def append_validation_command(self, cmd): + """ Assuming validation commands should be a list """ + logging.debug('append_validation(' + str(cmd) + ')') + self.output['validation_command'].append(cmd) + + def append_validation_result(self, result): + """ Assuming validation results should be a list """ + logging.debug('append_validation(' + str(result) + ')') + self.output['validation_expected_result'].append(result) + + def append_body(self, body): + """ Adding the meat of the work to the dict """ + logging.debug('append_body(' + str(body) + ')') + self.output['body'].append(body) + + def append_prereq(self, target): + """ Set Prereqs """ + logging.debug('append_prereq(' + target + ')') + self.output['prerequisites'].append(target) From 8df68266f68151d4ab72930a087902200a081418 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 17 Jan 2018 10:40:16 -0600 Subject: [PATCH 088/167] Remove deadcode Signed-off-by: Tommy Falgout --- simdem/parser/simdem1.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index da8ef42..e25e7c1 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -23,7 +23,6 @@ def parse_file(file_path): class SimDemMistletoeRenderer(BaseRenderer): """ Based off of https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 """ - links = [] section = None block = None @@ -123,7 +122,6 @@ def __getattr__(self, name): def reset_section(self): """ If we encounter a new section, reset everything we know """ - self.links = [] self.section = None self.block = None From 5b47283e26a2715b695b65ba1577b771407c7208 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 17 Jan 2018 10:57:44 -0600 Subject: [PATCH 089/167] Finishing touches on rm-ast refactor Signed-off-by: Tommy Falgout --- simdem/parser/simdem1.py | 16 +++-- simdem/parser/simdem1old.py | 140 ------------------------------------ 2 files changed, 11 insertions(+), 145 deletions(-) delete mode 100644 simdem/parser/simdem1old.py diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index e25e7c1..c0cdb64 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -78,13 +78,15 @@ def render_raw_text(self, token): """ Render raw text. The only thing to look for is the result text indicator """ if token.content == 'Results:': self.set_block('results') + return '' return token.content def render_paragraph(self, token): """ Render for Paragraph """ inner = self.render_inner(token) - body = {'type': 'text', 'content': inner} - self.append_body(body) + if inner: + body = {'type': 'text', 'content': inner} + self.append_body(body) return '' def render_block_code(self, token): @@ -100,9 +102,13 @@ def render_block_code(self, token): else: self.append_validation_command(content) else: - # Assume this is a normal code block to run block - content = {'type': 'commands', 'content': content.splitlines()} - self.append_body(content) + if self.block == 'results': + # Assume that the last body item is the command we're expecting results for + self.output['body'][-1]['expected_result'] = content + else: + # Assume this is a normal code block to run block + content = {'type': 'commands', 'content': content.splitlines()} + self.append_body(content) inner = self.render_inner(token) # After this code block, reset the block type (e.g. No longer a "Result block") diff --git a/simdem/parser/simdem1old.py b/simdem/parser/simdem1old.py deleted file mode 100644 index 895e757..0000000 --- a/simdem/parser/simdem1old.py +++ /dev/null @@ -1,140 +0,0 @@ -""" This module hosts the ContextParser -""" - -import logging - -import mistletoe.ast_renderer as renderer -import mistletoe.block_token - - -class SimDem1Parser(object): - """ This class parses the human readable markdown using a defined syntax to - know how to create the SimDem Execution Object - and uses the code block language to know which type of execution it is - """ - - mode = None - - def set_mode(self, mode): - """ Used mainly for debugging. No real need to have as a separate function """ - logging.debug('set_mode(' + str(mode) + ')') - self.mode = mode - - def parse_file(self, file_path): - """ The main meat for parsing the file. Uses mistletoe's AST parser - to create a tokenized object and then parses that tokenized object - into SimDem Execution Object - """ - with open(file_path, 'r') as fin: - ast = renderer.get_ast(mistletoe.block_token.Document(fin)) - res = [] - - for token in ast['children']: - logging.debug("parse_file():processing " + str(token)) - - # Heading - if token['type'] == 'Heading': - res = self.parse_heading(token, res) - elif token['type'] == 'List': - res = self.parse_list(token, res) - elif token['type'] == 'Paragraph': - res = self.parse_paragraph(token, res) - elif token['type'] == 'BlockCode': - res = self.parse_block_code(token, res) - else: - logging.debug('parse_file():unknown token type (type=' + token['type'] + - ';mode=' + self.mode + '). Igorning') - - return res - - def parse_heading(self, token, res): - """ Parse heading """ - res.append({'type': 'heading', 'level': token['level'], - 'content': token['children'][0]['content']}) - logging.debug("parse_file():found heading") - - # Prerequisite Heading - if token['children'][0]['content'].lower() == 'prerequisites': - self.set_mode('prerequisites') - - # Next Steps Heading - elif token['children'][0]['content'].lower() == 'next steps': - self.set_mode('next_steps') - - # Results Heading - elif token['children'][0]['content'].lower() == 'result': - self.set_mode('result') - - # Validation Heading - elif token['children'][0]['content'].lower() == 'validation': - self.set_mode('validation') - - else: - logging.debug("parse_file():unable to determing header type.") - self.set_mode(None) - return res - - def parse_list(self, token, res): - """ Parse list """ - logging.debug("parse_list()") - if self.mode == 'prerequisites': - logging.debug("parse_file():found prerequisites list") - titles = [x['children'][0]['children'][0]['content'] for x in token['children']] - content_t = "\n".join(' * ' + x for x in titles) - # Make sure to print out the page contents too - res.append({'type': 'text', 'content': content_t}) - - content_p = [x['children'][0]['target'] for x in token['children']] - res.append({'type': 'prerequisites', 'content': content_p}) - return res - - def parse_paragraph(self, token, res): - """ Parse Paragraph """ - logging.debug("parse_paragraph()") - if token['children'] and 'content' in token['children'][0]: - content = token['children'][0]['content'] - # Result Paragraph - if content == 'Results:': - if self.mode == 'validation': - self.set_mode('validation_result') - else: - self.set_mode('result') - #res.append({'type': 'result', 'content': content}) - # Regular Paragraph - else: - res.append({'type': 'text', 'content': content}) - return res - - def parse_block_code(self, token, res): - """ Parse block code """ - logging.debug("parse_block_code()") - - # Result Block Code - if self.mode == 'validation': - logging.debug("parse_block_code():found validation block code") - content = [line for line in token['children'][0]['content'].split("\n") if line] - res.append({'type': 'validation_command', 'content': content}) - - # Validation Result Block Code - elif self.mode == 'validation_result': - logging.debug("parse_block_code():found validation result block code") - content = token['children'][0]['content'].rstrip() - res.append({'type': 'validation_result', 'content': content}) - - # Result Block Code - elif self.mode == 'result': - logging.debug("parse_block_code():found result block code") - content = token['children'][0]['content'].rstrip() - res.append({'type': 'result', 'content': content}) - self.set_mode(None) - - # Block Code (regular) - elif self.mode is None: - logging.debug("parse_block_code():found default block code") - content = [line for line in token['children'][0]['content'].split("\n") if line] - res.append({'type': 'commands', 'content': content}) - - else: - logging.debug('parse_file():unexpected Block Code mode (' + self.mode + ')') - - return res From edd85661ab2a38fa2695e3b3dfc3726f00ed4564 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 17 Jan 2018 12:49:59 -0600 Subject: [PATCH 090/167] Fix all tests after refactoring Signed-off-by: Tommy Falgout --- content/prerequisite-run/expected_result.demo | 9 +-- content/simple/expected_result.demo | 2 + content/simple/expected_result.tutorial | 2 + tests/test_demo_mode.py | 5 +- tests/test_mistletoe.py | 2 +- tests/test_parser_simdem1.py | 66 ++++++++++--------- tests/test_tutorial_mode.py | 5 +- 7 files changed, 48 insertions(+), 43 deletions(-) diff --git a/content/prerequisite-run/expected_result.demo b/content/prerequisite-run/expected_result.demo index 9171799..13b07c4 100644 --- a/content/prerequisite-run/expected_result.demo +++ b/content/prerequisite-run/expected_result.demo @@ -1,6 +1,7 @@ -$ echo running-prereq -running-prereq +$ prereq_run=true +$ echo $prereq_run +true -$ echo post-prereq-run -post-prereq-run +$ echo $prereq_run +true diff --git a/content/simple/expected_result.demo b/content/simple/expected_result.demo index 3e08e67..8a8f518 100644 --- a/content/simple/expected_result.demo +++ b/content/simple/expected_result.demo @@ -1,3 +1,5 @@ $ echo foo foo +$ echo bar +bar diff --git a/content/simple/expected_result.tutorial b/content/simple/expected_result.tutorial index d54007a..9d3be1e 100644 --- a/content/simple/expected_result.tutorial +++ b/content/simple/expected_result.tutorial @@ -2,4 +2,6 @@ $ echo foo foo +$ echo bar +bar diff --git a/tests/test_demo_mode.py b/tests/test_demo_mode.py index 44ed51d..d89338c 100644 --- a/tests/test_demo_mode.py +++ b/tests/test_demo_mode.py @@ -33,9 +33,8 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable', 'results-block') - #@data('simple', 'simple-variable', 'results-block', - # 'results-block-fail', 'create-file', 'prerequisite-run') + @data('simple', 'simple-variable', 'results-block', + 'results-block-fail', 'create-file', 'prerequisite-run') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.out this allows us to test each of them easily diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index a4f9228..e4dd1d7 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -20,7 +20,7 @@ def test_ast(self): exp_res = {'children': [{'children': [{'content': 'Simple', 'type': 'RawText'}], 'level': 1, 'type': 'Heading'}, - {'children': [{'content': 'echo foo\n', + {'children': [{'content': 'echo foo\necho bar\n', 'type': 'RawText'}], 'language': 'shell', 'type': 'BlockCode'}], diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 72bc559..0ca0253 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# pylint: disable=C0330 """SimDem Test Case""" import configparser @@ -19,41 +19,43 @@ def setUp(self): def test_full(self): """Test parsing a document with all features in it""" - self.maxDiff = None + self.maxDiff = None # pylint: disable=C0103 file_path = 'content/simdem1/README.md' res = self.parser.parse_file(file_path) # This is pretty brittle. It might be valuable to have a test document with less content - exp_resl = [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, - {'content': 'This is the prerequisite section. SimDem looks for a set of ' - 'links to extract and run through first', - 'type': 'text'}, - {'content': ' * prereq-ignored', 'type': 'text'}, - {'content': ['content/simdem1/prereq-ignored.md'], 'type': 'prerequisites'}, - {'content': "They don't even need to be in the same list", 'type': 'text'}, - {'content': ' * prereq-processed', 'type': 'text'}, - {'content': ['content/simdem1/prereq-processed.md'], 'type': 'prerequisites'}, - {'content': 'By this point, the prerequisites have either run or have passed ' - 'their validation', - 'type': 'text'}, - {'content': 'Did our prerequisites run?', 'level': 1, 'type': 'heading'}, - {'content': ['echo prereq_ignored = $prereq_ignored', - 'echo prereq_processed = $prereq_processed'], - 'type': 'commands'}, - {'content': 'Do stuff here', 'level': 1, 'type': 'heading'}, - {'content': 'We want to execute this because the code type is shell', - 'type': 'text'}, - {'content': ['echo foo', 'var=bar'], 'type': 'commands'}, - {'content': 'Do more stuff here', 'level': 1, 'type': 'heading'}, - {'content': 'We assume the result is for the last command of the last code ' - 'block', - 'type': 'text'}, - {'content': ['echo baz', 'echo $var'], 'type': 'commands'}, - {'content': 'bar', 'type': 'result'}, - {'content': 'Next Steps', 'level': 1, 'type': 'heading'}, - {'content': 'The list inside this block are steps that could be followed when ' - 'performing an interactive tutorial', - 'type': 'text'}] + exp_resl = {'body': [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, + {'content': 'This is the prerequisite section. SimDem looks for a ' + 'set of links to extract and run through first', + 'type': 'text'}, + {'content': "They don't even need to be in the same list", + 'type': 'text'}, + {'content': 'By this point, the prerequisites have either run or ' + 'have passed their validation', + 'type': 'text'}, + {'content': 'Did our prerequisites run?', + 'level': 1, + 'type': 'heading'}, + {'content': ['echo prereq_ignored = $prereq_ignored', + 'echo prereq_processed = $prereq_processed'], + 'type': 'commands'}, + {'content': 'Do stuff here', 'level': 1, 'type': 'heading'}, + {'content': 'We want to execute this because the code type is shell', + 'type': 'text'}, + {'content': ['echo foo', 'var=bar'], 'type': 'commands'}, + {'content': 'Do more stuff here', 'level': 1, 'type': 'heading'}, + {'content': 'We assume the result is for the last command of the ' + 'last code block', + 'type': 'text'}, + {'content': ['echo baz', 'echo $var'], + 'expected_result': 'bar\n', + 'type': 'commands'}, + {'content': 'Next Steps', 'level': 1, 'type': 'heading'}, + {'content': 'The list inside this block are steps that could be ' + 'followed when performing an interactive tutorial', + 'type': 'text'}], + 'prerequisites': ['content/simdem1/prereq-ignored.md', + 'content/simdem1/prereq-processed.md']} pprint.pprint(res) self.assertEqual(res, exp_resl) diff --git a/tests/test_tutorial_mode.py b/tests/test_tutorial_mode.py index 02c12b5..7ce0f81 100644 --- a/tests/test_tutorial_mode.py +++ b/tests/test_tutorial_mode.py @@ -33,9 +33,8 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable') - #@data('simple', 'simple-variable', 'results-block', - # 'results-block-fail', 'create-file', 'prerequisite-run') + @data('simple', 'simple-variable', 'results-block', + 'results-block-fail', 'create-file', 'prerequisite-run') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily From cb2b26e87f6a5fd8fadc78a6dc31c457afd33160 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 17 Jan 2018 12:52:03 -0600 Subject: [PATCH 091/167] Remove file level example and rename test files Signed-off-by: Tommy Falgout --- content/create-file/README.md | 4 ---- content/create-file/expected_result.demo | 4 ---- content/create-file/expected_result.tutorial | 4 ---- tests/{test_demo_mode.py => test_mode_demo.py} | 2 +- tests/{test_tutorial_mode.py => test_mode_tutorial.py} | 2 +- 5 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 content/create-file/README.md delete mode 100644 content/create-file/expected_result.demo delete mode 100644 content/create-file/expected_result.tutorial rename tests/{test_demo_mode.py => test_mode_demo.py} (96%) rename tests/{test_tutorial_mode.py => test_mode_tutorial.py} (96%) diff --git a/content/create-file/README.md b/content/create-file/README.md deleted file mode 100644 index f3abbb6..0000000 --- a/content/create-file/README.md +++ /dev/null @@ -1,4 +0,0 @@ -```shell -echo 'abc' > scratch/foo -cat scratch/foo -``` diff --git a/content/create-file/expected_result.demo b/content/create-file/expected_result.demo deleted file mode 100644 index dd8004e..0000000 --- a/content/create-file/expected_result.demo +++ /dev/null @@ -1,4 +0,0 @@ -$ echo 'abc' > scratch/foo -$ cat scratch/foo -abc - diff --git a/content/create-file/expected_result.tutorial b/content/create-file/expected_result.tutorial deleted file mode 100644 index dd8004e..0000000 --- a/content/create-file/expected_result.tutorial +++ /dev/null @@ -1,4 +0,0 @@ -$ echo 'abc' > scratch/foo -$ cat scratch/foo -abc - diff --git a/tests/test_demo_mode.py b/tests/test_mode_demo.py similarity index 96% rename from tests/test_demo_mode.py rename to tests/test_mode_demo.py index d89338c..27080ce 100644 --- a/tests/test_demo_mode.py +++ b/tests/test_mode_demo.py @@ -34,7 +34,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', - 'results-block-fail', 'create-file', 'prerequisite-run') + 'results-block-fail', 'prerequisite-run') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.out this allows us to test each of them easily diff --git a/tests/test_tutorial_mode.py b/tests/test_mode_tutorial.py similarity index 96% rename from tests/test_tutorial_mode.py rename to tests/test_mode_tutorial.py index 7ce0f81..38453dc 100644 --- a/tests/test_tutorial_mode.py +++ b/tests/test_mode_tutorial.py @@ -34,7 +34,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', - 'results-block-fail', 'create-file', 'prerequisite-run') + 'results-block-fail', 'prerequisite-run') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily From 110990cf8e6fee79ab33cb43a113e49e08dc9642 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 18 Jan 2018 16:48:09 -0600 Subject: [PATCH 092/167] Fix prereq code Signed-off-by: Tommy Falgout --- .simdem.log.swp | Bin 0 -> 16384 bytes content/complete-features/codeblock.md | 38 --------- content/complete-features/context.md | 37 --------- content/prerequisite-run/README.md | 9 --- content/prerequisite-run/expected_result.demo | 7 -- .../prerequisite-run/expected_result.tutorial | 17 ---- content/prerequisite-run/prereq.md | 8 -- content/prerequisites/README.md | 18 +++++ content/prerequisites/expected_result.demo | 18 +++++ .../prerequisites/expected_result.tutorial | 42 ++++++++++ content/prerequisites/prereq-ignored.md | 31 ++++++++ content/prerequisites/prereq-processed.md | 31 ++++++++ docs/syntax.md | 10 +-- docs/syntax_codeblock.md | 18 ----- requirements.txt | 2 +- simdem/mode/automated.py | 73 +++++------------- simdem/mode/common.py | 43 ++++++++++- simdem/mode/demo.py | 26 +++++-- simdem/mode/tutorial.py | 27 +++++-- simdem/parser/simdem1.py | 19 +++-- tests/test_mode_demo.py | 2 +- tests/test_mode_tutorial.py | 2 +- 22 files changed, 255 insertions(+), 223 deletions(-) create mode 100644 .simdem.log.swp delete mode 100644 content/complete-features/codeblock.md delete mode 100644 content/complete-features/context.md delete mode 100644 content/prerequisite-run/README.md delete mode 100644 content/prerequisite-run/expected_result.demo delete mode 100644 content/prerequisite-run/expected_result.tutorial delete mode 100644 content/prerequisite-run/prereq.md create mode 100644 content/prerequisites/README.md create mode 100644 content/prerequisites/expected_result.demo create mode 100644 content/prerequisites/expected_result.tutorial create mode 100644 content/prerequisites/prereq-ignored.md create mode 100644 content/prerequisites/prereq-processed.md delete mode 100644 docs/syntax_codeblock.md diff --git a/.simdem.log.swp b/.simdem.log.swp new file mode 100644 index 0000000000000000000000000000000000000000..7b90bd97dfe8f22e4cd9bf7f6d458b3ff4e5cd1d GIT binary patch literal 16384 zcmeHNU2NM_6u!Xt8yg^oVB!JYbwE0irb+sNDwjqS8+ zC*T>x1`ZbIia@UP^kMAIX=8PCT2dD~?cH zFITf1JJ--_j^$c)`Lz+;{Yp#W@S^SwyLv+vjqojJSc^$hq}NgeQUtb)z+N^|EclQ5 z!GQyO-vbL<)|e_t5l9h85l9h85l9h85l9h85l9jE{~+MmyV$ER_twzl1L6BE>)!tf zUr&bP1M9B;9bV6c0|N#`%A%{(tK}#vX=zbuVLkAlLUYb`bK#9>(@TuHM7gFl2Q% zW0R0C?`Et9`Q|Rh&O;u7yniQSPeVSugRv(dKinrOSgi`o(U5aXgynPqJhx+ucCN&ao+Q=B`iU}=P zJOv>kBd_UMj;HirdhnItjcl!^J7bjletJ{J=*p7d1C_ioZv`h zNK27)BoiQl(H}AJKQLU*vi2J899{~F1XCu&O54dSR~$sC9cM~0HC2_qqU zjg!ysb_5BneG5Udk@%n|-dKFl_s1Kam{Wp1uoCXUe2PD|cBGwf!eQI7YQlA~Idy*8 z9!p3_w9z-|TIwpdnoh`3n!2ld!sSlWe1?ATX&>VkCMM|~TL}n)ZDFI&`Z+ZKzGZ<= zIfV|==}2ohlCeJ|X{RKmn$VpH9o>Sl!MuY6m1?g!XdAV8ZXJ(-vL;r*j?Ik6Q39CU z6an)FOVUjr9#_E*B?;C5xf%AfBiYpsKcyTUBNNqz7v8TIkP>Az5nb^X|3^tNuQ#Se zgBzB0DcFS-S%Ar{*U(K2`DxdaT-#VZF47LAt zNIC2Se1IOngOJbB1NZ~={|cl4SwatBKjb0Ex99`B4LJ!p0oe_C2R(tqkoV9JcocF2 z{eaVuKhY0Zg-k+jpdauiCbVcih&Is4GOjn@R2YZRXJzpeOzT{WgqQz37rNf;W zVNHU%HP)=gS~EIU*sL{@)f2k%q^^i5@i&hgp;7IIr9}@3P)3f>j5=ZoxMr*)Bcmgv zBi*+6OC)J4WlVy$h%rI7LXF?yD_(q|Kb|d4(Ebqy#;9*Gr}mVT2gM0`N9y*VL(ORW zN9gRAY=1|2%ldF6KB|fd6GbeKzqv3$$8)!p>j9pU-cyJssgBt7;;98bfBe+K$!R`c zD)I91nfcg&EfhD6XN+nBg#_Kb`Tuw}$+md5LGk~1gyVY($@4~b*uH9Zshv&{@m}>N z4)Y*K3b-&@WPayUwqbpmELoP%i{jZtvNao9qs@9lt&yJ$<6?_mrp-iAHWRus3HA$&(M+s8zNjR^mu=GALo}cQ7BU z8j40FEqdiULF)v^3Y)j2qax^CQ5tAA*6oWQ-C-K)xLA=fM7Y>f`?@nED!5T;$<)v~ zvWiq$GqRJ3%iR~l=b*Akb;B^r6PLyZcIP_zE ze0rvUml$p27){1N;xQUBrfuLP%UD_4Z0vUtwj=$}OB@eU@1^NFENhcX^0-VU6uJnB zxJeho!6P{oovXVn?mpyAo;kw9;+au?obE%A{4k`eiR>^xPSd%FRgrCW?o_$M5dQ@X CAYw)U literal 0 HcmV?d00001 diff --git a/content/complete-features/codeblock.md b/content/complete-features/codeblock.md deleted file mode 100644 index 80ff378..0000000 --- a/content/complete-features/codeblock.md +++ /dev/null @@ -1,38 +0,0 @@ -# Prerequisites - -```prerequisites -prereq.md -prereq-2.md -``` -# Do stuff here - -We want to execute this because the code type is shell - -```shell -echo foo -echo bar -``` - -# Do more stuff here - -```shell -echo baz -``` - -# Results - -The only thing that makes it a result is the code type is result. -We assume the result is for the last command of the last code block - -```result -baz -``` - -# Process these steps nexts - -The list inside this block are steps that could be followed when performing an interactive tutorial - -```next_steps -step-1.md -step-2.md -``` \ No newline at end of file diff --git a/content/complete-features/context.md b/content/complete-features/context.md deleted file mode 100644 index 795e38f..0000000 --- a/content/complete-features/context.md +++ /dev/null @@ -1,37 +0,0 @@ -# Prerequisites - -* [prereq-1](prereq.md) -* [prereq-2](prereq-2.md) - -# Do stuff here - -We want to execute this because the code type is shell - -```shell -echo foo -echo bar -``` - -# Do more stuff here - -```shell -echo baz -``` - -# Results - -The only thing that makes it a result is the code type is result. -We assume the result is for the last command of the last code block - -```result -baz -``` - -# Next Steps - -The list inside this block are steps that could be followed when performing an interactive tutorial - - 1. [Step #1](step-1.md) - 1. [Step #2](step-2.md) - - diff --git a/content/prerequisite-run/README.md b/content/prerequisite-run/README.md deleted file mode 100644 index 86b1888..0000000 --- a/content/prerequisite-run/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Prerequisites - -This is a [prereq](content/prerequisite-run/prereq.md) - -# Main - -```shell -echo $prereq_run -``` diff --git a/content/prerequisite-run/expected_result.demo b/content/prerequisite-run/expected_result.demo deleted file mode 100644 index 13b07c4..0000000 --- a/content/prerequisite-run/expected_result.demo +++ /dev/null @@ -1,7 +0,0 @@ -$ prereq_run=true -$ echo $prereq_run -true - -$ echo $prereq_run -true - diff --git a/content/prerequisite-run/expected_result.tutorial b/content/prerequisite-run/expected_result.tutorial deleted file mode 100644 index 7625dbf..0000000 --- a/content/prerequisite-run/expected_result.tutorial +++ /dev/null @@ -1,17 +0,0 @@ -# Inside prereq.md - -$ prereq_run=true -$ echo $prereq_run -true - -Completed running prereq.md - -# Prerequisites - -This is a prereq - -# Main - -$ echo $prereq_run -true - diff --git a/content/prerequisite-run/prereq.md b/content/prerequisite-run/prereq.md deleted file mode 100644 index db47629..0000000 --- a/content/prerequisite-run/prereq.md +++ /dev/null @@ -1,8 +0,0 @@ -# Inside prereq.md - -```shell -prereq_run=true -echo $prereq_run -``` - -Completed running prereq.md diff --git a/content/prerequisites/README.md b/content/prerequisites/README.md new file mode 100644 index 0000000..abb7367 --- /dev/null +++ b/content/prerequisites/README.md @@ -0,0 +1,18 @@ +# Prerequisites + +This is the prerequisite section. SimDem looks for a set of links to extract and run through first + +* [prereq-ignored](content/simdem1/prereq-ignored.md) + +They don't even need to be in the same list + +* [prereq-processed](content/simdem1/prereq-processed.md) + +By this point, the prerequisites have either run or have passed their validation + +# Did our prerequisites run? + +```shell +echo prereq_ignored = $prereq_ignored +echo prereq_processed = $prereq_processed +``` diff --git a/content/prerequisites/expected_result.demo b/content/prerequisites/expected_result.demo new file mode 100644 index 0000000..771dd0b --- /dev/null +++ b/content/prerequisites/expected_result.demo @@ -0,0 +1,18 @@ +$ echo prereq_validation_pass +prereq_validation_pass + +***PREREQUISITE VALIDATION PASSED*** +$ echo prereq_validation_fail +prereq_validation_fail + +***PREREQUISITE VALIDATION FAILED*** +$ echo YOU SHOULD SEE THIS +YOU SHOULD SEE THIS + +$ prereq_processed=true + +$ echo prereq_ignored = $prereq_ignored +prereq_ignored = +$ echo prereq_processed = $prereq_processed +prereq_processed = true + diff --git a/content/prerequisites/expected_result.tutorial b/content/prerequisites/expected_result.tutorial new file mode 100644 index 0000000..405a24e --- /dev/null +++ b/content/prerequisites/expected_result.tutorial @@ -0,0 +1,42 @@ +$ echo prereq_validation_pass +prereq_validation_pass + +***PREREQUISITE VALIDATION PASSED*** +$ echo prereq_validation_fail +prereq_validation_fail + +***PREREQUISITE VALIDATION FAILED*** +# WE ARE IN prereq-processed.md + +This file is designed to have the validation fail. This means that we should completely execute this file + +# Validation + +This is a validation section. If this validation section passes, we stop processing this file + +# Main area + +This should be run since the validation for the prereq has failed + +$ echo YOU SHOULD SEE THIS +YOU SHOULD SEE THIS + +# Set a variable that passes through + +$ prereq_processed=true + +# Prerequisites + +This is the prerequisite section. SimDem looks for a set of links to extract and run through first + +They don't even need to be in the same list + +By this point, the prerequisites have either run or have passed their validation + +# Did our prerequisites run? + +$ echo prereq_ignored = $prereq_ignored +prereq_ignored = +$ echo prereq_processed = $prereq_processed +prereq_processed = true + diff --git a/content/prerequisites/prereq-ignored.md b/content/prerequisites/prereq-ignored.md new file mode 100644 index 0000000..0566eab --- /dev/null +++ b/content/prerequisites/prereq-ignored.md @@ -0,0 +1,31 @@ +# WE ARE IN prereq-ignored.md + +This file is designed to have the validation pass. This means that we should stop executing this document after the results section + +# Validation + +This is a validation section. If this validation section passes, we stop processing this file. The command below should be the last text displayed + +``` +echo prereq_validation_pass +``` + +Results: + +``` +prereq_validation_pass +``` + +# Main area + +This should never be run since the validation for the prereq has been met + +``` +echo YOU SHOULD NOT SEE THIS +``` + +# Set a variable that passes through + +``` +prereq_ignored=true +``` diff --git a/content/prerequisites/prereq-processed.md b/content/prerequisites/prereq-processed.md new file mode 100644 index 0000000..019053e --- /dev/null +++ b/content/prerequisites/prereq-processed.md @@ -0,0 +1,31 @@ +# WE ARE IN prereq-processed.md + +This file is designed to have the validation fail. This means that we should completely execute this file + +# Validation + +This is a validation section. If this validation section passes, we stop processing this file + +``` +echo prereq_validation_fail +``` + +Results: + +``` +blah +``` + +# Main area + +This should be run since the validation for the prereq has failed + +``` +echo YOU SHOULD SEE THIS +``` + +# Set a variable that passes through + +``` +prereq_processed=true +``` diff --git a/docs/syntax.md b/docs/syntax.md index 4df6426..91e0012 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -2,19 +2,15 @@ One of the designs of SimDem is to allow multiple implementations of Markdown to support different use cases and documentation patterns. -Currently, there are two implementation of Markdown syntax supported. - -* Context (default) -* [Codeblock](syntax_codeblock.md) - +Currently, there is only one implementation of Markdown syntax supported. # Context Syntax Specification This is the syntax for the default codeblock format. It's design is to allow more natural, expressive, and readable documentation. It is based off of SimDem v1's syntax -## Context Based (default) +## SimDem V1 Based (default) -[Example Context Based Document](../content/complete-features/context.md) +[Example Context Based Document](../content/simdem1/README.md) Feature | Implementation --- | --- diff --git a/docs/syntax_codeblock.md b/docs/syntax_codeblock.md deleted file mode 100644 index e32074f..0000000 --- a/docs/syntax_codeblock.md +++ /dev/null @@ -1,18 +0,0 @@ -# Syntax - -This is the syntax for the alternative codeblock format - -## Codeblock Based - -[Example Codeblock Document](../content/complete-features/codeblock.md) - -## Features - -The only sections processed are code blocks and all features are determined by the code block type. - -Feature | Implementation ---- | --- -Command | \```shell -[Prerequisite](feature_prerequisite.md) | \```prerequisite -[Validation](feature_validation.md) | \```validation -Result | \```result diff --git a/requirements.txt b/requirements.txt index 41f28db..28e8897 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ nose -mistune +mistletoe pexpect ddt difflib diff --git a/simdem/mode/automated.py b/simdem/mode/automated.py index fdef4e0..b3f4d13 100644 --- a/simdem/mode/automated.py +++ b/simdem/mode/automated.py @@ -1,7 +1,6 @@ """ Automated mode for SimDem """ import logging -import difflib from simdem.mode.common import ModeCommon class AutomatedMode(ModeCommon): @@ -11,30 +10,36 @@ class AutomatedMode(ModeCommon): second keypress executes the command. """ - def process_file(self, file_path): + def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ - #print("*** Processing " + file_path + " ***") + logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) - last_command_result = None - + # Begin prereq body if 'prerequisites' in steps: for prereq_file in steps['prerequisites']: - self.process_file(prereq_file) - if 'validation_command' in steps: - last_command_result = self.process_commands(steps['validation_command']) - if 'validation_result' in steps: - if not self.is_result_valid(steps['validation_result'], last_command_result): - return + self.process_file(prereq_file, is_prereq=True) + if is_prereq and 'validation' in steps: + last_command_result = self.process_commands(steps['validation']['commands']) + if 'expected_result' in steps['validation']: + if self.is_result_valid(steps['validation']['expected_result'], + last_command_result): + print('***PREREQUISITE VALIDATION PASSED***') + return + else: + print('***PREREQUISITE VALIDATION FAILED***') + # End prereq body """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ for step in steps['body']: if step['type'] == 'commands': last_command_result = self.process_commands(step) - elif step['type'] == 'result': - self.is_result_valid(step['content'], last_command_result) - + if step['expected_result'] == 'result': + if self.is_result_valid(step['content'], last_command_result): + print('***VALIDATION FAILED***') + else: + print('***VALIDATION PASSED***') def process_commands(self, step): """ Pretend to type the command, run it and then display the output """ @@ -44,43 +49,3 @@ def process_commands(self, step): print(results, end="", flush=True) print() return results - - @staticmethod - def is_result_valid(expected_results, actual_results, expected_similarity=1.0): - """Checks to see if a command execution passes. - If actual results compared to expected results is within - the expected similarity level then it's considered a pass. - - expected_similarity = 1.0 could be a breaking change for older SimDem scripts. - explicit fails > implicit passes - Ross may disagree with me. Let's see how this story unfolds. - """ - - if not actual_results: - logging.error("is_result_valid(): actual_results is empty.") - return False - - logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ - "," + str(expected_similarity) + ")") - - expected_results_str = expected_results.rstrip() - actual_results_str = actual_results.rstrip() - logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ - "," + str(expected_similarity) + ")") - seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", - actual_results_str, - expected_results_str) - - is_pass = seq.ratio() >= expected_similarity - - if is_pass: - logging.info("is_result_valid passed") - print('***VALIDATION PASSED***') - - else: - logging.error("is_result_valid failed") - logging.error("actual_results = " + actual_results) - logging.error("expected_results = " + expected_results) - print('***VALIDATION FAILED***') - - return is_pass diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 8bb96b6..b78c5f7 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -1,6 +1,9 @@ """ Common mode for SimDem mode """ -class ModeCommon(object): +import logging +import difflib + +class ModeCommon(object): # pylint: disable=R0903 """ This class is designed to hold any shared code across modes """ config = None @@ -11,3 +14,41 @@ def __init__(self, config, parser, executor): self.config = config self.parser = parser self.executor = executor + + @staticmethod + def is_result_valid(expected_results, actual_results, expected_similarity=1.0): + """Checks to see if a command execution passes. + If actual results compared to expected results is within + the expected similarity level then it's considered a pass. + + expected_similarity = 1.0 could be a breaking change for older SimDem scripts. + explicit fails > implicit passes + Ross may disagree with me. Let's see how this story unfolds. + """ + + if not actual_results: + logging.error("is_result_valid(): actual_results is empty.") + return False + + logging.debug("is_result_valid(" + expected_results + "," + actual_results + \ + "," + str(expected_similarity) + ")") + + expected_results_str = expected_results.rstrip() + actual_results_str = actual_results.rstrip() + logging.debug("is_result_valid(" + expected_results_str + "," + actual_results_str + \ + "," + str(expected_similarity) + ")") + seq = difflib.SequenceMatcher(lambda x: x in " \t\n\r", + actual_results_str, + expected_results_str) + + is_pass = seq.ratio() >= expected_similarity + + if is_pass: + logging.info("is_result_valid passed") + + else: + logging.error("is_result_valid failed") + logging.error("actual_results = " + actual_results) + logging.error("expected_results = " + expected_results) + + return is_pass diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index e394804..f2213ac 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -2,6 +2,7 @@ import random import time +import logging from simdem.mode.common import ModeCommon class DemoMode(ModeCommon): @@ -9,24 +10,33 @@ class DemoMode(ModeCommon): It's designed for running files in a demo-able mode that looks like a human is typing it """ - def process_file(self, file_path): + def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ - #print("*** Processing " + file_path + " ***") + logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) - """ I'd like to use a dispatcher for this; however, we need to exit processing - if the validation fails. """ + # Begin prereq body if 'prerequisites' in steps: for prereq_file in steps['prerequisites']: - self.process_file(prereq_file) + self.process_file(prereq_file, is_prereq=True) + if is_prereq and 'validation' in steps: + last_command_result = self.process_commands(steps['validation']['commands']) + if 'expected_result' in steps['validation']: + if self.is_result_valid(steps['validation']['expected_result'], + last_command_result): + print('***PREREQUISITE VALIDATION PASSED***') + return + else: + print('***PREREQUISITE VALIDATION FAILED***') + # End prereq body for step in steps['body']: if step['type'] == 'commands': - self.process_commands(step) + self.process_commands(step['content']) - def process_commands(self, step): + def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ - for cmd in step['content']: + for cmd in cmds: self.type_command(cmd) results = self.executor.run_cmd(cmd) print(results, end="", flush=True) diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index a22b2c1..d662ccb 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -1,5 +1,6 @@ """ Tutorial mode for SimDem""" +import logging from simdem.mode.common import ModeCommon class TutorialMode(ModeCommon): @@ -8,16 +9,25 @@ class TutorialMode(ModeCommon): of the tutorial and pauses at code blocks to allow user interaction. """ - def process_file(self, file_path): + def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ - #print("*** Processing " + file_path + " ***") + logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) - """ I'd like to use a dispatcher for this; however, we need to exit processing - if the validation fails. """ + # Begin prereq body if 'prerequisites' in steps: for prereq_file in steps['prerequisites']: - self.process_file(prereq_file) + self.process_file(prereq_file, is_prereq=True) + if is_prereq and 'validation' in steps: + last_command_result = self.process_commands(steps['validation']['commands']) + if 'expected_result' in steps['validation']: + if self.is_result_valid(steps['validation']['expected_result'], + last_command_result): + print('***PREREQUISITE VALIDATION PASSED***') + return + else: + print('***PREREQUISITE VALIDATION FAILED***') + # End prereq body for step in steps['body']: if step['type'] == 'heading': @@ -25,7 +35,7 @@ def process_file(self, file_path): elif step['type'] == 'text': self.process_text(step) elif step['type'] == 'commands': - self.process_commands(step) + self.process_commands(step['content']) @staticmethod def process_heading(step): @@ -39,9 +49,9 @@ def process_text(step): print(step['content']) print() - def process_commands(self, step): + def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ - for cmd in step['content']: + for cmd in cmds: print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ' + cmd) results = self.executor.run_cmd(cmd) print(results, end="", flush=True) @@ -51,6 +61,7 @@ def process_commands(self, step): @staticmethod def process_next_steps(steps): """ Is there a good way to test this that doesn't involve lots of test code + expect? + Not fully tested yet. Low priority feature. """ idx = 1 if steps: diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index c0cdb64..6200719 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -22,7 +22,10 @@ def parse_file(file_path): class SimDemMistletoeRenderer(BaseRenderer): - """ Based off of https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 """ + """ Based off of https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 + Major Gotcha: If this class calls a function that does not exist, it will NOT error + This is due to the way that the Mistletoe Renderer works. Or at least how I think it works. + """ section = None block = None @@ -98,9 +101,9 @@ def render_block_code(self, token): # Validation blocks aren't run like normal blocks if self.block == 'results': # Validation result blocks are expecially not checked like normal blocks - self.append_validation_result(content) + self.set_validation_result(content) else: - self.append_validation_command(content) + self.set_validation_command(content) else: if self.block == 'results': # Assume that the last body item is the command we're expecting results for @@ -141,15 +144,15 @@ def set_block(self, block): logging.debug('set_block(' + str(block) + ')') self.block = block - def append_validation_command(self, cmd): + def set_validation_command(self, cmds): """ Assuming validation commands should be a list """ - logging.debug('append_validation(' + str(cmd) + ')') - self.output['validation_command'].append(cmd) + logging.debug('append_validation(' + str(cmds) + ')') + self.output['validation'] = {'commands': cmds.splitlines()} - def append_validation_result(self, result): + def set_validation_result(self, result): """ Assuming validation results should be a list """ logging.debug('append_validation(' + str(result) + ')') - self.output['validation_expected_result'].append(result) + self.output['validation']['expected_result'] = result def append_body(self, body): """ Adding the meat of the work to the dict """ diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index 27080ce..91afe13 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -34,7 +34,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', - 'results-block-fail', 'prerequisite-run') + 'results-block-fail', 'prerequisites') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.out this allows us to test each of them easily diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index 38453dc..ba45820 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -34,7 +34,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', - 'results-block-fail', 'prerequisite-run') + 'results-block-fail', 'prerequisites') def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily From ac6d3b6d01bda341b0d10c05ace33ad243a13424 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 18 Jan 2018 17:21:46 -0600 Subject: [PATCH 093/167] Moved duplciate code to common mode Signed-off-by: Tommy Falgout --- .simdem.log.swp | Bin 16384 -> 0 bytes content/prerequisites/expected_result.demo | 1 - .../prerequisites/expected_result.tutorial | 1 - simdem/mode/automated.py | 35 ++---------------- simdem/mode/common.py | 31 ++++++++++++++++ simdem/mode/demo.py | 22 ++--------- simdem/mode/dump.py | 4 +- simdem/mode/tutorial.py | 31 ++-------------- tests/test_mode_tutorial.py | 1 + 9 files changed, 45 insertions(+), 81 deletions(-) delete mode 100644 .simdem.log.swp diff --git a/.simdem.log.swp b/.simdem.log.swp deleted file mode 100644 index 7b90bd97dfe8f22e4cd9bf7f6d458b3ff4e5cd1d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHNU2NM_6u!Xt8yg^oVB!JYbwE0irb+sNDwjqS8+ zC*T>x1`ZbIia@UP^kMAIX=8PCT2dD~?cH zFITf1JJ--_j^$c)`Lz+;{Yp#W@S^SwyLv+vjqojJSc^$hq}NgeQUtb)z+N^|EclQ5 z!GQyO-vbL<)|e_t5l9h85l9h85l9h85l9h85l9jE{~+MmyV$ER_twzl1L6BE>)!tf zUr&bP1M9B;9bV6c0|N#`%A%{(tK}#vX=zbuVLkAlLUYb`bK#9>(@TuHM7gFl2Q% zW0R0C?`Et9`Q|Rh&O;u7yniQSPeVSugRv(dKinrOSgi`o(U5aXgynPqJhx+ucCN&ao+Q=B`iU}=P zJOv>kBd_UMj;HirdhnItjcl!^J7bjletJ{J=*p7d1C_ioZv`h zNK27)BoiQl(H}AJKQLU*vi2J899{~F1XCu&O54dSR~$sC9cM~0HC2_qqU zjg!ysb_5BneG5Udk@%n|-dKFl_s1Kam{Wp1uoCXUe2PD|cBGwf!eQI7YQlA~Idy*8 z9!p3_w9z-|TIwpdnoh`3n!2ld!sSlWe1?ATX&>VkCMM|~TL}n)ZDFI&`Z+ZKzGZ<= zIfV|==}2ohlCeJ|X{RKmn$VpH9o>Sl!MuY6m1?g!XdAV8ZXJ(-vL;r*j?Ik6Q39CU z6an)FOVUjr9#_E*B?;C5xf%AfBiYpsKcyTUBNNqz7v8TIkP>Az5nb^X|3^tNuQ#Se zgBzB0DcFS-S%Ar{*U(K2`DxdaT-#VZF47LAt zNIC2Se1IOngOJbB1NZ~={|cl4SwatBKjb0Ex99`B4LJ!p0oe_C2R(tqkoV9JcocF2 z{eaVuKhY0Zg-k+jpdauiCbVcih&Is4GOjn@R2YZRXJzpeOzT{WgqQz37rNf;W zVNHU%HP)=gS~EIU*sL{@)f2k%q^^i5@i&hgp;7IIr9}@3P)3f>j5=ZoxMr*)Bcmgv zBi*+6OC)J4WlVy$h%rI7LXF?yD_(q|Kb|d4(Ebqy#;9*Gr}mVT2gM0`N9y*VL(ORW zN9gRAY=1|2%ldF6KB|fd6GbeKzqv3$$8)!p>j9pU-cyJssgBt7;;98bfBe+K$!R`c zD)I91nfcg&EfhD6XN+nBg#_Kb`Tuw}$+md5LGk~1gyVY($@4~b*uH9Zshv&{@m}>N z4)Y*K3b-&@WPayUwqbpmELoP%i{jZtvNao9qs@9lt&yJ$<6?_mrp-iAHWRus3HA$&(M+s8zNjR^mu=GALo}cQ7BU z8j40FEqdiULF)v^3Y)j2qax^CQ5tAA*6oWQ-C-K)xLA=fM7Y>f`?@nED!5T;$<)v~ zvWiq$GqRJ3%iR~l=b*Akb;B^r6PLyZcIP_zE ze0rvUml$p27){1N;xQUBrfuLP%UD_4Z0vUtwj=$}OB@eU@1^NFENhcX^0-VU6uJnB zxJeho!6P{oovXVn?mpyAo;kw9;+au?obE%A{4k`eiR>^xPSd%FRgrCW?o_$M5dQ@X CAYw)U diff --git a/content/prerequisites/expected_result.demo b/content/prerequisites/expected_result.demo index 771dd0b..540ef52 100644 --- a/content/prerequisites/expected_result.demo +++ b/content/prerequisites/expected_result.demo @@ -1,7 +1,6 @@ $ echo prereq_validation_pass prereq_validation_pass -***PREREQUISITE VALIDATION PASSED*** $ echo prereq_validation_fail prereq_validation_fail diff --git a/content/prerequisites/expected_result.tutorial b/content/prerequisites/expected_result.tutorial index 405a24e..6a67b19 100644 --- a/content/prerequisites/expected_result.tutorial +++ b/content/prerequisites/expected_result.tutorial @@ -1,7 +1,6 @@ $ echo prereq_validation_pass prereq_validation_pass -***PREREQUISITE VALIDATION PASSED*** $ echo prereq_validation_fail prereq_validation_fail diff --git a/simdem/mode/automated.py b/simdem/mode/automated.py index b3f4d13..32cfee9 100644 --- a/simdem/mode/automated.py +++ b/simdem/mode/automated.py @@ -10,42 +10,15 @@ class AutomatedMode(ModeCommon): second keypress executes the command. """ - def process_file(self, file_path, is_prereq=False): - """ Parses the file and starts processing it """ - logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) - steps = self.parser.parse_file(file_path) - - # Begin prereq body - if 'prerequisites' in steps: - for prereq_file in steps['prerequisites']: - self.process_file(prereq_file, is_prereq=True) - if is_prereq and 'validation' in steps: - last_command_result = self.process_commands(steps['validation']['commands']) - if 'expected_result' in steps['validation']: - if self.is_result_valid(steps['validation']['expected_result'], - last_command_result): - print('***PREREQUISITE VALIDATION PASSED***') - return - else: - print('***PREREQUISITE VALIDATION FAILED***') - # End prereq body - + def process(self, steps): """ I'd like to use a dispatcher for this; however, we need to exit processing if the validation fails. """ + logging.debug("process()") for step in steps['body']: if step['type'] == 'commands': - last_command_result = self.process_commands(step) - if step['expected_result'] == 'result': + last_command_result = self.process_commands(step['content']) + if 'expected_results' in step: if self.is_result_valid(step['content'], last_command_result): print('***VALIDATION FAILED***') else: print('***VALIDATION PASSED***') - - def process_commands(self, step): - """ Pretend to type the command, run it and then display the output """ - for cmd in step['content']: - print(cmd, end="", flush=True) - results = self.executor.run_cmd(cmd) - print(results, end="", flush=True) - print() - return results diff --git a/simdem/mode/common.py b/simdem/mode/common.py index b78c5f7..d9cba01 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -15,6 +15,37 @@ def __init__(self, config, parser, executor): self.parser = parser self.executor = executor + def process_file(self, file_path, is_prereq=False): + """ Parses the file and starts processing it """ + logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) + steps = self.parser.parse_file(file_path) + + # Begin preqreq processing + if 'prerequisites' in steps: + for prereq_file in steps['prerequisites']: + self.process_file(prereq_file, is_prereq=True) + if is_prereq and 'validation' in steps: + last_command_result = self.process_commands(steps['validation']['commands']) # pylint: disable=no-member + if 'expected_result' in steps['validation']: + if self.is_result_valid(steps['validation']['expected_result'], + last_command_result): + #print('***PREREQUISITE VALIDATION PASSED***') + return + else: + print('***PREREQUISITE VALIDATION FAILED***') + # End prereq processing + + self.process(steps) # pylint: disable=no-member + + def process_commands(self, cmds): + """ Pretend to type the command, run it and then display the output """ + for cmd in cmds: + print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ' + cmd) + results = self.executor.run_cmd(cmd) + print(results, end="", flush=True) + print() + return results + @staticmethod def is_result_valid(expected_results, actual_results, expected_similarity=1.0): """Checks to see if a command execution passes. diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index f2213ac..1700825 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -10,25 +10,9 @@ class DemoMode(ModeCommon): It's designed for running files in a demo-able mode that looks like a human is typing it """ - def process_file(self, file_path, is_prereq=False): - """ Parses the file and starts processing it """ - logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) - steps = self.parser.parse_file(file_path) - - # Begin prereq body - if 'prerequisites' in steps: - for prereq_file in steps['prerequisites']: - self.process_file(prereq_file, is_prereq=True) - if is_prereq and 'validation' in steps: - last_command_result = self.process_commands(steps['validation']['commands']) - if 'expected_result' in steps['validation']: - if self.is_result_valid(steps['validation']['expected_result'], - last_command_result): - print('***PREREQUISITE VALIDATION PASSED***') - return - else: - print('***PREREQUISITE VALIDATION FAILED***') - # End prereq body + def process(self, steps): + """ Processes the steps from a processed file """ + logging.debug("process()") for step in steps['body']: if step['type'] == 'commands': diff --git a/simdem/mode/dump.py b/simdem/mode/dump.py index d4c2f5e..7f7f832 100644 --- a/simdem/mode/dump.py +++ b/simdem/mode/dump.py @@ -1,12 +1,14 @@ """ Debug renderer for SimDem""" import pprint +import logging from simdem.mode.common import ModeCommon class DumpMode(ModeCommon): # pylint: disable=R0903 """ This class is used to pretty print a parsed file """ - def process_file(self, file_path): + def process_file(self, file_path, is_prereq=False): """ Parse the file and print it. Not very exciting. """ + logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) pprint.pprint(steps) diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index d662ccb..fb54842 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -9,25 +9,9 @@ class TutorialMode(ModeCommon): of the tutorial and pauses at code blocks to allow user interaction. """ - def process_file(self, file_path, is_prereq=False): - """ Parses the file and starts processing it """ - logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) - steps = self.parser.parse_file(file_path) - - # Begin prereq body - if 'prerequisites' in steps: - for prereq_file in steps['prerequisites']: - self.process_file(prereq_file, is_prereq=True) - if is_prereq and 'validation' in steps: - last_command_result = self.process_commands(steps['validation']['commands']) - if 'expected_result' in steps['validation']: - if self.is_result_valid(steps['validation']['expected_result'], - last_command_result): - print('***PREREQUISITE VALIDATION PASSED***') - return - else: - print('***PREREQUISITE VALIDATION FAILED***') - # End prereq body + def process(self, steps): + """ Processes the steps from a processed file """ + logging.debug("process()") for step in steps['body']: if step['type'] == 'heading': @@ -49,15 +33,6 @@ def process_text(step): print(step['content']) print() - def process_commands(self, cmds): - """ Pretend to type the command, run it and then display the output """ - for cmd in cmds: - print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ' + cmd) - results = self.executor.run_cmd(cmd) - print(results, end="", flush=True) - print() - return results - @staticmethod def process_next_steps(steps): """ Is there a good way to test this that doesn't involve lots of test code + expect? diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index ba45820..ef3b0fe 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -39,6 +39,7 @@ def test_process(self, directory): """ Each content directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily """ + self.maxDiff = None # pylint: disable=C0103 self.simdem.process_file('./content/' + directory + '/README.md') # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. From bdc946f5ce9e7e0f5b9047467d7eb118709b5761 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 18 Jan 2018 17:32:07 -0600 Subject: [PATCH 094/167] removed unnecessary dirs --- .gitignore | 2 + hello_world/README.md | 1 - simdem-env/bin/activate | 78 -------------------------------- simdem-env/bin/activate.csh | 36 --------------- simdem-env/bin/activate.fish | 76 ------------------------------- simdem-env/bin/activate_this.py | 34 -------------- simdem-env/bin/autopep8 | 11 ----- simdem-env/bin/easy_install | 11 ----- simdem-env/bin/easy_install-3.6 | 11 ----- simdem-env/bin/flask | 11 ----- simdem-env/bin/pip | 11 ----- simdem-env/bin/pip3 | 11 ----- simdem-env/bin/pip3.6 | 11 ----- simdem-env/bin/pycodestyle | 11 ----- simdem-env/bin/python | 1 - simdem-env/bin/python-config | 78 -------------------------------- simdem-env/bin/python3 | 1 - simdem-env/bin/python3.6 | Bin 13068 -> 0 bytes simdem-env/bin/wheel | 11 ----- simdem-env/include/python3.6m | 1 - simdem-env/pip-selfcheck.json | 1 - 21 files changed, 2 insertions(+), 406 deletions(-) delete mode 100644 hello_world/README.md delete mode 100644 simdem-env/bin/activate delete mode 100644 simdem-env/bin/activate.csh delete mode 100644 simdem-env/bin/activate.fish delete mode 100644 simdem-env/bin/activate_this.py delete mode 100755 simdem-env/bin/autopep8 delete mode 100755 simdem-env/bin/easy_install delete mode 100755 simdem-env/bin/easy_install-3.6 delete mode 100755 simdem-env/bin/flask delete mode 100755 simdem-env/bin/pip delete mode 100755 simdem-env/bin/pip3 delete mode 100755 simdem-env/bin/pip3.6 delete mode 100755 simdem-env/bin/pycodestyle delete mode 120000 simdem-env/bin/python delete mode 100755 simdem-env/bin/python-config delete mode 120000 simdem-env/bin/python3 delete mode 100755 simdem-env/bin/python3.6 delete mode 100755 simdem-env/bin/wheel delete mode 120000 simdem-env/include/python3.6m delete mode 100644 simdem-env/pip-selfcheck.json diff --git a/.gitignore b/.gitignore index 3d3a126..c181d1e 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,5 @@ ENV/ # Other .vscode +simdem-env +hello_world diff --git a/hello_world/README.md b/hello_world/README.md deleted file mode 100644 index 1934e6f..0000000 --- a/hello_world/README.md +++ /dev/null @@ -1 +0,0 @@ -# Hello World Script diff --git a/simdem-env/bin/activate b/simdem-env/bin/activate deleted file mode 100644 index b84a1df..0000000 --- a/simdem-env/bin/activate +++ /dev/null @@ -1,78 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - unset -f pydoc >/dev/null 2>&1 - - # reset old environment variables - # ! [ -z ${VAR+_} ] returns true if VAR is declared at all - if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then - PATH="$_OLD_VIRTUAL_PATH" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then - PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then - hash -r 2>/dev/null - fi - - if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then - PS1="$_OLD_VIRTUAL_PS1" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "${1-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="/Users/thfalgou/git/simdem/venv" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -if ! [ -z "${PYTHONHOME+_}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then - _OLD_VIRTUAL_PS1="$PS1" - if [ "x" != x ] ; then - PS1="$PS1" - else - PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1" - fi - export PS1 -fi - -# Make sure to unalias pydoc if it's already there -alias pydoc 2>/dev/null >/dev/null && unalias pydoc - -pydoc () { - python -m pydoc "$@" -} - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then - hash -r 2>/dev/null -fi diff --git a/simdem-env/bin/activate.csh b/simdem-env/bin/activate.csh deleted file mode 100644 index 9b3e2a2..0000000 --- a/simdem-env/bin/activate.csh +++ /dev/null @@ -1,36 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi . - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/Users/thfalgou/git/simdem/venv" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - - - -if ("" != "") then - set env_name = "" -else - set env_name = `basename "$VIRTUAL_ENV"` -endif - -# Could be in a non-interactive environment, -# in which case, $prompt is undefined and we wouldn't -# care about the prompt anyway. -if ( $?prompt ) then - set _OLD_VIRTUAL_PROMPT="$prompt" - set prompt = "[$env_name] $prompt" -endif - -unset env_name - -alias pydoc python -m pydoc - -rehash - diff --git a/simdem-env/bin/activate.fish b/simdem-env/bin/activate.fish deleted file mode 100644 index cb52ddc..0000000 --- a/simdem-env/bin/activate.fish +++ /dev/null @@ -1,76 +0,0 @@ -# This file must be used using `. bin/activate.fish` *within a running fish ( http://fishshell.com ) session*. -# Do not run it directly. - -function deactivate -d 'Exit virtualenv mode and return to the normal environment.' - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`. - set -l fish_function_path - - # Erase virtualenv's `fish_prompt` and restore the original. - functions -e fish_prompt - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - end - - set -e VIRTUAL_ENV - - if test "$argv[1]" != 'nondestructive' - # Self-destruct! - functions -e pydoc - functions -e deactivate - end -end - -# Unset irrelevant variables. -deactivate nondestructive - -set -gx VIRTUAL_ENV "/Users/thfalgou/git/simdem/venv" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# Unset `$PYTHONHOME` if set. -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -function pydoc - python -m pydoc $argv -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # Copy the current `fish_prompt` function as `_old_fish_prompt`. - functions -c fish_prompt _old_fish_prompt - - function fish_prompt - # Save the current $status, for fish_prompts that display it. - set -l old_status $status - - # Prompt override provided? - # If not, just prepend the environment name. - if test -n "" - printf '%s%s' "" (set_color normal) - else - printf '%s(%s) ' (set_color normal) (basename "$VIRTUAL_ENV") - end - - # Restore the original $status - echo "exit $old_status" | source - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" -end diff --git a/simdem-env/bin/activate_this.py b/simdem-env/bin/activate_this.py deleted file mode 100644 index f18193b..0000000 --- a/simdem-env/bin/activate_this.py +++ /dev/null @@ -1,34 +0,0 @@ -"""By using execfile(this_file, dict(__file__=this_file)) you will -activate this virtualenv environment. - -This can be used when you must use an existing Python interpreter, not -the virtualenv bin/python -""" - -try: - __file__ -except NameError: - raise AssertionError( - "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))") -import sys -import os - -old_os_path = os.environ.get('PATH', '') -os.environ['PATH'] = os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path -base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if sys.platform == 'win32': - site_packages = os.path.join(base, 'Lib', 'site-packages') -else: - site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') -prev_sys_path = list(sys.path) -import site -site.addsitedir(site_packages) -sys.real_prefix = sys.prefix -sys.prefix = base -# Move the added items to the front of the path: -new_sys_path = [] -for item in list(sys.path): - if item not in prev_sys_path: - new_sys_path.append(item) - sys.path.remove(item) -sys.path[:0] = new_sys_path diff --git a/simdem-env/bin/autopep8 b/simdem-env/bin/autopep8 deleted file mode 100755 index 9af5c95..0000000 --- a/simdem-env/bin/autopep8 +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/simdem-env/bin/python - -# -*- coding: utf-8 -*- -import re -import sys - -from autopep8 import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/easy_install b/simdem-env/bin/easy_install deleted file mode 100755 index 375bf77..0000000 --- a/simdem-env/bin/easy_install +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python3.6 - -# -*- coding: utf-8 -*- -import re -import sys - -from setuptools.command.easy_install import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/easy_install-3.6 b/simdem-env/bin/easy_install-3.6 deleted file mode 100755 index 375bf77..0000000 --- a/simdem-env/bin/easy_install-3.6 +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python3.6 - -# -*- coding: utf-8 -*- -import re -import sys - -from setuptools.command.easy_install import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/flask b/simdem-env/bin/flask deleted file mode 100755 index 22e6196..0000000 --- a/simdem-env/bin/flask +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python3.6 - -# -*- coding: utf-8 -*- -import re -import sys - -from flask.cli import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/pip b/simdem-env/bin/pip deleted file mode 100755 index d9963b8..0000000 --- a/simdem-env/bin/pip +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python3.6 - -# -*- coding: utf-8 -*- -import re -import sys - -from pip import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/pip3 b/simdem-env/bin/pip3 deleted file mode 100755 index d9963b8..0000000 --- a/simdem-env/bin/pip3 +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python3.6 - -# -*- coding: utf-8 -*- -import re -import sys - -from pip import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/pip3.6 b/simdem-env/bin/pip3.6 deleted file mode 100755 index d9963b8..0000000 --- a/simdem-env/bin/pip3.6 +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python3.6 - -# -*- coding: utf-8 -*- -import re -import sys - -from pip import main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/simdem-env/bin/pycodestyle b/simdem-env/bin/pycodestyle deleted file mode 100755 index a532603..0000000 --- a/simdem-env/bin/pycodestyle +++ /dev/null @@ -1,11 +0,0 @@ -#!/Users/thfalgou/git/simdem/simdem-env/bin/python - -# -*- coding: utf-8 -*- -import re -import sys - -from pycodestyle import _main - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(_main()) diff --git a/simdem-env/bin/python b/simdem-env/bin/python deleted file mode 120000 index 039b719..0000000 --- a/simdem-env/bin/python +++ /dev/null @@ -1 +0,0 @@ -python3.6 \ No newline at end of file diff --git a/simdem-env/bin/python-config b/simdem-env/bin/python-config deleted file mode 100755 index f23f039..0000000 --- a/simdem-env/bin/python-config +++ /dev/null @@ -1,78 +0,0 @@ -#!/Users/thfalgou/git/simdem/venv/bin/python - -import sys -import getopt -import sysconfig - -valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', - 'ldflags', 'help'] - -if sys.version_info >= (3, 2): - valid_opts.insert(-1, 'extension-suffix') - valid_opts.append('abiflags') -if sys.version_info >= (3, 3): - valid_opts.append('configdir') - - -def exit_with_usage(code=1): - sys.stderr.write("Usage: {0} [{1}]\n".format( - sys.argv[0], '|'.join('--'+opt for opt in valid_opts))) - sys.exit(code) - -try: - opts, args = getopt.getopt(sys.argv[1:], '', valid_opts) -except getopt.error: - exit_with_usage() - -if not opts: - exit_with_usage() - -pyver = sysconfig.get_config_var('VERSION') -getvar = sysconfig.get_config_var - -opt_flags = [flag for (flag, val) in opts] - -if '--help' in opt_flags: - exit_with_usage(code=0) - -for opt in opt_flags: - if opt == '--prefix': - print(sysconfig.get_config_var('prefix')) - - elif opt == '--exec-prefix': - print(sysconfig.get_config_var('exec_prefix')) - - elif opt in ('--includes', '--cflags'): - flags = ['-I' + sysconfig.get_path('include'), - '-I' + sysconfig.get_path('platinclude')] - if opt == '--cflags': - flags.extend(getvar('CFLAGS').split()) - print(' '.join(flags)) - - elif opt in ('--libs', '--ldflags'): - abiflags = getattr(sys, 'abiflags', '') - libs = ['-lpython' + pyver + abiflags] - libs += getvar('LIBS').split() - libs += getvar('SYSLIBS').split() - # add the prefix/lib/pythonX.Y/config dir, but only if there is no - # shared library in prefix/lib/. - if opt == '--ldflags': - if not getvar('Py_ENABLE_SHARED'): - libs.insert(0, '-L' + getvar('LIBPL')) - if not getvar('PYTHONFRAMEWORK'): - libs.extend(getvar('LINKFORSHARED').split()) - print(' '.join(libs)) - - elif opt == '--extension-suffix': - ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') - if ext_suffix is None: - ext_suffix = sysconfig.get_config_var('SO') - print(ext_suffix) - - elif opt == '--abiflags': - if not getattr(sys, 'abiflags', None): - exit_with_usage() - print(sys.abiflags) - - elif opt == '--configdir': - print(sysconfig.get_config_var('LIBPL')) diff --git a/simdem-env/bin/python3 b/simdem-env/bin/python3 deleted file mode 120000 index 039b719..0000000 --- a/simdem-env/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -python3.6 \ No newline at end of file diff --git a/simdem-env/bin/python3.6 b/simdem-env/bin/python3.6 deleted file mode 100755 index 12874ce8175ef5e27e0834e9290154761ae65587..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13068 zcmeHOUu+ab7@wAc2UZLI2}cy%panGMjw{gAipeeX>Rz~Np#>{Y#=W~OcXD@o+1pFG zh@?qrbaQFK6B3_DjEShx7aj;0tO2Ax7z;6mM4%=_vj_-@1`-olzi)Q8-P;!R&6oMY zH{YN6_M4gC%=KaB2)qIjeg9~mf87Qoe0-e<%&<_lmjk}&-(nuz~=R2-tT zkfWtaKB?wtQP*bV3kdf7-07i>^Kl9DEpQExl;-4r$`=&ud@gQid-LIX8-ZOy-0a^~ zBL2#zSV!FbO{})_G`b&yjpW*x$2r=}zZ{M_3xNi)34mP1dhBL3*vCONLBc<_mqCz1 zgph|ocC-*O4)Q_ZZeS$L?=tQ^_T7(eL!QTH>3kS`0x+&QTrO$hTs9d_S8{2vk)8m? zc?`{~|NeYtjdkRsqr=PJA3ORC_!a__MnE6?lbWnm!kwC&R|ZvWPbs`n)s#-PTu94$RxN}E+@A0@MJs`)6c&9k zkbA~B^I)TZqsm?-Ro3NXPN79v&xAvvaHzYYXVd~|4Yhd3wY{^AwI`pOB0z=2ez_()t9;Ueb8++Z)fdJ*6w2M8>~&THqF`r)@Gnhj4oXQLhP}c=Yu?mZ${+W zFP3Fx@EPmcCDfir?E-2WQ5#2X3u>oO!>8M8$54A0wU1GI549txeTdp2(DHIFr>2Nn z)&+Gy$SZkOtH7I9r>x64!G7ft6irjLcA;FrcOyYpg|w1V)1YM(A*JT?av?3`vIRwu zwcX{sQqYB`o3oI_exG|16(Fy~w=aAfTMH8TB)l&pp*{z=5#Ep%0wbO8n`C4JZjsQo1Bft^h2yg^A0vrL307rl$z!BgGa0LDb z1Qsn1zK+8_8l_meo`xHVBH65wgHbHjWySF{3H9Po%zhB#wBRFA+Fj{V@^p(l*r_QB z_%*@n?6NODz`csOX$QO1NT?|}$D&u;?k+i7z>nL&oKD%JYw2p3? zr#F?IlWEgAA>gPz8|YxCV&-{pJ^qv6Iki*A!956`BQWDIYjRp^4~N&aM%J!d3pMaa Fe* Date: Thu, 18 Jan 2018 19:46:58 -0600 Subject: [PATCH 095/167] Fix documentation Signed-off-by: Tommy Falgout --- content/prerequisites/expected_result.dump | 17 +++++++ docs/README.md | 17 +++---- docs/development.md | 30 +++++------ docs/feature_validation.md | 7 ++- docs/modes.md | 58 ++++++++++++++++++++++ simdem/cli.py | 8 +-- simdem/mode/{automated.py => test.py} | 2 +- 7 files changed, 101 insertions(+), 38 deletions(-) create mode 100644 content/prerequisites/expected_result.dump create mode 100644 docs/modes.md rename simdem/mode/{automated.py => test.py} (96%) diff --git a/content/prerequisites/expected_result.dump b/content/prerequisites/expected_result.dump new file mode 100644 index 0000000..9169127 --- /dev/null +++ b/content/prerequisites/expected_result.dump @@ -0,0 +1,17 @@ +{'body': [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, + {'content': 'This is the prerequisite section. SimDem looks for a ' + 'set of links to extract and run through first', + 'type': 'text'}, + {'content': "They don't even need to be in the same list", + 'type': 'text'}, + {'content': 'By this point, the prerequisites have either run or ' + 'have passed their validation', + 'type': 'text'}, + {'content': 'Did our prerequisites run?', + 'level': 1, + 'type': 'heading'}, + {'content': ['echo prereq_ignored = $prereq_ignored', + 'echo prereq_processed = $prereq_processed'], + 'type': 'commands'}], + 'prerequisites': ['content/simdem1/prereq-ignored.md', + 'content/simdem1/prereq-processed.md']} diff --git a/docs/README.md b/docs/README.md index 3f50f52..409311b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ SimDem is: ## SimDem overview -Simdem allows you to wite a tutorial in markdown format and then run the commands as a simulated demo, interactive tutorial or even a test script. You can also generate executable shell scripts. +Simdem allows you to wite a tutorial in markdown format and then run the commands as a simulated demo, interactive tutorial, a test script or generate executable shell scripts. SimDem reads a Markdown file and executes the code block commands embedded within. It can even look like you are really typing the commands, which is great if you want to concentrate on explaining what you are doing but still run the demo live. @@ -20,16 +20,11 @@ It's easier to understand through example. If you are viewing this inside the in Tutorials can branch too, for example you can choose any of the following paths next: - 1. [Modes of operation](modes/README.md) - 2. [Hello World Demo](demo/README.md) - 3. [Build a Hello World script](tutorial/README.md) - 4. [Write SimDem documents](syntax/README.md) - 5. [Special Commands](special_commands/README.md) - 6. [Configure your scripts through variables](variables/README.md) - 7. [Write multi-part documents](multipart/README.md) - 8. [Use your documents as interactive tutorials or demos](running/README.md) - 9. [Use your documents as automated tests](test/README.md) - 10. [Build an SimDem container](building/README.md) + 1. [Modes of operation](modes.md) + 1. [Build Hello World Demo](build_hello_world.md) + 1. [SimDem document syntax](syntax.md) + 1. [Use your documents as interactive tutorials or demos](demo_mode.md) + 1. [Use your documents as automated tests](test_mode.md) diff --git a/docs/development.md b/docs/development.md index a537d22..961b3ae 100644 --- a/docs/development.md +++ b/docs/development.md @@ -15,7 +15,7 @@ We would love for you to contribute to SimDem. If there is anything that would Fetch the necessary packages ``` -make init +pip3 install -r requirements.txt ``` ## Validation @@ -23,7 +23,7 @@ make init Verify the tests pass ``` -make test +python3 setup.py nosetests ``` # Code Structure @@ -34,30 +34,24 @@ SimDem is broken into the following class types This class type parses the markdown document into a "SimDem Execution Object". This object has everything SimDem needs to know on how to run the document (e.g. prerequisites, commands, validations, etc.) -Implementations: -* Context -* CodeBlock +Follow the links to see an [example document](../content/prerequisites/README.md) with its output [SimDem Execution Object](../content/prerequisites/expected_output.dump) -Example SimDem Execution Object: -``` -{ 'prerequisites': ['prereq.md', 'prereq-2.md'], - 'commands': [ { 'command': 'echo foo' }, - { 'command': 'echo bar' }, - { 'command': 'echo baz', 'expected_result': 'baz' } ] - } -``` +Implementations: +* [SimDem1](../simdem/parser/simdem1.py) -## Render +## Mode -This class type formats the result of running the commands into the desired output +This class contains the logic for how the markdown document is processed. Implementations: -* Demonstration mode -* Automated mode (coming soon) +* [Demo mode](../simdem/mode/demo.py) +* [Automated mode](../simdem/mode/automated.py) +* [Dump](../simdem/mode/dump.py) +* [Tutorial](../simdem/mode/tutorial.py) ## Execute This class type executes the desired commands into the shell Implementations: -* Bash \ No newline at end of file +* [Bash](../simdem/executor/bash.py) \ No newline at end of file diff --git a/docs/feature_validation.md b/docs/feature_validation.md index c7a17bd..c1da762 100644 --- a/docs/feature_validation.md +++ b/docs/feature_validation.md @@ -65,7 +65,6 @@ cleanup/README.md # Next Steps - 1. [SimDem Index](../README.md) - 2. [Build a Hello World script](../tutorial/README.md) - 3. [Write SimDem documents](../syntax/README.md) - 4. [Configure your scripts through variables](../variables/README.md) + 1. [SimDem Index](README.md) + 2. [Build a Hello World script](hello_world.md) + 3. [Write SimDem documents](syntax.md) diff --git a/docs/modes.md b/docs/modes.md new file mode 100644 index 0000000..67554c8 --- /dev/null +++ b/docs/modes.md @@ -0,0 +1,58 @@ +# Modes of Operation + +SimDem and can be run one of the following modes: + + * Tutorial: Displays the descriptive text of the tutorial and pauses + at code blocks to allow user interaction. + * Demo: Does not display the descriptive text, but pauses at each + code block. When the user hits a key the command is "typed", a + second keypress executes the command. + * Test: Runs the commands and then verifies that the output is + sufficiently similar to the expected results (recorded in the + markdown file) to be considered correct. + * Dump: Prints out the internal SimDem object + +## Tutorial Mode + +Tutorial mode is ideal if you are using this as a learning or teaching +tool (see also learn mode below, which suits some learning styles +better. In this mode a description of what you are about to do is +shown on the screen, hit a key to see the command, hit another key to +execute the command. Tutorial mode is the default. + +To run a document in Tutorial Mode: `simdem content/simple/README.md` + + +## Demo (or Simulation) mode + +Demo mode is ideal if you are using this to teach or demonstrate how +to achive the goal. In this mode no descriptive text is shown, instead +when you press a key the next command is "typed", pressing another key +will execute the command. The idea is that you describe what is +happening as the application "types" the command for you. + +To run a document in Demo Mode: `simdem -m demo content/simple/README.md` + + +## Test Mode + +Test mode runs the commands and then verifies that the output is +sufficiently similar to the expected results (recorded in the markdown +file) to be considered correct. + +To run a document in Test Mode: `simdem -m test content/simple/README.md` + +## Dump Mode + +Dump mode is used for debugging to see how SimDem has parsed the document + +To run a document in Dump Mode: `simdem -m dump content/simple/README.md` + + +# Next Steps + + 1. [Beginning](README.md) + 1. [Build Hello World Demo](build_hello_world.md) + 1. [SimDem document syntax](syntax.md) + 1. [Use your documents as interactive tutorials or demos](demo_mode.md) + 1. [Use your documents as automated tests](test_mode.md) \ No newline at end of file diff --git a/simdem/cli.py b/simdem/cli.py index c789289..8c6f813 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -7,7 +7,7 @@ from simdem.executor import bash from simdem.parser import ast, simdem1 -from simdem.mode import demo, dump, automated, tutorial +from simdem.mode import demo, dump, test, tutorial def main(): @@ -20,7 +20,7 @@ def main(): argp.add_argument('--config-file', '-c', default="content/config/demo.ini", help="Config file to use") argp.add_argument('--mode', '-m', default="tutorial", - help="Mode to use", choices=['demo', 'dump', 'automated', 'tutorial']) + help="Mode to use", choices=['demo', 'dump', 'test', 'tutorial']) argp.add_argument('--parser', '-p', default="simdem1", help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", @@ -61,8 +61,8 @@ def get_mode(options, config): if options.mode == 'dump': return dump.DumpMode(config, parser, executor) - if options.mode == 'automated': - return automated.AutomatedMode(config, parser, executor) + if options.mode == 'test': + return test.TestMode(config, parser, executor) if options.mode == 'tutorial': return tutorial.TutorialMode(config, parser, executor) diff --git a/simdem/mode/automated.py b/simdem/mode/test.py similarity index 96% rename from simdem/mode/automated.py rename to simdem/mode/test.py index 32cfee9..e53d0f6 100644 --- a/simdem/mode/automated.py +++ b/simdem/mode/test.py @@ -3,7 +3,7 @@ import logging from simdem.mode.common import ModeCommon -class AutomatedMode(ModeCommon): +class TestMode(ModeCommon): """ This class is the automated SimDem mode Does not display the descriptive text, but pauses at each code block. When the user hits a key the command is "typed", a From 9e8f9bf43ca8a0b810c842a40fce21d4698a97bc Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 18 Jan 2018 21:14:50 -0600 Subject: [PATCH 096/167] Add next_steps functionality Signed-off-by: Tommy Falgout --- docs/README.md | 10 +++++----- simdem/mode/common.py | 2 ++ simdem/mode/tutorial.py | 23 +++++++++++++++++------ simdem/parser/simdem1.py | 13 ++++++++++++- tests/test_mode_demo.py | 1 - tests/test_parser_simdem1.py | 2 ++ 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/README.md b/docs/README.md index 409311b..42a830f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,11 +20,11 @@ It's easier to understand through example. If you are viewing this inside the in Tutorials can branch too, for example you can choose any of the following paths next: - 1. [Modes of operation](modes.md) - 1. [Build Hello World Demo](build_hello_world.md) - 1. [SimDem document syntax](syntax.md) - 1. [Use your documents as interactive tutorials or demos](demo_mode.md) - 1. [Use your documents as automated tests](test_mode.md) +1. [Modes of operation](modes.md) +1. [Build Hello World Demo](build_hello_world.md) +1. [SimDem document syntax](syntax.md) +1. [Use your documents as interactive tutorials or demos](demo_mode.md) +1. [Use your documents as automated tests](test_mode.md) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index d9cba01..8dbbe89 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -1,5 +1,6 @@ """ Common mode for SimDem mode """ +import os import logging import difflib @@ -9,6 +10,7 @@ class ModeCommon(object): # pylint: disable=R0903 config = None executor = None parser = None + cwd = './' def __init__(self, config, parser, executor): self.config = config diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index fb54842..eb3d11b 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -1,5 +1,6 @@ """ Tutorial mode for SimDem""" +import sys import logging from simdem.mode.common import ModeCommon @@ -21,6 +22,9 @@ def process(self, steps): elif step['type'] == 'commands': self.process_commands(step['content']) + if 'next_steps' in steps: + self.process_next_steps(steps['next_steps']) + @staticmethod def process_heading(step): """ Print out the heading exactly as we found it """ @@ -33,8 +37,7 @@ def process_text(step): print(step['content']) print() - @staticmethod - def process_next_steps(steps): + def process_next_steps(self, steps): """ Is there a good way to test this that doesn't involve lots of test code + expect? Not fully tested yet. Low priority feature. """ @@ -42,9 +45,17 @@ def process_next_steps(steps): if steps: print("Next steps available:") for step in steps: - print(idx + ".) " + step['title']) + print(str(idx) + ".) " + step['title']) idx += 1 - step_request = input("Which step do you want to take next?") - if step_request: - return steps[step_request+1]['target'] + print() + # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not + if sys.stdout.isatty(): + # You're running in a real terminal + step_request = input("Choose a step. " + + "Type the # or 'q' to quit and then press Enter: ") + #print('You chose:' + str(steps[int(step_request) - 1]['title'])) + self.process_file(steps[int(step_request) - 1]['target']) + else: + logging.info('Not connected to a TTY terminal Not requesting input.') + # You're being piped or redirected return diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 6200719..bc3b540 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -59,12 +59,16 @@ def render_heading(self, token): def render_list(self, token): """ Render a markdown list """ + logging.debug('render_list()') inner = self.render_inner(token) + logging.debug('render_list()::inner=' + inner) return inner def render_list_item(self, token): """ Render a markdown list item """ + logging.debug('render_list_item()') inner = self.render_inner(token) + logging.debug('render_list_item()::inner=' + inner) return inner def render_link(self, token): @@ -72,9 +76,11 @@ def render_link(self, token): We need to find another way to store link targets. Unfortunately, I can only think of storing them in an array right now """ + inner = self.render_inner(token) + if self.section and 'next_steps' in self.section: + self.append_next_step(inner, token.target) if self.section and 'prerequisites' in self.section: self.append_prereq(token.target) - inner = self.render_inner(token) return inner def render_raw_text(self, token): @@ -163,3 +169,8 @@ def append_prereq(self, target): """ Set Prereqs """ logging.debug('append_prereq(' + target + ')') self.output['prerequisites'].append(target) + + def append_next_step(self, name, target): + """ Add steps to next step section """ + logging.debug('append_next_step(' + name + ',' + target + ')') + self.output['next_steps'].append({'title': name, 'target': target}) diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index 91afe13..686ced6 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -20,7 +20,6 @@ class SimDemSystemTestSuite(unittest.TestCase): simdem = None def setUp(self): - config = configparser.ConfigParser() config.read("content/config/unit_test.ini") self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 0ca0253..9abb6dd 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -54,6 +54,8 @@ def test_full(self): {'content': 'The list inside this block are steps that could be ' 'followed when performing an interactive tutorial', 'type': 'text'}], + 'next_steps': [{'target': 'step-1.md', 'title': 'Step #1'}, + {'target': 'step-2.md', 'title': 'Step #2'}], 'prerequisites': ['content/simdem1/prereq-ignored.md', 'content/simdem1/prereq-processed.md']} pprint.pprint(res) From 96ffc9efc9c6c7228cceb8916cdac7a0923da817 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 18 Jan 2018 21:45:19 -0600 Subject: [PATCH 097/167] Add comment for file path Signed-off-by: Tommy Falgout --- simdem/mode/common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 8dbbe89..bcf66f8 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -10,7 +10,7 @@ class ModeCommon(object): # pylint: disable=R0903 config = None executor = None parser = None - cwd = './' + cwd = None def __init__(self, config, parser, executor): self.config = config @@ -21,6 +21,9 @@ def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) + # Change the working directory in case of any recursion + #print("changing path to:" + os.path.dirname(file_path)) + #os.chdir(os.path.dirname(file_path)) # Begin preqreq processing if 'prerequisites' in steps: From b8c2d04f3a8643277b5e717bbe95c918ed12b830 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 19 Jan 2018 10:06:43 -0600 Subject: [PATCH 098/167] Fix path issue for prereq/next_steps Signed-off-by: Tommy Falgout --- content/prerequisites/README.md | 4 ++-- simdem/mode/common.py | 34 ++++++++++++++++++++++++++++++--- simdem/mode/test.py | 4 ++++ simdem/mode/tutorial.py | 27 +------------------------- simdem/parser/simdem1.py | 4 ---- 5 files changed, 38 insertions(+), 35 deletions(-) diff --git a/content/prerequisites/README.md b/content/prerequisites/README.md index abb7367..7d96ef6 100644 --- a/content/prerequisites/README.md +++ b/content/prerequisites/README.md @@ -2,11 +2,11 @@ This is the prerequisite section. SimDem looks for a set of links to extract and run through first -* [prereq-ignored](content/simdem1/prereq-ignored.md) +* [prereq-ignored](./prereq-ignored.md) They don't even need to be in the same list -* [prereq-processed](content/simdem1/prereq-processed.md) +* [prereq-processed](./prereq-processed.md) By this point, the prerequisites have either run or have passed their validation diff --git a/simdem/mode/common.py b/simdem/mode/common.py index bcf66f8..459e041 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -1,6 +1,7 @@ """ Common mode for SimDem mode """ import os +import sys import logging import difflib @@ -10,7 +11,6 @@ class ModeCommon(object): # pylint: disable=R0903 config = None executor = None parser = None - cwd = None def __init__(self, config, parser, executor): self.config = config @@ -20,15 +20,17 @@ def __init__(self, config, parser, executor): def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) - steps = self.parser.parse_file(file_path) # Change the working directory in case of any recursion #print("changing path to:" + os.path.dirname(file_path)) #os.chdir(os.path.dirname(file_path)) + start_path = os.path.dirname(file_path) + steps = self.parser.parse_file(file_path) # Begin preqreq processing if 'prerequisites' in steps: for prereq_file in steps['prerequisites']: - self.process_file(prereq_file, is_prereq=True) + # Change the working directory in case of any recursion + self.process_file(start_path + '/' + prereq_file, is_prereq=True) if is_prereq and 'validation' in steps: last_command_result = self.process_commands(steps['validation']['commands']) # pylint: disable=no-member if 'expected_result' in steps['validation']: @@ -42,6 +44,9 @@ def process_file(self, file_path, is_prereq=False): self.process(steps) # pylint: disable=no-member + if 'next_steps' in steps: + self.process_next_steps(steps['next_steps'], start_path) + def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: @@ -88,3 +93,26 @@ def is_result_valid(expected_results, actual_results, expected_similarity=1.0): logging.error("expected_results = " + expected_results) return is_pass + + def process_next_steps(self, steps, start_path): + """ Is there a good way to test this that doesn't involve lots of test code + expect? + Not fully tested yet. Low priority feature. + """ + idx = 1 + if steps: + print("Next steps available:") + for step in steps: + print(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") + idx += 1 + print() + # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not + if sys.stdout.isatty(): + # You're running in a real terminal + step_request = input("Choose a step. " + + "Type the # or 'q' to quit and then press Enter: ") + #print('You chose:' + str(steps[int(step_request) - 1]['title'])) + self.process_file(start_path + '/' + steps[int(step_request) - 1]['target']) + else: + logging.info('Not connected to a TTY terminal Not requesting input.') + # You're being piped or redirected + return diff --git a/simdem/mode/test.py b/simdem/mode/test.py index e53d0f6..f2d6c32 100644 --- a/simdem/mode/test.py +++ b/simdem/mode/test.py @@ -22,3 +22,7 @@ def process(self, steps): print('***VALIDATION FAILED***') else: print('***VALIDATION PASSED***') + + def process_next_steps(self, steps, start_path): + """ No need to display next steps if in test mode """ + pass diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index eb3d11b..4d94750 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -22,8 +22,6 @@ def process(self, steps): elif step['type'] == 'commands': self.process_commands(step['content']) - if 'next_steps' in steps: - self.process_next_steps(steps['next_steps']) @staticmethod def process_heading(step): @@ -35,27 +33,4 @@ def process_heading(step): def process_text(step): """ Print out the text exactly as we found it """ print(step['content']) - print() - - def process_next_steps(self, steps): - """ Is there a good way to test this that doesn't involve lots of test code + expect? - Not fully tested yet. Low priority feature. - """ - idx = 1 - if steps: - print("Next steps available:") - for step in steps: - print(str(idx) + ".) " + step['title']) - idx += 1 - print() - # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not - if sys.stdout.isatty(): - # You're running in a real terminal - step_request = input("Choose a step. " + - "Type the # or 'q' to quit and then press Enter: ") - #print('You chose:' + str(steps[int(step_request) - 1]['title'])) - self.process_file(steps[int(step_request) - 1]['target']) - else: - logging.info('Not connected to a TTY terminal Not requesting input.') - # You're being piped or redirected - return + print() \ No newline at end of file diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index bc3b540..8597a28 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -59,16 +59,12 @@ def render_heading(self, token): def render_list(self, token): """ Render a markdown list """ - logging.debug('render_list()') inner = self.render_inner(token) - logging.debug('render_list()::inner=' + inner) return inner def render_list_item(self, token): """ Render a markdown list item """ - logging.debug('render_list_item()') inner = self.render_inner(token) - logging.debug('render_list_item()::inner=' + inner) return inner def render_link(self, token): From ab64ac56b02723974c970fc132061379b78d70d1 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 19 Jan 2018 11:43:02 -0600 Subject: [PATCH 099/167] Implement input for demo mode Signed-off-by: Tommy Falgout --- content/simdem1/README.md | 4 ++-- simdem/mode/common.py | 2 -- simdem/mode/demo.py | 4 +++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/content/simdem1/README.md b/content/simdem1/README.md index 4192c87..7777bf2 100644 --- a/content/simdem1/README.md +++ b/content/simdem1/README.md @@ -2,11 +2,11 @@ This is the prerequisite section. SimDem looks for a set of links to extract and run through first -* [prereq-ignored](content/simdem1/prereq-ignored.md) +* [prereq-ignored](./prereq-ignored.md) They don't even need to be in the same list -* [prereq-processed](content/simdem1/prereq-processed.md) +* [prereq-processed](./prereq-processed.md) By this point, the prerequisites have either run or have passed their validation diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 459e041..0e3936a 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -21,8 +21,6 @@ def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) # Change the working directory in case of any recursion - #print("changing path to:" + os.path.dirname(file_path)) - #os.chdir(os.path.dirname(file_path)) start_path = os.path.dirname(file_path) steps = self.parser.parse_file(file_path) diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index 1700825..96ead27 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -21,6 +21,8 @@ def process(self, steps): def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: + # Request enter from user to know when to proceed + input() self.type_command(cmd) results = self.executor.run_cmd(cmd) print(results, end="", flush=True) @@ -35,7 +37,7 @@ def type_command(self, cmd): print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ', end="", flush=True) for _, char in enumerate(cmd): if char != "\n": - typing_delay = None #float(self.config.get('RENDER', 'TYPING_DELAY')) + typing_delay = float(self.config.get('RENDER', 'TYPING_DELAY')) if typing_delay: delay = random.uniform(0.02, typing_delay) time.sleep(delay) From fc4beb627acb000fc91bf2f30b2709d4de7392d0 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 19 Jan 2018 13:47:56 -0600 Subject: [PATCH 100/167] Add prompting for a character when in demo mode. Created interactive class Signed-off-by: Tommy Falgout --- simdem/misc/getch.py | 40 +++++++++++++++++++++++++++++++++ simdem/mode/common.py | 26 +--------------------- simdem/mode/demo.py | 6 ++--- simdem/mode/interactive.py | 43 ++++++++++++++++++++++++++++++++++++ simdem/mode/tutorial.py | 7 +++--- tests/test_parser_simdem1.py | 4 ++-- 6 files changed, 92 insertions(+), 34 deletions(-) create mode 100644 simdem/misc/getch.py create mode 100644 simdem/mode/interactive.py diff --git a/simdem/misc/getch.py b/simdem/misc/getch.py new file mode 100644 index 0000000..86f9f48 --- /dev/null +++ b/simdem/misc/getch.py @@ -0,0 +1,40 @@ +# pylint: disable=R0903,E0401,W0612 +""" Get Character Class """ + +# https://stackoverflow.com/a/510404/8475874 +class Getch: + """Gets a single character from standard input. Does not echo to the screen.""" + def __init__(self): + try: + self.impl = GetchWindows() + except ImportError: + self.impl = GetchUnix() + + def __call__(self): + return self.impl() + + +class GetchUnix: + """ Get character impl for *nix """ + def __call__(self): + import sys + import tty + import termios + filedesc = sys.stdin.fileno() + old_settings = termios.tcgetattr(filedesc) + try: + tty.setraw(sys.stdin.fileno()) + char = sys.stdin.read(1) + finally: + termios.tcsetattr(filedesc, termios.TCSADRAIN, old_settings) + return char + + +class GetchWindows: + """ Get character impl for *nix """ + def __init__(self): + import msvcrt + + def __call__(self): + import msvcrt + return msvcrt.getch() diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 0e3936a..3d0516a 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -1,7 +1,6 @@ """ Common mode for SimDem mode """ import os -import sys import logging import difflib @@ -43,7 +42,7 @@ def process_file(self, file_path, is_prereq=False): self.process(steps) # pylint: disable=no-member if 'next_steps' in steps: - self.process_next_steps(steps['next_steps'], start_path) + self.process_next_steps(steps['next_steps'], start_path) # pylint: disable=no-member def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ @@ -91,26 +90,3 @@ def is_result_valid(expected_results, actual_results, expected_similarity=1.0): logging.error("expected_results = " + expected_results) return is_pass - - def process_next_steps(self, steps, start_path): - """ Is there a good way to test this that doesn't involve lots of test code + expect? - Not fully tested yet. Low priority feature. - """ - idx = 1 - if steps: - print("Next steps available:") - for step in steps: - print(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") - idx += 1 - print() - # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not - if sys.stdout.isatty(): - # You're running in a real terminal - step_request = input("Choose a step. " + - "Type the # or 'q' to quit and then press Enter: ") - #print('You chose:' + str(steps[int(step_request) - 1]['title'])) - self.process_file(start_path + '/' + steps[int(step_request) - 1]['target']) - else: - logging.info('Not connected to a TTY terminal Not requesting input.') - # You're being piped or redirected - return diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index 96ead27..e0356e0 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -3,9 +3,9 @@ import random import time import logging -from simdem.mode.common import ModeCommon +from simdem.mode.interactive import InteractiveMode -class DemoMode(ModeCommon): +class DemoMode(InteractiveMode): """ This class is the default SimDem file processor. It's designed for running files in a demo-able mode that looks like a human is typing it """ @@ -22,7 +22,7 @@ def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: # Request enter from user to know when to proceed - input() + self.get_single_key_input() self.type_command(cmd) results = self.executor.run_cmd(cmd) print(results, end="", flush=True) diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py new file mode 100644 index 0000000..f5287c7 --- /dev/null +++ b/simdem/mode/interactive.py @@ -0,0 +1,43 @@ +""" Interactive Mode Class """ +import sys +import logging +from simdem.mode.common import ModeCommon +from simdem.misc.getch import Getch + +class InteractiveMode(ModeCommon): + """ Interactive Mode subclass """ + + @staticmethod + def get_single_key_input(): + """ SimDem1 uses this method: + https://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key + For SimDem2, I'm trying this alternative to allow for Windows compatibility + https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user/510404#510404 + """ + if sys.stdout.isatty(): + getch = Getch() + return getch.impl() + return + + def process_next_steps(self, steps, start_path): + """ Is there a good way to test this that doesn't involve lots of test code + expect? + Not fully tested yet. Low priority feature. + """ + idx = 1 + if steps: + print("Next steps available:") + for step in steps: + print(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") + idx += 1 + print() + # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not + if sys.stdout.isatty(): + # You're running in a real terminal + step_request = input("Choose a step. " + + "Type the # or 'q' to quit and then press Enter: ") + #print('You chose:' + str(steps[int(step_request) - 1]['title'])) + self.process_file(start_path + '/' + steps[int(step_request) - 1]['target']) + else: + logging.info('Not connected to a TTY terminal Not requesting input.') + # You're being piped or redirected + return diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 4d94750..9d24ffc 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -1,10 +1,9 @@ """ Tutorial mode for SimDem""" -import sys import logging -from simdem.mode.common import ModeCommon +from simdem.mode.interactive import InteractiveMode -class TutorialMode(ModeCommon): +class TutorialMode(InteractiveMode): """ This class is the tutorial mode class for SimDem. It's designed for running files in a tutorial mode which Displays the descriptive text of the tutorial and pauses at code blocks to allow user interaction. @@ -33,4 +32,4 @@ def process_heading(step): def process_text(step): """ Print out the text exactly as we found it """ print(step['content']) - print() \ No newline at end of file + print() diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 9abb6dd..59aa09e 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -56,8 +56,8 @@ def test_full(self): 'type': 'text'}], 'next_steps': [{'target': 'step-1.md', 'title': 'Step #1'}, {'target': 'step-2.md', 'title': 'Step #2'}], - 'prerequisites': ['content/simdem1/prereq-ignored.md', - 'content/simdem1/prereq-processed.md']} + 'prerequisites': ['./prereq-ignored.md', + './prereq-processed.md']} pprint.pprint(res) self.assertEqual(res, exp_resl) From d08ac1e88eefa1cf293daa1d29dd797b2340e02f Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 22 Jan 2018 13:17:03 -0600 Subject: [PATCH 101/167] Upgrade mistletoe to 0.5 Signed-off-by: Tommy Falgout --- .requirements.txt.swp | Bin 0 -> 12288 bytes requirements.txt | 9 ++++----- simdem/mode/tutorial.py | 1 - simdem/parser/simdem1.py | 2 +- tests/test_mistletoe.py | 2 +- tests/test_parser_simdem1.py | 12 ++++++------ 6 files changed, 12 insertions(+), 14 deletions(-) create mode 100644 .requirements.txt.swp diff --git a/.requirements.txt.swp b/.requirements.txt.swp new file mode 100644 index 0000000000000000000000000000000000000000..325a4fd1c17d2a18870a95960982b24020c63411 GIT binary patch literal 12288 zcmeI%F-yZh6u|KpaTP^ZhpJnlX;qMpt|E>)xFyY{9^`Uq@=`kK=kmMg=-}w=;#Jbc zp+hGN{|Ar9UE}5C}KzwzJ(+ZKmY** r5I_I{1Q0*~0R#|0;GYR(HqQ&2N|vcqW>J}minEOtM!k{JH%5K{Z;U_+ literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt index 28e8897..47efaa7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -nose -mistletoe -pexpect -ddt -difflib +nose==1.3.7 +mistletoe==0.5 +pexpect==4.2.1 +ddt==1.1.1 diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 9d24ffc..3e6110e 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -32,4 +32,3 @@ def process_heading(step): def process_text(step): """ Print out the text exactly as we found it """ print(step['content']) - print() diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 8597a28..21a74bb 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -81,7 +81,7 @@ def render_link(self, token): def render_raw_text(self, token): """ Render raw text. The only thing to look for is the result text indicator """ - if token.content == 'Results:': + if token.content.rstrip() == 'Results:': self.set_block('results') return '' return token.content diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index e4dd1d7..832f2ef 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -23,7 +23,7 @@ def test_ast(self): {'children': [{'content': 'echo foo\necho bar\n', 'type': 'RawText'}], 'language': 'shell', - 'type': 'BlockCode'}], + 'type': 'CodeFence'}], 'footnotes': {}, 'type': 'Document'} diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 59aa09e..d23cfd5 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -26,12 +26,12 @@ def test_full(self): # This is pretty brittle. It might be valuable to have a test document with less content exp_resl = {'body': [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, {'content': 'This is the prerequisite section. SimDem looks for a ' - 'set of links to extract and run through first', + 'set of links to extract and run through first\n', 'type': 'text'}, - {'content': "They don't even need to be in the same list", + {'content': "They don't even need to be in the same list\n", 'type': 'text'}, {'content': 'By this point, the prerequisites have either run or ' - 'have passed their validation', + 'have passed their validation\n', 'type': 'text'}, {'content': 'Did our prerequisites run?', 'level': 1, @@ -40,19 +40,19 @@ def test_full(self): 'echo prereq_processed = $prereq_processed'], 'type': 'commands'}, {'content': 'Do stuff here', 'level': 1, 'type': 'heading'}, - {'content': 'We want to execute this because the code type is shell', + {'content': 'We want to execute this because the code type is shell\n', 'type': 'text'}, {'content': ['echo foo', 'var=bar'], 'type': 'commands'}, {'content': 'Do more stuff here', 'level': 1, 'type': 'heading'}, {'content': 'We assume the result is for the last command of the ' - 'last code block', + 'last code block\n', 'type': 'text'}, {'content': ['echo baz', 'echo $var'], 'expected_result': 'bar\n', 'type': 'commands'}, {'content': 'Next Steps', 'level': 1, 'type': 'heading'}, {'content': 'The list inside this block are steps that could be ' - 'followed when performing an interactive tutorial', + 'followed when performing an interactive tutorial\n', 'type': 'text'}], 'next_steps': [{'target': 'step-1.md', 'title': 'Step #1'}, {'target': 'step-2.md', 'title': 'Step #2'}], From 60e4b71ba9c54fd0e596bf3ad42267e000b5f244 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 22 Jan 2018 13:18:18 -0600 Subject: [PATCH 102/167] Fix install docs Signed-off-by: Tommy Falgout --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b6b08af..8f4aafe 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Currently, it's only available for installation in development mode: ``` git clone git@github.com:Azure/simdem.git +pip3 install -r requirements.txt pip3 install -v -e . ``` From b8a0f91c56108cb409fbe0275f794b5bfc453b83 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 22 Jan 2018 19:39:04 -0600 Subject: [PATCH 103/167] Allow ability to override config setting via cli Signed-off-by: Tommy Falgout --- .requirements.txt.swp | Bin 12288 -> 0 bytes content/config/demo.ini | 18 +++++++++--------- content/config/unit_test.ini | 18 +++++++++--------- simdem/cli.py | 16 +++++++++++++--- simdem/mode/common.py | 2 +- simdem/mode/demo.py | 4 ++-- simdem/mode/dump.py | 11 +++++++++++ tests/test_mode_demo.py | 6 +++--- tests/test_mode_tutorial.py | 6 +++--- 9 files changed, 51 insertions(+), 30 deletions(-) delete mode 100644 .requirements.txt.swp diff --git a/.requirements.txt.swp b/.requirements.txt.swp deleted file mode 100644 index 325a4fd1c17d2a18870a95960982b24020c63411..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI%F-yZh6u|KpaTP^ZhpJnlX;qMpt|E>)xFyY{9^`Uq@=`kK=kmMg=-}w=;#Jbc zp+hGN{|Ar9UE}5C}KzwzJ(+ZKmY** r5I_I{1Q0*~0R#|0;GYR(HqQ&2N|vcqW>J}minEOtM!k{JH%5K{Z;U_+ diff --git a/content/config/demo.ini b/content/config/demo.ini index c50d152..c38722b 100644 --- a/content/config/demo.ini +++ b/content/config/demo.ini @@ -1,16 +1,16 @@ -[DEFAULT] -SIMDEM_VERSION = 2.0.0 +[meta] +simdem_version = 2.0.0 # Logging -[LOG] -FILE = simdem.log -LEVEL = DEBUG -FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s +[log] +file = simdem.log +level = DEBUG +format = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s -[RENDER] +[render] # When in demo mode we insert a small random delay between characters. # TYPING DELAY is the upper bound of this delay. -TYPING_DELAY = 0.2 +typing_delay = 0.2 # Prompt to use in the console -CONSOLE_PROMPT = $ \ No newline at end of file +console_prompt = $ \ No newline at end of file diff --git a/content/config/unit_test.ini b/content/config/unit_test.ini index e1f8af5..d618800 100644 --- a/content/config/unit_test.ini +++ b/content/config/unit_test.ini @@ -1,16 +1,16 @@ -[DEFAULT] -SIMDEM_VERSION = 2.0.0 +[meta] +simdem_version = 2.0.0 # Logging -[LOG] -FILE = simdem.log -LEVEL = DEBUG -FORMAT = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s +[log] +file = simdem.log +level = DEBUG +format = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s -[RENDER] +[render] # When in demo mode we insert a small random delay between characters. # TYPING DELAY is the upper bound of this delay. -TYPING_DELAY = 0 +typing_delay = 0 # Prompt to use in the console -CONSOLE_PROMPT = $ \ No newline at end of file +console_prompt = $ \ No newline at end of file diff --git a/simdem/cli.py b/simdem/cli.py index 8c6f813..56c7bd7 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -25,6 +25,8 @@ def main(): help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", help="Executor class to use", choices=['bash']) + argp.add_argument('--setting', '-s', metavar='setting', + help="Setting to override in config file") options = argp.parse_args() file_path = options.file @@ -35,12 +37,20 @@ def main(): config = configparser.ConfigParser() config.read(options.config_file) + inject_arguments(options, config) setup_logging(config, options) mode = get_mode(options, config) mode.process_file(file_path) +def inject_arguments(options, config): + """ Injects CLI arguments into config settings """ + if options.setting: + [key, value] = options.setting.split('=') + [section, option] = key.split('.') + config.set(section, option, value) + def validate(options, file_path): """ validate all passed in arguments """ if not os.path.isfile(file_path): @@ -81,11 +91,11 @@ def get_executor(options): def setup_logging(config, options): """ Establishes logging level and format """ - log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() - root_logger.setLevel(config.get('LOG', 'LEVEL')) + root_logger.setLevel(config.get('log', 'level')) - file_handler = logging.FileHandler(config.get('LOG', 'FILE')) + file_handler = logging.FileHandler(config.get('log', 'file')) file_handler.setFormatter(log_formatter) root_logger.addHandler(file_handler) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 3d0516a..0b095bb 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -47,7 +47,7 @@ def process_file(self, file_path, is_prereq=False): def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: - print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ' + cmd) + print(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd) results = self.executor.run_cmd(cmd) print(results, end="", flush=True) print() diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index e0356e0..aec865f 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -34,10 +34,10 @@ def type_command(self, cmd): # Must add ' ' when typing command because whitespaces are removed from configparser # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure - print(self.config.get('RENDER', 'CONSOLE_PROMPT', raw=True) + ' ', end="", flush=True) + print(self.config.get('render', 'console_prompt', raw=True) + ' ', end="", flush=True) for _, char in enumerate(cmd): if char != "\n": - typing_delay = float(self.config.get('RENDER', 'TYPING_DELAY')) + typing_delay = float(self.config.get('render', 'typing_delay')) if typing_delay: delay = random.uniform(0.02, typing_delay) time.sleep(delay) diff --git a/simdem/mode/dump.py b/simdem/mode/dump.py index 7f7f832..2f4be10 100644 --- a/simdem/mode/dump.py +++ b/simdem/mode/dump.py @@ -2,6 +2,7 @@ import pprint import logging +import configparser from simdem.mode.common import ModeCommon class DumpMode(ModeCommon): # pylint: disable=R0903 @@ -11,4 +12,14 @@ def process_file(self, file_path, is_prereq=False): """ Parse the file and print it. Not very exciting. """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) + print("Config=") + for section in self.config.sections(): + for option in self.config.options(section): + try: + print(section + '.' + option + '=' + str(self.config.get(section, option))) + except configparser.InterpolationMissingOptionError: + pass + print() + + print("Parsed file=") pprint.pprint(steps) diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index 686ced6..eed7d08 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -24,10 +24,10 @@ def setUp(self): config.read("content/config/unit_test.ini") self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) - log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() - root_logger.setLevel(config.get('LOG', 'LEVEL')) - file_handler = logging.FileHandler(config.get('LOG', 'FILE')) + root_logger.setLevel(config.get('log', 'level')) + file_handler = logging.FileHandler(config.get('log', 'file')) file_handler.setFormatter(log_formatter) root_logger.addHandler(file_handler) diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index ef3b0fe..3550cc8 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -25,10 +25,10 @@ def setUp(self): config.read("content/config/unit_test.ini") self.simdem = tutorial.TutorialMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) - log_formatter = logging.Formatter(config.get('LOG', 'FORMAT', raw=True)) + log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() - root_logger.setLevel(config.get('LOG', 'LEVEL')) - file_handler = logging.FileHandler(config.get('LOG', 'FILE')) + root_logger.setLevel(config.get('log', 'level')) + file_handler = logging.FileHandler(config.get('log', 'file')) file_handler.setFormatter(log_formatter) root_logger.addHandler(file_handler) From e0c8edc1f464c5309cf05507acf5e0d6e5dff0b9 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 22 Jan 2018 19:56:19 -0600 Subject: [PATCH 104/167] Moved print_config_data to own function --- simdem/mode/dump.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/simdem/mode/dump.py b/simdem/mode/dump.py index 2f4be10..9d2bff7 100644 --- a/simdem/mode/dump.py +++ b/simdem/mode/dump.py @@ -12,6 +12,10 @@ def process_file(self, file_path, is_prereq=False): """ Parse the file and print it. Not very exciting. """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) + pprint.pprint(steps) + + def print_config_data(self): + """ Dead code for now, but useful for debugging config """ print("Config=") for section in self.config.sections(): for option in self.config.options(section): @@ -20,6 +24,3 @@ def process_file(self, file_path, is_prereq=False): except configparser.InterpolationMissingOptionError: pass print() - - print("Parsed file=") - pprint.pprint(steps) From 3ec9d702be7e20d42444ae754f50c6749c87922a Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 24 Jan 2018 09:27:06 -0800 Subject: [PATCH 105/167] Add SEO tests Signed-off-by: Tommy Falgout --- content/next-steps/expected_result.seo | 33 +++++++ content/prerequisites/expected_result.seo | 37 ++++++++ .../results-block-fail/expected_result.seo | 30 +++++++ content/results-block/expected_result.seo | 20 +++++ content/simdem1/expected_result.seo | 89 +++++++++++++++++++ content/simple-variable/expected_result.seo | 25 ++++++ content/simple/expected_result.seo | 2 + simdem/mode/dump.py | 4 +- simdem/mode/interactive.py | 5 ++ tests/test_parser_simdem1.py | 63 ++++--------- 10 files changed, 262 insertions(+), 46 deletions(-) create mode 100644 content/next-steps/expected_result.seo create mode 100644 content/prerequisites/expected_result.seo create mode 100644 content/results-block-fail/expected_result.seo create mode 100644 content/results-block/expected_result.seo create mode 100644 content/simdem1/expected_result.seo create mode 100644 content/simple-variable/expected_result.seo create mode 100644 content/simple/expected_result.seo diff --git a/content/next-steps/expected_result.seo b/content/next-steps/expected_result.seo new file mode 100644 index 0000000..29ec0e6 --- /dev/null +++ b/content/next-steps/expected_result.seo @@ -0,0 +1,33 @@ +{ + "body": [ + { + "content": "this is text\n", + "type": "text" + }, + { + "content": [ + "echo main" + ], + "type": "commands" + }, + { + "content": "Next steps", + "level": 1, + "type": "heading" + }, + { + "content": "The list inside this block are steps that could be followed when performing an interactive tutorial\n", + "type": "text" + } + ], + "next_steps": [ + { + "target": "step-1.md", + "title": "Step #1" + }, + { + "target": "step-2.md", + "title": "Step #2" + } + ] +} diff --git a/content/prerequisites/expected_result.seo b/content/prerequisites/expected_result.seo new file mode 100644 index 0000000..b3874f9 --- /dev/null +++ b/content/prerequisites/expected_result.seo @@ -0,0 +1,37 @@ +{ + "body": [ + { + "content": "Prerequisites", + "level": 1, + "type": "heading" + }, + { + "content": "This is the prerequisite section. SimDem looks for a set of links to extract and run through first\n", + "type": "text" + }, + { + "content": "They don't even need to be in the same list\n", + "type": "text" + }, + { + "content": "By this point, the prerequisites have either run or have passed their validation\n", + "type": "text" + }, + { + "content": "Did our prerequisites run?", + "level": 1, + "type": "heading" + }, + { + "content": [ + "echo prereq_ignored = $prereq_ignored", + "echo prereq_processed = $prereq_processed" + ], + "type": "commands" + } + ], + "prerequisites": [ + "./prereq-ignored.md", + "./prereq-processed.md" + ] +} diff --git a/content/results-block-fail/expected_result.seo b/content/results-block-fail/expected_result.seo new file mode 100644 index 0000000..ef5c3bc --- /dev/null +++ b/content/results-block-fail/expected_result.seo @@ -0,0 +1,30 @@ +{ + "body": [ + { + "content": "this is text\n", + "type": "text" + }, + { + "content": [ + "echo foo", + "echo bar" + ], + "expected_result": "barr\n", + "type": "commands" + }, + { + "content": "In demo mode, we will continue processing.\n", + "type": "text" + }, + { + "content": [ + "echo post_result_block_fail" + ], + "type": "commands" + }, + { + "content": "even more text\n", + "type": "text" + } + ] +} diff --git a/content/results-block/expected_result.seo b/content/results-block/expected_result.seo new file mode 100644 index 0000000..5076507 --- /dev/null +++ b/content/results-block/expected_result.seo @@ -0,0 +1,20 @@ +{ + "body": [ + { + "content": "this is text\n", + "type": "text" + }, + { + "content": [ + "echo foo", + "echo bar" + ], + "expected_result": "bar\n", + "type": "commands" + }, + { + "content": "even more text\n", + "type": "text" + } + ] +} diff --git a/content/simdem1/expected_result.seo b/content/simdem1/expected_result.seo new file mode 100644 index 0000000..02815a3 --- /dev/null +++ b/content/simdem1/expected_result.seo @@ -0,0 +1,89 @@ +{ + "body": [ + { + "content": "Prerequisites", + "level": 1, + "type": "heading" + }, + { + "content": "This is the prerequisite section. SimDem looks for a set of links to extract and run through first\n", + "type": "text" + }, + { + "content": "They don't even need to be in the same list\n", + "type": "text" + }, + { + "content": "By this point, the prerequisites have either run or have passed their validation\n", + "type": "text" + }, + { + "content": "Did our prerequisites run?", + "level": 1, + "type": "heading" + }, + { + "content": [ + "echo prereq_ignored = $prereq_ignored", + "echo prereq_processed = $prereq_processed" + ], + "type": "commands" + }, + { + "content": "Do stuff here", + "level": 1, + "type": "heading" + }, + { + "content": "We want to execute this because the code type is shell\n", + "type": "text" + }, + { + "content": [ + "echo foo", + "var=bar" + ], + "type": "commands" + }, + { + "content": "Do more stuff here", + "level": 1, + "type": "heading" + }, + { + "content": "We assume the result is for the last command of the last code block\n", + "type": "text" + }, + { + "content": [ + "echo baz", + "echo $var" + ], + "expected_result": "bar\n", + "type": "commands" + }, + { + "content": "Next Steps", + "level": 1, + "type": "heading" + }, + { + "content": "The list inside this block are steps that could be followed when performing an interactive tutorial\n", + "type": "text" + } + ], + "next_steps": [ + { + "target": "step-1.md", + "title": "Step #1" + }, + { + "target": "step-2.md", + "title": "Step #2" + } + ], + "prerequisites": [ + "./prereq-ignored.md", + "./prereq-processed.md" + ] +} diff --git a/content/simple-variable/expected_result.seo b/content/simple-variable/expected_result.seo new file mode 100644 index 0000000..b11a28d --- /dev/null +++ b/content/simple-variable/expected_result.seo @@ -0,0 +1,25 @@ +{ + "body": [ + { + "content": "this is text", + "level": 1, + "type": "heading" + }, + { + "content": [ + "FOO=bar" + ], + "type": "commands" + }, + { + "content": [ + "echo $FOO" + ], + "type": "commands" + }, + { + "content": "more text\n", + "type": "text" + } + ] +} diff --git a/content/simple/expected_result.seo b/content/simple/expected_result.seo new file mode 100644 index 0000000..efdd9ee --- /dev/null +++ b/content/simple/expected_result.seo @@ -0,0 +1,2 @@ +{"body": [{"content": "Simple", "level": 1, "type": "heading"}, + {"content": ["echo foo", "echo bar"], "type": "commands"}]} diff --git a/simdem/mode/dump.py b/simdem/mode/dump.py index 9d2bff7..89474be 100644 --- a/simdem/mode/dump.py +++ b/simdem/mode/dump.py @@ -1,7 +1,7 @@ """ Debug renderer for SimDem""" -import pprint import logging +import json import configparser from simdem.mode.common import ModeCommon @@ -12,7 +12,7 @@ def process_file(self, file_path, is_prereq=False): """ Parse the file and print it. Not very exciting. """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) - pprint.pprint(steps) + print(json.dumps(steps, indent=4, sort_keys=True)) def print_config_data(self): """ Dead code for now, but useful for debugging config """ diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index f5287c7..66141db 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -13,7 +13,12 @@ def get_single_key_input(): https://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key For SimDem2, I'm trying this alternative to allow for Windows compatibility https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user/510404#510404 + + https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not + Might need to allow config override of the conditional by allowing a config variable. + Experienced an issue where it hung running in a container and I didn't completely debug """ + if sys.stdout.isatty(): getch = Getch() return getch.impl() diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index d23cfd5..eef604a 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -1,11 +1,15 @@ -# pylint: disable=C0330 -"""SimDem Test Case""" +# -*- coding: utf-8 -*- +""" system level test class """ import configparser import unittest -import pprint +import json + +from ddt import data, ddt + from simdem.parser import simdem1 +@ddt class SimDem1ParserTestSuite(unittest.TestCase): """Advanced test cases.""" @@ -17,49 +21,20 @@ def setUp(self): self.parser = simdem1.SimDem1Parser() - def test_full(self): - """Test parsing a document with all features in it""" + + # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer + @data('simple', 'simple-variable', 'results-block', + 'results-block-fail', 'prerequisites') + def test_process(self, directory): + """ Each content directory is expected to have a README.md and an expected_result.tutorial + this allows us to test each of them easily + """ self.maxDiff = None # pylint: disable=C0103 - file_path = 'content/simdem1/README.md' - res = self.parser.parse_file(file_path) + res = self.parser.parse_file('./content/' + directory + '/README.md') - # This is pretty brittle. It might be valuable to have a test document with less content - exp_resl = {'body': [{'content': 'Prerequisites', 'level': 1, 'type': 'heading'}, - {'content': 'This is the prerequisite section. SimDem looks for a ' - 'set of links to extract and run through first\n', - 'type': 'text'}, - {'content': "They don't even need to be in the same list\n", - 'type': 'text'}, - {'content': 'By this point, the prerequisites have either run or ' - 'have passed their validation\n', - 'type': 'text'}, - {'content': 'Did our prerequisites run?', - 'level': 1, - 'type': 'heading'}, - {'content': ['echo prereq_ignored = $prereq_ignored', - 'echo prereq_processed = $prereq_processed'], - 'type': 'commands'}, - {'content': 'Do stuff here', 'level': 1, 'type': 'heading'}, - {'content': 'We want to execute this because the code type is shell\n', - 'type': 'text'}, - {'content': ['echo foo', 'var=bar'], 'type': 'commands'}, - {'content': 'Do more stuff here', 'level': 1, 'type': 'heading'}, - {'content': 'We assume the result is for the last command of the ' - 'last code block\n', - 'type': 'text'}, - {'content': ['echo baz', 'echo $var'], - 'expected_result': 'bar\n', - 'type': 'commands'}, - {'content': 'Next Steps', 'level': 1, 'type': 'heading'}, - {'content': 'The list inside this block are steps that could be ' - 'followed when performing an interactive tutorial\n', - 'type': 'text'}], - 'next_steps': [{'target': 'step-1.md', 'title': 'Step #1'}, - {'target': 'step-2.md', 'title': 'Step #2'}], - 'prerequisites': ['./prereq-ignored.md', - './prereq-processed.md']} - pprint.pprint(res) - self.assertEqual(res, exp_resl) + # Research how to read dict from file + exp_res = json.load(open('./content/' + directory + '/expected_result.seo', 'r')) + self.assertEqual(exp_res, res) if __name__ == '__main__': From bcbf8a7843454ee761117317d4b95f72a9b9a372 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 26 Jan 2018 11:39:49 -0600 Subject: [PATCH 106/167] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8f4aafe..cd02c83 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Currently, it's only available for installation in development mode: ``` git clone git@github.com:Azure/simdem.git +git checkout -b simdem2 pip3 install -r requirements.txt pip3 install -v -e . ``` From cd39e482066d4da87cb84dd7c573a01c372bc803 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sat, 27 Jan 2018 12:12:55 -0600 Subject: [PATCH 107/167] Rename content dir to examples Signed-off-by: Tommy Falgout --- {content => examples}/config/demo.ini | 0 {content => examples}/config/unit_test.ini | 0 {content => examples}/next-steps/README.md | 0 {content => examples}/next-steps/expected_result.out | 0 {content => examples}/next-steps/expected_result.seo | 0 {content => examples}/next-steps/step-1.md | 0 {content => examples}/next-steps/step-2.md | 0 {content => examples}/prerequisites/README.md | 0 {content => examples}/prerequisites/expected_result.demo | 0 {content => examples}/prerequisites/expected_result.dump | 0 {content => examples}/prerequisites/expected_result.seo | 0 .../prerequisites/expected_result.tutorial | 0 {content => examples}/prerequisites/prereq-ignored.md | 0 {content => examples}/prerequisites/prereq-processed.md | 0 {content => examples}/results-block-fail/README.md | 0 .../results-block-fail/expected_result.demo | 0 .../results-block-fail/expected_result.seo | 0 .../results-block-fail/expected_result.tutorial | 0 {content => examples}/results-block/README.md | 0 {content => examples}/results-block/expected_result.demo | 0 {content => examples}/results-block/expected_result.seo | 0 .../results-block/expected_result.tutorial | 0 {content => examples}/simdem1/README.md | 0 {content => examples}/simdem1/expected_result.seo | 0 {content => examples}/simdem1/prereq-ignored.md | 0 {content => examples}/simdem1/prereq-processed.md | 0 {content => examples}/simple-variable/README.md | 0 .../simple-variable/expected_result.demo | 0 {content => examples}/simple-variable/expected_result.seo | 0 .../simple-variable/expected_result.tutorial | 0 {content => examples}/simple/README.md | 0 {content => examples}/simple/expected_result.demo | 0 {content => examples}/simple/expected_result.seo | 0 {content => examples}/simple/expected_result.tutorial | 0 tests/test_mistletoe.py | 2 +- tests/test_mode_demo.py | 8 ++++---- tests/test_mode_tutorial.py | 8 ++++---- tests/test_parser_simdem1.py | 8 ++++---- 38 files changed, 13 insertions(+), 13 deletions(-) rename {content => examples}/config/demo.ini (100%) rename {content => examples}/config/unit_test.ini (100%) rename {content => examples}/next-steps/README.md (100%) rename {content => examples}/next-steps/expected_result.out (100%) rename {content => examples}/next-steps/expected_result.seo (100%) rename {content => examples}/next-steps/step-1.md (100%) rename {content => examples}/next-steps/step-2.md (100%) rename {content => examples}/prerequisites/README.md (100%) rename {content => examples}/prerequisites/expected_result.demo (100%) rename {content => examples}/prerequisites/expected_result.dump (100%) rename {content => examples}/prerequisites/expected_result.seo (100%) rename {content => examples}/prerequisites/expected_result.tutorial (100%) rename {content => examples}/prerequisites/prereq-ignored.md (100%) rename {content => examples}/prerequisites/prereq-processed.md (100%) rename {content => examples}/results-block-fail/README.md (100%) rename {content => examples}/results-block-fail/expected_result.demo (100%) rename {content => examples}/results-block-fail/expected_result.seo (100%) rename {content => examples}/results-block-fail/expected_result.tutorial (100%) rename {content => examples}/results-block/README.md (100%) rename {content => examples}/results-block/expected_result.demo (100%) rename {content => examples}/results-block/expected_result.seo (100%) rename {content => examples}/results-block/expected_result.tutorial (100%) rename {content => examples}/simdem1/README.md (100%) rename {content => examples}/simdem1/expected_result.seo (100%) rename {content => examples}/simdem1/prereq-ignored.md (100%) rename {content => examples}/simdem1/prereq-processed.md (100%) rename {content => examples}/simple-variable/README.md (100%) rename {content => examples}/simple-variable/expected_result.demo (100%) rename {content => examples}/simple-variable/expected_result.seo (100%) rename {content => examples}/simple-variable/expected_result.tutorial (100%) rename {content => examples}/simple/README.md (100%) rename {content => examples}/simple/expected_result.demo (100%) rename {content => examples}/simple/expected_result.seo (100%) rename {content => examples}/simple/expected_result.tutorial (100%) diff --git a/content/config/demo.ini b/examples/config/demo.ini similarity index 100% rename from content/config/demo.ini rename to examples/config/demo.ini diff --git a/content/config/unit_test.ini b/examples/config/unit_test.ini similarity index 100% rename from content/config/unit_test.ini rename to examples/config/unit_test.ini diff --git a/content/next-steps/README.md b/examples/next-steps/README.md similarity index 100% rename from content/next-steps/README.md rename to examples/next-steps/README.md diff --git a/content/next-steps/expected_result.out b/examples/next-steps/expected_result.out similarity index 100% rename from content/next-steps/expected_result.out rename to examples/next-steps/expected_result.out diff --git a/content/next-steps/expected_result.seo b/examples/next-steps/expected_result.seo similarity index 100% rename from content/next-steps/expected_result.seo rename to examples/next-steps/expected_result.seo diff --git a/content/next-steps/step-1.md b/examples/next-steps/step-1.md similarity index 100% rename from content/next-steps/step-1.md rename to examples/next-steps/step-1.md diff --git a/content/next-steps/step-2.md b/examples/next-steps/step-2.md similarity index 100% rename from content/next-steps/step-2.md rename to examples/next-steps/step-2.md diff --git a/content/prerequisites/README.md b/examples/prerequisites/README.md similarity index 100% rename from content/prerequisites/README.md rename to examples/prerequisites/README.md diff --git a/content/prerequisites/expected_result.demo b/examples/prerequisites/expected_result.demo similarity index 100% rename from content/prerequisites/expected_result.demo rename to examples/prerequisites/expected_result.demo diff --git a/content/prerequisites/expected_result.dump b/examples/prerequisites/expected_result.dump similarity index 100% rename from content/prerequisites/expected_result.dump rename to examples/prerequisites/expected_result.dump diff --git a/content/prerequisites/expected_result.seo b/examples/prerequisites/expected_result.seo similarity index 100% rename from content/prerequisites/expected_result.seo rename to examples/prerequisites/expected_result.seo diff --git a/content/prerequisites/expected_result.tutorial b/examples/prerequisites/expected_result.tutorial similarity index 100% rename from content/prerequisites/expected_result.tutorial rename to examples/prerequisites/expected_result.tutorial diff --git a/content/prerequisites/prereq-ignored.md b/examples/prerequisites/prereq-ignored.md similarity index 100% rename from content/prerequisites/prereq-ignored.md rename to examples/prerequisites/prereq-ignored.md diff --git a/content/prerequisites/prereq-processed.md b/examples/prerequisites/prereq-processed.md similarity index 100% rename from content/prerequisites/prereq-processed.md rename to examples/prerequisites/prereq-processed.md diff --git a/content/results-block-fail/README.md b/examples/results-block-fail/README.md similarity index 100% rename from content/results-block-fail/README.md rename to examples/results-block-fail/README.md diff --git a/content/results-block-fail/expected_result.demo b/examples/results-block-fail/expected_result.demo similarity index 100% rename from content/results-block-fail/expected_result.demo rename to examples/results-block-fail/expected_result.demo diff --git a/content/results-block-fail/expected_result.seo b/examples/results-block-fail/expected_result.seo similarity index 100% rename from content/results-block-fail/expected_result.seo rename to examples/results-block-fail/expected_result.seo diff --git a/content/results-block-fail/expected_result.tutorial b/examples/results-block-fail/expected_result.tutorial similarity index 100% rename from content/results-block-fail/expected_result.tutorial rename to examples/results-block-fail/expected_result.tutorial diff --git a/content/results-block/README.md b/examples/results-block/README.md similarity index 100% rename from content/results-block/README.md rename to examples/results-block/README.md diff --git a/content/results-block/expected_result.demo b/examples/results-block/expected_result.demo similarity index 100% rename from content/results-block/expected_result.demo rename to examples/results-block/expected_result.demo diff --git a/content/results-block/expected_result.seo b/examples/results-block/expected_result.seo similarity index 100% rename from content/results-block/expected_result.seo rename to examples/results-block/expected_result.seo diff --git a/content/results-block/expected_result.tutorial b/examples/results-block/expected_result.tutorial similarity index 100% rename from content/results-block/expected_result.tutorial rename to examples/results-block/expected_result.tutorial diff --git a/content/simdem1/README.md b/examples/simdem1/README.md similarity index 100% rename from content/simdem1/README.md rename to examples/simdem1/README.md diff --git a/content/simdem1/expected_result.seo b/examples/simdem1/expected_result.seo similarity index 100% rename from content/simdem1/expected_result.seo rename to examples/simdem1/expected_result.seo diff --git a/content/simdem1/prereq-ignored.md b/examples/simdem1/prereq-ignored.md similarity index 100% rename from content/simdem1/prereq-ignored.md rename to examples/simdem1/prereq-ignored.md diff --git a/content/simdem1/prereq-processed.md b/examples/simdem1/prereq-processed.md similarity index 100% rename from content/simdem1/prereq-processed.md rename to examples/simdem1/prereq-processed.md diff --git a/content/simple-variable/README.md b/examples/simple-variable/README.md similarity index 100% rename from content/simple-variable/README.md rename to examples/simple-variable/README.md diff --git a/content/simple-variable/expected_result.demo b/examples/simple-variable/expected_result.demo similarity index 100% rename from content/simple-variable/expected_result.demo rename to examples/simple-variable/expected_result.demo diff --git a/content/simple-variable/expected_result.seo b/examples/simple-variable/expected_result.seo similarity index 100% rename from content/simple-variable/expected_result.seo rename to examples/simple-variable/expected_result.seo diff --git a/content/simple-variable/expected_result.tutorial b/examples/simple-variable/expected_result.tutorial similarity index 100% rename from content/simple-variable/expected_result.tutorial rename to examples/simple-variable/expected_result.tutorial diff --git a/content/simple/README.md b/examples/simple/README.md similarity index 100% rename from content/simple/README.md rename to examples/simple/README.md diff --git a/content/simple/expected_result.demo b/examples/simple/expected_result.demo similarity index 100% rename from content/simple/expected_result.demo rename to examples/simple/expected_result.demo diff --git a/content/simple/expected_result.seo b/examples/simple/expected_result.seo similarity index 100% rename from content/simple/expected_result.seo rename to examples/simple/expected_result.seo diff --git a/content/simple/expected_result.tutorial b/examples/simple/expected_result.tutorial similarity index 100% rename from content/simple/expected_result.tutorial rename to examples/simple/expected_result.tutorial diff --git a/tests/test_mistletoe.py b/tests/test_mistletoe.py index 832f2ef..70c607e 100644 --- a/tests/test_mistletoe.py +++ b/tests/test_mistletoe.py @@ -14,7 +14,7 @@ def test_ast(self): """Verify we understand how the Mistletoe AST parsers work""" #self.maxDiff = None - file_path = 'content/simple/README.md' + file_path = 'examples/simple/README.md' with open(file_path, 'r') as fin: res = renderer.get_ast(token.Document(fin)) exp_res = {'children': [{'children': [{'content': 'Simple', 'type': 'RawText'}], diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index eed7d08..eaf2e18 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -21,7 +21,7 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") + config.read("examples/config/unit_test.ini") self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) @@ -35,14 +35,14 @@ def setUp(self): @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'prerequisites') def test_process(self, directory): - """ Each content directory is expected to have a README.md and an expected_result.out + """ Each examples directory is expected to have a README.md and an expected_result.out this allows us to test each of them easily """ - self.demo.process_file('./content/' + directory + '/README.md') + self.demo.process_file('./examples/' + directory + '/README.md') # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. res = sys.stdout.getvalue() # pylint: disable=E1101 - exp_res = open('./content/' + directory + '/expected_result.demo', 'r').read() + exp_res = open('./examples/' + directory + '/expected_result.demo', 'r').read() self.assertEqual(exp_res, res) diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index 3550cc8..d7664ca 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -22,7 +22,7 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") + config.read("examples/config/unit_test.ini") self.simdem = tutorial.TutorialMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) @@ -36,15 +36,15 @@ def setUp(self): @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'prerequisites') def test_process(self, directory): - """ Each content directory is expected to have a README.md and an expected_result.tutorial + """ Each examples directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily """ self.maxDiff = None # pylint: disable=C0103 - self.simdem.process_file('./content/' + directory + '/README.md') + self.simdem.process_file('./examples/' + directory + '/README.md') # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. res = sys.stdout.getvalue() # pylint: disable=E1101 - exp_res = open('./content/' + directory + '/expected_result.tutorial', 'r').read() + exp_res = open('./examples/' + directory + '/expected_result.tutorial', 'r').read() self.assertEqual(exp_res, res) diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index eef604a..dd939fd 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -17,7 +17,7 @@ class SimDem1ParserTestSuite(unittest.TestCase): def setUp(self): config = configparser.ConfigParser() - config.read("content/config/unit_test.ini") + config.read("examples/config/unit_test.ini") self.parser = simdem1.SimDem1Parser() @@ -26,14 +26,14 @@ def setUp(self): @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'prerequisites') def test_process(self, directory): - """ Each content directory is expected to have a README.md and an expected_result.tutorial + """ Each examples directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily """ self.maxDiff = None # pylint: disable=C0103 - res = self.parser.parse_file('./content/' + directory + '/README.md') + res = self.parser.parse_file('./examples/' + directory + '/README.md') # Research how to read dict from file - exp_res = json.load(open('./content/' + directory + '/expected_result.seo', 'r')) + exp_res = json.load(open('./examples/' + directory + '/expected_result.seo', 'r')) self.assertEqual(exp_res, res) From 192e75f1c8a1cd173d0186531d01a2f82b11bc66 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sat, 27 Jan 2018 12:14:17 -0600 Subject: [PATCH 108/167] remove scratch dir --- scratch/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 scratch/.gitignore diff --git a/scratch/.gitignore b/scratch/.gitignore deleted file mode 100644 index 72e8ffc..0000000 --- a/scratch/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* From ecd531e042ba5386279409aa3576c00d14a6d820 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sat, 27 Jan 2018 12:15:37 -0600 Subject: [PATCH 109/167] Update syntax.md --- docs/syntax.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/syntax.md b/docs/syntax.md index 91e0012..84b9fee 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -10,11 +10,11 @@ This is the syntax for the default codeblock format. It's design is to allow mo ## SimDem V1 Based (default) -[Example Context Based Document](../content/simdem1/README.md) +[Example Context Based Document](../examples/simdem1/README.md) Feature | Implementation --- | --- Command | \```shell [Prerequisite](feature_prerequisite.md) | `# Prerequisite` followed by natural language text containing links to local or remote SimDem documents taht that should be executed prior to the main body of the current document. See [Prerequisites](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/prerequisites) for more details. [Validation](feature_validation.md) | `# Validation` followed by descriptive natural language and code-blocks to be run as tests prior to running the main content in this document. If all tests pass then there is no need to run the main document. See the [validation](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/prerequisites#validation) section of the prerequisites documentation for more details. -Result | `# Result` followed by a code block with the expected result of the code block immediately before the Results header. See the [testing](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/test) documentation for more details. \ No newline at end of file +Result | `# Result` followed by a code block with the expected result of the code block immediately before the Results header. See the [testing](https://github.com/Azure/simdem/tree/master/demo_scripts/simdem/test) documentation for more details. From 9b6804c37870c725b2ab77fda289b600677f3d86 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sat, 27 Jan 2018 12:17:28 -0600 Subject: [PATCH 110/167] Update README.md --- examples/simdem1/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/simdem1/README.md b/examples/simdem1/README.md index 7777bf2..f2dbd8b 100644 --- a/examples/simdem1/README.md +++ b/examples/simdem1/README.md @@ -1,10 +1,12 @@ # Prerequisites -This is the prerequisite section. SimDem looks for a set of links to extract and run through first +This is the prerequisite section. SimDem extracts links to run through prior to executing the main steps. + +Here is an example of a prerequisite that will be ignored because it's conditions are already met. * [prereq-ignored](./prereq-ignored.md) -They don't even need to be in the same list +Here is an example of a prerequisites that will be run because it's conditions are not met. * [prereq-processed](./prereq-processed.md) From 3a2e6dee21110ca2f249402c1b085bf6a2e69a42 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sat, 27 Jan 2018 12:20:40 -0600 Subject: [PATCH 111/167] add step files to simdem example --- examples/simdem1/step-1.md | 4 ++++ examples/simdem1/step-2.md | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 examples/simdem1/step-1.md create mode 100644 examples/simdem1/step-2.md diff --git a/examples/simdem1/step-1.md b/examples/simdem1/step-1.md new file mode 100644 index 0000000..b07c629 --- /dev/null +++ b/examples/simdem1/step-1.md @@ -0,0 +1,4 @@ + +```shell +echo step-1 +``` \ No newline at end of file diff --git a/examples/simdem1/step-2.md b/examples/simdem1/step-2.md new file mode 100644 index 0000000..9e5fc4d --- /dev/null +++ b/examples/simdem1/step-2.md @@ -0,0 +1,4 @@ + +```shell +echo step-2 +``` \ No newline at end of file From f0969a052c5f588bdc2e789c8d61f2f0a917a46c Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 28 Jan 2018 18:01:49 -0600 Subject: [PATCH 112/167] Add ability to use default ini file Signed-off-by: Tommy Falgout --- setup.py | 10 +++++----- simdem/cli.py | 27 +++++++++++++++++---------- simdem/simdem.ini | 16 ++++++++++++++++ 3 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 simdem/simdem.ini diff --git a/setup.py b/setup.py index de5c16f..5544cb4 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +""" Setup.py """ # Created from https://github.com/kennethreitz/setup.py @@ -6,20 +7,20 @@ with open('README.md') as f: - readme = f.read() + README = f.read() with open('LICENSE') as f: - license = f.read() + LICENSE = f.read() setup( name='simdem', version='0.1.0', description='SimDem', - long_description=readme, + long_description=README, author='Tommy Falgout/Ross Gardler', author_email='thfalgou@microsoft.com', url='https://github.com/Azure/simdem', - license=license, + license=LICENSE, packages=find_packages(exclude=('tests', 'docs')), entry_points={ 'console_scripts': [ @@ -27,4 +28,3 @@ ] } ) - diff --git a/simdem/cli.py b/simdem/cli.py index 56c7bd7..06ec3d1 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -4,6 +4,7 @@ import logging import argparse import os +import pkg_resources from simdem.executor import bash from simdem.parser import ast, simdem1 @@ -17,7 +18,7 @@ def main(): help='file to process') argp.add_argument('--debug', '-d', action="store_true", help="Turn on logging to console") - argp.add_argument('--config-file', '-c', default="content/config/demo.ini", + argp.add_argument('--config-file', '-c', help="Config file to use") argp.add_argument('--mode', '-m', default="tutorial", help="Mode to use", choices=['demo', 'dump', 'test', 'tutorial']) @@ -30,13 +31,11 @@ def main(): options = argp.parse_args() file_path = options.file - validate_error = validate(options, file_path) - if validate_error: - print(validate_error) - exit(1) + config_file_path = get_config_file_path(options) + validate(file_path, config_file_path) config = configparser.ConfigParser() - config.read(options.config_file) + config.read(config_file_path) inject_arguments(options, config) setup_logging(config, options) @@ -44,6 +43,14 @@ def main(): mode = get_mode(options, config) mode.process_file(file_path) +def get_config_file_path(options): + """ Returns the found config file path """ + options_config_file = options.config_file + if options_config_file: + return options_config_file + file_path = pkg_resources.resource_filename(__name__, 'simdem.ini') + return file_path + def inject_arguments(options, config): """ Injects CLI arguments into config settings """ if options.setting: @@ -51,13 +58,13 @@ def inject_arguments(options, config): [section, option] = key.split('.') config.set(section, option, value) -def validate(options, file_path): +def validate(file_path, config_file_path): """ validate all passed in arguments """ if not os.path.isfile(file_path): - return "Unable to find file: " + file_path + raise FileNotFoundError('Unable to find file: ' + file_path) - if not os.path.isfile(options.config_file): - return "Unable to find config file: " + options.config_file + if not os.path.isfile(config_file_path): + raise FileNotFoundError('Unable to find config file: ' + config_file_path) def get_mode(options, config): """ Returns correct renderer object """ diff --git a/simdem/simdem.ini b/simdem/simdem.ini new file mode 100644 index 0000000..c38722b --- /dev/null +++ b/simdem/simdem.ini @@ -0,0 +1,16 @@ +[meta] +simdem_version = 2.0.0 + +# Logging +[log] +file = simdem.log +level = DEBUG +format = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s + +[render] +# When in demo mode we insert a small random delay between characters. +# TYPING DELAY is the upper bound of this delay. +typing_delay = 0.2 + +# Prompt to use in the console +console_prompt = $ \ No newline at end of file From a90ad4a0d5452107d0825272881f2b02696daf1c Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 28 Jan 2018 20:06:36 -0600 Subject: [PATCH 113/167] More refactoring to not duplicate code --- simdem/executor/bash.py | 3 +++ simdem/mode/common.py | 1 + simdem/mode/demo.py | 13 +------------ simdem/mode/interactive.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py index b7af93f..fb1d3c0 100644 --- a/simdem/executor/bash.py +++ b/simdem/executor/bash.py @@ -26,6 +26,9 @@ def run_cmd(self, command): command = command.strip() logging.debug("Execute command: '" + command + "'") + if not command: + logging.debug('Empty command. Not executing') + return '' response = self.get_shell().run_command(command) # https://pexpect.readthedocs.io/en/stable/overview.html#find-the-end-of-line-cr-lf-conventions # Because pexpect respects TTY (which uses CRLF) instead of UNIX, we must swap out. diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 0b095bb..26c8dbb 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -39,6 +39,7 @@ def process_file(self, file_path, is_prereq=False): print('***PREREQUISITE VALIDATION FAILED***') # End prereq processing + self.executor.run_cmd('cd ' + start_path) self.process(steps) # pylint: disable=no-member if 'next_steps' in steps: diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index aec865f..4512f80 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -18,18 +18,7 @@ def process(self, steps): if step['type'] == 'commands': self.process_commands(step['content']) - def process_commands(self, cmds): - """ Pretend to type the command, run it and then display the output """ - for cmd in cmds: - # Request enter from user to know when to proceed - self.get_single_key_input() - self.type_command(cmd) - results = self.executor.run_cmd(cmd) - print(results, end="", flush=True) - print() - return results - - def type_command(self, cmd): + def display_command(self, cmd): """ Displays the command on the screen """ # Must add ' ' when typing command because whitespaces are removed from configparser diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index 66141db..beb380a 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -7,6 +7,21 @@ class InteractiveMode(ModeCommon): """ Interactive Mode subclass """ + def process_commands(self, cmds): + """ Pretend to type the command, run it and then display the output """ + for cmd in cmds: + # Request enter from user to know when to proceed + self.get_single_key_input() + self.display_command(cmd) + results = self.executor.run_cmd(cmd) + print(results, end="", flush=True) + print() + return results + + def display_command(self, cmd): + """ Default result for displaying a command """ + print(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd, flush=True) + @staticmethod def get_single_key_input(): """ SimDem1 uses this method: From 91dd10a00749f153e12dd5dcaa08bfff24067beb Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 28 Jan 2018 22:50:07 -0600 Subject: [PATCH 114/167] Fix text not being displayed and clearing screen on steps Signed-off-by: Tommy Falgout --- docs/README.md | 7 ++--- docs/hello_world.md | 31 +++++++++++++++++++++ docs/mode_demo.md | 9 +++++++ docs/mode_dump.md | 5 ++++ docs/mode_test.md | 8 ++++++ docs/mode_tutorial.md | 9 +++++++ docs/modes.md | 58 ++++++++++------------------------------ simdem/mode/test.py | 8 +++--- simdem/mode/tutorial.py | 10 ++++++- simdem/parser/simdem1.py | 2 ++ 10 files changed, 95 insertions(+), 52 deletions(-) create mode 100644 docs/hello_world.md create mode 100644 docs/mode_demo.md create mode 100644 docs/mode_dump.md create mode 100644 docs/mode_test.md create mode 100644 docs/mode_tutorial.md diff --git a/docs/README.md b/docs/README.md index 42a830f..1341fc9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,10 +21,11 @@ It's easier to understand through example. If you are viewing this inside the in Tutorials can branch too, for example you can choose any of the following paths next: 1. [Modes of operation](modes.md) -1. [Build Hello World Demo](build_hello_world.md) +1. [Build Hello World Demo](hello_world.md) 1. [SimDem document syntax](syntax.md) -1. [Use your documents as interactive tutorials or demos](demo_mode.md) -1. [Use your documents as automated tests](test_mode.md) +1. [Use your documents as an interactive tutorial](mode_tutorial.md) +1. [Use your documents as an interactive demo](mode_demo.md) +1. [Use your documents as an automated test](mode_test.md) diff --git a/docs/hello_world.md b/docs/hello_world.md new file mode 100644 index 0000000..18e2f45 --- /dev/null +++ b/docs/hello_world.md @@ -0,0 +1,31 @@ +# Hello World + +This is the start of a hello-world example. + +## Running commands + +To run a set of commands, insert them inside a code block. Here's an example: + +```shell +echo "hello world" +``` + +To verify output, add a "Results:" paragraph immediately after your command: + +```shell +echo "hello again" +``` + +Results: + +``` +hello again +``` + +# Next Steps + +Tutorials can branch too, for example you can choose any of the following paths next: + +1. [SimDem document syntax](syntax.md) +1. [Use your documents as interactive tutorials or demos](demo_mode.md) +1. [Use your documents as automated tests](test_mode.md) diff --git a/docs/mode_demo.md b/docs/mode_demo.md new file mode 100644 index 0000000..efbf5ae --- /dev/null +++ b/docs/mode_demo.md @@ -0,0 +1,9 @@ +# Demo (or Simulation) mode + +Demo mode is ideal if you are using this to teach or demonstrate how +to achive the goal. In this mode no descriptive text is shown, instead +when you press a key the next command is "typed", pressing another key +will execute the command. The idea is that you describe what is +happening as the application "types" the command for you. + +To run a document in Demo Mode: `simdem -m demo examples/simple/README.md` diff --git a/docs/mode_dump.md b/docs/mode_dump.md new file mode 100644 index 0000000..bdcda84 --- /dev/null +++ b/docs/mode_dump.md @@ -0,0 +1,5 @@ +# Dump Mode + +Dump mode is used for debugging to see how SimDem has parsed the document + +To run a document in Dump Mode: `simdem -m dump examples/simple/README.md` diff --git a/docs/mode_test.md b/docs/mode_test.md new file mode 100644 index 0000000..23dbd08 --- /dev/null +++ b/docs/mode_test.md @@ -0,0 +1,8 @@ +# Test Mode + +Test mode runs the commands and then verifies that the output is +sufficiently similar to the expected results (recorded in the markdown +file) to be considered correct. + +To run a document in Test Mode: `simdem -m test examples/simple/README.md` + diff --git a/docs/mode_tutorial.md b/docs/mode_tutorial.md new file mode 100644 index 0000000..d14f74d --- /dev/null +++ b/docs/mode_tutorial.md @@ -0,0 +1,9 @@ +# Tutorial Mode + +Tutorial mode is ideal if you are using this as a learning or teaching +tool (see also learn mode below, which suits some learning styles +better. In this mode a description of what you are about to do is +shown on the screen, hit a key to see the command, hit another key to +execute the command. Tutorial mode is the default. + +To run a document in Tutorial Mode: `simdem examples/simple/README.md` diff --git a/docs/modes.md b/docs/modes.md index 67554c8..dfbfcd9 100644 --- a/docs/modes.md +++ b/docs/modes.md @@ -2,57 +2,27 @@ SimDem and can be run one of the following modes: - * Tutorial: Displays the descriptive text of the tutorial and pauses - at code blocks to allow user interaction. - * Demo: Does not display the descriptive text, but pauses at each - code block. When the user hits a key the command is "typed", a - second keypress executes the command. - * Test: Runs the commands and then verifies that the output is - sufficiently similar to the expected results (recorded in the - markdown file) to be considered correct. - * Dump: Prints out the internal SimDem object +* Tutorial: Displays the descriptive text of the tutorial and pauses + at code blocks to allow user interaction. +* Demo: Does not display the descriptive text, but pauses at each + code block. When the user hits a key the command is "typed", a + second keypress executes the command. +* Test: Runs the commands and then verifies that the output is + sufficiently similar to the expected results (recorded in the + markdown file) to be considered correct. +* Dump: Prints out the internal SimDem object -## Tutorial Mode -Tutorial mode is ideal if you are using this as a learning or teaching -tool (see also learn mode below, which suits some learning styles -better. In this mode a description of what you are about to do is -shown on the screen, hit a key to see the command, hit another key to -execute the command. Tutorial mode is the default. -To run a document in Tutorial Mode: `simdem content/simple/README.md` -## Demo (or Simulation) mode - -Demo mode is ideal if you are using this to teach or demonstrate how -to achive the goal. In this mode no descriptive text is shown, instead -when you press a key the next command is "typed", pressing another key -will execute the command. The idea is that you describe what is -happening as the application "types" the command for you. - -To run a document in Demo Mode: `simdem -m demo content/simple/README.md` - - -## Test Mode - -Test mode runs the commands and then verifies that the output is -sufficiently similar to the expected results (recorded in the markdown -file) to be considered correct. - -To run a document in Test Mode: `simdem -m test content/simple/README.md` - -## Dump Mode - -Dump mode is used for debugging to see how SimDem has parsed the document - -To run a document in Dump Mode: `simdem -m dump content/simple/README.md` - # Next Steps 1. [Beginning](README.md) - 1. [Build Hello World Demo](build_hello_world.md) + 1. [Build Hello World Demo](hello_world.md) 1. [SimDem document syntax](syntax.md) - 1. [Use your documents as interactive tutorials or demos](demo_mode.md) - 1. [Use your documents as automated tests](test_mode.md) \ No newline at end of file + 1. [Use your documents as an interactive tutorial](mode_tutorial.md) + 1. [Use your documents as an interactive demo](mode_demo.md) + 1. [Use your documents as an automated test](mode_test.md) + 1. [Dump your parsed documentation](mode_dump.md) \ No newline at end of file diff --git a/simdem/mode/test.py b/simdem/mode/test.py index f2d6c32..e564c1f 100644 --- a/simdem/mode/test.py +++ b/simdem/mode/test.py @@ -17,11 +17,11 @@ def process(self, steps): for step in steps['body']: if step['type'] == 'commands': last_command_result = self.process_commands(step['content']) - if 'expected_results' in step: - if self.is_result_valid(step['content'], last_command_result): - print('***VALIDATION FAILED***') - else: + if 'expected_result' in step: + if self.is_result_valid(step['expected_result'], last_command_result): print('***VALIDATION PASSED***') + else: + print('***VALIDATION FAILED***') def process_next_steps(self, steps, start_path): """ No need to display next steps if in test mode """ diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 3e6110e..a89b95d 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -13,13 +13,21 @@ def process(self, steps): """ Processes the steps from a processed file """ logging.debug("process()") + # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python + print("\033[H\033[J") for step in steps['body']: if step['type'] == 'heading': self.process_heading(step) elif step['type'] == 'text': self.process_text(step) elif step['type'] == 'commands': - self.process_commands(step['content']) + last_command_result = self.process_commands(step['content']) + logging.debug(step) + if 'expected_result' in step: + if self.is_result_valid(step['expected_result'], last_command_result): + print('*** SIMDEM RESULT PASSED ***') + else: + print('*** SIMDEM RESULT FAILED ***') @staticmethod diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 21a74bb..1c85245 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -65,6 +65,8 @@ def render_list(self, token): def render_list_item(self, token): """ Render a markdown list item """ inner = self.render_inner(token) + if self.section is None: + self.output['body'].append({'type': 'text', 'content': ' * ' + inner}) return inner def render_link(self, token): From 8f608d75fb641a08d3365f30a07d7d6449e5bc4c Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 28 Jan 2018 23:49:26 -0600 Subject: [PATCH 115/167] Update for SimDem demo for OScon Signed-off-by: Tommy Falgout --- docs/README.md | 12 ++++++------ docs/fin.md | 5 +++++ docs/hello_world.md | 7 +++++-- docs/modes.md | 5 ----- simdem/parser/simdem1.py | 1 + 5 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 docs/fin.md diff --git a/docs/README.md b/docs/README.md index 1341fc9..be3a577 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,12 +2,12 @@ SimDem is: - * Documentation - * An interactive tutorial - * A live demo - * An automated test script - * A Shell script - +* Documentation +* An interactive tutorial +* A live demo +* An automated test script +* A Shell script + ## SimDem overview Simdem allows you to wite a tutorial in markdown format and then run the commands as a simulated demo, interactive tutorial, a test script or generate executable shell scripts. diff --git a/docs/fin.md b/docs/fin.md new file mode 100644 index 0000000..6482e96 --- /dev/null +++ b/docs/fin.md @@ -0,0 +1,5 @@ +# Fin + +```shell +cowsay "Thanks for trying SimDem! https://github.com/Azure/simdem/tree/simdem2" +``` diff --git a/docs/hello_world.md b/docs/hello_world.md index 18e2f45..d6c16af 100644 --- a/docs/hello_world.md +++ b/docs/hello_world.md @@ -6,6 +6,8 @@ This is the start of a hello-world example. To run a set of commands, insert them inside a code block. Here's an example: +NOTE: If running in tutorial or demo mode, you will need to press a key to tell SimDem to execute the command + ```shell echo "hello world" ``` @@ -27,5 +29,6 @@ hello again Tutorials can branch too, for example you can choose any of the following paths next: 1. [SimDem document syntax](syntax.md) -1. [Use your documents as interactive tutorials or demos](demo_mode.md) -1. [Use your documents as automated tests](test_mode.md) +1. [Use your documents as interactive tutorials or demos](mode_demo.md) +1. [Use your documents as automated tests](mode_test.md) +1. [FIN](fin.md) diff --git a/docs/modes.md b/docs/modes.md index dfbfcd9..c4da7d8 100644 --- a/docs/modes.md +++ b/docs/modes.md @@ -12,11 +12,6 @@ SimDem and can be run one of the following modes: markdown file) to be considered correct. * Dump: Prints out the internal SimDem object - - - - - # Next Steps 1. [Beginning](README.md) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 1c85245..e43f6b2 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -60,6 +60,7 @@ def render_heading(self, token): def render_list(self, token): """ Render a markdown list """ inner = self.render_inner(token) + self.output['body'].append({'type': 'text', 'content': ' '}) return inner def render_list_item(self, token): From 99e03a82c9fe1995ee8a01017631c00e336827ab Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 29 Jan 2018 18:55:53 -0600 Subject: [PATCH 116/167] Add Dockerfile and fix issues for CircleCI integration Signed-off-by: Tommy Falgout --- Dockerfile | 18 ++++++++++++++++++ simdem/mode/common.py | 6 ++++-- simdem/mode/test.py | 5 +++-- 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aaf1913 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3 + +WORKDIR /usr/src/app +RUN apt-get update + +COPY requirements.txt ./ + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN pip install -v -e . + +# The default prompt is # which throws off the initialization +RUN echo 'export PS1="$ "' >> /root/.bashrc + +CMD [ "simdem" ] + diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 26c8dbb..85f6ce8 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -20,7 +20,8 @@ def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) # Change the working directory in case of any recursion - start_path = os.path.dirname(file_path) + start_path = os.path.dirname(os.path.abspath(file_path)) + logging.debug('parse_file::start_path=' + start_path) steps = self.parser.parse_file(file_path) # Begin preqreq processing @@ -39,7 +40,8 @@ def process_file(self, file_path, is_prereq=False): print('***PREREQUISITE VALIDATION FAILED***') # End prereq processing - self.executor.run_cmd('cd ' + start_path) + if start_path: + self.executor.run_cmd('cd ' + start_path) self.process(steps) # pylint: disable=no-member if 'next_steps' in steps: diff --git a/simdem/mode/test.py b/simdem/mode/test.py index e564c1f..474f685 100644 --- a/simdem/mode/test.py +++ b/simdem/mode/test.py @@ -19,9 +19,10 @@ def process(self, steps): last_command_result = self.process_commands(step['content']) if 'expected_result' in step: if self.is_result_valid(step['expected_result'], last_command_result): - print('***VALIDATION PASSED***') + print('*** SIMDEM TEST RESULT PASSED ***') else: - print('***VALIDATION FAILED***') + print('*** SIMDEM TEST RESULT FAILED ***') + exit(1) def process_next_steps(self, steps, start_path): """ No need to display next steps if in test mode """ From 6483fa24c3c291d71a3999ce5fff00474287f44f Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 2 Feb 2018 04:28:51 -0600 Subject: [PATCH 117/167] Add better next-steps logic --- simdem/mode/interactive.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index beb380a..c7f9652 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -11,13 +11,19 @@ def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: # Request enter from user to know when to proceed - self.get_single_key_input() + key = self.get_single_key_input() + self.process_command_input(key) self.display_command(cmd) results = self.executor.run_cmd(cmd) print(results, end="", flush=True) print() return results + def process_command_input(self, key): + """ Process the command input. It's 4AM and I'm sleepy + For now, just return. We'll implement that later """ + return key + def display_command(self, cmd): """ Default result for displaying a command """ print(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd, flush=True) @@ -39,24 +45,36 @@ def get_single_key_input(): return getch.impl() return - def process_next_steps(self, steps, start_path): + def process_next_steps(self, next_steps, start_path): """ Is there a good way to test this that doesn't involve lots of test code + expect? Not fully tested yet. Low priority feature. """ idx = 1 - if steps: + if next_steps: print("Next steps available:") - for step in steps: + for step in next_steps: print(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") idx += 1 print() # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not if sys.stdout.isatty(): # You're running in a real terminal - step_request = input("Choose a step. " + - "Type the # or 'q' to quit and then press Enter: ") + in_string = "" + in_value = 0 + + while in_value < 1 or in_value > len(next_steps): + in_string = input("Choose a step. " + + "Enter a value between 1 and " + + str(len(next_steps)) + " or 'quit' ") + if in_string.lower() == "quit" or in_string.lower() == "q": + return + try: + in_value = int(in_string) + except ValueError: + pass + #print('You chose:' + str(steps[int(step_request) - 1]['title'])) - self.process_file(start_path + '/' + steps[int(step_request) - 1]['target']) + self.process_file(start_path + '/' + next_steps[int(in_string) - 1]['target']) else: logging.info('Not connected to a TTY terminal Not requesting input.') # You're being piped or redirected From f5483510cecbb89306fe81568d4e45a53b873cfa Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 2 Feb 2018 05:52:02 -0600 Subject: [PATCH 118/167] Got the demo + break to work as I want it to. Signed-off-by: Tommy Falgout --- docs/demo.md | 2 +- simdem/mode/demo.py | 1 - simdem/mode/interactive.py | 65 ++++++++++++++++++++++++++++++-------- simdem/simdem.ini | 2 +- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/docs/demo.md b/docs/demo.md index c46c51a..249624e 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -15,7 +15,7 @@ echo "Hello World" That's cool, lets try again: ``` -echo "It might look like this was typed into the terminal (even more so if you ran SimDem with the '--style simulate' flag, that simulates a person typing), bit it really comes from a markdown file." +echo "It might look like this was typed into the terminal, but it really comes from a markdown file." ``` The date command will show that these commands are being executed in real time. diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index 4512f80..fb1babc 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -23,7 +23,6 @@ def display_command(self, cmd): # Must add ' ' when typing command because whitespaces are removed from configparser # https://docs.python.org/3/library/configparser.html#supported-ini-file-structure - print(self.config.get('render', 'console_prompt', raw=True) + ' ', end="", flush=True) for _, char in enumerate(cmd): if char != "\n": typing_delay = float(self.config.get('render', 'typing_delay')) diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index c7f9652..36e8d3f 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -8,25 +8,64 @@ class InteractiveMode(ModeCommon): """ Interactive Mode subclass """ def process_commands(self, cmds): - """ Pretend to type the command, run it and then display the output """ - for cmd in cmds: - # Request enter from user to know when to proceed + """ Loop through the commands to run as well as expect interrupt logic from the user """ + result = None + cmd = None + from collections import deque + cmd_deque = deque(cmds) + # https://twitter.com/sandwich_cool/status/956932558847176704 + # The "I smell danger" picture is never truer than now + # This while True statement is written in VSCode while I work for MSFT + while True: + self.display_prompt() key = self.get_single_key_input() - self.process_command_input(key) - self.display_command(cmd) - results = self.executor.run_cmd(cmd) - print(results, end="", flush=True) - print() + result = self.process_command_input(key, last_command=cmd) + logging.debug(cmd_deque) + if result: + continue + elif cmd_deque: + cmd = cmd_deque.popleft() + result = self.run_command(cmd) + if not cmd_deque: + break + return result + + def run_command(self, cmd): + """ Pretend to type the command, run it and then display the output """ + # Request enter from user to know when to proceed + logging.debug('run_command(' + cmd + ')') + self.display_command(cmd) + results = self.executor.run_cmd(cmd) + print(results, end="", flush=True) return results - def process_command_input(self, key): + def process_command_input(self, key, last_command=None): """ Process the command input. It's 4AM and I'm sleepy For now, just return. We'll implement that later """ - return key + result = None + if key == 'b': + logging.debug('Received Break request') + print("\nshell> ", end='') + command = input() + if command != "": + result = self.executor.run_cmd(command) + print(result, end="", flush=True) + elif key == 'r': + logging.debug('Received Run last command request') + if last_command: + result = self.run_command(last_command) + logging.debug('Output=' + str(result)) + return result + # Otherwise, we will return and assume the user wants to continue - def display_command(self, cmd): + def display_prompt(self): + """ Default result for displaying a command """ + print(self.config.get('render', 'console_prompt', raw=True) + ' ', end="", flush=True) + + @staticmethod + def display_command(cmd): """ Default result for displaying a command """ - print(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd, flush=True) + print(cmd, flush=True) @staticmethod def get_single_key_input(): @@ -65,7 +104,7 @@ def process_next_steps(self, next_steps, start_path): while in_value < 1 or in_value > len(next_steps): in_string = input("Choose a step. " + "Enter a value between 1 and " + - str(len(next_steps)) + " or 'quit' ") + str(len(next_steps)) + " or 'q' to quit: ") if in_string.lower() == "quit" or in_string.lower() == "q": return try: diff --git a/simdem/simdem.ini b/simdem/simdem.ini index c38722b..57123e3 100644 --- a/simdem/simdem.ini +++ b/simdem/simdem.ini @@ -10,7 +10,7 @@ format = %(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s [render] # When in demo mode we insert a small random delay between characters. # TYPING DELAY is the upper bound of this delay. -typing_delay = 0.2 +typing_delay = 0.1 # Prompt to use in the console console_prompt = $ \ No newline at end of file From 3d09504a9cec8952a95e7f3dd88b84af103873ce Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 2 Feb 2018 05:52:28 -0600 Subject: [PATCH 119/167] traiing whitespace is the enemy Signed-off-by: Tommy Falgout --- simdem/mode/interactive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index 36e8d3f..cb5198e 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -13,7 +13,7 @@ def process_commands(self, cmds): cmd = None from collections import deque cmd_deque = deque(cmds) - # https://twitter.com/sandwich_cool/status/956932558847176704 + # https://twitter.com/sandwich_cool/status/956932558847176704 # The "I smell danger" picture is never truer than now # This while True statement is written in VSCode while I work for MSFT while True: From 20116920ff31e3f135676e238774f33987bd752e Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 2 Feb 2018 06:19:44 -0600 Subject: [PATCH 120/167] Fix broken testcases Signed-off-by: Tommy Falgout --- docs/development.md | 14 +++++++++++++- examples/prerequisites/expected_result.demo | 5 ----- .../prerequisites/expected_result.tutorial | 5 ----- .../results-block-fail/expected_result.demo | 2 -- .../expected_result.tutorial | 3 +-- examples/results-block/expected_result.demo | 1 - .../results-block/expected_result.tutorial | 2 +- examples/simple-variable/expected_result.demo | 2 -- .../simple-variable/expected_result.tutorial | 2 -- examples/simple/expected_result.demo | 1 - examples/simple/expected_result.seo | 18 ++++++++++++++++-- examples/simple/expected_result.tutorial | 1 - simdem/mode/tutorial.py | 4 +++- simdem/parser/simdem1.py | 3 ++- 14 files changed, 36 insertions(+), 27 deletions(-) diff --git a/docs/development.md b/docs/development.md index 961b3ae..07dd33e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -54,4 +54,16 @@ Implementations: This class type executes the desired commands into the shell Implementations: -* [Bash](../simdem/executor/bash.py) \ No newline at end of file +* [Bash](../simdem/executor/bash.py) + +## Brittle testcases + +Because I'm a bad developer and the test cases for SimDem are brittle, here's how to regenerate all of the .demo/.tutorial files. + +This might happen because adding a newline to the logic breaks the testcases. +``` +for dir in simple simple-variable results-block results-block-fail prerequisites; do simdem -m tutorial examples/$dir/README.md > examples/$dir/expected_result.tutorial; done; +for dir in simple simple-variable results-block results-block-fail prerequisites; do simdem -m dump examples/$dir/README.md > examples/$dir/expected_result.seo; done; +for dir in simple simple-variable results-block results-block-fail prerequisites; do simdem -m demo examples/$dir/README.md > examples/$dir/expected_result.demo; done; + +``` \ No newline at end of file diff --git a/examples/prerequisites/expected_result.demo b/examples/prerequisites/expected_result.demo index 540ef52..b279e81 100644 --- a/examples/prerequisites/expected_result.demo +++ b/examples/prerequisites/expected_result.demo @@ -1,17 +1,12 @@ $ echo prereq_validation_pass prereq_validation_pass - $ echo prereq_validation_fail prereq_validation_fail - ***PREREQUISITE VALIDATION FAILED*** $ echo YOU SHOULD SEE THIS YOU SHOULD SEE THIS - $ prereq_processed=true - $ echo prereq_ignored = $prereq_ignored prereq_ignored = $ echo prereq_processed = $prereq_processed prereq_processed = true - diff --git a/examples/prerequisites/expected_result.tutorial b/examples/prerequisites/expected_result.tutorial index 6a67b19..95ec75d 100644 --- a/examples/prerequisites/expected_result.tutorial +++ b/examples/prerequisites/expected_result.tutorial @@ -1,9 +1,7 @@ $ echo prereq_validation_pass prereq_validation_pass - $ echo prereq_validation_fail prereq_validation_fail - ***PREREQUISITE VALIDATION FAILED*** # WE ARE IN prereq-processed.md @@ -19,11 +17,9 @@ This should be run since the validation for the prereq has failed $ echo YOU SHOULD SEE THIS YOU SHOULD SEE THIS - # Set a variable that passes through $ prereq_processed=true - # Prerequisites This is the prerequisite section. SimDem looks for a set of links to extract and run through first @@ -38,4 +34,3 @@ $ echo prereq_ignored = $prereq_ignored prereq_ignored = $ echo prereq_processed = $prereq_processed prereq_processed = true - diff --git a/examples/results-block-fail/expected_result.demo b/examples/results-block-fail/expected_result.demo index 5566f95..44b24a7 100644 --- a/examples/results-block-fail/expected_result.demo +++ b/examples/results-block-fail/expected_result.demo @@ -2,7 +2,5 @@ $ echo foo foo $ echo bar bar - $ echo post_result_block_fail post_result_block_fail - diff --git a/examples/results-block-fail/expected_result.tutorial b/examples/results-block-fail/expected_result.tutorial index 6e279c2..a608a0a 100644 --- a/examples/results-block-fail/expected_result.tutorial +++ b/examples/results-block-fail/expected_result.tutorial @@ -4,11 +4,10 @@ $ echo foo foo $ echo bar bar - +*** SIMDEM RESULT FAILED *** In demo mode, we will continue processing. $ echo post_result_block_fail post_result_block_fail - even more text diff --git a/examples/results-block/expected_result.demo b/examples/results-block/expected_result.demo index 8a8f518..ac40788 100644 --- a/examples/results-block/expected_result.demo +++ b/examples/results-block/expected_result.demo @@ -2,4 +2,3 @@ $ echo foo foo $ echo bar bar - diff --git a/examples/results-block/expected_result.tutorial b/examples/results-block/expected_result.tutorial index 635d3c6..d24443d 100644 --- a/examples/results-block/expected_result.tutorial +++ b/examples/results-block/expected_result.tutorial @@ -4,6 +4,6 @@ $ echo foo foo $ echo bar bar - +*** SIMDEM RESULT PASSED *** even more text diff --git a/examples/simple-variable/expected_result.demo b/examples/simple-variable/expected_result.demo index 0ca9529..2b21e48 100644 --- a/examples/simple-variable/expected_result.demo +++ b/examples/simple-variable/expected_result.demo @@ -1,5 +1,3 @@ $ FOO=bar - $ echo $FOO bar - diff --git a/examples/simple-variable/expected_result.tutorial b/examples/simple-variable/expected_result.tutorial index 43278d6..f603dad 100644 --- a/examples/simple-variable/expected_result.tutorial +++ b/examples/simple-variable/expected_result.tutorial @@ -1,9 +1,7 @@ # this is text $ FOO=bar - $ echo $FOO bar - more text diff --git a/examples/simple/expected_result.demo b/examples/simple/expected_result.demo index 8a8f518..ac40788 100644 --- a/examples/simple/expected_result.demo +++ b/examples/simple/expected_result.demo @@ -2,4 +2,3 @@ $ echo foo foo $ echo bar bar - diff --git a/examples/simple/expected_result.seo b/examples/simple/expected_result.seo index efdd9ee..a7bd765 100644 --- a/examples/simple/expected_result.seo +++ b/examples/simple/expected_result.seo @@ -1,2 +1,16 @@ -{"body": [{"content": "Simple", "level": 1, "type": "heading"}, - {"content": ["echo foo", "echo bar"], "type": "commands"}]} +{ + "body": [ + { + "content": "Simple", + "level": 1, + "type": "heading" + }, + { + "content": [ + "echo foo", + "echo bar" + ], + "type": "commands" + } + ] +} diff --git a/examples/simple/expected_result.tutorial b/examples/simple/expected_result.tutorial index 9d3be1e..f47410d 100644 --- a/examples/simple/expected_result.tutorial +++ b/examples/simple/expected_result.tutorial @@ -4,4 +4,3 @@ $ echo foo foo $ echo bar bar - diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index a89b95d..271c500 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -13,8 +13,10 @@ def process(self, steps): """ Processes the steps from a processed file """ logging.debug("process()") + # This breaks testcases right now. Blarg # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python - print("\033[H\033[J") + #print("\033[H\033[J") + #os.system('clear') for step in steps['body']: if step['type'] == 'heading': self.process_heading(step) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index e43f6b2..591c9f0 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -60,7 +60,8 @@ def render_heading(self, token): def render_list(self, token): """ Render a markdown list """ inner = self.render_inner(token) - self.output['body'].append({'type': 'text', 'content': ' '}) + if self.section is None: + self.output['body'].append({'type': 'text', 'content': ' '}) return inner def render_list_item(self, token): From 6d5874d1cf2ebd9caeefcb0710d7d589c079c87b Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 2 Feb 2018 06:22:39 -0600 Subject: [PATCH 121/167] Fix clear for tutorial mode Signed-off-by: Tommy Falgout --- simdem/mode/tutorial.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 271c500..c901d5d 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -1,5 +1,7 @@ """ Tutorial mode for SimDem""" +import os +import sys import logging from simdem.mode.interactive import InteractiveMode @@ -13,10 +15,10 @@ def process(self, steps): """ Processes the steps from a processed file """ logging.debug("process()") - # This breaks testcases right now. Blarg # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python #print("\033[H\033[J") - #os.system('clear') + if sys.stdout.isatty(): + os.system('clear') for step in steps['body']: if step['type'] == 'heading': self.process_heading(step) From bb31e757e8a1bf7b36ddd8378bcfe5ef2d492734 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 2 Feb 2018 06:28:07 -0600 Subject: [PATCH 122/167] Fix collections Signed-off-by: Tommy Falgout --- simdem/mode/interactive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index cb5198e..919c76d 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -1,6 +1,7 @@ """ Interactive Mode Class """ import sys import logging +from collections import deque from simdem.mode.common import ModeCommon from simdem.misc.getch import Getch @@ -11,11 +12,10 @@ def process_commands(self, cmds): """ Loop through the commands to run as well as expect interrupt logic from the user """ result = None cmd = None - from collections import deque cmd_deque = deque(cmds) # https://twitter.com/sandwich_cool/status/956932558847176704 # The "I smell danger" picture is never truer than now - # This while True statement is written in VSCode while I work for MSFT + # This statement is written in VSCode while I work for MSFT while True: self.display_prompt() key = self.get_single_key_input() From 331708c117ad92ce8af831605ab07c81231a25db Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 4 Feb 2018 21:21:01 -0800 Subject: [PATCH 123/167] Create Basic UI class Signed-off-by: Tommy Falgout --- .../expected_result.tutorial | 2 +- .../results-block/expected_result.tutorial | 2 +- simdem/cli.py | 18 ++++-- simdem/mode/common.py | 13 +++-- simdem/mode/demo.py | 4 +- simdem/mode/dump.py | 8 +-- simdem/mode/interactive.py | 48 +++++----------- simdem/mode/test.py | 4 +- simdem/mode/tutorial.py | 21 +++---- simdem/ui/basic.py | 57 +++++++++++++++++++ tests/test_mode_demo.py | 5 +- tests/test_mode_tutorial.py | 5 +- 12 files changed, 115 insertions(+), 72 deletions(-) create mode 100644 simdem/ui/basic.py diff --git a/examples/results-block-fail/expected_result.tutorial b/examples/results-block-fail/expected_result.tutorial index a608a0a..94b21dc 100644 --- a/examples/results-block-fail/expected_result.tutorial +++ b/examples/results-block-fail/expected_result.tutorial @@ -4,7 +4,7 @@ $ echo foo foo $ echo bar bar -*** SIMDEM RESULT FAILED *** +*** SIMDEM TEST RESULT FAILED *** In demo mode, we will continue processing. $ echo post_result_block_fail diff --git a/examples/results-block/expected_result.tutorial b/examples/results-block/expected_result.tutorial index d24443d..2b468c7 100644 --- a/examples/results-block/expected_result.tutorial +++ b/examples/results-block/expected_result.tutorial @@ -4,6 +4,6 @@ $ echo foo foo $ echo bar bar -*** SIMDEM RESULT PASSED *** +*** SIMDEM TEST RESULT PASSED *** even more text diff --git a/simdem/cli.py b/simdem/cli.py index 06ec3d1..130b70d 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -9,7 +9,7 @@ from simdem.executor import bash from simdem.parser import ast, simdem1 from simdem.mode import demo, dump, test, tutorial - +from simdem.ui import basic def main(): """ Main execution function """ @@ -26,6 +26,8 @@ def main(): help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", help="Executor class to use", choices=['bash']) + argp.add_argument('--render', '-r', default="basic", + help="Render class to use", choices=['basic']) argp.add_argument('--setting', '-s', metavar='setting', help="Setting to override in config file") options = argp.parse_args() @@ -71,18 +73,24 @@ def get_mode(options, config): parser = get_parser(options) executor = get_executor(options) + ui = get_ui(options) if options.mode == 'demo': - return demo.DemoMode(config, parser, executor) + return demo.DemoMode(config, parser, executor, ui) if options.mode == 'dump': - return dump.DumpMode(config, parser, executor) + return dump.DumpMode(config, parser, executor, ui) if options.mode == 'test': - return test.TestMode(config, parser, executor) + return test.TestMode(config, parser, executor, ui) if options.mode == 'tutorial': - return tutorial.TutorialMode(config, parser, executor) + return tutorial.TutorialMode(config, parser, executor, ui) + +def get_ui(options): + """ return UI object """ + if options.ui == 'basic': + return basic.BasicUI() def get_parser(options): """ Returns correct parser object """ diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 85f6ce8..9dd4cc6 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -10,11 +10,13 @@ class ModeCommon(object): # pylint: disable=R0903 config = None executor = None parser = None + render = None - def __init__(self, config, parser, executor): + def __init__(self, config, parser, executor, ui): self.config = config self.parser = parser self.executor = executor + self.ui = ui def process_file(self, file_path, is_prereq=False): """ Parses the file and starts processing it """ @@ -34,10 +36,9 @@ def process_file(self, file_path, is_prereq=False): if 'expected_result' in steps['validation']: if self.is_result_valid(steps['validation']['expected_result'], last_command_result): - #print('***PREREQUISITE VALIDATION PASSED***') return else: - print('***PREREQUISITE VALIDATION FAILED***') + self.ui.print_validation_failed() # End prereq processing if start_path: @@ -50,10 +51,10 @@ def process_file(self, file_path, is_prereq=False): def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: - print(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd) + self.ui.println(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd) results = self.executor.run_cmd(cmd) - print(results, end="", flush=True) - print() + self.ui.print_flush(results) + self.ui.print() return results @staticmethod diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index fb1babc..94cdfaf 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -29,5 +29,5 @@ def display_command(self, cmd): if typing_delay: delay = random.uniform(0.02, typing_delay) time.sleep(delay) - print(char, end="", flush=True) - print("", flush=True) + self.ui.print(char) + self.ui.println('') diff --git a/simdem/mode/dump.py b/simdem/mode/dump.py index 89474be..ef87640 100644 --- a/simdem/mode/dump.py +++ b/simdem/mode/dump.py @@ -12,15 +12,15 @@ def process_file(self, file_path, is_prereq=False): """ Parse the file and print it. Not very exciting. """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) steps = self.parser.parse_file(file_path) - print(json.dumps(steps, indent=4, sort_keys=True)) + self.ui.println(json.dumps(steps, indent=4, sort_keys=True)) def print_config_data(self): """ Dead code for now, but useful for debugging config """ - print("Config=") + self.ui.println("Config=") for section in self.config.sections(): for option in self.config.options(section): try: - print(section + '.' + option + '=' + str(self.config.get(section, option))) + self.ui.println(section + '.' + option + '=' + str(self.config.get(section, option))) except configparser.InterpolationMissingOptionError: pass - print() + self.ui.println() diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index 919c76d..990de6d 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -3,7 +3,6 @@ import logging from collections import deque from simdem.mode.common import ModeCommon -from simdem.misc.getch import Getch class InteractiveMode(ModeCommon): """ Interactive Mode subclass """ @@ -18,7 +17,7 @@ def process_commands(self, cmds): # This statement is written in VSCode while I work for MSFT while True: self.display_prompt() - key = self.get_single_key_input() + key = self.ui.get_single_key_input() result = self.process_command_input(key, last_command=cmd) logging.debug(cmd_deque) if result: @@ -36,7 +35,7 @@ def run_command(self, cmd): logging.debug('run_command(' + cmd + ')') self.display_command(cmd) results = self.executor.run_cmd(cmd) - print(results, end="", flush=True) + self.ui.print(results) return results def process_command_input(self, key, last_command=None): @@ -45,11 +44,11 @@ def process_command_input(self, key, last_command=None): result = None if key == 'b': logging.debug('Received Break request') - print("\nshell> ", end='') - command = input() + self.ui.print("\nshell> ") + command = self.ui.get_line_input() if command != "": result = self.executor.run_cmd(command) - print(result, end="", flush=True) + self.ui.print(result) elif key == 'r': logging.debug('Received Run last command request') if last_command: @@ -60,29 +59,11 @@ def process_command_input(self, key, last_command=None): def display_prompt(self): """ Default result for displaying a command """ - print(self.config.get('render', 'console_prompt', raw=True) + ' ', end="", flush=True) + self.ui.print(self.config.get('render', 'console_prompt', raw=True) + ' ') - @staticmethod - def display_command(cmd): + def display_command(self, cmd): """ Default result for displaying a command """ - print(cmd, flush=True) - - @staticmethod - def get_single_key_input(): - """ SimDem1 uses this method: - https://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key - For SimDem2, I'm trying this alternative to allow for Windows compatibility - https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user/510404#510404 - - https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not - Might need to allow config override of the conditional by allowing a config variable. - Experienced an issue where it hung running in a container and I didn't completely debug - """ - - if sys.stdout.isatty(): - getch = Getch() - return getch.impl() - return + self.ui.println(cmd) def process_next_steps(self, next_steps, start_path): """ Is there a good way to test this that doesn't involve lots of test code + expect? @@ -90,11 +71,11 @@ def process_next_steps(self, next_steps, start_path): """ idx = 1 if next_steps: - print("Next steps available:") + self.ui.println("Next steps available:") for step in next_steps: - print(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") + self.ui.println(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") idx += 1 - print() + self.ui.println() # https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not if sys.stdout.isatty(): # You're running in a real terminal @@ -102,9 +83,9 @@ def process_next_steps(self, next_steps, start_path): in_value = 0 while in_value < 1 or in_value > len(next_steps): - in_string = input("Choose a step. " + - "Enter a value between 1 and " + - str(len(next_steps)) + " or 'q' to quit: ") + prompt = "Choose a step. Enter a value between 1 and " + \ + str(len(next_steps)) + " or 'q' to quit: " + in_string = self.ui.get_line_input(prompt) if in_string.lower() == "quit" or in_string.lower() == "q": return try: @@ -112,7 +93,6 @@ def process_next_steps(self, next_steps, start_path): except ValueError: pass - #print('You chose:' + str(steps[int(step_request) - 1]['title'])) self.process_file(start_path + '/' + next_steps[int(in_string) - 1]['target']) else: logging.info('Not connected to a TTY terminal Not requesting input.') diff --git a/simdem/mode/test.py b/simdem/mode/test.py index 474f685..ca6a6c6 100644 --- a/simdem/mode/test.py +++ b/simdem/mode/test.py @@ -19,9 +19,9 @@ def process(self, steps): last_command_result = self.process_commands(step['content']) if 'expected_result' in step: if self.is_result_valid(step['expected_result'], last_command_result): - print('*** SIMDEM TEST RESULT PASSED ***') + self.ui.print_test_passed() else: - print('*** SIMDEM TEST RESULT FAILED ***') + self.ui.print_test_failed() exit(1) def process_next_steps(self, steps, start_path): diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index c901d5d..082661a 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -15,10 +15,7 @@ def process(self, steps): """ Processes the steps from a processed file """ logging.debug("process()") - # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python - #print("\033[H\033[J") - if sys.stdout.isatty(): - os.system('clear') + self.ui.clear() for step in steps['body']: if step['type'] == 'heading': self.process_heading(step) @@ -29,18 +26,16 @@ def process(self, steps): logging.debug(step) if 'expected_result' in step: if self.is_result_valid(step['expected_result'], last_command_result): - print('*** SIMDEM RESULT PASSED ***') + self.ui.print_test_passed() else: - print('*** SIMDEM RESULT FAILED ***') + self.ui.print_test_failed() - @staticmethod - def process_heading(step): + def process_heading(self, step): """ Print out the heading exactly as we found it """ - print(step['level'] * '#' + ' ' + step['content']) - print() + self.ui.println(step['level'] * '#' + ' ' + step['content']) + self.ui.println() - @staticmethod - def process_text(step): + def process_text(self, step): """ Print out the text exactly as we found it """ - print(step['content']) + self.ui.println(step['content']) diff --git a/simdem/ui/basic.py b/simdem/ui/basic.py new file mode 100644 index 0000000..3db6f94 --- /dev/null +++ b/simdem/ui/basic.py @@ -0,0 +1,57 @@ +""" Basic Render Class """ + +import os +import sys +from simdem.misc.getch import Getch + +class BasicUI(object): + """ No frills, no thrills render object """ + + @staticmethod + def get_single_key_input(): + """ SimDem1 uses this method: + https://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key + For SimDem2, I'm trying this alternative to allow for Windows compatibility + https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user/510404#510404 + + https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not + Might need to allow config override of the conditional by allowing a config variable. + Experienced an issue where it hung running in a container and I didn't completely debug + """ + + if sys.stdout.isatty(): + getch = Getch() + return getch.impl() + return + + @staticmethod + def get_line_input(prompt): + """ Request single line from user """ + return input(prompt) + + @staticmethod + def println(output=''): + print(output, flush=True) + + @staticmethod + def print(output=''): + print(output, end="", flush=True) + + @staticmethod + def print_validation_failed(): + print('***PREREQUISITE VALIDATION FAILED***') + + @staticmethod + def clear(): + # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python + #print("\033[H\033[J") + if sys.stdout.isatty(): + os.system('clear') + + @staticmethod + def print_test_passed(): + print('*** SIMDEM TEST RESULT PASSED ***') + + @staticmethod + def print_test_failed(): + print('*** SIMDEM TEST RESULT FAILED ***') diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index eaf2e18..9acbb4b 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -11,7 +11,7 @@ from simdem.parser import simdem1 from simdem.executor import bash from simdem.mode import demo - +from simdem.ui import basic @ddt class SimDemSystemTestSuite(unittest.TestCase): @@ -22,7 +22,8 @@ class SimDemSystemTestSuite(unittest.TestCase): def setUp(self): config = configparser.ConfigParser() config.read("examples/config/unit_test.ini") - self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) + self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor(), + basic.BasicUI()) log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index d7664ca..079804f 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -11,7 +11,7 @@ from simdem.parser import simdem1 from simdem.executor import bash from simdem.mode import tutorial - +from simdem.ui import basic @ddt class SimDemSystemTestSuite(unittest.TestCase): @@ -23,7 +23,8 @@ def setUp(self): config = configparser.ConfigParser() config.read("examples/config/unit_test.ini") - self.simdem = tutorial.TutorialMode(config, simdem1.SimDem1Parser(), bash.BashExecutor()) + self.simdem = tutorial.TutorialMode(config, simdem1.SimDem1Parser(), bash.BashExecutor(), + basic.BasicUI()) log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() From db26e725bc6eaa341bec193566c7493d67a6b3a7 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 4 Feb 2018 21:22:03 -0800 Subject: [PATCH 124/167] Fix cli after refactor of ui class Signed-off-by: Tommy Falgout --- simdem/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simdem/cli.py b/simdem/cli.py index 130b70d..2d39600 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -26,8 +26,8 @@ def main(): help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", help="Executor class to use", choices=['bash']) - argp.add_argument('--render', '-r', default="basic", - help="Render class to use", choices=['basic']) + argp.add_argument('--ui', '-u', default="basic", + help="UI class to use", choices=['basic']) argp.add_argument('--setting', '-s', metavar='setting', help="Setting to override in config file") options = argp.parse_args() From b58e1965fc0e0e0096a42e7d3efdb8dd4995f5a7 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 4 Feb 2018 21:28:30 -0800 Subject: [PATCH 125/167] Fix refactor issue Signed-off-by: Tommy Falgout --- simdem/mode/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 9dd4cc6..ef5ecf9 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -53,8 +53,8 @@ def process_commands(self, cmds): for cmd in cmds: self.ui.println(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd) results = self.executor.run_cmd(cmd) - self.ui.print_flush(results) - self.ui.print() + self.ui.println(results) + self.ui.println() return results @staticmethod From ebf5a6844a542f53feaf18cc38c59625072f7ee6 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 4 Feb 2018 21:55:40 -0800 Subject: [PATCH 126/167] Refactor UI class to not just print raw text. Future planning Signed-off-by: Tommy Falgout --- simdem/cli.py | 6 +++--- simdem/mode/common.py | 11 ++++++----- simdem/mode/demo.py | 2 +- simdem/mode/interactive.py | 23 ++++++++--------------- simdem/mode/tutorial.py | 14 ++------------ simdem/ui/basic.py | 35 ++++++++++++++++++++++++++--------- tests/test_mode_demo.py | 2 +- tests/test_mode_tutorial.py | 2 +- 8 files changed, 48 insertions(+), 47 deletions(-) diff --git a/simdem/cli.py b/simdem/cli.py index 2d39600..03e86ec 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -73,7 +73,7 @@ def get_mode(options, config): parser = get_parser(options) executor = get_executor(options) - ui = get_ui(options) + ui = get_ui(options, config) if options.mode == 'demo': return demo.DemoMode(config, parser, executor, ui) @@ -87,10 +87,10 @@ def get_mode(options, config): if options.mode == 'tutorial': return tutorial.TutorialMode(config, parser, executor, ui) -def get_ui(options): +def get_ui(options, config): """ return UI object """ if options.ui == 'basic': - return basic.BasicUI() + return basic.BasicUI(config) def get_parser(options): """ Returns correct parser object """ diff --git a/simdem/mode/common.py b/simdem/mode/common.py index ef5ecf9..0cb8f32 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -51,11 +51,12 @@ def process_file(self, file_path, is_prereq=False): def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: - self.ui.println(self.config.get('render', 'console_prompt', raw=True) + ' ' + cmd) - results = self.executor.run_cmd(cmd) - self.ui.println(results) - self.ui.println() - return results + self.ui.print_prompt() + self.ui.print_cmd(cmd) + result = self.executor.run_cmd(cmd) + self.ui.print_result(result) + self.ui.print_break() + return result @staticmethod def is_result_valid(expected_results, actual_results, expected_similarity=1.0): diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index 94cdfaf..f4d44b8 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -30,4 +30,4 @@ def display_command(self, cmd): delay = random.uniform(0.02, typing_delay) time.sleep(delay) self.ui.print(char) - self.ui.println('') + self.ui.print_break() diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index 990de6d..ca0b8c1 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -16,7 +16,7 @@ def process_commands(self, cmds): # The "I smell danger" picture is never truer than now # This statement is written in VSCode while I work for MSFT while True: - self.display_prompt() + self.ui.print_prompt() key = self.ui.get_single_key_input() result = self.process_command_input(key, last_command=cmd) logging.debug(cmd_deque) @@ -33,10 +33,11 @@ def run_command(self, cmd): """ Pretend to type the command, run it and then display the output """ # Request enter from user to know when to proceed logging.debug('run_command(' + cmd + ')') - self.display_command(cmd) - results = self.executor.run_cmd(cmd) - self.ui.print(results) - return results + self.ui.print_cmd(cmd) + self.ui.print_break() + result = self.executor.run_cmd(cmd) + self.ui.print_result(result) + return result def process_command_input(self, key, last_command=None): """ Process the command input. It's 4AM and I'm sleepy @@ -45,10 +46,10 @@ def process_command_input(self, key, last_command=None): if key == 'b': logging.debug('Received Break request') self.ui.print("\nshell> ") - command = self.ui.get_line_input() + command = self.ui.get_line_input('') if command != "": result = self.executor.run_cmd(command) - self.ui.print(result) + self.ui.print_result(result) elif key == 'r': logging.debug('Received Run last command request') if last_command: @@ -57,14 +58,6 @@ def process_command_input(self, key, last_command=None): return result # Otherwise, we will return and assume the user wants to continue - def display_prompt(self): - """ Default result for displaying a command """ - self.ui.print(self.config.get('render', 'console_prompt', raw=True) + ' ') - - def display_command(self, cmd): - """ Default result for displaying a command """ - self.ui.println(cmd) - def process_next_steps(self, next_steps, start_path): """ Is there a good way to test this that doesn't involve lots of test code + expect? Not fully tested yet. Low priority feature. diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 082661a..cf2d6ce 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -18,9 +18,9 @@ def process(self, steps): self.ui.clear() for step in steps['body']: if step['type'] == 'heading': - self.process_heading(step) + self.ui.print_heading(step['content'], step['level']) elif step['type'] == 'text': - self.process_text(step) + self.ui.println(step['content']) elif step['type'] == 'commands': last_command_result = self.process_commands(step['content']) logging.debug(step) @@ -29,13 +29,3 @@ def process(self, steps): self.ui.print_test_passed() else: self.ui.print_test_failed() - - - def process_heading(self, step): - """ Print out the heading exactly as we found it """ - self.ui.println(step['level'] * '#' + ' ' + step['content']) - self.ui.println() - - def process_text(self, step): - """ Print out the text exactly as we found it """ - self.ui.println(step['content']) diff --git a/simdem/ui/basic.py b/simdem/ui/basic.py index 3db6f94..a570ae0 100644 --- a/simdem/ui/basic.py +++ b/simdem/ui/basic.py @@ -7,6 +7,9 @@ class BasicUI(object): """ No frills, no thrills render object """ + def __init__(self, config): + self.config = config + @staticmethod def get_single_key_input(): """ SimDem1 uses this method: @@ -37,9 +40,8 @@ def println(output=''): def print(output=''): print(output, end="", flush=True) - @staticmethod - def print_validation_failed(): - print('***PREREQUISITE VALIDATION FAILED***') + def print_validation_failed(self): + self.println('***PREREQUISITE VALIDATION FAILED***') @staticmethod def clear(): @@ -48,10 +50,25 @@ def clear(): if sys.stdout.isatty(): os.system('clear') - @staticmethod - def print_test_passed(): - print('*** SIMDEM TEST RESULT PASSED ***') + def print_test_passed(self): + self.println('*** SIMDEM TEST RESULT PASSED ***') - @staticmethod - def print_test_failed(): - print('*** SIMDEM TEST RESULT FAILED ***') + def print_test_failed(self): + self.println('*** SIMDEM TEST RESULT FAILED ***') + + def print_heading(self, content, level): + """ Print out the heading exactly as we found it """ + self.println(level * '#' + ' ' + content) + self.print_break() + + def print_prompt(self): + self.print(self.config.get('render', 'console_prompt', raw=True) + ' ') + + def print_cmd(self, cmd): + self.print(cmd) + + def print_result(self, result): + self.print(result) + + def print_break(self): + self.println() \ No newline at end of file diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index 9acbb4b..d717a30 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -23,7 +23,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("examples/config/unit_test.ini") self.demo = demo.DemoMode(config, simdem1.SimDem1Parser(), bash.BashExecutor(), - basic.BasicUI()) + basic.BasicUI(config)) log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index 079804f..fc55795 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -24,7 +24,7 @@ def setUp(self): config = configparser.ConfigParser() config.read("examples/config/unit_test.ini") self.simdem = tutorial.TutorialMode(config, simdem1.SimDem1Parser(), bash.BashExecutor(), - basic.BasicUI()) + basic.BasicUI(config)) log_formatter = logging.Formatter(config.get('log', 'format', raw=True)) root_logger = logging.getLogger() From a6d790559c10a304ff12ddabc67ff813908773f6 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 6 Feb 2018 17:58:19 -0800 Subject: [PATCH 127/167] Fix issue where text did not render --- examples/markdown-syntax/README.md | 18 ++++++++++++++++++ examples/markdown-syntax/expected_result.seo | 16 ++++++++++++++++ simdem/mode/common.py | 1 + simdem/mode/interactive.py | 1 + simdem/parser/simdem1.py | 12 ++++++++++++ tests/test_parser_simdem1.py | 1 + 6 files changed, 49 insertions(+) create mode 100644 examples/markdown-syntax/README.md create mode 100644 examples/markdown-syntax/expected_result.seo diff --git a/examples/markdown-syntax/README.md b/examples/markdown-syntax/README.md new file mode 100644 index 0000000..dc7716a --- /dev/null +++ b/examples/markdown-syntax/README.md @@ -0,0 +1,18 @@ +# Heading 1 + +## Code Block + +This is a code block: + +```shell +echo foo +echo bar +``` + +## Emphasis + +*This is emphasis* + +## Inline code + +Precode `this is inline code` Postcode \ No newline at end of file diff --git a/examples/markdown-syntax/expected_result.seo b/examples/markdown-syntax/expected_result.seo new file mode 100644 index 0000000..a7bd765 --- /dev/null +++ b/examples/markdown-syntax/expected_result.seo @@ -0,0 +1,16 @@ +{ + "body": [ + { + "content": "Simple", + "level": 1, + "type": "heading" + }, + { + "content": [ + "echo foo", + "echo bar" + ], + "type": "commands" + } + ] +} diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 0cb8f32..38495b0 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -53,6 +53,7 @@ def process_commands(self, cmds): for cmd in cmds: self.ui.print_prompt() self.ui.print_cmd(cmd) + self.ui.print_break() result = self.executor.run_cmd(cmd) self.ui.print_result(result) self.ui.print_break() diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index ca0b8c1..a5de248 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -34,6 +34,7 @@ def run_command(self, cmd): # Request enter from user to know when to proceed logging.debug('run_command(' + cmd + ')') self.ui.print_cmd(cmd) + # For some reason this requires a print_break() while common does not. Too late to debug self.ui.print_break() result = self.executor.run_cmd(cmd) self.ui.print_result(result) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 591c9f0..1316a47 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -90,6 +90,18 @@ def render_raw_text(self, token): return '' return token.content + def render_emphasis(self, token): + """ Render for inline code """ + inner = self.render_inner(token) + if inner: + body = {'type': 'text', 'content': '*' + str(inner) + '*'} + self.append_body(body) + return '' + + def render_inline_code(self, token): + """ Render for inline code """ + return '`' + self.render_inner(token) + '`' + def render_paragraph(self, token): """ Render for Paragraph """ inner = self.render_inner(token) diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index dd939fd..290f93e 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -23,6 +23,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer + #@data('markdown-syntax') @data('simple', 'simple-variable', 'results-block', 'results-block-fail', 'prerequisites') def test_process(self, directory): From 59737d5e73d0a9a02069dc5a14b28d2461c3903c Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 8 Feb 2018 12:11:16 -0800 Subject: [PATCH 128/167] update docs Signed-off-by: Tommy Falgout --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index cd02c83..236896c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,16 @@ A great place to start working on SimDem is to run SimDem on its own documentati simdem docs/README.md ``` +## Documentation + +It's more fun to run SimDem on it's own documentation; however, you can learn how how SimDem works by [reading the docs](https://github.com/Azure/simdem/tree/simdem2/docs). + +We provide a [simple hello-world example](https://github.com/Azure/simdem/blob/simdem2/docs/hello_world.md). + +### Examples + +If you want to see existing examples, with expected output, check out the [examples](https://github.com/Azure/simdem/tree/simdem2/examples) + # Syntax Currently, SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). From d862cd03dc948305bbcd95fc1b8b30c3bb02d761 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 9 Feb 2018 15:51:31 -0800 Subject: [PATCH 129/167] Change validation to 0.8 Signed-off-by: Tommy Falgout --- simdem/mode/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 38495b0..4cc1cae 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -60,7 +60,7 @@ def process_commands(self, cmds): return result @staticmethod - def is_result_valid(expected_results, actual_results, expected_similarity=1.0): + def is_result_valid(expected_results, actual_results, expected_similarity=0.8): """Checks to see if a command execution passes. If actual results compared to expected results is within the expected similarity level then it's considered a pass. From ec755da0b73a14febb2cdefde1b1790712ba0bc3 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 12 Feb 2018 12:50:14 -0600 Subject: [PATCH 130/167] fix broken test --- examples/results-block-fail/README.md | 2 +- examples/results-block-fail/expected_result.seo | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/results-block-fail/README.md b/examples/results-block-fail/README.md index 5c8918a..70f1b4f 100644 --- a/examples/results-block-fail/README.md +++ b/examples/results-block-fail/README.md @@ -8,7 +8,7 @@ echo bar Results: ```result -barr +barrrrr ``` In demo mode, we will continue processing. diff --git a/examples/results-block-fail/expected_result.seo b/examples/results-block-fail/expected_result.seo index ef5c3bc..e53e5f7 100644 --- a/examples/results-block-fail/expected_result.seo +++ b/examples/results-block-fail/expected_result.seo @@ -9,7 +9,7 @@ "echo foo", "echo bar" ], - "expected_result": "barr\n", + "expected_result": "barrrrr\n", "type": "commands" }, { From 14b2fafdd32ad3817181c37b4a90d2296f9138d2 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 12 Feb 2018 14:06:04 -0600 Subject: [PATCH 131/167] Add processing of ToC Signed-off-by: Tommy Falgout --- examples/toc/README.md | 11 +++++++++ examples/toc/expected_result.demo | 6 +++++ examples/toc/expected_result.seo | 32 +++++++++++++++++++++++++++ examples/toc/expected_result.test | 0 examples/toc/expected_result.tutorial | 11 +++++++++ examples/toc/one.md | 1 + examples/toc/two.md | 1 + simdem/mode/common.py | 12 +++++++--- simdem/mode/interactive.py | 3 ++- simdem/parser/simdem1.py | 24 +++++++++++--------- tests/test_mode_demo.py | 2 +- tests/test_mode_tutorial.py | 2 +- tests/test_parser_simdem1.py | 2 +- 13 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 examples/toc/README.md create mode 100644 examples/toc/expected_result.demo create mode 100644 examples/toc/expected_result.seo create mode 100644 examples/toc/expected_result.test create mode 100644 examples/toc/expected_result.tutorial create mode 100644 examples/toc/one.md create mode 100644 examples/toc/two.md diff --git a/examples/toc/README.md b/examples/toc/README.md new file mode 100644 index 0000000..c164c3d --- /dev/null +++ b/examples/toc/README.md @@ -0,0 +1,11 @@ +# This the table of contents test + +## ToC + +* [Main](README.md) +* [One](one.md) +* [Two](two.md) + +## Main content + +Foo \ No newline at end of file diff --git a/examples/toc/expected_result.demo b/examples/toc/expected_result.demo new file mode 100644 index 0000000..99d9dd2 --- /dev/null +++ b/examples/toc/expected_result.demo @@ -0,0 +1,6 @@ + +Next steps available: +1. Main (README.md) +2. One (one.md) +3. Two (two.md) + diff --git a/examples/toc/expected_result.seo b/examples/toc/expected_result.seo new file mode 100644 index 0000000..b5cd60c --- /dev/null +++ b/examples/toc/expected_result.seo @@ -0,0 +1,32 @@ +{ + "body": [ + { + "content": "This the table of contents test", + "level": 1, + "type": "heading" + }, + { + "content": "Main content", + "level": 2, + "type": "heading" + }, + { + "content": "Foo", + "type": "text" + } + ], + "toc": [ + { + "target": "README.md", + "title": "Main" + }, + { + "target": "one.md", + "title": "One" + }, + { + "target": "two.md", + "title": "Two" + } + ] +} diff --git a/examples/toc/expected_result.test b/examples/toc/expected_result.test new file mode 100644 index 0000000..e69de29 diff --git a/examples/toc/expected_result.tutorial b/examples/toc/expected_result.tutorial new file mode 100644 index 0000000..e02b67c --- /dev/null +++ b/examples/toc/expected_result.tutorial @@ -0,0 +1,11 @@ +# This the table of contents test + +## Main content + +Foo + +Next steps available: +1. Main (README.md) +2. One (one.md) +3. Two (two.md) + diff --git a/examples/toc/one.md b/examples/toc/one.md new file mode 100644 index 0000000..55f8b72 --- /dev/null +++ b/examples/toc/one.md @@ -0,0 +1 @@ +# one \ No newline at end of file diff --git a/examples/toc/two.md b/examples/toc/two.md new file mode 100644 index 0000000..fe9a18f --- /dev/null +++ b/examples/toc/two.md @@ -0,0 +1 @@ +# two \ No newline at end of file diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 4cc1cae..bd4e281 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -18,7 +18,7 @@ def __init__(self, config, parser, executor, ui): self.executor = executor self.ui = ui - def process_file(self, file_path, is_prereq=False): + def process_file(self, file_path, is_prereq=False, toc={}): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) # Change the working directory in case of any recursion @@ -26,6 +26,12 @@ def process_file(self, file_path, is_prereq=False): logging.debug('parse_file::start_path=' + start_path) steps = self.parser.parse_file(file_path) + # We want to inherit the parent's TOC to reduce the # of copies needed + if toc: + logging.debug('Adding parent POC') + steps['toc'] = toc + logging.debug(steps) + # Begin preqreq processing if 'prerequisites' in steps: for prereq_file in steps['prerequisites']: @@ -45,8 +51,8 @@ def process_file(self, file_path, is_prereq=False): self.executor.run_cmd('cd ' + start_path) self.process(steps) # pylint: disable=no-member - if 'next_steps' in steps: - self.process_next_steps(steps['next_steps'], start_path) # pylint: disable=no-member + if 'toc' in steps: + self.process_next_steps(steps['toc'], start_path) # pylint: disable=no-member def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index a5de248..f883d73 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -65,6 +65,7 @@ def process_next_steps(self, next_steps, start_path): """ idx = 1 if next_steps: + self.ui.println() self.ui.println("Next steps available:") for step in next_steps: self.ui.println(str(idx) + ". " + step['title'] + " (" + step['target'] + ") ") @@ -87,7 +88,7 @@ def process_next_steps(self, next_steps, start_path): except ValueError: pass - self.process_file(start_path + '/' + next_steps[int(in_string) - 1]['target']) + self.process_file(start_path + '/' + next_steps[int(in_string) - 1]['target'], toc=next_steps) else: logging.info('Not connected to a TTY terminal Not requesting input.') # You're being piped or redirected diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 1316a47..01a81b0 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -37,16 +37,15 @@ def __init__(self): def render_heading(self, token): """ Render for Heading: # """ inner = self.render_inner(token) - content = {'type': 'heading', 'level': token.level, 'content': inner} - self.append_body(content) # Prerequisite Heading if inner.lower() == 'prerequisites': self.set_section('prerequisites') - # Next Steps Heading - elif inner.lower() == 'next steps': - self.set_section('next_steps') + # ToC (legacy: Next Steps) Heading + elif inner.lower() == 'next steps' or inner.lower() == 'toc': + self.set_section('toc') + return inner # Validation Heading elif inner.lower() == 'validation': @@ -55,6 +54,9 @@ def render_heading(self, token): else: logging.debug("parse_file():unable to determing header type.") self.reset_section() + + content = {'type': 'heading', 'level': token.level, 'content': inner} + self.append_body(content) return inner def render_list(self, token): @@ -77,10 +79,10 @@ def render_link(self, token): Unfortunately, I can only think of storing them in an array right now """ inner = self.render_inner(token) - if self.section and 'next_steps' in self.section: - self.append_next_step(inner, token.target) if self.section and 'prerequisites' in self.section: self.append_prereq(token.target) + if self.section and 'toc' in self.section: + self.append_toc(inner, token.target) return inner def render_raw_text(self, token): @@ -182,7 +184,7 @@ def append_prereq(self, target): logging.debug('append_prereq(' + target + ')') self.output['prerequisites'].append(target) - def append_next_step(self, name, target): - """ Add steps to next step section """ - logging.debug('append_next_step(' + name + ',' + target + ')') - self.output['next_steps'].append({'title': name, 'target': target}) + def append_toc(self, name, target): + """ Add steps to table of contents section """ + logging.debug('append_toc(' + name + ',' + target + ')') + self.output['toc'].append({'title': name, 'target': target}) diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index d717a30..349102b 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -33,7 +33,7 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable', 'results-block', + @data('simple', 'simple-variable', 'results-block', 'toc', 'results-block-fail', 'prerequisites') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.out diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index fc55795..9928215 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -34,7 +34,7 @@ def setUp(self): root_logger.addHandler(file_handler) # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer - @data('simple', 'simple-variable', 'results-block', + @data('simple', 'simple-variable', 'results-block', 'toc', 'results-block-fail', 'prerequisites') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.tutorial diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 290f93e..5e1f08f 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -24,7 +24,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer #@data('markdown-syntax') - @data('simple', 'simple-variable', 'results-block', + @data('simple', 'simple-variable', 'results-block', 'toc', 'results-block-fail', 'prerequisites') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.tutorial From efeb0bd2a906ecc37c34bc677994087fe42f670d Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 15 Feb 2018 20:53:34 -0600 Subject: [PATCH 132/167] Add cleanup mode Signed-off-by: Tommy Falgout --- simdem/cli.py | 7 +++++-- simdem/mode/cleanup.py | 19 +++++++++++++++++++ simdem/parser/simdem1.py | 12 ++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 simdem/mode/cleanup.py diff --git a/simdem/cli.py b/simdem/cli.py index 03e86ec..2d189ab 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -8,7 +8,7 @@ from simdem.executor import bash from simdem.parser import ast, simdem1 -from simdem.mode import demo, dump, test, tutorial +from simdem.mode import demo, dump, test, tutorial, cleanup from simdem.ui import basic def main(): @@ -21,7 +21,7 @@ def main(): argp.add_argument('--config-file', '-c', help="Config file to use") argp.add_argument('--mode', '-m', default="tutorial", - help="Mode to use", choices=['demo', 'dump', 'test', 'tutorial']) + help="Mode to use", choices=['demo', 'dump', 'test', 'tutorial', 'cleanup']) argp.add_argument('--parser', '-p', default="simdem1", help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", @@ -81,6 +81,9 @@ def get_mode(options, config): if options.mode == 'dump': return dump.DumpMode(config, parser, executor, ui) + if options.mode == 'cleanup': + return cleanup.CleanupMode(config, parser, executor, ui) + if options.mode == 'test': return test.TestMode(config, parser, executor, ui) diff --git a/simdem/mode/cleanup.py b/simdem/mode/cleanup.py new file mode 100644 index 0000000..ba67feb --- /dev/null +++ b/simdem/mode/cleanup.py @@ -0,0 +1,19 @@ +""" Automated mode for SimDem """ + +import logging +from simdem.mode.common import ModeCommon + +class CleanupMode(ModeCommon): + """ This class is the SimDem Cleanup mode + """ + + def process(self, steps): + """ I'd like to use a dispatcher for this; however, we need to exit processing + if the validation fails. """ + logging.debug("process()") + if 'cleanup' in steps: + self.process_commands(steps['cleanup']['commands']) + + def process_next_steps(self, steps, start_path): + """ No need to display next steps if in test mode """ + pass diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 01a81b0..51f6f4a 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -51,6 +51,11 @@ def render_heading(self, token): elif inner.lower() == 'validation': self.set_section('validation') + # Cleanup + elif inner.lower() == 'cleanup': + self.set_section('cleanup') + return inner + else: logging.debug("parse_file():unable to determing header type.") self.reset_section() @@ -124,6 +129,8 @@ def render_block_code(self, token): self.set_validation_result(content) else: self.set_validation_command(content) + elif self.section == 'cleanup': + self.set_cleanup(content) else: if self.block == 'results': # Assume that the last body item is the command we're expecting results for @@ -188,3 +195,8 @@ def append_toc(self, name, target): """ Add steps to table of contents section """ logging.debug('append_toc(' + name + ',' + target + ')') self.output['toc'].append({'title': name, 'target': target}) + + def set_cleanup(self, cmds): + """ Assuming cleanup commands are a list """ + logging.debug('set_cleanup_command(' + str(cmds) + ')') + self.output['cleanup'] = {'commands': cmds.splitlines()} From f6aef9cac57d80c7906002222bd40c79ab4e8424 Mon Sep 17 00:00:00 2001 From: Ross Gardler Date: Fri, 23 Feb 2018 00:59:04 +0000 Subject: [PATCH 133/167] Add license to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5544cb4..a3ebc5e 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ author='Tommy Falgout/Ross Gardler', author_email='thfalgou@microsoft.com', url='https://github.com/Azure/simdem', - license=LICENSE, + license='MIT', packages=find_packages(exclude=('tests', 'docs')), entry_points={ 'console_scripts': [ From e1e7492ea85b4dd9e4e14e8185ff34d3152e8ded Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 4 Mar 2018 12:32:08 -0600 Subject: [PATCH 134/167] Fix expected result for syntax --- examples/markdown-syntax/expected_result.seo | 33 +++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/examples/markdown-syntax/expected_result.seo b/examples/markdown-syntax/expected_result.seo index a7bd765..9853d2a 100644 --- a/examples/markdown-syntax/expected_result.seo +++ b/examples/markdown-syntax/expected_result.seo @@ -1,16 +1,47 @@ { "body": [ { - "content": "Simple", + "content": "Heading 1", "level": 1, "type": "heading" }, + { + "content": "Code Block", + "level": 2, + "type": "heading" + }, + { + "content": "This is a code block:\n", + "type": "text" + }, { "content": [ "echo foo", "echo bar" ], "type": "commands" + }, + { + "content": "Emphasis", + "level": 2, + "type": "heading" + }, + { + "content": "*This is emphasis*", + "type": "text" + }, + { + "content": "\n", + "type": "text" + }, + { + "content": "Inline code", + "level": 2, + "type": "heading" + }, + { + "content": "Precode `this is inline code` Postcode", + "type": "text" } ] } From c1c7f99121ccabdeeea62eb04398754f440b631c Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 4 Mar 2018 12:56:54 -0600 Subject: [PATCH 135/167] Treat validation as normal case if not a prepreq. https://github.com/Azure/simdem/issues/100 --- examples/prerequisites/README.md | 2 +- examples/prerequisites/expected_result.dump | 2 +- examples/prerequisites/expected_result.seo | 2 +- examples/prerequisites/expected_result.tutorial | 2 +- simdem/mode/common.py | 2 +- simdem/parser/simdem1.py | 10 ++++++---- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/prerequisites/README.md b/examples/prerequisites/README.md index 7d96ef6..74f3028 100644 --- a/examples/prerequisites/README.md +++ b/examples/prerequisites/README.md @@ -10,7 +10,7 @@ They don't even need to be in the same list By this point, the prerequisites have either run or have passed their validation -# Did our prerequisites run? +# Validation ```shell echo prereq_ignored = $prereq_ignored diff --git a/examples/prerequisites/expected_result.dump b/examples/prerequisites/expected_result.dump index 9169127..6b92bbe 100644 --- a/examples/prerequisites/expected_result.dump +++ b/examples/prerequisites/expected_result.dump @@ -7,7 +7,7 @@ {'content': 'By this point, the prerequisites have either run or ' 'have passed their validation', 'type': 'text'}, - {'content': 'Did our prerequisites run?', + {'content': 'Validation', 'level': 1, 'type': 'heading'}, {'content': ['echo prereq_ignored = $prereq_ignored', diff --git a/examples/prerequisites/expected_result.seo b/examples/prerequisites/expected_result.seo index b3874f9..6c90d1c 100644 --- a/examples/prerequisites/expected_result.seo +++ b/examples/prerequisites/expected_result.seo @@ -18,7 +18,7 @@ "type": "text" }, { - "content": "Did our prerequisites run?", + "content": "Validation", "level": 1, "type": "heading" }, diff --git a/examples/prerequisites/expected_result.tutorial b/examples/prerequisites/expected_result.tutorial index 95ec75d..c84a938 100644 --- a/examples/prerequisites/expected_result.tutorial +++ b/examples/prerequisites/expected_result.tutorial @@ -28,7 +28,7 @@ They don't even need to be in the same list By this point, the prerequisites have either run or have passed their validation -# Did our prerequisites run? +# Validation $ echo prereq_ignored = $prereq_ignored prereq_ignored = diff --git a/simdem/mode/common.py b/simdem/mode/common.py index bd4e281..dfaa508 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -24,7 +24,7 @@ def process_file(self, file_path, is_prereq=False, toc={}): # Change the working directory in case of any recursion start_path = os.path.dirname(os.path.abspath(file_path)) logging.debug('parse_file::start_path=' + start_path) - steps = self.parser.parse_file(file_path) + steps = self.parser.parse_file(file_path, is_prereq) # We want to inherit the parent's TOC to reduce the # of copies needed if toc: diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 51f6f4a..2f6beed 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -11,10 +11,10 @@ class SimDem1Parser(object): # pylint: disable=R0903 """ Parses markdown based off of Mistletoe renderer """ @staticmethod - def parse_file(file_path): + def parse_file(file_path, is_prerequisite=False): """ The main meat for parsing the file. """ with open(file_path, 'r') as fin: - with SimDemMistletoeRenderer() as renderer: + with SimDemMistletoeRenderer(is_prerequisite) as renderer: rendered = renderer.render(Document(fin)) # Do stuff here @@ -28,11 +28,13 @@ class SimDemMistletoeRenderer(BaseRenderer): """ section = None block = None + _is_prerequisite = False - def __init__(self): + def __init__(self, is_prerequisite): super().__init__() self.output = defaultdict(list) self.reset_section() + self._is_prerequisite = is_prerequisite def render_heading(self, token): """ Render for Heading: # """ @@ -48,7 +50,7 @@ def render_heading(self, token): return inner # Validation Heading - elif inner.lower() == 'validation': + elif self._is_prerequisite and inner.lower() == 'validation': self.set_section('validation') # Cleanup From 477beee65c3740b894d8dd73b378d2311c67ecd2 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 09:26:30 -0600 Subject: [PATCH 136/167] Raise error when calling render for non-supported section --- simdem/parser/simdem1.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/simdem/parser/simdem1.py b/simdem/parser/simdem1.py index 2f6beed..7277095 100644 --- a/simdem/parser/simdem1.py +++ b/simdem/parser/simdem1.py @@ -23,8 +23,6 @@ def parse_file(file_path, is_prerequisite=False): class SimDemMistletoeRenderer(BaseRenderer): """ Based off of https://gist.github.com/miyuchina/a06bd90d91b70be0906266760547da62 - Major Gotcha: If this class calls a function that does not exist, it will NOT error - This is due to the way that the Mistletoe Renderer works. Or at least how I think it works. """ section = None block = None @@ -154,7 +152,13 @@ def render_document(self, token): return dict(self.output) def __getattr__(self, name): - return lambda token: '' + """ Kudos to @miyuchina for this suggestion + https://github.com/Azure/simdem/pull/89#issuecomment-370445949 + """ + if name.startswith('render'): + return lambda token: '' + raise AttributeError('{} has no attribute {}'.format(repr(type(self).__name__), repr(name))) + # Everything below here is boring setters From b485b5c1465bc2d7a35daec3c81711cea49ac849 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 16:36:22 -0600 Subject: [PATCH 137/167] Add ability to specify setup script --- docs/README.md | 2 ++ docs/feature_validation.md | 70 -------------------------------------- simdem/cli.py | 18 ++++++---- simdem/mode/common.py | 6 ++++ 4 files changed, 20 insertions(+), 76 deletions(-) delete mode 100644 docs/feature_validation.md diff --git a/docs/README.md b/docs/README.md index be3a577..207737c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,6 +26,8 @@ Tutorials can branch too, for example you can choose any of the following paths 1. [Use your documents as an interactive tutorial](mode_tutorial.md) 1. [Use your documents as an interactive demo](mode_demo.md) 1. [Use your documents as an automated test](mode_test.md) +1. [Features](features.md) +1. [Advanced Features](feature_advanced.md) diff --git a/docs/feature_validation.md b/docs/feature_validation.md deleted file mode 100644 index c1da762..0000000 --- a/docs/feature_validation.md +++ /dev/null @@ -1,70 +0,0 @@ -# Automated Testing - -When running with the `--test` flag will verify that the output of each command is as expected. It does this by comparing the output of the command with the subsequent `Results:` section in the script. - -``` -cat docs/feature_validation.md -``` - -## Running in test mode - -Running with demo renderer will not check the results against expectations. However, running with the `test` command will do so. - -``` -echo "This test is expected to fail" -``` - -Results: - -``` -It fails because the results we have in the script are significantly -different to the output of the command. -``` - -By default a 66% or more match indicates a pass. However, in some -cases a much lower similarity is expected, for example, the output of -`date` will vary considerably each time it is run. In these situations -you can provide an expected similarity as part of the three backticks -that start a code block, for example ```Expected_Similarity=0.2 which -is low enough for the test to be recorded as a pass. Note, it is -important that you do not insert any spaces in this notation. - -``` -date -``` - -Results: - -```Expected_Similarity=0.2 -Tue Jun 6 15:23:53 UTC 2017 -``` - -# Fast Fail - -The default setting is for SimDem to stop the test run on the first -test failure. This can be overridden by setting the command line flag -`--fastfail` to any value other than `True`. - -# Test Plans - -It is often a good idea to split tests into separate files. SimDem -will allow you to do this by providing a `test_plan.txt` file. Each -line in this file is either a comment (lines starting with '#') or a -filename for a SimDem script to be used in testing. Each of these -files will be concatenated together to create a complete test plan. - -For example, the following example `test_plan.txt` will run all the -code and tests in `preparation/README.md` followed by those in -`main/README.md` and finally those in `cleanup/README.md`. - -` -preparation/README.md -main/README.md -cleanup/README.md -` - -# Next Steps - - 1. [SimDem Index](README.md) - 2. [Build a Hello World script](hello_world.md) - 3. [Write SimDem documents](syntax.md) diff --git a/simdem/cli.py b/simdem/cli.py index 2d189ab..2e24cb3 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -26,10 +26,12 @@ def main(): help="Parser class to use", choices=['simdem1', 'ast']) argp.add_argument('--executor', '-e', default="bash", help="Executor class to use", choices=['bash']) + argp.add_argument('--setup-script', '-s', default=None, + help="Setup script to execute") argp.add_argument('--ui', '-u', default="basic", help="UI class to use", choices=['basic']) - argp.add_argument('--setting', '-s', metavar='setting', - help="Setting to override in config file") + argp.add_argument('--override-config', '-o', metavar='override', + help="Override setting in config file") options = argp.parse_args() file_path = options.file @@ -38,11 +40,15 @@ def main(): config = configparser.ConfigParser() config.read(config_file_path) - inject_arguments(options, config) + inject_config_options(options, config) setup_logging(config, options) mode = get_mode(options, config) + + if options.setup_script: + mode.run_setup_script(options.setup_script) + mode.process_file(file_path) def get_config_file_path(options): @@ -53,10 +59,10 @@ def get_config_file_path(options): file_path = pkg_resources.resource_filename(__name__, 'simdem.ini') return file_path -def inject_arguments(options, config): +def inject_config_options(options, config): """ Injects CLI arguments into config settings """ - if options.setting: - [key, value] = options.setting.split('=') + if options.override: + [key, value] = options.override.split('=') [section, option] = key.split('.') config.set(section, option, value) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index dfaa508..96ede27 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -18,6 +18,12 @@ def __init__(self, config, parser, executor, ui): self.executor = executor self.ui = ui + def run_setup_script(self, file_path): + """ Runs setup script """ + logging.debug("run_setup_script(" + file_path + ")") + self.executor.run_cmd('. ' + file_path) + + def process_file(self, file_path, is_prereq=False, toc={}): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) From 0d4072dabbe6a01c5aa2116adfeeb5ab94aa3067 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 16:36:33 -0600 Subject: [PATCH 138/167] Add ability to specify setup script --- docs/feature_advanced.md | 15 +++++++++++++++ examples/setup-script/README.md | 11 +++++++++++ examples/setup-script/setup.sh | 1 + 3 files changed, 27 insertions(+) create mode 100644 docs/feature_advanced.md create mode 100644 examples/setup-script/README.md create mode 100644 examples/setup-script/setup.sh diff --git a/docs/feature_advanced.md b/docs/feature_advanced.md new file mode 100644 index 0000000..eb0be63 --- /dev/null +++ b/docs/feature_advanced.md @@ -0,0 +1,15 @@ +# Advanced Features + +## Setup Script + +A setup script is a simple way of bootstrapping your environment prior to running the document. + +You might use this feature if you want to do the following before running the main documentation: +* Set environment variables +* Script prerequisite commands + +Example usage: + +```shell +simdem -s examples/setup-script/setup.sh examples/setup-script/README.md +``` \ No newline at end of file diff --git a/examples/setup-script/README.md b/examples/setup-script/README.md new file mode 100644 index 0000000..d3331ee --- /dev/null +++ b/examples/setup-script/README.md @@ -0,0 +1,11 @@ +# Setup script + +```shell +echo $FOO +``` + +Results: + +```shell +bar +``` \ No newline at end of file diff --git a/examples/setup-script/setup.sh b/examples/setup-script/setup.sh new file mode 100644 index 0000000..1566bb1 --- /dev/null +++ b/examples/setup-script/setup.sh @@ -0,0 +1 @@ +FOO=bar \ No newline at end of file From 86a4bde0180cce1296325912d603f88082b28e21 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 16:38:09 -0600 Subject: [PATCH 139/167] Update README.md --- examples/setup-script/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/setup-script/README.md b/examples/setup-script/README.md index d3331ee..6a3dacf 100644 --- a/examples/setup-script/README.md +++ b/examples/setup-script/README.md @@ -1,5 +1,7 @@ # Setup script +To execute: `simdem -s examples/setup-script/setup.sh examples/setup-script/README.md` + ```shell echo $FOO ``` @@ -8,4 +10,4 @@ Results: ```shell bar -``` \ No newline at end of file +``` From a83095c028196641813a7374ef86b4aec3cb903a Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 16:47:33 -0600 Subject: [PATCH 140/167] cleanup setup script output --- simdem/cli.py | 4 ++-- simdem/mode/common.py | 9 ++++++++- simdem/mode/tutorial.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/simdem/cli.py b/simdem/cli.py index 2e24cb3..f7e0102 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -61,8 +61,8 @@ def get_config_file_path(options): def inject_config_options(options, config): """ Injects CLI arguments into config settings """ - if options.override: - [key, value] = options.override.split('=') + if options.override_config: + [key, value] = options.override_config.split('=') [section, option] = key.split('.') config.set(section, option, value) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 96ede27..46f62de 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -21,7 +21,14 @@ def __init__(self, config, parser, executor, ui): def run_setup_script(self, file_path): """ Runs setup script """ logging.debug("run_setup_script(" + file_path + ")") - self.executor.run_cmd('. ' + file_path) + + cmd = '. ' + file_path + self.ui.print_prompt() + self.ui.print_cmd(cmd) + self.ui.print_break() + result = self.executor.run_cmd(cmd) + self.ui.print_result(result) + self.ui.print_break() def process_file(self, file_path, is_prereq=False, toc={}): diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index cf2d6ce..59863c0 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -15,7 +15,7 @@ def process(self, steps): """ Processes the steps from a processed file """ logging.debug("process()") - self.ui.clear() +# self.ui.clear() for step in steps['body']: if step['type'] == 'heading': self.ui.print_heading(step['content'], step['level']) From c8b7c0581b0a1a485d14ecd207f2c649a88c8fab Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 18:12:46 -0600 Subject: [PATCH 141/167] update version to 0.9.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a3ebc5e..cf14acf 100755 --- a/setup.py +++ b/setup.py @@ -14,10 +14,10 @@ setup( name='simdem', - version='0.1.0', + version='0.9.0', description='SimDem', long_description=README, - author='Tommy Falgout/Ross Gardler', + author='Tommy Falgout, Ross Gardler', author_email='thfalgou@microsoft.com', url='https://github.com/Azure/simdem', license='MIT', From cbf6da92506faaff2364298445cf38cc7b3b9810 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 18:17:05 -0600 Subject: [PATCH 142/167] fix mistletoe version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47efaa7..150fe1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ nose==1.3.7 -mistletoe==0.5 +mistletoe==0.5.3 pexpect==4.2.1 ddt==1.1.1 From 9336b292d912f7aab1c4f2fe02912c9fbcbef76f Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 18:29:39 -0600 Subject: [PATCH 143/167] fix output --- tests/test_mode_demo.py | 2 +- tests/test_parser_simdem1.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index 349102b..eab5af6 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -36,7 +36,7 @@ def setUp(self): @data('simple', 'simple-variable', 'results-block', 'toc', 'results-block-fail', 'prerequisites') def test_process(self, directory): - """ Each examples directory is expected to have a README.md and an expected_result.out + """ Each examples directory is expected to have a README.md and an expected_result.demo this allows us to test each of them easily """ self.demo.process_file('./examples/' + directory + '/README.md') diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index 5e1f08f..e25376f 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -27,7 +27,7 @@ def setUp(self): @data('simple', 'simple-variable', 'results-block', 'toc', 'results-block-fail', 'prerequisites') def test_process(self, directory): - """ Each examples directory is expected to have a README.md and an expected_result.tutorial + """ Each examples directory is expected to have a README.md and an expected_result.seo this allows us to test each of them easily """ self.maxDiff = None # pylint: disable=C0103 From 452b98f6849898d3fffec3f8d39c12530c8d05f9 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 20:57:57 -0600 Subject: [PATCH 144/167] fix unit tests not working as python setup.py tests --- simdem/executor/bash.py | 2 +- tests/test_mode_demo.py | 7 ++++++- tests/test_mode_tutorial.py | 9 +++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/simdem/executor/bash.py b/simdem/executor/bash.py index fb1d3c0..10bd9bc 100644 --- a/simdem/executor/bash.py +++ b/simdem/executor/bash.py @@ -14,7 +14,7 @@ class BashExecutor(object): """ _shell = None - _env = None + _env = {'PS1': '"$ "'} def __init__(self): pass diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index eab5af6..8ae1226 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -6,6 +6,7 @@ import sys import unittest +from io import StringIO from ddt import data, ddt from simdem.parser import simdem1 @@ -31,6 +32,7 @@ def setUp(self): file_handler = logging.FileHandler(config.get('log', 'file')) file_handler.setFormatter(log_formatter) root_logger.addHandler(file_handler) + sys.stdout = StringIO() # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', 'toc', @@ -42,7 +44,10 @@ def test_process(self, directory): self.demo.process_file('./examples/' + directory + '/README.md') # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. - res = sys.stdout.getvalue() # pylint: disable=E1101 + res = sys.stdout.getvalue() + exp_file = open('./examples/' + directory + '/expected_result.tutorial', 'r') + exp_res = exp_file.read() + exp_file.close() exp_res = open('./examples/' + directory + '/expected_result.demo', 'r').read() self.assertEqual(exp_res, res) diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index 9928215..14826a9 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -6,6 +6,7 @@ import sys import unittest +from io import StringIO from ddt import data, ddt from simdem.parser import simdem1 @@ -32,6 +33,7 @@ def setUp(self): file_handler = logging.FileHandler(config.get('log', 'file')) file_handler.setFormatter(log_formatter) root_logger.addHandler(file_handler) + sys.stdout = StringIO() # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', 'toc', @@ -41,11 +43,14 @@ def test_process(self, directory): this allows us to test each of them easily """ self.maxDiff = None # pylint: disable=C0103 + self.simdem.process_file('./examples/' + directory + '/README.md') # Unsure why Pylint complains that 'TextIOWrapper' has no 'getvalue' member. # I'm not Python smart enough yet to know why this works, but Pylint says it shouldn't. - res = sys.stdout.getvalue() # pylint: disable=E1101 - exp_res = open('./examples/' + directory + '/expected_result.tutorial', 'r').read() + res = sys.stdout.getvalue() + exp_file = open('./examples/' + directory + '/expected_result.tutorial', 'r') + exp_res = exp_file.read() + exp_file.close() self.assertEqual(exp_res, res) From b1aebbe9a83a341ffec1692add6f8b2fe4df9588 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 5 Mar 2018 20:58:59 -0600 Subject: [PATCH 145/167] add circleci files --- .circleci/config.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..8c6c3b5 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,33 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/python:3.6.1 + + working_directory: ~/simdem + + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + - run: + name: run tests + command: | + . venv/bin/activate + python setup.py test From 0f51e513b73dff2e718d9a28bed9e566945b29eb Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 8 Mar 2018 15:34:58 -0600 Subject: [PATCH 146/167] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 236896c..88ce359 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Currently, it's only available for installation in development mode: ``` git clone git@github.com:Azure/simdem.git -git checkout -b simdem2 +git checkout -b simdem2 remotes/origin/simdem2 pip3 install -r requirements.txt pip3 install -v -e . ``` From 6b8243bec5cf9a0ec8938fbeb87ec6d578dd2948 Mon Sep 17 00:00:00 2001 From: Ross Gardler Date: Fri, 9 Mar 2018 00:52:58 +0000 Subject: [PATCH 147/167] document potential instalation problem - fixes #99 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 88ce359..ced7f24 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ pip3 install -r requirements.txt pip3 install -v -e . ``` +Note: in rare machine configuations you may get a "simdem: command not +found" error after installing. If you see this error take a look at +the workaround in [issue #99](https://github.com/Azure/simdem/issues/99). + ## Running A great place to start working on SimDem is to run SimDem on its own documentation From 4840505ca30557ddb41639b8edd9c14b5435ad31 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Tue, 13 Mar 2018 20:41:54 -0500 Subject: [PATCH 148/167] Add -e functionality --- simdem/cli.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/simdem/cli.py b/simdem/cli.py index f7e0102..6bd7fd1 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -24,13 +24,15 @@ def main(): help="Mode to use", choices=['demo', 'dump', 'test', 'tutorial', 'cleanup']) argp.add_argument('--parser', '-p', default="simdem1", help="Parser class to use", choices=['simdem1', 'ast']) - argp.add_argument('--executor', '-e', default="bash", - help="Executor class to use", choices=['bash']) - argp.add_argument('--setup-script', '-s', default=None, - help="Setup script to execute") + argp.add_argument('--shell', '-s', default="bash", + help="Shell class to use", choices=['bash']) + argp.add_argument('--environment', '-e', default='', + help="Environment variables to inject (Example: -e FOO=foo1,BAR=bar1)") + argp.add_argument('--boot-strap', '-b', default=None, + help="Boot strap script to execute") argp.add_argument('--ui', '-u', default="basic", help="UI class to use", choices=['basic']) - argp.add_argument('--override-config', '-o', metavar='override', + argp.add_argument('--override-config', metavar='override', help="Override setting in config file") options = argp.parse_args() @@ -46,7 +48,11 @@ def main(): mode = get_mode(options, config) - if options.setup_script: + if options.environment: + commands = options.environment.split(',') + mode.process_commands(commands) + + if options.boot_strap: mode.run_setup_script(options.setup_script) mode.process_file(file_path) @@ -78,23 +84,23 @@ def get_mode(options, config): """ Returns correct renderer object """ parser = get_parser(options) - executor = get_executor(options) + shell = get_shell(options) ui = get_ui(options, config) if options.mode == 'demo': - return demo.DemoMode(config, parser, executor, ui) + return demo.DemoMode(config, parser, shell, ui) if options.mode == 'dump': - return dump.DumpMode(config, parser, executor, ui) + return dump.DumpMode(config, parser, shell, ui) if options.mode == 'cleanup': - return cleanup.CleanupMode(config, parser, executor, ui) + return cleanup.CleanupMode(config, parser, shell, ui) if options.mode == 'test': - return test.TestMode(config, parser, executor, ui) + return test.TestMode(config, parser, shell, ui) if options.mode == 'tutorial': - return tutorial.TutorialMode(config, parser, executor, ui) + return tutorial.TutorialMode(config, parser, shell, ui) def get_ui(options, config): """ return UI object """ @@ -108,9 +114,9 @@ def get_parser(options): elif options.parser == 'simdem1': return simdem1.SimDem1Parser() -def get_executor(options): - """ Returns correct executor object """ - if options.executor == 'bash': +def get_shell(options): + """ Returns correct shell object """ + if options.shell == 'bash': return bash.BashExecutor() def setup_logging(config, options): From 60cb5a71587746ca85275a4e55a5dbe9311f85a0 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 15 Mar 2018 11:37:59 -0500 Subject: [PATCH 149/167] Fix issue in https://github.com/Azure/simdem/issues/106 --- simdem/mode/common.py | 7 +++++-- simdem/mode/demo.py | 3 +-- simdem/mode/interactive.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 46f62de..23f392f 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -24,7 +24,7 @@ def run_setup_script(self, file_path): cmd = '. ' + file_path self.ui.print_prompt() - self.ui.print_cmd(cmd) + self.print_command(cmd) self.ui.print_break() result = self.executor.run_cmd(cmd) self.ui.print_result(result) @@ -71,13 +71,16 @@ def process_commands(self, cmds): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: self.ui.print_prompt() - self.ui.print_cmd(cmd) + self.print_command(cmd) self.ui.print_break() result = self.executor.run_cmd(cmd) self.ui.print_result(result) self.ui.print_break() return result + def print_command(self, cmd): + self.ui.print_cmd(cmd) + @staticmethod def is_result_valid(expected_results, actual_results, expected_similarity=0.8): """Checks to see if a command execution passes. diff --git a/simdem/mode/demo.py b/simdem/mode/demo.py index f4d44b8..796da5d 100644 --- a/simdem/mode/demo.py +++ b/simdem/mode/demo.py @@ -18,7 +18,7 @@ def process(self, steps): if step['type'] == 'commands': self.process_commands(step['content']) - def display_command(self, cmd): + def print_command(self, cmd): """ Displays the command on the screen """ # Must add ' ' when typing command because whitespaces are removed from configparser @@ -30,4 +30,3 @@ def display_command(self, cmd): delay = random.uniform(0.02, typing_delay) time.sleep(delay) self.ui.print(char) - self.ui.print_break() diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index f883d73..cd48b1e 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -33,7 +33,7 @@ def run_command(self, cmd): """ Pretend to type the command, run it and then display the output """ # Request enter from user to know when to proceed logging.debug('run_command(' + cmd + ')') - self.ui.print_cmd(cmd) + self.print_command(cmd) # For some reason this requires a print_break() while common does not. Too late to debug self.ui.print_break() result = self.executor.run_cmd(cmd) From c36443103bbf81103b34dc3f86eba90975a93e02 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 15 Mar 2018 12:41:24 -0500 Subject: [PATCH 150/167] Add --version --- simdem/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/simdem/cli.py b/simdem/cli.py index 6bd7fd1..dc84955 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -13,7 +13,7 @@ def main(): """ Main execution function """ - argp = argparse.ArgumentParser() + argp = argparse.ArgumentParser(prog='simdem') argp.add_argument('file', metavar='file', help='file to process') argp.add_argument('--debug', '-d', action="store_true", @@ -34,6 +34,9 @@ def main(): help="UI class to use", choices=['basic']) argp.add_argument('--override-config', metavar='override', help="Override setting in config file") + version = pkg_resources.require("simdem")[0].version + argp.add_argument('--version', action='version', version='%(prog)s ' + version, + help="Display SimDem's version number") options = argp.parse_args() file_path = options.file From 9d1b4e416a8333c0abd035bacd551d7df43daff6 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 16 Mar 2018 11:20:48 -0500 Subject: [PATCH 151/167] Add ability to suppress display when running commands --- simdem/mode/common.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 23f392f..294878b 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -67,15 +67,18 @@ def process_file(self, file_path, is_prereq=False, toc={}): if 'toc' in steps: self.process_next_steps(steps['toc'], start_path) # pylint: disable=no-member - def process_commands(self, cmds): + def process_commands(self, cmds, display=True): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: - self.ui.print_prompt() - self.print_command(cmd) - self.ui.print_break() + if display: + self.ui.print_prompt() + self.print_command(cmd) + self.ui.print_break() result = self.executor.run_cmd(cmd) - self.ui.print_result(result) - self.ui.print_break() + if display: + self.ui.print_result(result) + if display: + self.ui.print_break() return result def print_command(self, cmd): From b89400f23f39cef3b77dcdbce963d208626742b0 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 16 Mar 2018 13:26:34 -0500 Subject: [PATCH 152/167] Add 1st pass for SIMDEM_TEMP_DIR --- .gitignore | 1 + examples/config/demo.ini | 5 ++++- examples/config/unit_test.ini | 5 ++++- simdem/mode/common.py | 29 +++++++++++++++++++++-------- simdem/simdem.ini | 5 ++++- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index c181d1e..48a1535 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,4 @@ ENV/ .vscode simdem-env hello_world +.simdem diff --git a/examples/config/demo.ini b/examples/config/demo.ini index c38722b..cd00d97 100644 --- a/examples/config/demo.ini +++ b/examples/config/demo.ini @@ -1,5 +1,8 @@ [meta] -simdem_version = 2.0.0 +simdem_version = 0.9.0 + +[main] +temp_dir = .simdem # Logging [log] diff --git a/examples/config/unit_test.ini b/examples/config/unit_test.ini index d618800..05bc13d 100644 --- a/examples/config/unit_test.ini +++ b/examples/config/unit_test.ini @@ -1,5 +1,8 @@ [meta] -simdem_version = 2.0.0 +simdem_version = 0.9.0 + +[main] +temp_dir = .simdem # Logging [log] diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 294878b..ad0f222 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -3,6 +3,7 @@ import os import logging import difflib +import pathlib class ModeCommon(object): # pylint: disable=R0903 """ This class is designed to hold any shared code across modes @@ -30,12 +31,13 @@ def run_setup_script(self, file_path): self.ui.print_result(result) self.ui.print_break() - def process_file(self, file_path, is_prereq=False, toc={}): """ Parses the file and starts processing it """ logging.debug("parse_file(file_path=" + file_path + ", is_prereq=" + str(is_prereq)) # Change the working directory in case of any recursion start_path = os.path.dirname(os.path.abspath(file_path)) + + self.setup_temp_dir() logging.debug('parse_file::start_path=' + start_path) steps = self.parser.parse_file(file_path, is_prereq) @@ -70,17 +72,22 @@ def process_file(self, file_path, is_prereq=False, toc={}): def process_commands(self, cmds, display=True): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: - if display: - self.ui.print_prompt() - self.print_command(cmd) - self.ui.print_break() - result = self.executor.run_cmd(cmd) - if display: - self.ui.print_result(result) + result = self.process_command(cmd, display) if display: self.ui.print_break() return result + def process_command(self, cmd, display=True): + """ Process single command """ + if display: + self.ui.print_prompt() + self.print_command(cmd) + self.ui.print_break() + result = self.executor.run_cmd(cmd) + if display: + self.ui.print_result(result) + return result + def print_command(self, cmd): self.ui.print_cmd(cmd) @@ -121,3 +128,9 @@ def is_result_valid(expected_results, actual_results, expected_similarity=0.8): logging.error("expected_results = " + expected_results) return is_pass + + def setup_temp_dir(self): + logging.info("temp_dir=" + self.config.get('main', 'temp_dir', raw=True)) + directory = os.path.abspath(self.config.get('main', 'temp_dir', raw=True)) + self.process_command("mkdir -p " + directory, display=False) + self.process_command("export SIMDEM_TEMP_DIR=" + directory, display=False) \ No newline at end of file diff --git a/simdem/simdem.ini b/simdem/simdem.ini index 57123e3..e6bfa37 100644 --- a/simdem/simdem.ini +++ b/simdem/simdem.ini @@ -1,5 +1,8 @@ [meta] -simdem_version = 2.0.0 +simdem_version = 0.9.0 + +[main] +temp_dir = .simdem # Logging [log] From d6646ebaca117baeea5589deedb1e1ee274cd1a1 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 16 Mar 2018 14:02:36 -0500 Subject: [PATCH 153/167] Add reference to users home dir --- simdem/mode/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index ad0f222..a0c0675 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -130,7 +130,7 @@ def is_result_valid(expected_results, actual_results, expected_similarity=0.8): return is_pass def setup_temp_dir(self): - logging.info("temp_dir=" + self.config.get('main', 'temp_dir', raw=True)) - directory = os.path.abspath(self.config.get('main', 'temp_dir', raw=True)) + directory = str(pathlib.Path.home()) + '/' + self.config.get('main', 'temp_dir', raw=True) + logging.info("temp_dir=" + directory) self.process_command("mkdir -p " + directory, display=False) self.process_command("export SIMDEM_TEMP_DIR=" + directory, display=False) \ No newline at end of file From a658d6fc52ba0eebf97649127cb4c42263758638 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 19 Mar 2018 11:51:40 -0500 Subject: [PATCH 154/167] Add relevant comments --- simdem/mode/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index a0c0675..783c5dc 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -89,6 +89,7 @@ def process_command(self, cmd, display=True): return result def print_command(self, cmd): + """ Default action to print the command is to just call the UI. """ self.ui.print_cmd(cmd) @staticmethod @@ -130,6 +131,7 @@ def is_result_valid(expected_results, actual_results, expected_similarity=0.8): return is_pass def setup_temp_dir(self): + """ https://github.com/Azure/simdem/issues/104 """ directory = str(pathlib.Path.home()) + '/' + self.config.get('main', 'temp_dir', raw=True) logging.info("temp_dir=" + directory) self.process_command("mkdir -p " + directory, display=False) From 71f306fe56778efe77f173ce0cb8d8cfd7fc6124 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 19 Mar 2018 11:54:48 -0500 Subject: [PATCH 155/167] Add support for "d" https://github.com/Azure/simdem/issues/107 --- simdem/mode/interactive.py | 15 ++++++++++++--- simdem/mode/tutorial.py | 4 +++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index cd48b1e..5da2cc2 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -7,7 +7,7 @@ class InteractiveMode(ModeCommon): """ Interactive Mode subclass """ - def process_commands(self, cmds): + def process_commands(self, cmds, last_text=None): """ Loop through the commands to run as well as expect interrupt logic from the user """ result = None cmd = None @@ -18,14 +18,18 @@ def process_commands(self, cmds): while True: self.ui.print_prompt() key = self.ui.get_single_key_input() - result = self.process_command_input(key, last_command=cmd) + result = self.process_command_input(key, last_command=cmd, last_text=last_text) logging.debug(cmd_deque) + if result: + # We received a special key. Loop through again continue elif cmd_deque: + # Are there still commands left to be run? cmd = cmd_deque.popleft() result = self.run_command(cmd) if not cmd_deque: + # We've completed all commands break return result @@ -40,7 +44,7 @@ def run_command(self, cmd): self.ui.print_result(result) return result - def process_command_input(self, key, last_command=None): + def process_command_input(self, key, last_command=None, last_text=None): """ Process the command input. It's 4AM and I'm sleepy For now, just return. We'll implement that later """ result = None @@ -55,6 +59,11 @@ def process_command_input(self, key, last_command=None): logging.debug('Received Run last command request') if last_command: result = self.run_command(last_command) + elif key == 'd': + logging.debug('Received Show last description request') + if last_text: + self.ui.println(last_text) + result = last_text logging.debug('Output=' + str(result)) return result # Otherwise, we will return and assume the user wants to continue diff --git a/simdem/mode/tutorial.py b/simdem/mode/tutorial.py index 59863c0..78c3f82 100644 --- a/simdem/mode/tutorial.py +++ b/simdem/mode/tutorial.py @@ -15,14 +15,16 @@ def process(self, steps): """ Processes the steps from a processed file """ logging.debug("process()") + last_text = None # self.ui.clear() for step in steps['body']: if step['type'] == 'heading': self.ui.print_heading(step['content'], step['level']) elif step['type'] == 'text': self.ui.println(step['content']) + last_text = step['content'] elif step['type'] == 'commands': - last_command_result = self.process_commands(step['content']) + last_command_result = self.process_commands(step['content'], last_text=last_text) logging.debug(step) if 'expected_result' in step: if self.is_result_valid(step['expected_result'], last_command_result): From f96afc5d625ed6d2114c873feacbdbbca3fb8ef1 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Mon, 19 Mar 2018 13:06:42 -0500 Subject: [PATCH 156/167] Add color to output --- simdem/cli.py | 8 +++-- simdem/ui/basic.py | 66 ++-------------------------------------- simdem/ui/color.py | 64 +++++++++++++++++++++++++++++++++++++++ simdem/ui/common.py | 74 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 67 deletions(-) create mode 100644 simdem/ui/color.py create mode 100644 simdem/ui/common.py diff --git a/simdem/cli.py b/simdem/cli.py index dc84955..4a80a3a 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -9,7 +9,7 @@ from simdem.executor import bash from simdem.parser import ast, simdem1 from simdem.mode import demo, dump, test, tutorial, cleanup -from simdem.ui import basic +from simdem.ui import basic, color def main(): """ Main execution function """ @@ -30,8 +30,8 @@ def main(): help="Environment variables to inject (Example: -e FOO=foo1,BAR=bar1)") argp.add_argument('--boot-strap', '-b', default=None, help="Boot strap script to execute") - argp.add_argument('--ui', '-u', default="basic", - help="UI class to use", choices=['basic']) + argp.add_argument('--ui', '-u', default='color', + help="UI class to use", choices=['basic', 'color']) argp.add_argument('--override-config', metavar='override', help="Override setting in config file") version = pkg_resources.require("simdem")[0].version @@ -109,6 +109,8 @@ def get_ui(options, config): """ return UI object """ if options.ui == 'basic': return basic.BasicUI(config) + elif options.ui == 'color': + return color.ColorUI(config) def get_parser(options): """ Returns correct parser object """ diff --git a/simdem/ui/basic.py b/simdem/ui/basic.py index a570ae0..25188f7 100644 --- a/simdem/ui/basic.py +++ b/simdem/ui/basic.py @@ -3,72 +3,10 @@ import os import sys from simdem.misc.getch import Getch +from simdem.ui.common import CommonUI -class BasicUI(object): +class BasicUI(CommonUI): """ No frills, no thrills render object """ def __init__(self, config): self.config = config - - @staticmethod - def get_single_key_input(): - """ SimDem1 uses this method: - https://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key - For SimDem2, I'm trying this alternative to allow for Windows compatibility - https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user/510404#510404 - - https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not - Might need to allow config override of the conditional by allowing a config variable. - Experienced an issue where it hung running in a container and I didn't completely debug - """ - - if sys.stdout.isatty(): - getch = Getch() - return getch.impl() - return - - @staticmethod - def get_line_input(prompt): - """ Request single line from user """ - return input(prompt) - - @staticmethod - def println(output=''): - print(output, flush=True) - - @staticmethod - def print(output=''): - print(output, end="", flush=True) - - def print_validation_failed(self): - self.println('***PREREQUISITE VALIDATION FAILED***') - - @staticmethod - def clear(): - # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python - #print("\033[H\033[J") - if sys.stdout.isatty(): - os.system('clear') - - def print_test_passed(self): - self.println('*** SIMDEM TEST RESULT PASSED ***') - - def print_test_failed(self): - self.println('*** SIMDEM TEST RESULT FAILED ***') - - def print_heading(self, content, level): - """ Print out the heading exactly as we found it """ - self.println(level * '#' + ' ' + content) - self.print_break() - - def print_prompt(self): - self.print(self.config.get('render', 'console_prompt', raw=True) + ' ') - - def print_cmd(self, cmd): - self.print(cmd) - - def print_result(self, result): - self.print(result) - - def print_break(self): - self.println() \ No newline at end of file diff --git a/simdem/ui/color.py b/simdem/ui/color.py new file mode 100644 index 0000000..1e7eaa7 --- /dev/null +++ b/simdem/ui/color.py @@ -0,0 +1,64 @@ +""" Basic Render Class """ + +import os +import sys +import colorama +from simdem.misc.getch import Getch +from simdem.ui.common import CommonUI + +colorama.init(strip=None) + +class ColorUI(CommonUI): + """ No frills, no thrills render object """ + + def __init__(self, config): + self.config = config + + def println(self, output='', color=colorama.Fore.WHITE + colorama.Style.BRIGHT): + self.display(output, color=color) + + def print(self, output='', color=None): + self.display(output, end="", flush=True, color=color) + + def print_validation_failed(self): + self.println('***PREREQUISITE VALIDATION FAILED***', color=colorama.Fore.RED + colorama.Style.BRIGHT) + + @staticmethod + def clear(): + # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python + #print("\033[H\033[J") + if sys.stdout.isatty(): + os.system('clear') + + def print_test_passed(self): + self.println('*** SIMDEM TEST RESULT PASSED ***', color=colorama.Fore.GREEN + colorama.Style.BRIGHT) + + def print_test_failed(self): + self.println('*** SIMDEM TEST RESULT FAILED ***', color=colorama.Fore.RED + colorama.Style.BRIGHT) + + def print_heading(self, content, level): + """ Print out the heading exactly as we found it """ + self.println(level * '#' + ' ' + content, color=colorama.Fore.CYAN + colorama.Style.BRIGHT) + self.print_break() + + def print_prompt(self): + self.print(self.config.get('render', 'console_prompt', raw=True) + ' ', color=colorama.Fore.WHITE) + + def print_cmd(self, cmd): + self.print(cmd, color=colorama.Fore.WHITE + colorama.Style.BRIGHT) + + def print_result(self, result): + self.print(result, color=colorama.Fore.GREEN + colorama.Style.BRIGHT) + + def print_break(self): + self.println() + + def display(self, text, color=None, flush=False, end='\n'): + """ Display some text in a given color. Do not print a new line unless + new_line is set to True. + """ + if color: + print(color, end="") + print(text, end=end, flush=flush) + if color: + print(colorama.Style.RESET_ALL, end="") \ No newline at end of file diff --git a/simdem/ui/common.py b/simdem/ui/common.py new file mode 100644 index 0000000..a00d328 --- /dev/null +++ b/simdem/ui/common.py @@ -0,0 +1,74 @@ +""" Basic Render Class """ + +import os +import sys +from simdem.misc.getch import Getch + +class CommonUI(object): + """ No frills, no thrills render object """ + + def __init__(self, config): + self.config = config + + @staticmethod + def get_single_key_input(): + """ SimDem1 uses this method: + https://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key + For SimDem2, I'm trying this alternative to allow for Windows compatibility + https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user/510404#510404 + + https://stackoverflow.com/questions/1077113/how-do-i-detect-whether-sys-stdout-is-attached-to-terminal-or-not + Might need to allow config override of the conditional by allowing a config variable. + Experienced an issue where it hung running in a container and I didn't completely debug + """ + + if sys.stdout.isatty(): + getch = Getch() + return getch.impl() + return + + @staticmethod + def get_line_input(prompt): + """ Request single line from user """ + return input(prompt) + + @staticmethod + def println(output=''): + print(output, flush=True) + + @staticmethod + def print(output=''): + print(output, end="", flush=True) + + def print_validation_failed(self): + self.println('***PREREQUISITE VALIDATION FAILED***') + + @staticmethod + def clear(): + # https://www.quora.com/Is-there-a-Clear-screen-function-in-Python + #print("\033[H\033[J") + if sys.stdout.isatty(): + os.system('clear') + + def print_test_passed(self): + self.println('*** SIMDEM TEST RESULT PASSED ***') + + def print_test_failed(self): + self.println('*** SIMDEM TEST RESULT FAILED ***') + + def print_heading(self, content, level): + """ Print out the heading exactly as we found it """ + self.println(level * '#' + ' ' + content) + self.print_break() + + def print_prompt(self): + self.print(self.config.get('render', 'console_prompt', raw=True) + ' ') + + def print_cmd(self, cmd): + self.print(cmd) + + def print_result(self, result): + self.print(result) + + def print_break(self): + self.println() \ No newline at end of file From 8185a02330349748f54da0c31a75157f22a35e09 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 12 Apr 2018 10:19:14 -0500 Subject: [PATCH 157/167] Add support for env.sh --- simdem/mode/common.py | 15 ++++++++++++++- simdem/mode/interactive.py | 17 ++++++++++------- tests/test_mode_demo.py | 2 +- tests/test_mode_tutorial.py | 2 +- tests/test_parser_simdem1.py | 2 +- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/simdem/mode/common.py b/simdem/mode/common.py index 783c5dc..7ea36f9 100644 --- a/simdem/mode/common.py +++ b/simdem/mode/common.py @@ -41,6 +41,11 @@ def process_file(self, file_path, is_prereq=False, toc={}): logging.debug('parse_file::start_path=' + start_path) steps = self.parser.parse_file(file_path, is_prereq) + # https://github.com/Azure/simdem/issues/92 + env_file = start_path + '/./env.sh' + if os.path.isfile(env_file): + self.process_env_file(env_file) + # We want to inherit the parent's TOC to reduce the # of copies needed if toc: logging.debug('Adding parent POC') @@ -69,10 +74,18 @@ def process_file(self, file_path, is_prereq=False, toc={}): if 'toc' in steps: self.process_next_steps(steps['toc'], start_path) # pylint: disable=no-member + def process_env_file(self, env_file): + """ Run the env file. Assumes it exists """ + logging.debug('process_env_file(' + env_file + ')') + env_fh = open(env_file) + env_contents = env_fh.readlines() + env_fh.close() + self.process_commands(env_contents) + def process_commands(self, cmds, display=True): """ Pretend to type the command, run it and then display the output """ for cmd in cmds: - result = self.process_command(cmd, display) + result = self.process_command(cmd, display=display) if display: self.ui.print_break() return result diff --git a/simdem/mode/interactive.py b/simdem/mode/interactive.py index 5da2cc2..31a48f1 100644 --- a/simdem/mode/interactive.py +++ b/simdem/mode/interactive.py @@ -7,7 +7,7 @@ class InteractiveMode(ModeCommon): """ Interactive Mode subclass """ - def process_commands(self, cmds, last_text=None): + def process_commands(self, cmds, last_text=None, display=True): """ Loop through the commands to run as well as expect interrupt logic from the user """ result = None cmd = None @@ -27,21 +27,24 @@ def process_commands(self, cmds, last_text=None): elif cmd_deque: # Are there still commands left to be run? cmd = cmd_deque.popleft() - result = self.run_command(cmd) + result = self.run_command(cmd, display=display) if not cmd_deque: # We've completed all commands break return result - def run_command(self, cmd): + def run_command(self, cmd, display=True): """ Pretend to type the command, run it and then display the output """ # Request enter from user to know when to proceed logging.debug('run_command(' + cmd + ')') - self.print_command(cmd) - # For some reason this requires a print_break() while common does not. Too late to debug - self.ui.print_break() + + if display: + self.print_command(cmd) + # For some reason this requires a print_break() while common does not. Too late to debug + self.ui.print_break() result = self.executor.run_cmd(cmd) - self.ui.print_result(result) + if display: + self.ui.print_result(result) return result def process_command_input(self, key, last_command=None, last_text=None): diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index 8ae1226..e768156 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -36,7 +36,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', 'toc', - 'results-block-fail', 'prerequisites') + 'results-block-fail', 'prerequisites', 'env') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.demo this allows us to test each of them easily diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index 14826a9..8daa8ed 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -37,7 +37,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', 'toc', - 'results-block-fail', 'prerequisites') + 'results-block-fail', 'prerequisites', 'env') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index e25376f..b324683 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -25,7 +25,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer #@data('markdown-syntax') @data('simple', 'simple-variable', 'results-block', 'toc', - 'results-block-fail', 'prerequisites') + 'results-block-fail', 'prerequisites', 'env') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.seo this allows us to test each of them easily From 0ce78ed55972a423751daae76d803d669e05482f Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 12 Apr 2018 10:26:03 -0500 Subject: [PATCH 158/167] Fix env-files dir name which was excluded due to gitignore --- examples/environment-files/README.md | 9 ++++++++ examples/environment-files/env.sh | 2 ++ .../environment-files/expected_result.demo | 4 ++++ .../environment-files/expected_result.seo | 23 +++++++++++++++++++ .../expected_result.tutorial | 11 +++++++++ tests/test_mode_demo.py | 2 +- tests/test_mode_tutorial.py | 2 +- tests/test_parser_simdem1.py | 2 +- 8 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 examples/environment-files/README.md create mode 100644 examples/environment-files/env.sh create mode 100644 examples/environment-files/expected_result.demo create mode 100644 examples/environment-files/expected_result.seo create mode 100644 examples/environment-files/expected_result.tutorial diff --git a/examples/environment-files/README.md b/examples/environment-files/README.md new file mode 100644 index 0000000..d3d9795 --- /dev/null +++ b/examples/environment-files/README.md @@ -0,0 +1,9 @@ +# testing importing env.sh file + +text1 + +```shell +echo $FOO +``` + +text2 diff --git a/examples/environment-files/env.sh b/examples/environment-files/env.sh new file mode 100644 index 0000000..a4b9c23 --- /dev/null +++ b/examples/environment-files/env.sh @@ -0,0 +1,2 @@ +FOO=foo +BAR=bar \ No newline at end of file diff --git a/examples/environment-files/expected_result.demo b/examples/environment-files/expected_result.demo new file mode 100644 index 0000000..ed09d86 --- /dev/null +++ b/examples/environment-files/expected_result.demo @@ -0,0 +1,4 @@ +$ FOO=foo +$ BAR=bar +$ echo $FOO +foo diff --git a/examples/environment-files/expected_result.seo b/examples/environment-files/expected_result.seo new file mode 100644 index 0000000..97d5cad --- /dev/null +++ b/examples/environment-files/expected_result.seo @@ -0,0 +1,23 @@ +{ + "body": [ + { + "content": "testing importing env.sh file", + "level": 1, + "type": "heading" + }, + { + "content": "text1\n", + "type": "text" + }, + { + "content": [ + "echo $FOO" + ], + "type": "commands" + }, + { + "content": "text2\n", + "type": "text" + } + ] +} diff --git a/examples/environment-files/expected_result.tutorial b/examples/environment-files/expected_result.tutorial new file mode 100644 index 0000000..af3b1ca --- /dev/null +++ b/examples/environment-files/expected_result.tutorial @@ -0,0 +1,11 @@ +$ FOO=foo + +$ BAR=bar +# testing importing env.sh file + +text1 + +$ echo $FOO +foo +text2 + diff --git a/tests/test_mode_demo.py b/tests/test_mode_demo.py index e768156..1c8b953 100644 --- a/tests/test_mode_demo.py +++ b/tests/test_mode_demo.py @@ -36,7 +36,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', 'toc', - 'results-block-fail', 'prerequisites', 'env') + 'results-block-fail', 'prerequisites', 'environment-files') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.demo this allows us to test each of them easily diff --git a/tests/test_mode_tutorial.py b/tests/test_mode_tutorial.py index 8daa8ed..e487d77 100644 --- a/tests/test_mode_tutorial.py +++ b/tests/test_mode_tutorial.py @@ -37,7 +37,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer @data('simple', 'simple-variable', 'results-block', 'toc', - 'results-block-fail', 'prerequisites', 'env') + 'results-block-fail', 'prerequisites', 'environment-files') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.tutorial this allows us to test each of them easily diff --git a/tests/test_parser_simdem1.py b/tests/test_parser_simdem1.py index b324683..be085b6 100644 --- a/tests/test_parser_simdem1.py +++ b/tests/test_parser_simdem1.py @@ -25,7 +25,7 @@ def setUp(self): # https://docs.python.org/3/library/unittest.html#unittest.TestResult.buffer #@data('markdown-syntax') @data('simple', 'simple-variable', 'results-block', 'toc', - 'results-block-fail', 'prerequisites', 'env') + 'results-block-fail', 'prerequisites', 'environment-files') def test_process(self, directory): """ Each examples directory is expected to have a README.md and an expected_result.seo this allows us to test each of them easily From 9f267ab1f39e2eb9b5fbf09c6a824e204b091967 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Wed, 16 May 2018 15:49:42 -0500 Subject: [PATCH 159/167] fix https://github.com/Azure/simdem/issues/109 --- simdem/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simdem/cli.py b/simdem/cli.py index 4a80a3a..93805c4 100755 --- a/simdem/cli.py +++ b/simdem/cli.py @@ -51,6 +51,9 @@ def main(): mode = get_mode(options, config) + # Add user's home directory to path + mode.process_command("export HOME=" + os.path.expanduser("~")) + if options.environment: commands = options.environment.split(',') mode.process_commands(commands) From 84a9946edfa54edf7c278bd5eda5abe0759a0056 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 17 May 2018 15:26:51 -0500 Subject: [PATCH 160/167] Update features.md --- docs/features.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features.md b/docs/features.md index bf00ce1..7709129 100644 --- a/docs/features.md +++ b/docs/features.md @@ -1,6 +1,6 @@ # Features -This document is intended to be a list of all features supported in SimDem. To see examples on how to write documents that use these features, please see the [syntax documentation](syntad.md). +This document is intended to be a list of all features supported in SimDem. To see examples on how to write documents that use these features, please see the [syntax documentation](syntax.md). ## Commands @@ -38,4 +38,4 @@ Validation is used to verify that the expected result is matched. The two main * Result Pattern Match * Exit code -For details, see the [validation feature documentation](feature_validation.md) \ No newline at end of file +For details, see the [validation feature documentation](feature_validation.md) From f9333cc22ccc66c359742177449a237693e92fdf Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 18 May 2018 15:28:10 -0500 Subject: [PATCH 161/167] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ced7f24..4faf441 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,7 @@ pip3 install -r requirements.txt pip3 install -v -e . ``` -Note: in rare machine configuations you may get a "simdem: command not -found" error after installing. If you see this error take a look at -the workaround in [issue #99](https://github.com/Azure/simdem/issues/99). +Note: In rare machine configuations you may get a "simdem: command not found" error after installing. If you see this error take a look at the workaround in [issue #99](https://github.com/Azure/simdem/issues/99). ## Running From 42ebfa762751aa7e30b793ffb70003204e8ea0f3 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 18 May 2018 15:28:57 -0500 Subject: [PATCH 162/167] Update syntax.md --- docs/syntax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/syntax.md b/docs/syntax.md index 84b9fee..b3cdbda 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -6,7 +6,7 @@ Currently, there is only one implementation of Markdown syntax supported. # Context Syntax Specification -This is the syntax for the default codeblock format. It's design is to allow more natural, expressive, and readable documentation. It is based off of SimDem v1's syntax +This is the syntax for the default codeblock format. It's design is to allow more natural, expressive, and readable documentation. It is based off of SimDem v1's syntax. ## SimDem V1 Based (default) From 3a6f05344b95fabf5d463b8bd54cd9eda84d4c6d Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 18 May 2018 15:30:24 -0500 Subject: [PATCH 163/167] Update README.md --- examples/simdem1/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simdem1/README.md b/examples/simdem1/README.md index f2dbd8b..c69ab81 100644 --- a/examples/simdem1/README.md +++ b/examples/simdem1/README.md @@ -2,11 +2,11 @@ This is the prerequisite section. SimDem extracts links to run through prior to executing the main steps. -Here is an example of a prerequisite that will be ignored because it's conditions are already met. +Here is an example of a prerequisite that will be ignored because its conditions are already met. * [prereq-ignored](./prereq-ignored.md) -Here is an example of a prerequisites that will be run because it's conditions are not met. +Here is an example of a prerequisites that will be run because its conditions are not met. * [prereq-processed](./prereq-processed.md) From 0a8b2557e0de8f14c9200013eb01cedb008ab885 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 18 May 2018 15:30:52 -0500 Subject: [PATCH 164/167] Update README.md --- examples/simdem1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simdem1/README.md b/examples/simdem1/README.md index c69ab81..220ee54 100644 --- a/examples/simdem1/README.md +++ b/examples/simdem1/README.md @@ -10,7 +10,7 @@ Here is an example of a prerequisites that will be run because its conditions ar * [prereq-processed](./prereq-processed.md) -By this point, the prerequisites have either run or have passed their validation +By this point, the prerequisites have either run or have passed their validation. # Did our prerequisites run? From 76e339e1e278bece264b665fcca9c38556341866 Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Fri, 18 May 2018 15:31:47 -0500 Subject: [PATCH 165/167] Update README.md --- examples/simdem1/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/simdem1/README.md b/examples/simdem1/README.md index 220ee54..99cadd7 100644 --- a/examples/simdem1/README.md +++ b/examples/simdem1/README.md @@ -21,7 +21,7 @@ echo prereq_processed = $prereq_processed # Do stuff here -We want to execute this because the code type is shell +We want to execute this because the code type is shell. ```shell echo foo @@ -31,7 +31,7 @@ var=bar # Do more stuff here -We assume the result is for the last command of the last code block +We assume the result is for the last command of the last code block. ```shell echo baz From 8ffe9966bed994a8d665a85cc2b7eb9c14b2e7ad Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Sun, 20 May 2018 18:49:42 -0500 Subject: [PATCH 166/167] Improve main README.md Signed-off-by: Tommy Falgout --- README.md | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ced7f24..e862f8f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # SimDem -This project provides an easy way to convert tutorials written in markdown into interactive demos and automated tests. +SimDem provides an easy way to convert tutorials written in markdown into interactive demos and automated tests. -# Features +## Features SimDem supports the following features: * Command execution @@ -12,11 +12,11 @@ SimDem supports the following features: Details on the complete feature list can be found in the [feature documentation](docs/features.md). -# Getting Started +## Getting Started -## Installation +### Installation -Currently, it's only available for installation in development mode: +Currently, only available for installation in development mode: ``` git clone git@github.com:Azure/simdem.git @@ -25,13 +25,11 @@ pip3 install -r requirements.txt pip3 install -v -e . ``` -Note: in rare machine configuations you may get a "simdem: command not -found" error after installing. If you see this error take a look at -the workaround in [issue #99](https://github.com/Azure/simdem/issues/99). +Note: in rare machine configuations you may get a "simdem: command not found" error after installing. If so, see [this workaround (issue #99)](https://github.com/Azure/simdem/issues/99). -## Running +### Running -A great place to start working on SimDem is to run SimDem on its own documentation +After installing, a great place to start is to run SimDem on its own documentation. ``` simdem docs/README.md @@ -39,23 +37,37 @@ simdem docs/README.md ## Documentation -It's more fun to run SimDem on it's own documentation; however, you can learn how how SimDem works by [reading the docs](https://github.com/Azure/simdem/tree/simdem2/docs). +You can learn how how SimDem works by [reading the docs](https://github.com/Azure/simdem/tree/simdem2/docs). -We provide a [simple hello-world example](https://github.com/Azure/simdem/blob/simdem2/docs/hello_world.md). +Here is a [simple hello-world example](https://github.com/Azure/simdem/blob/simdem2/docs/hello_world.md). ### Examples If you want to see existing examples, with expected output, check out the [examples](https://github.com/Azure/simdem/tree/simdem2/examples) -# Syntax +## Syntax Currently, SimDem supports Markdown as the source document. Details on how to compose Markdown documents can be found in the [syntax documentation](docs/syntax.md). -# Development +## Built With + +* [Mistletoe](https://github.com/miyuchina/mistletoe) + +## Contributing We would love to have you be a part of the SimDem development team. For details, see the [development documentation](docs/development.md). -# History +## History SimDem v2 is a complete rewrite of SimDem v1. The latest commit for v1 can be found at: https://github.com/Azure/simdem/tree/cb1caf17fd684e125789c26817f43eeae0e1c523 + +## License + +This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details. + +## Acknowledgements + +* [Ross Gardler](https://twitter.com/rgardler) - The original creator of SimDem +* [Mi Yu](https://github.com/miyuchina) - Author of Mistletoe who provided guidance on Markdown parsing +* [Tommy Falgout](https://lastcoolnameleft.com) - Author of SimDem v2 \ No newline at end of file From 6b5d40fcb38649584de4e54953c96b861ea709ee Mon Sep 17 00:00:00 2001 From: Tommy Falgout Date: Thu, 16 Aug 2018 11:49:12 -0500 Subject: [PATCH 167/167] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8ebbe72..fb61e81 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ pip3 install -r requirements.txt pip3 install -v -e . ``` -Note: in rare machine configuations you may get a "simdem: command not found" error after installing. If so, see [this workaround (issue #99)](https://github.com/Azure/simdem/issues/99). +If you are experiencing issues with installation, please see these issues: +* ["simdem: command not found" error](https://github.com/Azure/simdem/issues/99) +* [cannot import name 'main'](https://github.com/Azure/simdem/issues/111) ### Running @@ -71,4 +73,4 @@ This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md * [Ross Gardler](https://twitter.com/rgardler) - The original creator of SimDem * [Mi Yu](https://github.com/miyuchina) - Author of Mistletoe who provided guidance on Markdown parsing -* [Tommy Falgout](https://lastcoolnameleft.com) - Author of SimDem v2 \ No newline at end of file +* [Tommy Falgout](https://lastcoolnameleft.com) - Author of SimDem v2