diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e53c0a --- /dev/null +++ b/.gitignore @@ -0,0 +1,218 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +# Others +nohup.out + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/gitcproc.iml b/.idea/gitcproc.iml new file mode 100644 index 0000000..4f2c9af --- /dev/null +++ b/.idea/gitcproc.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d1e22ec --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..62ea834 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 1f917f0..de04323 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: # We don't actually use the Travis Python, but this keeps it organized. - - "2.7" + - "3.8.6" before_script: cd src/logChunk @@ -24,13 +24,13 @@ install: # weird travis-ci python paths - export PYTHONPATH=$PYTHONPATH:./src/util/ - export PYTHONPATH=$PYTHONPATH:./src/logChunk/ - - pip install psycopg2 - - pip install PyYAML - - pip install nltk - - pip install gitPython + - pip3 install psycopg2 + - pip3 install PyYAML + - pip3 install nltk==3.4 + - pip3 install gitPython #Set up nltk - - python -m nltk.downloader stopwords - - python -m nltk.downloader wordnet + - python3 -m nltk.downloader stopwords + - python3 -m nltk.downloader wordnet # Replace dep1 dep2 ... with your dependencies # - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION dep1 dep2 ... @@ -39,15 +39,14 @@ install: script: # Your test script goes here - - python ./scopeTrackerTest.py - - python ./logChunkTestC.py - - python ./logChunkTestJAVA.py - - python ./logChunkTestPython.py - - python ./ghLogDbTest.py - - python ./ghLogDbTestPython.py - #- python ./gitcprocTest.py + - python3 ./scopeTrackerTest.py + - python3 ./logChunkTestC.py + - python3 ./logChunkTestJAVA.py + - python3 ./logChunkTestPython.py + - python3 ./ghLogDbTest.py + - python3 ./ghLogDbTestPython.py + #- python3 ./gitcprocTest.py notifications: email: - yvsuchak@ucdavis.edu - caseycasal@gmail.com - diff --git a/README.md b/README.md index fe7e7a4..61df089 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # GitCProc + GitCProc is a tool to extract changes to elements in code and associate them with their surrounding functions. It takes git diff logs as input and produces a table mapping the additions and deletions of the elements to a 4-tuple (project, sha, file, function/method) specifying the commit and location in the code of changed elements. @@ -6,21 +7,20 @@ It also analyzes commit messages to estimate if the commits were bug fixing or n Currently, we have designed it to work with C/C++/Java/Python, but we have designed the framework to be extensible to other languages that make use of the concept of scope and source blocks. -# Video Demo +## Video Demo -There is a video walkthrough of running the tool on a simple example at: https://youtu.be/shugzDjxj0w -A shortened version can be found here: https://youtu.be/5sOUoMHuP9s +There is a video walkthrough of running the tool on a simple example at: https://youtu.be/shugzDjxj0w. A shortened version can be found here: https://youtu.be/5sOUoMHuP9s. -# Walkthrough +## Walkthrough There is also a text version of the video walkthrough in the file "ExampleWalkthrough". -# Docker +## Docker A docker to handle the installation of these libraries and install postgres can be obtained with: -``` +```bash docker pull caseycas/gitcproc-docker docker tag caseycas/gitcproc-docker gitcproc-docker docker run --name gitcproc-docker -e POSTGRES_PASSWORD=postgres -d gitcproc-docker @@ -30,7 +30,7 @@ docker exec -it gitcproc-docker bash An example project and search can be run in the docker with (password prompt needs 'postgres'): This should take about a minute to run. -``` +```bash cd src/logChunk/ python gitcproc.py -d -pl -wl ../util/docker_conf.ini psql -U postgres @@ -38,32 +38,32 @@ SELECT * FROM public.change_summary limit 5; SELECT * FROM public.method_change_detail limit 5; ``` -# Required Libraries +## Required Libraries + GitCProc runs on python 2.7 and requires the following libraries: --psycopg2 --nltk --PyYAML --GitPython -The python script src/logChunk/installDependencies.py will install these for you. Be aware that psycopg2 requires postgres to be installed on your machine with necessary supporting libraries. For Ubuntu, make sure you have libpq-dev installed. +- `psycopg2-binary` +- `nltk` +- `PyYAML` +- `GitPython` -If you wish to output your results to a database instead of a csv file, you will need a postgres server installed. +The python script `src/logChunk/installDependencies.py` will install these for you. Be aware that psycopg2 requires postgres to be installed on your machine with necessary supporting libraries. For Ubuntu, make sure you have `libpq-dev` installed. +If you wish to output your results to a database instead of a csv file, you will need a postgres server installed. -# Running Instructions +## Running Instructions -The master script is located at src/logChunk/gitcproc.py, which depends on 3 input files to run. +The master script is located at `src/logChunk/gitcproc.py`, which depends on 3 input files to run. -1) First, if you want to download your projects from GitHub, you need a file to specify the list of projects -from GitHub. The file format is a list of full project names, e.g. caseycas/gitcproc, one on each line. +1) First, if you want to download your projects from GitHub, you need a file to specify the list of projects from GitHub. The file format is a list of full project names, e.g. caseycas/gitcproc, one on each line. 2) The keywords file specifies what structures you wish to track. It is formatted in the following manner: -
+ +```txt [keyword_1], [INCLUDED/EXCLUDED], [SINGLE/BLOCK] -
... -
[keyword_N],[INCLUDED/EXCLUDED], [SINGLE/BLOCK] +``` The first part is the keyword you wish to track. For instance, if you are tracking assert statements, one keyword of interest would be "assert", if you were tracking try-catch blocks, you might have "try" and "catch". @@ -98,39 +98,36 @@ currently, we support C, C++, Java, and Python file types. [Flags] - Finally, this section contains several output and debugging options: - SEP: -> A string used to flatten out downloaded project names. We recommend using ___ as we + - SEP: -> A string used to flatten out downloaded project names. We recommend using ___ as we have not observed GitHub projects using this in their names. To illustrate what it does, if this tool downloads caseycas/gitcproc with SEP: "___" the directory name it will be saved to is caseycas___gitcproc. - DEBUG: -> True or False, this outputs extensive debug information. It is highly recommended + - DEBUG: -> True or False, this outputs extensive debug information. It is highly recommended you set this to False when running on a real git project. - DEBUGLITE: -> True or False, this outputs very lightweight debug information. - DATABASE: -> True or False, this signals to write output to your Database. The tool will + - DEBUGLITE: -> True or False, this outputs very lightweight debug information. + - DATABASE: -> True or False, this signals to write output to your Database. The tool will create the tables if they don't exist yet, but you must have postgres installed will an available database and schema specified in [Database]. This option will prompt you for your database password once the script gitcproc.py starts. - CSV: -> True or False, this signals to write to a CSV file. This option is recommended if you - want to miminize set up and test the tool out. - LOGTIME: -> True or False, this signals to write performance time information. Currently it - measure time to parse the log and time to write out to the Database or CSV file. + - CSV: -> True or False, this signals to write to a CSV file. This option is recommended if you want to miminize set up and test the tool out. + - LOGTIME: -> True or False, this signals to write performance time information. Currently it measure time to parse the log and time to write out to the Database or CSV file. With these files created, to run the full process you can do something like: -python gitcproc.py -d -wl -pl ../util/sample_conf.ini +`python gitcproc.py -d -wl -pl ../util/sample_conf.ini` The config file is mandantory, and option -d runs the project download step, -wl creates the git log for downloaded projects, and -pl parses the logs and sends the output to your choosen format. - -# Running Tests +## Running Tests This repository is set up to run travis CI on each commit at: https://travis-ci.org/caseycas/gitcproc The test scripts are as follows: -``` +```bash src/logChunk/logChunkTestC.py src/logChunk/logChunkTestJAVA.py src/logChunk/logChunkTestPython src/logChunk/ghLogDbTest.py src/logChunk/ghLogDbTestPython.py src/logChunk/scopeTrackerTest.py -``` \ No newline at end of file +``` diff --git a/evaluation/log_files/.gitignore b/evaluation/log_files/.gitignore new file mode 100644 index 0000000..50c40a1 --- /dev/null +++ b/evaluation/log_files/.gitignore @@ -0,0 +1,2 @@ +*.err +*.out diff --git a/evaluation/repos/.gitignore b/evaluation/repos/.gitignore new file mode 100644 index 0000000..0dfa47f --- /dev/null +++ b/evaluation/repos/.gitignore @@ -0,0 +1,2 @@ +StreamProjects +FuncProjects diff --git a/evaluation/scripts/sql/select_execution_mode_changes.sql b/evaluation/scripts/sql/select_execution_mode_changes.sql new file mode 100644 index 0000000..a328692 --- /dev/null +++ b/evaluation/scripts/sql/select_execution_mode_changes.sql @@ -0,0 +1,31 @@ +COPY ( + SELECT method_change_detail.project, + method_change_detail.sha, + method_change_detail.language, + method_change_detail.file_name, + method_change_detail.is_test, + method_change_detail.method_name, + method_change_detail.parallel___adds, + method_change_detail.parallel___dels, + method_change_detail."parallelStream___adds", + method_change_detail."parallelStream___dels", + method_change_detail.sequential___adds, + method_change_detail.sequential___dels, + method_change_detail.total_adds, + method_change_detail.total_dels, + method_change_detail.warning_alert, + change_summary.is_bug + FROM method_change_detail + INNER JOIN change_summary + ON method_change_detail.sha = change_summary.sha + WHERE method_change_detail.parallel___adds > 0 + OR method_change_detail.parallel___dels > 0 + OR method_change_detail.sequential___adds > 0 + OR method_change_detail.sequential___dels > 0 + OR method_change_detail."parallelStream___adds" > 0 + OR method_change_detail."parallelStream___dels" > 0 + ORDER BY method_change_detail.project DESC, + method_change_detail.sha, + file_name, + method_name +) TO STDOUT with CSV HEADER diff --git a/evaluation/scripts/sql/select_function_bug_fixes_details.sql b/evaluation/scripts/sql/select_function_bug_fixes_details.sql new file mode 100644 index 0000000..dc52cb4 --- /dev/null +++ b/evaluation/scripts/sql/select_function_bug_fixes_details.sql @@ -0,0 +1,13 @@ +COPY ( + SELECT method_change_detail.* + FROM method_change_detail + INNER JOIN change_summary + ON method_change_detail.sha = change_summary.sha + WHERE change_summary.is_bug + AND ( "tf.function_adds" > 0 + OR "tf.function_dels" > 0 ) + ORDER BY method_change_detail.project DESC, + method_change_detail.sha, + file_name, + method_name +) TO STDOUT with CSV HEADER diff --git a/evaluation/scripts/sql/select_function_bug_fixes_summary.sql b/evaluation/scripts/sql/select_function_bug_fixes_summary.sql new file mode 100644 index 0000000..273a6c5 --- /dev/null +++ b/evaluation/scripts/sql/select_function_bug_fixes_summary.sql @@ -0,0 +1,18 @@ +COPY ( + SELECT DISTINCT On (change_summary.sha) change_summary.sha, + change_summary.project, + author, + author_email, + commit_date, + is_bug + FROM change_summary + INNER JOIN method_change_detail + ON method_change_detail.sha = change_summary.sha + WHERE change_summary.is_bug + AND ( "tf.function_adds" > 0 + OR "tf.function_dels" > 0 ) + ORDER BY change_summary.sha, + change_summary.project DESC, + file_name, + method_name +) TO STDOUT with CSV HEADER diff --git a/evaluation/scripts/sql/select_functions.sql b/evaluation/scripts/sql/select_functions.sql new file mode 100644 index 0000000..75a216b --- /dev/null +++ b/evaluation/scripts/sql/select_functions.sql @@ -0,0 +1,26 @@ +select + mcd.project, + mcd.sha, + cs.author, + cs.author_email, + cs.commit_date, + cs.is_bug, + mcd.language, + mcd.file_name, + mcd.is_test, + mcd.method_name, + mcd."tf.function_adds", + mcd."tf.function_dels", + mcd.total_adds, + mcd.total_dels, + mcd.warning_alert +from method_change_detail as mcd + inner join change_summary as cs + on mcd.sha = cs.sha +where + mcd."tf.function_adds" > 0 or + mcd."tf.function_dels" > 0 +order by + cs.project desc, + file_name, + method_name \ No newline at end of file diff --git a/evaluation/scripts/sql/select_stream_bug_fixes_details.sql b/evaluation/scripts/sql/select_stream_bug_fixes_details.sql new file mode 100644 index 0000000..39deaed --- /dev/null +++ b/evaluation/scripts/sql/select_stream_bug_fixes_details.sql @@ -0,0 +1,268 @@ +COPY ( + SELECT method_change_detail.* + FROM method_change_detail + INNER JOIN change_summary + ON method_change_detail.sha = change_summary.sha + WHERE change_summary.is_bug + AND change_summary.commit_date >= '2014-03-18' + AND ( "BaseStream_adds" > 0 + OR "BaseStream_dels" > 0 + OR "Collector_adds" > 0 + OR "Collector_dels" > 0 + OR "Characteristics_adds" > 0 + OR "Characteristics_dels" > 0 + OR "Collectors_adds" > 0 + OR "Collectors_dels" > 0 + OR "DoubleStream_adds" > 0 + OR "DoubleStream_dels" > 0 + OR "range__adds" > 0 + OR "range__dels" > 0 + OR "rangeClosed__adds" > 0 + OR "rangeClosed__dels" > 0 + OR "IntStream_adds" > 0 + OR "IntStream_dels" > 0 + OR "LongStream_adds" > 0 + OR "LongStream_dels" > 0 + OR "Spliterator_adds" > 0 + OR "Spliterator_dels" > 0 + OR "OfDouble_adds" > 0 + OR "OfDouble_dels" > 0 + OR "OfInt_adds" > 0 + OR "OfInt_dels" > 0 + OR "OfLong_adds" > 0 + OR "OfLong_dels" > 0 + OR "OfPrimitive_adds" > 0 + OR "OfPrimitive_dels" > 0 + OR "Stream_adds" > 0 + OR "Stream_dels" > 0 + OR "StreamSupport_adds" > 0 + OR "StreamSupport_dels" > 0 + OR "accumulator___adds" > 0 + OR "accumulator___dels" > 0 + OR "allMatch__adds" > 0 + OR "allMatch__dels" > 0 + OR "allProcesses___adds" > 0 + OR "allProcesses___dels" > 0 + OR "anyMatch__adds" > 0 + OR "anyMatch__dels" > 0 + OR "asDoubleStream___adds" > 0 + OR "asDoubleStream___dels" > 0 + OR "asIntStream___adds" > 0 + OR "asIntStream___dels" > 0 + OR "asLongStream___adds" > 0 + OR "asLongStream___dels" > 0 + OR "average___adds" > 0 + OR "average___dels" > 0 + OR "averagingDouble__adds" > 0 + OR "averagingDouble__dels" > 0 + OR "averagingInt__adds" > 0 + OR "averagingInt__dels" > 0 + OR "averagingLong__adds" > 0 + OR "averagingLong__dels" > 0 + OR "boxed___adds" > 0 + OR "boxed___dels" > 0 + OR "build___adds" > 0 + OR "build___dels" > 0 + OR "builder___adds" > 0 + OR "builder___dels" > 0 + OR "catalogs___adds" > 0 + OR "catalogs___dels" > 0 + OR "characteristics___adds" > 0 + OR "characteristics___dels" > 0 + OR "chars___adds" > 0 + OR "chars___dels" > 0 + OR "children___adds" > 0 + OR "children___dels" > 0 + OR "codePoints___adds" > 0 + OR "codePoints___dels" > 0 + OR "collect__adds" > 0 + OR "collect__dels" > 0 + OR "collectingAndThen__adds" > 0 + OR "collectingAndThen__dels" > 0 + OR "combiner___adds" > 0 + OR "combiner___dels" > 0 + OR "concat__adds" > 0 + OR "concat__dels" > 0 + OR "count___adds" > 0 + OR "count___dels" > 0 + OR "counting___adds" > 0 + OR "counting___dels" > 0 + OR "datesUntil__adds" > 0 + OR "datesUntil__dels" > 0 + OR "descendants___adds" > 0 + OR "descendants___dels" > 0 + OR "distinct___adds" > 0 + OR "distinct___dels" > 0 + OR "doubleStream__adds" > 0 + OR "doubleStream__dels" > 0 + OR "doubles__adds" > 0 + OR "doubles__dels" > 0 + OR "doubles___adds" > 0 + OR "doubles___dels" > 0 + OR "drivers___adds" > 0 + OR "drivers___dels" > 0 + OR "dropWhile__adds" > 0 + OR "dropWhile__dels" > 0 + OR "elementsAsStream___adds" > 0 + OR "elementsAsStream___dels" > 0 + OR "estimateSize___adds" > 0 + OR "estimateSize___dels" > 0 + OR "filter__adds" > 0 + OR "filter__dels" > 0 + OR "filtering__adds" > 0 + OR "filtering__dels" > 0 + OR "findAny___adds" > 0 + OR "findAny___dels" > 0 + OR "findFirst___adds" > 0 + OR "findFirst___dels" > 0 + OR "finisher___adds" > 0 + OR "finisher___dels" > 0 + OR "flatMap__adds" > 0 + OR "flatMap__dels" > 0 + OR "flatMapToDouble__adds" > 0 + OR "flatMapToDouble__dels" > 0 + OR "flatMapToInt__adds" > 0 + OR "flatMapToInt__dels" > 0 + OR "flatMapping__adds" > 0 + OR "flatMapping__dels" > 0 + OR "forEach__adds" > 0 + OR "forEach__dels" > 0 + OR "forEachOrdered__adds" > 0 + OR "forEachOrdered__dels" > 0 + OR "forEachRemaining__adds" > 0 + OR "forEachRemaining__dels" > 0 + OR "generate__adds" > 0 + OR "generate__dels" > 0 + OR "getComparator___adds" > 0 + OR "getComparator___dels" > 0 + OR "getExactSizeIfKnown___adds" > 0 + OR "getExactSizeIfKnown___dels" > 0 + OR "groupingBy__adds" > 0 + OR "groupingBy__dels" > 0 + OR "groupingByConcurrent__adds" > 0 + OR "groupingByConcurrent__dels" > 0 + OR "hasCharacteristics__adds" > 0 + OR "hasCharacteristics__dels" > 0 + OR "inetAddresses___adds" > 0 + OR "inetAddresses___dels" > 0 + OR "intStream__adds" > 0 + OR "intStream__dels" > 0 + OR "ints___adds" > 0 + OR "ints___dels" > 0 + OR "isParallel___adds" > 0 + OR "isParallel___dels" > 0 + OR "joining___adds" > 0 + OR "joining___dels" > 0 + OR "limit__adds" > 0 + OR "limit__dels" > 0 + OR "lines___adds" > 0 + OR "lines___dels" > 0 + OR "longStream__adds" > 0 + OR "longStream__dels" > 0 + OR "longs__adds" > 0 + OR "longs__dels" > 0 + OR "longs___adds" > 0 + OR "longs___dels" > 0 + OR "map__adds" > 0 + OR "map__dels" > 0 + OR "mapToInt__adds" > 0 + OR "mapToInt__dels" > 0 + OR "mapToLong__adds" > 0 + OR "mapToLong__dels" > 0 + OR "mapToObj__adds" > 0 + OR "mapToObj__dels" > 0 + OR "mapping__adds" > 0 + OR "mapping__dels" > 0 + OR "max___adds" > 0 + OR "max___dels" > 0 + OR "maxBy__adds" > 0 + OR "maxBy__dels" > 0 + OR "min___adds" > 0 + OR "min___dels" > 0 + OR "minBy__adds" > 0 + OR "minBy__dels" > 0 + OR "networkInterfaces___adds" > 0 + OR "networkInterfaces___dels" > 0 + OR "noneMatch__adds" > 0 + OR "noneMatch__dels" > 0 + OR "parallel___adds" > 0 + OR "parallel___dels" > 0 + OR "parallelStream___adds" > 0 + OR "parallelStream___dels" > 0 + OR "partitioningBy__adds" > 0 + OR "partitioningBy__dels" > 0 + OR "peek__adds" > 0 + OR "peek__dels" > 0 + OR "reduce__adds" > 0 + OR "reduce__dels" > 0 + OR "reducing__adds" > 0 + OR "reducing__dels" > 0 + OR "results___adds" > 0 + OR "results___dels" > 0 + OR "sequential___adds" > 0 + OR "sequential___dels" > 0 + OR "skip__adds" > 0 + OR "skip__dels" > 0 + OR "sorted___adds" > 0 + OR "sorted___dels" > 0 + OR "splitAsStream__adds" > 0 + OR "splitAsStream__dels" > 0 + OR "spliterator___adds" > 0 + OR "spliterator___dels" > 0 + OR "stream___adds" > 0 + OR "stream___dels" > 0 + OR "subInterfaces___adds" > 0 + OR "subInterfaces___dels" > 0 + OR "sum___adds" > 0 + OR "sum___dels" > 0 + OR "summarizingDouble__adds" > 0 + OR "summarizingDouble__dels" > 0 + OR "summarizingInt__adds" > 0 + OR "summarizingInt__dels" > 0 + OR "summarizingLong__adds" > 0 + OR "summarizingLong__dels" > 0 + OR "summaryStatistics___adds" > 0 + OR "summaryStatistics___dels" > 0 + OR "summingDouble__adds" > 0 + OR "summingDouble__dels" > 0 + OR "summingInt__adds" > 0 + OR "summingInt__dels" > 0 + OR "summingLong__adds" > 0 + OR "summingLong__dels" > 0 + OR "supplier___adds" > 0 + OR "supplier___dels" > 0 + OR "takeWhile__adds" > 0 + OR "takeWhile__dels" > 0 + OR "toArray___adds" > 0 + OR "toArray___dels" > 0 + OR "toCollection__adds" > 0 + OR "toCollection__dels" > 0 + OR "toConcurrentMap__adds" > 0 + OR "toConcurrentMap__dels" > 0 + OR "toList___adds" > 0 + OR "toList___dels" > 0 + OR "toMap__adds" > 0 + OR "toMap__dels" > 0 + OR "toSet___adds" > 0 + OR "toSet___dels" > 0 + OR "toUnmodifiableList___adds" > 0 + OR "toUnmodifiableList___dels" > 0 + OR "toUnmodifiableMap__adds" > 0 + OR "toUnmodifiableMap__dels" > 0 + OR "toUnmodifiableSet___adds" > 0 + OR "toUnmodifiableSet___dels" > 0 + OR "tokens___adds" > 0 + OR "tokens___dels" > 0 + OR "tryAdvance__adds" > 0 + OR "tryAdvance__dels" > 0 + OR "trySplit___adds" > 0 + OR "trySplit___dels" > 0 + OR "unordered___adds" > 0 + OR "unordered___dels" > 0 + OR "versionedStream___adds" > 0 + OR "versionedStream___dels" > 0 ) + ORDER BY method_change_detail.project DESC, + method_change_detail.sha, + file_name, + method_name +) TO STDOUT with CSV HEADER diff --git a/evaluation/scripts/sql/select_stream_bug_fixes_summary.sql b/evaluation/scripts/sql/select_stream_bug_fixes_summary.sql new file mode 100644 index 0000000..8d28b0a --- /dev/null +++ b/evaluation/scripts/sql/select_stream_bug_fixes_summary.sql @@ -0,0 +1,273 @@ +COPY ( + SELECT DISTINCT On (change_summary.sha) change_summary.sha, + change_summary.project, + author, + author_email, + commit_date, + is_bug + FROM change_summary + INNER JOIN method_change_detail + ON method_change_detail.sha = change_summary.sha + WHERE change_summary.is_bug + AND change_summary.commit_date >= '2014-03-18' + AND ( "BaseStream_adds" > 0 + OR "BaseStream_dels" > 0 + OR "Collector_adds" > 0 + OR "Collector_dels" > 0 + OR "Characteristics_adds" > 0 + OR "Characteristics_dels" > 0 + OR "Collectors_adds" > 0 + OR "Collectors_dels" > 0 + OR "DoubleStream_adds" > 0 + OR "DoubleStream_dels" > 0 + OR "range__adds" > 0 + OR "range__dels" > 0 + OR "rangeClosed__adds" > 0 + OR "rangeClosed__dels" > 0 + OR "IntStream_adds" > 0 + OR "IntStream_dels" > 0 + OR "LongStream_adds" > 0 + OR "LongStream_dels" > 0 + OR "Spliterator_adds" > 0 + OR "Spliterator_dels" > 0 + OR "OfDouble_adds" > 0 + OR "OfDouble_dels" > 0 + OR "OfInt_adds" > 0 + OR "OfInt_dels" > 0 + OR "OfLong_adds" > 0 + OR "OfLong_dels" > 0 + OR "OfPrimitive_adds" > 0 + OR "OfPrimitive_dels" > 0 + OR "Stream_adds" > 0 + OR "Stream_dels" > 0 + OR "StreamSupport_adds" > 0 + OR "StreamSupport_dels" > 0 + OR "accumulator___adds" > 0 + OR "accumulator___dels" > 0 + OR "allMatch__adds" > 0 + OR "allMatch__dels" > 0 + OR "allProcesses___adds" > 0 + OR "allProcesses___dels" > 0 + OR "anyMatch__adds" > 0 + OR "anyMatch__dels" > 0 + OR "asDoubleStream___adds" > 0 + OR "asDoubleStream___dels" > 0 + OR "asIntStream___adds" > 0 + OR "asIntStream___dels" > 0 + OR "asLongStream___adds" > 0 + OR "asLongStream___dels" > 0 + OR "average___adds" > 0 + OR "average___dels" > 0 + OR "averagingDouble__adds" > 0 + OR "averagingDouble__dels" > 0 + OR "averagingInt__adds" > 0 + OR "averagingInt__dels" > 0 + OR "averagingLong__adds" > 0 + OR "averagingLong__dels" > 0 + OR "boxed___adds" > 0 + OR "boxed___dels" > 0 + OR "build___adds" > 0 + OR "build___dels" > 0 + OR "builder___adds" > 0 + OR "builder___dels" > 0 + OR "catalogs___adds" > 0 + OR "catalogs___dels" > 0 + OR "characteristics___adds" > 0 + OR "characteristics___dels" > 0 + OR "chars___adds" > 0 + OR "chars___dels" > 0 + OR "children___adds" > 0 + OR "children___dels" > 0 + OR "codePoints___adds" > 0 + OR "codePoints___dels" > 0 + OR "collect__adds" > 0 + OR "collect__dels" > 0 + OR "collectingAndThen__adds" > 0 + OR "collectingAndThen__dels" > 0 + OR "combiner___adds" > 0 + OR "combiner___dels" > 0 + OR "concat__adds" > 0 + OR "concat__dels" > 0 + OR "count___adds" > 0 + OR "count___dels" > 0 + OR "counting___adds" > 0 + OR "counting___dels" > 0 + OR "datesUntil__adds" > 0 + OR "datesUntil__dels" > 0 + OR "descendants___adds" > 0 + OR "descendants___dels" > 0 + OR "distinct___adds" > 0 + OR "distinct___dels" > 0 + OR "doubleStream__adds" > 0 + OR "doubleStream__dels" > 0 + OR "doubles__adds" > 0 + OR "doubles__dels" > 0 + OR "doubles___adds" > 0 + OR "doubles___dels" > 0 + OR "drivers___adds" > 0 + OR "drivers___dels" > 0 + OR "dropWhile__adds" > 0 + OR "dropWhile__dels" > 0 + OR "elementsAsStream___adds" > 0 + OR "elementsAsStream___dels" > 0 + OR "estimateSize___adds" > 0 + OR "estimateSize___dels" > 0 + OR "filter__adds" > 0 + OR "filter__dels" > 0 + OR "filtering__adds" > 0 + OR "filtering__dels" > 0 + OR "findAny___adds" > 0 + OR "findAny___dels" > 0 + OR "findFirst___adds" > 0 + OR "findFirst___dels" > 0 + OR "finisher___adds" > 0 + OR "finisher___dels" > 0 + OR "flatMap__adds" > 0 + OR "flatMap__dels" > 0 + OR "flatMapToDouble__adds" > 0 + OR "flatMapToDouble__dels" > 0 + OR "flatMapToInt__adds" > 0 + OR "flatMapToInt__dels" > 0 + OR "flatMapping__adds" > 0 + OR "flatMapping__dels" > 0 + OR "forEach__adds" > 0 + OR "forEach__dels" > 0 + OR "forEachOrdered__adds" > 0 + OR "forEachOrdered__dels" > 0 + OR "forEachRemaining__adds" > 0 + OR "forEachRemaining__dels" > 0 + OR "generate__adds" > 0 + OR "generate__dels" > 0 + OR "getComparator___adds" > 0 + OR "getComparator___dels" > 0 + OR "getExactSizeIfKnown___adds" > 0 + OR "getExactSizeIfKnown___dels" > 0 + OR "groupingBy__adds" > 0 + OR "groupingBy__dels" > 0 + OR "groupingByConcurrent__adds" > 0 + OR "groupingByConcurrent__dels" > 0 + OR "hasCharacteristics__adds" > 0 + OR "hasCharacteristics__dels" > 0 + OR "inetAddresses___adds" > 0 + OR "inetAddresses___dels" > 0 + OR "intStream__adds" > 0 + OR "intStream__dels" > 0 + OR "ints___adds" > 0 + OR "ints___dels" > 0 + OR "isParallel___adds" > 0 + OR "isParallel___dels" > 0 + OR "joining___adds" > 0 + OR "joining___dels" > 0 + OR "limit__adds" > 0 + OR "limit__dels" > 0 + OR "lines___adds" > 0 + OR "lines___dels" > 0 + OR "longStream__adds" > 0 + OR "longStream__dels" > 0 + OR "longs__adds" > 0 + OR "longs__dels" > 0 + OR "longs___adds" > 0 + OR "longs___dels" > 0 + OR "map__adds" > 0 + OR "map__dels" > 0 + OR "mapToInt__adds" > 0 + OR "mapToInt__dels" > 0 + OR "mapToLong__adds" > 0 + OR "mapToLong__dels" > 0 + OR "mapToObj__adds" > 0 + OR "mapToObj__dels" > 0 + OR "mapping__adds" > 0 + OR "mapping__dels" > 0 + OR "max___adds" > 0 + OR "max___dels" > 0 + OR "maxBy__adds" > 0 + OR "maxBy__dels" > 0 + OR "min___adds" > 0 + OR "min___dels" > 0 + OR "minBy__adds" > 0 + OR "minBy__dels" > 0 + OR "networkInterfaces___adds" > 0 + OR "networkInterfaces___dels" > 0 + OR "noneMatch__adds" > 0 + OR "noneMatch__dels" > 0 + OR "parallel___adds" > 0 + OR "parallel___dels" > 0 + OR "parallelStream___adds" > 0 + OR "parallelStream___dels" > 0 + OR "partitioningBy__adds" > 0 + OR "partitioningBy__dels" > 0 + OR "peek__adds" > 0 + OR "peek__dels" > 0 + OR "reduce__adds" > 0 + OR "reduce__dels" > 0 + OR "reducing__adds" > 0 + OR "reducing__dels" > 0 + OR "results___adds" > 0 + OR "results___dels" > 0 + OR "sequential___adds" > 0 + OR "sequential___dels" > 0 + OR "skip__adds" > 0 + OR "skip__dels" > 0 + OR "sorted___adds" > 0 + OR "sorted___dels" > 0 + OR "splitAsStream__adds" > 0 + OR "splitAsStream__dels" > 0 + OR "spliterator___adds" > 0 + OR "spliterator___dels" > 0 + OR "stream___adds" > 0 + OR "stream___dels" > 0 + OR "subInterfaces___adds" > 0 + OR "subInterfaces___dels" > 0 + OR "sum___adds" > 0 + OR "sum___dels" > 0 + OR "summarizingDouble__adds" > 0 + OR "summarizingDouble__dels" > 0 + OR "summarizingInt__adds" > 0 + OR "summarizingInt__dels" > 0 + OR "summarizingLong__adds" > 0 + OR "summarizingLong__dels" > 0 + OR "summaryStatistics___adds" > 0 + OR "summaryStatistics___dels" > 0 + OR "summingDouble__adds" > 0 + OR "summingDouble__dels" > 0 + OR "summingInt__adds" > 0 + OR "summingInt__dels" > 0 + OR "summingLong__adds" > 0 + OR "summingLong__dels" > 0 + OR "supplier___adds" > 0 + OR "supplier___dels" > 0 + OR "takeWhile__adds" > 0 + OR "takeWhile__dels" > 0 + OR "toArray___adds" > 0 + OR "toArray___dels" > 0 + OR "toCollection__adds" > 0 + OR "toCollection__dels" > 0 + OR "toConcurrentMap__adds" > 0 + OR "toConcurrentMap__dels" > 0 + OR "toList___adds" > 0 + OR "toList___dels" > 0 + OR "toMap__adds" > 0 + OR "toMap__dels" > 0 + OR "toSet___adds" > 0 + OR "toSet___dels" > 0 + OR "toUnmodifiableList___adds" > 0 + OR "toUnmodifiableList___dels" > 0 + OR "toUnmodifiableMap__adds" > 0 + OR "toUnmodifiableMap__dels" > 0 + OR "toUnmodifiableSet___adds" > 0 + OR "toUnmodifiableSet___dels" > 0 + OR "tokens___adds" > 0 + OR "tokens___dels" > 0 + OR "tryAdvance__adds" > 0 + OR "tryAdvance__dels" > 0 + OR "trySplit___adds" > 0 + OR "trySplit___dels" > 0 + OR "unordered___adds" > 0 + OR "unordered___dels" > 0 + OR "versionedStream___adds" > 0 + OR "versionedStream___dels" > 0 ) + ORDER BY change_summary.sha, + change_summary.project DESC, + file_name, + method_name +) TO STDOUT with CSV HEADER diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..0c5da9c --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +Results/ diff --git a/src/logChunk/.gitignore b/src/logChunk/.gitignore new file mode 100644 index 0000000..d10a5fe --- /dev/null +++ b/src/logChunk/.gitignore @@ -0,0 +1,3 @@ +CloneErrorCases.txt +MissingGitRepos.txt +OtherGitRepos.txt diff --git a/src/logChunk/BracketScopeTracker.py b/src/logChunk/BracketScopeTracker.py index d292684..6fa2c73 100644 --- a/src/logChunk/BracketScopeTracker.py +++ b/src/logChunk/BracketScopeTracker.py @@ -146,8 +146,8 @@ def decreaseNewBrackets(self, line): if(self.newVerStack != []): removed = self.newVerStack.pop() if(self.config_info.DEBUG): - print("Removing: " + str(removed)) - print("Context: " + str(self.lastNewBlockContext)) + print(("Removing: " + str(removed))) + print(("Context: " + str(self.lastNewBlockContext))) if(removed[LABELINDEX] == FUNC): self.lastNewFuncContext = self.getTopType(self.newVerStack, FUNC) elif(removed[LABELINDEX] == SBLOCK): @@ -163,8 +163,8 @@ def decreaseOldBrackets(self, line): if(self.oldVerStack != []): removed = self.oldVerStack.pop() if(self.config_info.DEBUG): - print("Removing: " + str(removed)) - print("Context: " + str(self.lastOldBlockContext)) + print(("Removing: " + str(removed))) + print(("Context: " + str(self.lastOldBlockContext))) if(removed[LABELINDEX] == FUNC): self.lastOldFuncContext = self.getTopType(self.oldVerStack, FUNC) elif(removed[LABELINDEX] == SBLOCK): diff --git a/src/logChunk/ExceptionExtractor.py b/src/logChunk/ExceptionExtractor.py index c32b171..1d46f4d 100644 --- a/src/logChunk/ExceptionExtractor.py +++ b/src/logChunk/ExceptionExtractor.py @@ -21,7 +21,7 @@ def extractException(projPath): log_file = projPath + os.sep + LOG_FILE if not os.path.isfile(log_file): - print("!! %s does not exist" % (log_file)) + print(("!! %s does not exist" % (log_file))) return [] #else: # print("Going to process %s " % (log_file)) @@ -34,10 +34,10 @@ def extractException(projPath): def main(): - print "Utility to BULK process github logs" + print("Utility to BULK process github logs") if len(sys.argv) < 2: - print "!!! Usage: python ExceptionExtractor.py top_project directory" + print("!!! Usage: python ExceptionExtractor.py top_project directory") sys.exit() if not os.path.isdir("../Results"): os.mkdir("../Results") diff --git a/src/logChunk/JavaTestKeywords.txt b/src/logChunk/JavaTestKeywords.txt index 3d92dd0..bcf796c 100644 --- a/src/logChunk/JavaTestKeywords.txt +++ b/src/logChunk/JavaTestKeywords.txt @@ -1,8 +1,8 @@ -"try",INCLUDED,BLOCK -"catch",INCLUDED,BLOCK -"throw",INCLUDED,SINGLE -"finally",INCLUDED,BLOCK -"raise",INCLUDED,BLOCK -Exception,INCLUDED,SINGLE -"while",INCLUDED,BLOCK -"for",INCLUDED,BLOCK \ No newline at end of file +"try",included,block +"catch",included,block +"throw",included,single +"finally",included,block +"raise",included,block +Exception,included,single +"while",included,block +"for",included,block diff --git a/src/logChunk/LanguageSwitcherFactory.py b/src/logChunk/LanguageSwitcherFactory.py index 2a15d36..f70d9f2 100644 --- a/src/logChunk/LanguageSwitcherFactory.py +++ b/src/logChunk/LanguageSwitcherFactory.py @@ -44,7 +44,7 @@ def determineLanguage(language): #Replace these with tokens? elif(language.lower() == "python" or language.lower() in LanguageSwitcherFactory.extMap["Python"]["extensions"]): return PythonLanguageSwitcher.PythonLanguageSwitcher(set(LanguageSwitcherFactory.extMap["Python"]["extensions"])) else: - print(LanguageSwitcherFactory.extMap["C"]["extensions"]) + print((LanguageSwitcherFactory.extMap["C"]["extensions"])) raise UnsupportedLanguageException(language + " not yet supported.") @staticmethod diff --git a/src/logChunk/PatchMethod.py b/src/logChunk/PatchMethod.py index 140e303..1e74b2b 100644 --- a/src/logChunk/PatchMethod.py +++ b/src/logChunk/PatchMethod.py @@ -1,12 +1,12 @@ def toStr(text): try: - text1 = str(text).encode('iso-8859-1') + text1 = str(text) temp_text = text1.replace("\'","\"") temp_text = temp_text.strip() return "\'" + str(temp_text) + "\'" except: - print type(text) + print(type(text)) return "\'NA\'" @@ -37,21 +37,21 @@ def printPatch(self): def dumpMethod(self): dictStr= toStr(self.method) - for key, value in self.keywordDictionary.iteritems(): + for key, value in self.keywordDictionary.items(): dictStr= dictStr+","+ toStr(value) dictStr += "," + toStr(self.total_add) + "," + toStr(self.total_del) + "," + toStr(self.warning) - + return dictStr #Get the Header string for inserting into a database. def getTitleString(self): dictStr= "(project, sha, language, file_name, is_test, method_name" - for key, value in self.keywordDictionary.iteritems(): - dictStr= dictStr+","+ str(key).replace(" ", "_").lower() #ToStr will add ' around the strings... + for key, value in self.keywordDictionary.items(): + dictStr= dictStr+",\""+ str(key).replace(" ", "_").replace("(", "_").replace(")", "_") + "\"" #ToStr will add ' around the strings... dictStr += ",total_adds,total_dels,warning_alert)" - + return dictStr def getFullTitleString(self): @@ -62,11 +62,12 @@ def getFullTitleString(self): ''' dictStr = "(project character varying(500), sha text, language character varying(500)," + \ " file_name text, is_test boolean, method_name text" - for key, value in self.keywordDictionary.iteritems(): - dictStr= dictStr+", "+ str(key).replace(" ", "_").lower() + "integer" #ToStr will add ' around the strings... + for key, value in self.keywordDictionary.items(): + dictStr= dictStr+", "+ str(key).replace(" ", "_").replace("(", "_").replace(")", "_") + \ + "integer" #ToStr will add ' around the strings... dictStr += ", total_adds integer, total_dels integer, warning_alert boolean)" - + return dictStr def dictToCsv(self): diff --git a/src/logChunk/PythonScopeTracker.py b/src/logChunk/PythonScopeTracker.py index c57759c..e0d6643 100644 --- a/src/logChunk/PythonScopeTracker.py +++ b/src/logChunk/PythonScopeTracker.py @@ -76,11 +76,11 @@ def indentDepth(self, whiteSpace): #Make sure there is no mixing of tabs and spaces if(self.config_info.DEBUG): try: - print("Indent Token: \"" + self.indentToken + "\"") - print("WhiteSpace: \"" + whiteSpace + "\"") + print(("Indent Token: \"" + self.indentToken + "\"")) + print(("WhiteSpace: \"" + whiteSpace + "\"")) except: - print("Indent Token: \"" + unicode(self.indentToken, 'utf-8', errors='ignore') + "\"") - print("WhiteSpace: \"" + unicode(whiteSpace, 'utf-8', errors='ignore') + "\"") + print(("Indent Token: \"" + str(self.indentToken, 'utf-8', errors='ignore') + "\"")) + print(("WhiteSpace: \"" + str(whiteSpace, 'utf-8', errors='ignore') + "\"")) assert(self.indentToken != "") if(self.indentToken == "\t"): @@ -137,8 +137,8 @@ def isScopeIncrease(self, line, lineType): def isScopeDecrease(self, line, lineType): if(self.config_info.DEBUG): print("IN isScopeDecrease") - print("Indent Token: \"" + self.indentToken + "\"") - print("Line: \"" + line + "\"") + print(("Indent Token: \"" + self.indentToken + "\"")) + print(("Line: \"" + line + "\"")) if(self.indentToken == ""): #If Scope is decreasing, if must have increased at some point return S_NO #We need to ignore blank lines for scope Decreases? @@ -151,9 +151,9 @@ def isScopeDecrease(self, line, lineType): indent = re.match(r"\s*", line).group() depth = self.indentDepth(indent) if(self.config_info.DEBUG): - print("Depth:" + str(depth)) - print("Line: \"" + line + "\"") - print("Old Stack: " + str(self.oldVerStack)) + print(("Depth:" + str(depth))) + print(("Line: \"" + line + "\"")) + print(("Old Stack: " + str(self.oldVerStack))) if(lineType == ADD): if(len(self.newVerStack) > depth): return S_YES @@ -164,8 +164,8 @@ def isScopeDecrease(self, line, lineType): oldDiff = len(self.oldVerStack) > depth newDiff = len(self.newVerStack) > depth if(self.config_info.DEBUG): - print("Old Diff:" + str(oldDiff)) - print("New Diff:" + str(newDiff)) + print(("Old Diff:" + str(oldDiff))) + print(("New Diff:" + str(newDiff))) if(oldDiff != newDiff): #Scope is decreasing from the perspective of one stack and increasing from the other. return S_SIMUL elif(oldDiff == True): @@ -213,10 +213,10 @@ def handleFunctionNameEnding(self, line, functionName, lineType, funcIdentFunc): #increase counterpart is likely also not behaving correctly. if(self.config_info.DEBUG): print("Handle Ending") - print("FunctionName: " + functionName) - print("Line: " + line) - print("Old Context:" + self.lastOldFuncContext) - print("New Context:" + self.lastNewFuncContext) + print(("FunctionName: " + functionName)) + print(("Line: " + line)) + print(("Old Context:" + self.lastOldFuncContext)) + print(("New Context:" + self.lastNewFuncContext)) if(funcIdentFunc(functionName) != ""): #This is the line containing the followup if(self.config_info.DEBUG): @@ -309,15 +309,15 @@ def simulScopeChange(self, stackValue, lineType, changeType, depth, lineDiff): #if(not ((oldChange < 0 and newChange > 0) or (oldChange > 0 and newChange < 0))): if(self.config_info.DEBUG): print("SIMUL SCOPE CHANGE:") - print("Line: " + stackValue) - print("Depth: " + str(depth)) - print("OldChange: " + str(oldChange)) - print("NewChange: " + str(newChange)) - print("Func Old Line: " + str(self.funcOldLine)) - print("Func New Line: " + str(self.funcNewLine)) - print("Block Old Line: " + str(self.blockOldLine)) - print("Block New Line: " + str(self.blockNewLine)) - print("Arguments: " + " ".join([str(stackValue), str(lineType), str(changeType), str(depth), str(lineDiff)])) + print(("Line: " + stackValue)) + print(("Depth: " + str(depth))) + print(("OldChange: " + str(oldChange))) + print(("NewChange: " + str(newChange))) + print(("Func Old Line: " + str(self.funcOldLine))) + print(("Func New Line: " + str(self.funcNewLine))) + print(("Block Old Line: " + str(self.blockOldLine))) + print(("Block New Line: " + str(self.blockNewLine))) + print(("Arguments: " + " ".join([str(stackValue), str(lineType), str(changeType), str(depth), str(lineDiff)]))) print("STACKVALUES") self.printScope() # #assert(0) @@ -355,9 +355,9 @@ def simulScopeChange(self, stackValue, lineType, changeType, depth, lineDiff): def increaseNewIndent(self, stackValue, changeType, lineDiff): if(self.config_info.DEBUG): print("New Indent Increase") - print("Adding: " + str(stackValue)) - print("Type: " + str(changeType)) - print("Stack: " + str(self.newVerStack)) + print(("Adding: " + str(stackValue))) + print(("Type: " + str(changeType))) + print(("Stack: " + str(self.newVerStack))) if(changeType == GENERIC): if(self.funcNewLine != 1 and self.blockNewLine == 0): @@ -401,20 +401,20 @@ def increaseNewIndent(self, stackValue, changeType, lineDiff): assert("Not a valid change type.") if(self.config_info.DEBUG): - print("Stack (After): " + str(self.newVerStack)) + print(("Stack (After): " + str(self.newVerStack))) def increaseOldIndent(self, stackValue, changeType, lineDiff): if(self.config_info.DEBUG): print("Old Indent Increase") - print("Adding: " + str(stackValue)) - print("Type: " + str(changeType)) - print("Stack: " + str(self.oldVerStack)) + print(("Adding: " + str(stackValue))) + print(("Type: " + str(changeType))) + print(("Stack: " + str(self.oldVerStack))) if(changeType == GENERIC): if(self.config_info.DEBUG): print("GENERIC!") - print("Func Old Line: " + str(self.funcOldLine)) - print("Block Old Line: " + str(self.blockOldLine)) + print(("Func Old Line: " + str(self.funcOldLine))) + print(("Block Old Line: " + str(self.blockOldLine))) if(self.funcOldLine != 1 and self.blockOldLine == 0): self.oldVerStack.append((self.indentToken, GENERIC)) #Should be able to increase only 1 level at a time? elif(self.blockOldLine == 1): @@ -435,7 +435,7 @@ def increaseOldIndent(self, stackValue, changeType, lineDiff): self.lastOldFuncContext = stackValue self.funcOldLine = 1 if(self.config_info.DEBUG): - print("Func Old Line: " + str(self.funcOldLine)) + print(("Func Old Line: " + str(self.funcOldLine))) elif(self.funcOldLine == 1): #Indent after a func line self.oldVerStack.append((self.lastOldFuncContext, FUNC)) self.funcOldLine = -1 @@ -460,7 +460,7 @@ def increaseOldIndent(self, stackValue, changeType, lineDiff): assert("Not a valid change type.") if(self.config_info.DEBUG): - print("Stack (After): " + str(self.oldVerStack)) + print(("Stack (After): " + str(self.oldVerStack))) #string, [ADD|REMOVE|OTHER], [GENERIC|FUNC|BLOCK] -> -- #Increase the depth of our tracker and add in function or block contexts if they have been discovered. @@ -468,9 +468,9 @@ def increaseOldIndent(self, stackValue, changeType, lineDiff): def increaseScope(self, stackValue, line, lineType, changeType, lineDiff = -1, isSimul = False): if(self.config_info.DEBUG): try: - print("Scope Increasing Line: " + line) + print(("Scope Increasing Line: " + line)) except: - print("Scope Increasing Line: " + unicode(line, 'utf-8', errors='ignore')) + print(("Scope Increasing Line: " + str(line, 'utf-8', errors='ignore'))) if(lineType == ADD): self.increaseNewIndent(stackValue, changeType, lineDiff) @@ -496,9 +496,9 @@ def decreaseNewIndent(self): if(self.newVerStack != []): removed = self.newVerStack.pop() if(self.config_info.DEBUG): - print("Removing: " + str(removed)) + print(("Removing: " + str(removed))) #print("Context: " + str(self.lastNewBlockContext)) - print("Stack: " + str(self.newVerStack)) + print(("Stack: " + str(self.newVerStack))) if(removed[LABELINDEX] == FUNC): self.lastNewFuncContext = self.getTopType(self.newVerStack, FUNC) #elif(removed[LABELINDEX] == SBLOCK): @@ -512,9 +512,9 @@ def decreaseOldIndent(self): if(self.oldVerStack != []): removed = self.oldVerStack.pop() if(self.config_info.DEBUG): - print("Removing: " + str(removed)) + print(("Removing: " + str(removed))) #print("Context: " + str(self.lastOldBlockContext)) - print("Stack: " + str(self.oldVerStack)) + print(("Stack: " + str(self.oldVerStack))) if(removed[LABELINDEX] == FUNC): self.lastOldFuncContext = self.getTopType(self.oldVerStack, FUNC) #elif(removed[LABELINDEX] == SBLOCK): @@ -631,17 +631,17 @@ def afterIncrease(self, line): #No need to do anything here. We can't have code def printScope(self): print("------------------------------------") print("Language:") - print(self.language) + print((self.language)) print("Old Stack:") - print(self.oldVerStack) + print((self.oldVerStack)) print("Old Func Cache:") - print(self.lastOldFuncContext) + print((self.lastOldFuncContext)) print("Old Block Keyword:") - print(self.oldBlockKeyword) + print((self.oldBlockKeyword)) print("New Stack:") - print(self.newVerStack) + print((self.newVerStack)) print("New Func Cache:") - print(self.lastNewFuncContext) + print((self.lastNewFuncContext)) print("New Block Keyword:") - print(self.newBlockKeyword) + print((self.newBlockKeyword)) print("------------------------------------") \ No newline at end of file diff --git a/src/logChunk/allRunn.py b/src/logChunk/allRunn.py index 4d8ebe1..1c33dcf 100644 --- a/src/logChunk/allRunn.py +++ b/src/logChunk/allRunn.py @@ -15,10 +15,10 @@ def main(): - print "Utility to BULK process github logs" + print("Utility to BULK process github logs") if len(sys.argv) < 2: - print "!!! Usage: python allRun.py top_project directory" + print("!!! Usage: python allRun.py top_project directory") sys.exit() if not os.path.isdir("../Results"): os.mkdir("../Results") @@ -50,7 +50,7 @@ def main(): for dir in os.listdir(rootdir): path= os.path.join(rootdir,dir) - print path + print(path) os.system('python ghProc.py %s'%path) if __name__ == '__main__': diff --git a/src/logChunk/dictUtil.py b/src/logChunk/dictUtil.py index 2a7da4f..7ac6040 100644 --- a/src/logChunk/dictUtil.py +++ b/src/logChunk/dictUtil.py @@ -13,7 +13,7 @@ def incrementDict(dictKey, dictionary, inc=1): #Given a dictionary of counts return true if at least one is non zero #and false otherwise def nonZeroCount(dictionary): - for k,v in dictionary.iteritems(): + for k,v in dictionary.items(): assert(v >= 0) if(v > 0): return True diff --git a/src/logChunk/dumpLogs.py b/src/logChunk/dumpLogs.py index af27722..d034a95 100644 --- a/src/logChunk/dumpLogs.py +++ b/src/logChunk/dumpLogs.py @@ -29,8 +29,9 @@ def getFullTitleString(keywordDictionary): dictStr = "(project character varying(500), sha text, language character varying(500)," + \ " file_name text, is_test boolean, method_name text" - for key, value in keywordDictionary.iteritems(): - dictStr= dictStr+", "+ str(key).replace(" ", "_").lower() + " integer" #ToStr will add ' around the strings... + for key, value in keywordDictionary.items(): + dictStr= dictStr+", \""+ str(key).replace(" ", "_").replace("(", "_").replace(")", "_") + \ + "\" integer" #ToStr will add ' around the strings... dictStr += ", total_adds integer, total_dels integer, warning_alert boolean)" @@ -56,7 +57,7 @@ def cleanDb(self): if response.lower().startswith('y'): for table in tables: - print("Deleting table %r \n" % table) + print(("Deleting table %r \n" % table)) sql_command = "DELETE FROM " + table self.dbCon.insert(sql_command) diff --git a/src/logChunk/func_keywords.txt b/src/logChunk/func_keywords.txt new file mode 100644 index 0000000..fdb4589 --- /dev/null +++ b/src/logChunk/func_keywords.txt @@ -0,0 +1 @@ +"tf.function",included,single diff --git a/src/logChunk/func_repos.txt b/src/logChunk/func_repos.txt new file mode 120000 index 0000000..6a9580d --- /dev/null +++ b/src/logChunk/func_repos.txt @@ -0,0 +1 @@ +func_repos_from_boa.txt \ No newline at end of file diff --git a/src/logChunk/func_repos_from_GH_commit_search.txt b/src/logChunk/func_repos_from_GH_commit_search.txt new file mode 100644 index 0000000..106a43d --- /dev/null +++ b/src/logChunk/func_repos_from_GH_commit_search.txt @@ -0,0 +1,204 @@ +Albert-Z-Guo/Product-Recommendation +AlumiK/bagel-tensorflow +AndPotap/igr +AndPotap/tf2_primer +Andiry/tensorflow2 +Anieca/deep-learning-models +AstroViking/tf-kde +BlueFisher/Advanced-Soft-Actor-Critic +BrikerMan/Kashgari +CUN-bjy/gym-ddpg-keras +CaseyLeeDowdle/TransportNets +Chiranjeevi731/Machine-Learning +Chocomunk/tetris-bot +DavidDB33/drling +Forgotten/BabyDeepMD +Ghostman2016/tensorflow +GoogleCloudPlatform/training-data-analyst +GyuyoulKim/RLSandbox +HJSang/TensorflowExamples +Harrypotterrrr/EDVR +HumanCompatibleAI/better-adversarial-defenses +HwanGoh/uq-vae +Jeyalakshmi-nethajy/jupy +JohnHush/tf2_implementation +JulienBrochier/MONet-implementation +KDD2018/Machine-Learning +KatharinaHermann/tum-Advanced-DL-for-robotics-RL +LalinduWenasara/project_elefante +LongmaoTeamTf/DeepRecSys +LuchnikovI/QGOpt +Madanfeng/advGAN_tf2 +ManideepBangaru/TensorFlow_DeepLearning +Marshelix/GalaClusterMod +MoatazNegm/GCPAdvanced +NKNY/TowardControlledGenerationOfText +P-Schumacher/research +PAIR-code/recommendation-rudders +Pengjp/gene_research +PiotrStyla/ObjectDetection +ReggieVW/TFLiteFlowerClassificationApp +StanislasBertrand/RetinaFace-tf2 +Steven-N-Hart/Ganomaly +TappeiHorikawa/AddExpGANs +TensorSpeech/TensorFlowASR +TensorSpeech/TensorFlowTTS +UM-ARM-Lab/contact_shape_completion +UM-ARM-Lab/probabilistic_shape_completion +VikramRadhakrishnan/LittleEngineThatCould +Ximilar-com/tf-image +ahmed-gharib89/TensorFlow_2_for_Deep_Learning +ahsanbari/GoogleCloud +ali-tny/deep-topology +alpha-davidson/TensorBNN +ankagrgit/training-data-analyst +antonpetkoff/identifier-suggestion +apoluekt/AmpliTF +arbiter1elegantiae/Robustness-of-deep-neural-networks-with-trainable-activation-functions +arnomoonens/yarll +artbees/probability +artbees/tensorflow +artbees/trax +athena-team/athena +awslabs/sagemaker-debugger +bark-simulator/bark-ml +bazukas/mtcnn-tf2 +bentoml/BentoML +beringresearch/datasets +bmstu-iu8-g4-2020-project/code2var +borundev/tensorflow_docs +brendanhasz/probflow +capreolus-ir/capreolus +chinganc/librl +chjort/chambers +comtalyst/generative-facial-cosmetics +coyote009/samples +danielwatson6/capstone +ddt666/tf2_learning +deepmind/acme +dionhaefner/pyhpc-benchmarks +dlzou/dnlp-chatbot +dmitry-kabanov/dmitry-kabanov.github.io +dpp109/training-data-analyst +dpressel/mead-baseline +dylanpeifer/deepgroebner +edgarjimenez1996/Google_ML_Course +eric-erki/tensorflow +erickcfarias/GAN-CIRCLE-keras +ermekaitygulov/RobotArm +esmjah/NeuralControl +filipbartek/vampire-ml +galeone/tfgo +galv/lingvo-copy +gergely-flamich/BVAE +ghl3/Pokermon +google/automl +google/iree +google/tf-quant-finance +google/trax +google/uncertainty-metrics +gowtham-prudhvi/learning-full +gretelai/gretel-synthetics +gwendalF/progressive_gan +hauser-group/mlff +hxtruong6/dynamic-graph-embedding +hyLiu1994/HelpGirlfriend +hyunho1027/T-RexRL +inailuig/tensorflow-upstream +ivanCanaveral/tensorflow-tips +jaentrouble/Adipos_tf_train2 +jaentrouble/mouse_test_10 +jamborta/tffm2 +jason9075/tf2_arcface +jiangyi15/tf-pwa +jscant/gnina_tensorflow +kennethAsher/google_tensorflow2 +keras-team/keras-io +kingusiu/vande +kokonstyn/gan_tenserflow +korneelvdbroek/audiocodec +kynk94/TF2-Image-Generation +l123456789jy/TFLite_Flower +linZHank/rl-lunarlander +liushui263/TF2.0in30days +ljk423/ctgan-tf +lordtt13/tensorflow2-advanced-operations +loreloc/spnflow +lucasmiranda42/deepof +lyhue1991/eat_tensorflow2_in_30_days +madhavi2491/training-data-analyst +markpinnock/CTVirtualContrast +mfarghall/tensorflow_1 +michaelaerni/eth-cil-project +microsoft/tf2-gnn +mikeurbach/tensorflow +milesgray/augtistic +ming-hu427/tensorflow +mlfpm/deepof +moono/moco-tf2.x +moono/stylegan2-tf-2.x +na018/awesome.skating.ai +nicojahn/open-neural-apc +nihil21/cnn-deblur +noo-rashbass/synthetic-data-service +nothinn/tensorflow +nyikovicsmate/thesis +onnx/onnx-tensorflow +oscar-carlsson/wasp_learning_feature_representations +peerdavid/conflicting-bundles +piyop/tfaug +plum78/clone1 +rafaelsf80/notebooks +rankir/k8s_training +revsic/tf-alae +rlatjcj/Paper-code-review +rncm-prism/prism-samplernn +roads/psiz +rshu/Adversarial-Evasion-Defense +saArbabi/models +saahiluppal/langtret +saforem2/l2hmc-qcd +sagarpant1/tensorflow +sascha-senko/TensorflowCourse +sce-tts/TensorflowTTS +seunghwan1228/Transfomer-MachineTranslation +shizukanaskytree/tensorflow +shuiruge/neural-ode +siavash-khodadadeh/MetaLearning-TF2.0 +slidemorning/deep-learning +slidemorning/tensorflow-2.0 +sofiabroome/painface-recognition +sunithajai/glcapstone +tanzhenyu/image_augmentation +taotesea/tensorflow +tensorflow/addons +tensorflow/community +tensorflow/docs +tensorflow/estimator +tensorflow/examples +tensorflow/federated +tensorflow/java +tensorflow/lingvo +tensorflow/probability +tensorflow/runtime +tensorflow/tensorflow +tensorflow/tfx +tensorflow/tfx-bsl +tensorflow/transform +thetianshuhuang/l2o +toonsquare/gan +tum-adlr-ss20-06/project +tygu1004/promedius-summer-intern_cyclegan +uchendui/rl-tf2 +ufal/npfl122 +v-sivak/gkp-rl +vaecole/SIAT-AUTOS-ITS +werelaxe/spbu-machine-learning +wjkaufman/rl_pulse +yardenas/tf2-mbpo +yearhasfish/gcp +ymd-h/b4tf +yoshihikoueno/DNNCancerAnnotator +yxd886/jax +yxd886/trax +zhanglab/ReadsClassification diff --git a/src/logChunk/func_repos_from_boa.txt b/src/logChunk/func_repos_from_boa.txt new file mode 100644 index 0000000..31b60b3 --- /dev/null +++ b/src/logChunk/func_repos_from_boa.txt @@ -0,0 +1,885 @@ +A-bone1/Attention-ocr-Chinese-Version +A2Zadeh/CMU-MultimodalSDK +AIChallenger/AI_Challenger_2017 +AIChallenger/AI_Challenger_2018 +AITTSMD/MTCNN-Tensorflow +AKSHAYUBHAT/ComputationalHealthcare +AKSHAYUBHAT/DeepVideoAnalytics +ANSSI-FR/SecuML +ARM-DOE/pyart +AlexEMG/DeepLabCut +Ao-Lee/Vgg-Face-Fine-tune +Arsey/keras-transfer-learning-for-oxford102 +AxeldeRomblay/MLBox +Azure/kubeflow-labs +BRML/climin +Bartzi/stn-ocr +BenWhetton/keras-surgeon +BichenWuUCB/squeezeDet +Bihaqo/t3f +Cadene/skip-thoughts.torch +CamDavidsonPilon/tdigest +CartoDB/cartoframes +CharlesShang/FastMaskRCNN +CharlesShang/TFFRCNN +ChenglongChen/tensorflow-XNN +Chung-I/Variational-Recurrent-Autoencoder-Tensorflow +ClimbsRocks/auto_ml +Cloud-CV/Fabrik +Coder-Yu/RecQ +CogComp/cogcomp-nlpy +Conchylicultor/DeepQA +CuriousAI/ladder +DLTK/DLTK +DeNeutoy/act-tensorflow +DeepVisionTeam/TensorFlowBook +DistrictDataLabs/yellowbrick +DjangoPeng/tensorflow-in-depth +DrSleep/tensorflow-deeplab-resnet +DrewNF/Tensorflow_Object_Tracking_Video +DwangoMediaVillage/keras_compressor +EderSantana/seya +EdinburghNLP/nematus +EducationalTestingService/skll +EdwardTyantov/ultrasound-nerve-segmentation +EliotAndres/pretrained.ml +EpistasisLab/scikit-rebate +EpistasisLab/tpot +EricLingRui/NLP-tools +EricZgw/PyramidBox +FuZer/StudyTensorFlow +FudanNLP/nlpcc2017_news_headline_categorization +GPflow/GPflow +GRAAL-Research/pytoune +GSByeon/multi-speaker-tacotron-tensorflow +GenTang/intro_ds +GeorgeSeif/Semantic-Segmentation-Suite +Goldesel23/Siamese-Networks-for-One-Shot-Learning +GoogleCloudPlatform/cloudml-samples +Grzego/handwriting-generation +GustavZ/realtime_object_detection +HDI-Project/ATM +HIPS/Kayak +HKUST-KnowComp/R-Net +HRLTY/TP-GAN +HaloOrangeWang/NoiseMaker +Hananel-Hazan/bindsnet +HariSekhon/devops-python-tools +HealthCatalyst/healthcareai-py +HendrikStrobelt/Seq2Seq-Vis +HiKapok/tf.fashionAI +HouJP/kaggle-quora-question-pairs +HunterMcGushion/hyperparameter_hunter +IBM/AIF360 +IBM/tensorflow-hangul-recognition +IDSIA/brainstorm +InFoCusp/tf_cnnvis +IndicoDataSolutions/finetune +IntelLabs/hpat +IsaacChanghau/neural_sequence_labeling +Jack-Cherish/Deep-Learning +JacksonWuxs/DaPy +JasperSnoek/spearmint +JayParks/tf-seq2seq +JianWenJun/MLDemo +JihongJu/keras-fcn +JonathanRaiman/theano_lstm +Justin-Tan/generative-compression +JustinhoCHN/SRGAN_Wasserstein +KelvinZx/sfcn-opi +KeplerGO/pyke +Khan/guacamole +Kyubyong/neural_japanese_transliterator +Kyubyong/tacotron +Kyubyong/word_prediction +Lasagne/Lasagne +LexPredict/lexpredict-lexnlp +Li-Ming-Fan/OCR-DETECTION-CTPN +LoSealL/VideoSuperResolution +LunaBlack/RNN-Classification +LxMLS/lxmls-toolkit +LynnHo/DCGAN-LSGAN-WGAN-WGAN-GP-Tensorflow +MIC-DKFZ/trixi +MadryLab/mnist_challenge +MajorTal/DeepSpell +MarekKowalski/DeepAlignmentNetwork +Mariewelt/OpenChem +MarkPKCollier/NeuralTuringMachine +MarvinTeichmann/KittiBox +MashiMaroLjc/ML-and-DM-in-action +MaxHalford/xam +MaybeShewill-CV/CRNN_Tensorflow +MaybeShewill-CV/lanenet-lane-detection +MichalDanielDobrzanski/DeepLearningPython35 +Microsoft/LQ-Nets +Microsoft/MMdnn +Microsoft/NimbusML +Microsoft/gated-graph-neural-network-samples +Microsoft/samples-for-ai +MikeTheWatchGuy/PySimpleGUI +MillionIntegrals/vel +Morphl-AI/MorphL-Community-Edition +MorvanZhou/Tensorflow-Computer-Vision-Tutorial +MurtyShikhar/Question-Answering +NLPLearn/QANet +NLPLearn/R-net +NVIDIA/DeepLearningExamples +NVIDIA/FastPhotoStyle +NervanaSystems/neon +NervanaSystems/nlp-architect +NifTK/NiftyNet +OlavHN/bnlstm +OpenMined/Grid +OpenMined/PySyft +OpenNMT/OpenNMT-tf +PacktPublishing/Advanced-Deep-Learning-with-Keras +PacktPublishing/Artificial-Intelligence-with-Python +PacktPublishing/Getting-Started-with-TensorFlow +PaddlePaddle/models +PavlosMelissinos/enet-keras +PetrochukM/PyTorch-NLP +PiSimo/PiCamNN +QUVA-Lab/artemis +Qidian213/deep_sort_yolov3 +QuantumLiu/wechat_callback +RJT1990/mantra +RaRe-Technologies/gensim +RandolphVI/Multi-Label-Text-Classification +RandolphVI/Text-Pairs-Relation-Classification +RaphaelMeudec/deblur-gan +RasaHQ/rasa_core +RasaHQ/rasa_nlu +Rayhane-mamah/Tacotron-2 +Rochester-NRT/RocAlphaGo +ScottfreeLLC/AlphaPy +Sentimentron/Dracula +SeoSangwoo/Attention-Based-BiLSTM-relation-extraction +SiyuanQi/gpnn +Speedml/speedml +SpiderClub/haipproxy +THUNLP-MT/THUMT +TargetHolding/pyspark-cassandra +TarrySingh/Artificial-Intelligence-Deep-Learning-Machine-Learning-Tutorials +TeamHG-Memex/sklearn-crfsuite +TeamHG-Memex/tensorboard_logger +Tencent/FeatherCNN +TensorLab/tensorfx +TensorMSA/tensormsa +Theano/Theano +Thenerdstation/mltest +TomAugspurger/engarde +UKPLab/deeplearning4nlp-tutorial +Vetal1977/tf_serving_example +VisTrails/VisTrails +WZBSocialScienceCenter/pdftabextract +XanaduAI/pennylane +XifengGuo/DEC-keras +XuezheMax/NeuroNLP2 +YCG09/chinese_ocr +Yelp/mrjob +YerevaNN/R-NET-in-Keras +YingZhangDUT/Deep-Mutual-Learning +Yorko/mlcourse.ai +Zardinality/TF_Deformable_Net +Zehaos/MobileNet +aboSamoor/polyglot +achillesrasquinha/bulbea +adatao/tensorspark +adbrebs/taxi +adobe/NLP-Cube +aetros/aetros-cli +ahmetozlu/tensorflow_object_counting_api +ahmetozlu/vehicle_counting_tensorflow +aigamedev/scikit-neuralnetwork +airbnb/knowledge-repo +airbnb/omniduct +ajtulloch/sklearn-compiledtrees +albermax/innvestigate +albertaparicio/tfg-voice-conversion +albertbup/deep-belief-network +aleju/imgaug +alexjc/neural-doodle +alfredfrancis/ai-chatbot-framework +aliyun/aliyun-odps-python-sdk +allanzelener/YAD2K +allenai/PeerRead +allenai/allennlp +allenai/bilm-tf +allenai/deep_qa +alno/kaggle-allstate-claims-severity +alshedivat/keras-gp +amaas/stanford-ctc +analysiscenter/batchflow +anantzoid/VQA-Keras-Visual-Question-Answering +anayebi/keras-extra +andabi/deep-voice-conversion +andabi/music-source-separation +andabi/voice-vector +andersbll/deeppy +andreiapostoae/dota2-predictor +andrejonasson/dynamic-coattention-network-plus +andrewliao11/gail-tf +annoviko/pyclustering +antoine77340/Youtube-8M-WILLOW +anttttti/Wordbatch +anymail/django-anymail +apachecn/AiLearning +arogozhnikov/einops +artpar/languagecrunch +asanakoy/deeppose_tf +ashnkumar/sketch-code +aspuru-guzik-group/chemical_vae +astorfi/3D-convolutional-speaker-recognition +astorfi/TensorFlow-World +astroML/astroML +atbaker/wikipedia-question-generator +atpaino/deep-text-corrector +attardi/deepnl +automl/auto-sklearn +autonomio/talos +aws/sagemaker-python-sdk +aws/sagemaker-tensorflow-container +awslabs/aws-glue-libs +awslabs/mxnet-lambda +ayushoriginal/Sentiment-Analysis-Twitter +azavea/raster-vision +baldassarreFe/deep-koalarization +barronalex/Dynamic-Memory-Networks-in-TensorFlow +barronalex/Tacotron +bashtage/linearmodels +basveeling/wavenet +batzner/indrnn +bbengfort/hadoop-fundamentals +benanne/morb +bethgelab/foolbox +bfelbo/DeepMoji +bgavran/DNC +bhansconnect/alpha_zero_othello +billzorn/mtgencode +bilylee/SiamFC-TensorFlow +biolab/orange +biolab/orange3 +blackecho/Deep-Learning-TensorFlow +blaze/blaze +blue-oil/blueoil +bnpy/bnpy +bonzanini/Book-SocialMediaMiningPython +bourdakos1/Custom-Object-Detection +brian-team/brian2 +brightmart/text_classification +broadinstitute/keras-rcnn +broadinstitute/keras-resnet +brucechou1983/CheXNet-Keras +bshao001/ChatLearner +bstriner/keras-adversarial +buppt/ChineseNER +buriburisuri/ByteNet +buriburisuri/ebgan +buriburisuri/sugartensor +burness/tensorflow-101 +caffe2/tutorials +calclavia/DeepJ +calvinschmdt/EasyTensorflow +cangermueller/deepcpg +carpedm20/NAF-tensorflow +carpedm20/simulated-unsupervised-tensorflow +castorini/Castor +castorini/honk +cbaziotis/datastories-semeval2017-task4 +cbyn/bitpredict +cerndb/dist-keras +cesc-park/CRCN +cesium-ml/cesium +cgarciae/tensorbuilder +chainer/chainer +chainer/chainercv +chakki-works/doccano +changun/CollMetric +chartbeat-labs/textacy +chen0040/keras-anomaly-detection +chen0040/keras-text-summarization +chenyilun95/tf-cpn +cheungdaven/DeepRec +chiphuyen/stanford-tensorflow-tutorials +chribsen/simple-machine-learning-examples +chrischoy/3D-R2N2 +chrisdonahue/nesmdb +christopherjenness/ML-lib +chuyangliu/snake +cmgreen210/TensorFlowDeepAutoencoder +cmusphinx/g2p-seq2seq +codekansas/keras-language-modeling +conan7882/CNN-Visualization +conan7882/adversarial-autoencoders-tf +crazydonkey200/neural-symbolic-machines +crowsonkb/style_transfer +csxeba/brainforge +danduncan/HappyNet +danielfrg/copper +danijar/layered +darkonhub/darkon +darksigma/Fundamentals-of-Deep-Learning-Book +dask/dask-ml +data61/aboleth +databricks/spark-deep-learning +databricks/spark-sklearn +datitran/object_detector_app +datmo/datmo +davidadamojr/TextRank +davidsandberg/facenet +daxiongshu/tradeshift-text-classification +deepchem/deepchem +deepgram/kur +deepmind/dnc +deepmind/learning-to-learn +deepmind/scalable_agent +deepmind/sonnet +deepmipt/DeepPavlov +deeppomf/DeepCreamPy +dementrock/tensorfuse +developmentseed/label-maker +devsisters/DQN-tensorflow +dinghanshen/SWEM +dirkneumann/deepdist +divamgupta/image-segmentation-keras +dmarnerides/pydlt +dmbee/seglearn +dmlc/dgl +dmlc/gluon-nlp +dmlc/tvm +dnouri/nolearn +dnouri/skorch +domluna/memn2n +donnemartin/data-science-ipython-notebooks +douban/dpark +douban/tfmesos +dpressel/baseline +drscotthawley/panotti +dvornikita/blitznet +dzitkowskik/StockPredictionRNN +eBay/Sequence-Semantic-Embedding +edublancas/sklearn-evaluation +ekampf/PySpark-Boilerplate +ematvey/hierarchical-attention-networks +emedvedev/attention-ocr +endernewton/tf-faster-rcnn +equalitie/learn2ban +eragonruan/text-detection-ctpn +erickrf/nlpnet +eridgd/WCT-TF +erikbern/deep-fonts +ethancaballero/Improved-Dynamic-Memory-Networks-DMN-plus +ethereon/caffe-tensorflow +explosion/spaCy +explosion/spacy-models +explosion/thinc +fabianbormann/Tensorflow-DeconvNet-Segmentation +faizann24/Using-machine-learning-to-detect-malicious-URLs +farizrahman4u/qlearning4k +farizrahman4u/recurrentshop +farizrahman4u/seq2seq +farzaa/DeepLeague +fastnlp/fastNLP +felipessalvatore/self_driving_pi_car +femioladeji/License-Plate-Recognition-Nigerian-vehicles +fengju514/Face-Pose-Net +fephsun/neuralnetmusic +feynmanliang/bachbot +fferroni/PhasedLSTM-Keras +fizyr/keras-retinanet +flipkart-incubator/optimus +flomlo/ntm_keras +flowlight0/talkingdata-adtracking-fraud-detection +floydhub/dockerfiles +flyyufelix/Direct-Future-Prediction-Keras +for-ai/CipherGAN +freeman-lab/spark-ml-streaming +ftramer/Steal-ML +fukatani/stacked_generalization +fukuball/fuku-ml +fvisin/reseg +galeone/dynamic-training-bench +geometalab/OSMDeepOD +giacbrd/ShallowLearn +gitlimlab/Relation-Network-Tensorflow +google-research/batch-ppo +google/deepvariant +google/prettytensor +google/seq2seq +googlei18n/language-resources +greatgeekgrace/Technical-Analysis-And-Practice-in-TensorFlow +guillaumegenthial/im2latex +gunthercox/ChatterBot +habrman/FaceRecognition +hadyelsahar/CNN-RelationExtraction +hahnyuan/nn_tools +hanbt/learn_dl +hankcs/CS224n +hannes-brt/hebel +hardmaru/sketch-rnn +harpribot/deep-summarization +harvitronix/five-video-classification-methods +harvitronix/neural-network-genetic-algorithm +hassony2/kinetics_i3d_pytorch +hclhkbu/dlbench +hellochick/PSPNet-tensorflow +hengluchang/deep-news-summarization +hexahedria/biaxial-rnn-music-composition +hidasib/GRU4Rec +hirofumi0810/tensorflow_end2end_speech_recognition +hiveml/tensorflow-grad-cam +hmmlearn/hmmlearn +honeynet/cuckooml +hqli/face_recognition +htm-community/nupic.studio +huggingface/neuralcoref +huggingface/torchMoji +hugorut/neural-cli +hundredblocks/semantic-search +huxiaoman7/PaddlePaddle_code +hwalsuklee/tensorflow-generative-model-collections +hwwang55/GraphGAN +hycis/Mozi +hyperspy/hyperspy +hzy46/Deep-Learning-21-Examples +hzy46/fast-neural-style-tensorflow +ibab/tensorflow-wavenet +icoxfog417/tensorflow_qrnn +idealo/image-quality-assessment +idealo/image-super-resolution +idiap/importance-sampling +igormq/asr-study +ilblackdragon/tf_examples +ildoonet/deepface +ilovin/lstm_ctc_ocr +imatge-upc/skiprnn-2017-telecombcn +indrajithi/mgc-django +inejc/painters +inspirehep/magpie +instacart/lore +iperov/DeepFaceLab +isseu/emotion-recognition-neural-networks +itdxer/neupy +iterative/dvc +jaberg/skdata +jacobandreas/nmn2 +jacobgil/keras-grad-cam +jacoxu/encoder_decoder +jadianes/winerama-recommender-tutorial +jakeret/tf_unet +jantic/DeOldify +jarfo/kchar +jay-mahadeokar/pynetbuilder +jeetsukumaran/DendroPy +jeongyoonlee/Kaggler +jfkirk/tensorrec +jhfjhfj1/autokeras +jiamings/cramer-gan +jiamings/wgan +jimfleming/recurrent-entity-networks +jisungk/RIDDLE +jmcarpenter2/parfit +joelgrus/data-science-from-scratch +jorditorresBCN/FirstContactWithTensorFlow +jorditorresBCN/LibroTensorFlow +jorgenkg/python-neural-network +jostmey/rwa +jparkhill/TensorMol +jpmml/sklearn2pmml +jremillard/images-to-osm +jrosebr1/bat-country +jrosebr1/simple-keras-rest-api +jsilter/parametric_tsne +juliandewit/kaggle_ndsb2 +juliandewit/kaggle_ndsb2017 +jupyter-incubator/sparkmagic +jupyter/enterprise_gateway +jxwufan/AssociativeRetrieval +jzadeh/aktaion +jzilly/RecurrentHighwayNetworks +karpathy/neuraltalk +kayousterhout/trace-analysis +keeganmccallum/sql4pandas +keithito/tacotron +kengz/openai_lab +kentsommer/tensorflow-posenet +keras-rl/keras-rl +keras-team/keras +keras-team/keras-applications +keras-team/keras-contrib +kermitt2/delft +keunwoochoi/kapre +kevinhughes27/TensorKart +kevinjliang/tf-Faster-RCNN +khanhnamle1994/natural-language-processing +kingfengji/gcForest +klaussinani/moviebox +konlpy/konlpy +kootenpv/neural_complete +kopopt/fast_tffm +kozistr/Awesome-GANs +kubeflow/kubeflow +kuleshov/audio-super-res +kundajelab/dragonn +kuz/caffe-with-spearmint +kvfrans/deepcolor +kvh/ramp +kwotsin/TensorFlow-ENet +kykiefer/depression-detect +l11x0m7/lightnn +lancopku/SU4MLC +lanwuwei/SPM_toolkit +lazyprogrammer/machine_learning_examples +lensacom/sparkit-learn +lifeomic/sparkflow +linkedin/WhereHows +lisa-lab/DeepLearningTutorials +liu-nlper/DocumentClassification +liu-nlper/NER-LSTM-CRF +liuyuemaicha/Deep-Reinforcement-Learning-for-Dialogue-Generation-in-tensorflow +liyaguang/DCRNN +llSourcell/AI_Artist +llSourcell/Machine-Learning-API-Tutorial +llSourcell/Make_Money_with_Tensorflow +llSourcell/Neural_Network_Voices +llSourcell/how_to_convert_text_to_images +llSourcell/how_to_deploy_a_keras_model_to_production +llSourcell/tensorflow_chatbot +llSourcell/tensorflow_demo +llSourcell/tensorflow_speech_recognition_demo +lmjohns3/theanets +locationtech-labs/geopyspark +locuslab/convex_adversarial +lopuhin/kaggle-dstl +lpigou/Theano-3D-ConvNet +lpty/tensorflow_tutorial +lsy17096535/face-landmark +luispedro/BuildingMachineLearningSystemsWithPython +luispedro/milk +luoyetx/deep-landmark +luyishisi/tensorflow +lvapeab/nmt-keras +macanv/BERT-BiLSTM-CRF-NER +machinalis/featureforge +maciejkula/glove-python +maiminh1996/YOLOv3-tensorflow +marekrei/sequence-labeler +marian-margeta/gait-recognition +marionleborgne/cloudbrain +martin-gorner/tensorflow-rnn-shakespeare +marvin-ai/marvin-python-toolbox +mathDR/reading-text-in-the-wild +matroid/dlwithtf +matterport/Mask_RCNN +matthewearl/deep-anpr +maxim5/time-series-machine-learning +maxpumperla/deep_learning_and_the_game_of_go +maxpumperla/elephas +mckinziebrandon/DeepChatModels +meereeum/lda2vec-tf +meownoid/face-identification-tpe +mikelynn2/sentimentAPI +mila-udem/blocks +mila-udem/fuel +mila-udem/platoon +miloharper/neural-network-animation +mindsdb/mindsdb +minimaxir/person-blocker +miranthajayatilake/YOLOw-Keras +mirceamironenco/BayesianRecurrentNN +mitmul/ssai-cnn +mlflow/mlflow +mnielsen/neural-networks-and-deep-learning +mohammadpz/Recurrent-Neural-Networks +mokemokechicken/keras_npi +mondejar/ecg-classification +mortendahl/tf-encrypted +mpastell/Pweave +mrlittlepig/ImageProcessing +mtth/hdfs +mxbi/mlcrate +nanopony/keras-convautoencoder +naokishibuya/car-behavioral-cloning +ncoudray/DeepPATH +ne7ermore/torch-light +neeilan/DeepPlayByPlay +neptune-ml/open-solution-data-science-bowl-2018 +nerox8664/pytorch2keras +neubig/nn4nlp-code +nextml/NEXT +nicholaslocascio/deep-regex +nickliqian/cnn_captcha +nicola-decao/s-vae-tf +nicolov/segmentation_keras +niitsuma/word2vec-keras-in-gensim +nilearn/nilearn +nl8590687/ASRT_SpeechRecognition +nlpinaction/learning-nlp +nok/sklearn-porter +nservant/HiC-Pro +nusnlp/nea +nyukat/breast_density_classifier +oarriaga/face_classification +odashi/chainer_examples +okfn-brasil/serenata-de-amor +omni-us/squeezedet-keras +omoindrot/tensorflow-triplet-loss +onnx/onnx-caffe2 +onnx/onnx-tensorflow +onnx/onnxmltools +onnx/tensorflow-onnx +openai/gradient-checkpointing +openai/vime +openai/weightnorm +openstack/sahara +osforscience/TensorFlow-Course +oujago/NumpyDL +outcastofmusic/quick-nlp +ozan-oktay/Attention-Gated-Networks +pandas-dev/pandas +pandas-ml/pandas-ml +pannous/tensorflow-ocr +pannous/tensorflow-speech-recognition +parasdahal/deepnet +pbhatia243/tf-layer-norm +pender/chatbot-rnn +persistforever/cifar10-tensorflow +persistforever/yolo-tensorflow +petercunha/Emotion +peterldowns/clickbait-classifier +pfnet-research/chainer-chemistry +pfnet-research/nips17-adversarial-attack +philipperemy/Deep-Learning-Tinder +philipperemy/keras-activations +philipperemy/keras-tcn +philipperemy/tensorflow-1.4-billion-password-analysis +philipperemy/tensorflow-phased-lstm +philipxjm/Deep-Convolution-Stock-Technical-Analysis +phreeza/tensorflow-vrnn +pjankiewicz/mercari-solution +pkmital/pycadl +plotly/plotly.py +polyaxon/polyaxon +princewen/tensorflow_practice +productml/blurr +pydata/pyrallel +pymc-devs/pymc3 +pymetrics/audit-ai +pytorch/ignite +pytorch/text +pytroll/satpy +qfgaohao/pytorch-ssd +qqwweee/keras-yolo3 +qubvel/classification_models +qubvel/segmentation_models +r9y9/deepvoice3_pytorch +raghakot/keras-resnet +raghakot/keras-text +raghakot/keras-vis +rakeshvar/rnn_ctc +randaller/cnn-rtlsdr +rasbt/mlxtend +ray-project/ray +rcmalli/keras-mobilenet +reiinakano/scikit-plot +reinforceio/tensorforce +renmengye/tensorflow-forward-ad +rezoo/illustration2vec +riga/tfdeploy +rinuboney/ladder +rizkiarm/LipNet +robertmartin8/MachineLearningStocks +robintibor/braindecode +robmsmt/KerasDeepSpeech +rockingdingo/deepnlp +romanorac/discomll +ronghanghu/tensorflow_compact_bilinear_pooling +roseperrone/video-object-detection +rtavenar/tslearn +rubenvillegas/cvpr2018nkn +rushter/MLAlgorithms +rushter/heamy +rusty1s/pytorch_geometric +rwth-i6/returnn +ry/tensorflow-resnet +sadeepj/crfasrnn_keras +salesforce/decaNLP +samim23/NeuralTalkAnimator +sanghoon/tf-exercise-gan +santi-pdp/segan +sapruash/RecursiveNN +scikit-garden/scikit-garden +scikit-learn-contrib/imbalanced-learn +scikit-learn-contrib/project-template +scikit-learn/scikit-learn +scikit-multilearn/scikit-multilearn +scofield7419/sequence-labeling-BiLSTM-CRF +sebp/scikit-survival +sentinel-hub/eo-learn +shanren7/real_time_face_recognition +shaohua0116/Group-Normalization-Tensorflow +shaohua0116/Multiview2Novelview +shawnxu1318/Google-Neural-Machine-Translation-GNMT +shaypal5/skift +shenweichen/DeepCTR +shubhomoydas/ad_examples +shuyo/iir +simpleai-team/simpleai +sjvasquez/instacart-basket-prediction +skggm/skggm +sklearn-theano/sklearn-theano +smallcorgi/Faster-RCNN_TF +sniklaus/pytorch-pwc +snipsco/ntm-lasagne +snowkylin/TensorFlow-cn +snowkylin/ntm +soeaver/Pytorch_Mask_RCNN +solivr/tf-crnn +sony/nnabla +sonyisme/keras-recommendation +sparklingpandas/sparklingpandas +spotify/chartify +spotify/luigi +spragunr/deep_q_rl +src-d/ml +sreyafrancis/PFNN +stanfordmlgroup/nlc +stanfordnlp/spinn +stanfordnlp/stanza +stared/livelossplot +stas-semeniuta/textvae +stellargraph/stellargraph +stephenhky/PyShortTextCategorization +stonycat/ML-in-Action-Code-and-Note +strubell/LISA +supervisely/supervisely +sveitser/kaggle_diabetic +svenkreiss/databench +svenkreiss/pysparkling +sxjscience/HKO-7 +syang1993/gst-tacotron +tadejmagajna/HereIsWally +taehoonlee/tensornets +taki0112/MUNIT-Tensorflow +taki0112/Self-Attention-GAN-Tensorflow +taki0112/StarGAN-Tensorflow +takiyu/hyperface +taolei87/rcnn +tatsy/keras-generative +tdda/tdda +tdrussell/IllustrationGAN +tencia/stocks_rnn +tensorflow/agents +tensorflow/compression +tensorflow/data-validation +tensorflow/datasets +tensorflow/fold +tensorflow/hub +tensorflow/kfac +tensorflow/lattice +tensorflow/mesh +tensorflow/models +tensorflow/moonlight +tensorflow/nmt +tensorflow/ranking +tensorflow/skflow +tensorflow/tensor2tensor +tensorflow/tensorboard +tensorflowkorea/tensorflow-kr +tensorlang/tensorlang +tensorlayer/openpose-plus +tensorlayer/tensorlayer +tensorpack/tensorpack +tensorprob/tensorprob +terrytangyuan/tensorflow-in-practice-code +tesserai/tfi +tf-coreml/tf-coreml +tflearn/tflearn +tgsmith61591/pmdarima +thaines/helit +thatbrguy/Pedestrian-Detection +the-new-sky/Kadot +therne/dmn-tensorflow +thoschm/START-Summit-2017-Blockchain-Machine-Learning-Workshop +thtrieu/darkflow +thu-ml/zhusuan +thulab/DeepHash +thunlp/OpenNRE +tiberiu44/TTS-Cube +timediv/speechT +titu1994/Image-Super-Resolution +titu1994/MobileNetworks +tkamishima/mlmpy +tkuanlun350/Tensorflow-SegNet +tobegit3hub/deep_image_model +tobegit3hub/tensorflow_template_application +tobyyouup/conv_seq2seq +tokestermw/tensorflow-shakespeare +tomrunia/TF_FeatureExtraction +tonybeltramelli/Deep-Spying +tostq/Caffe-Python-Tutorial +tperol/ConvNetQuake +trailbehind/DeepOSM +trevorstephens/gplearn +tryolabs/luminoth +tscohen/GrouPy +tuxchow/ecm +u39kun/deep-learning-benchmark +uTensor/uTensor +uber/petastorm +uchicago-cs/deepdish +ufal/neuralmonkey +ufora/ufora +ufoym/deepo +una-dinosauria/human-motion-prediction +undertheseanlp/underthesea +uoguelph-mlrg/theano_alexnet +upul/Aurora +vahidk/TensorflowFramework +vatlab/SoS +vecxoz/vecstack +vergeml/vergeml +visipedia/tf_classification +vitruvianscience/OpenDeep +voicy-ai/DialogStateTracking +vonclites/squeezenet +vrenkens/nabu +vrenkens/tfkaldi +vudung45/FaceRec +waleedka/hiddenlayer +wangheda/youtube-8m +wendykan/DeepLearningMovies +wenwei202/terngrad +wiseodd/hipsternet +wnzhang/deep-ctr +wookayin/tensorflow-plot +wuaalb/keras_extensions +wuzheng-sjtu/FastFPN +xdever/RFCN-tensorflow +xiaofengShi/CHINESE-OCR +xiaohan2012/twitter-sent-dnn +xingyul/Sparse-Winograd-CNN +xionghc/Facial-Expression-Recognition +yahoo/TensorFlowOnSpark +yahoo/lopq +yangxue0827/FPN_Tensorflow +yangxue0827/R-DFPN_FPN_Tensorflow +yangxue0827/R2CNN_FPN_Tensorflow +yao62995/A3C +yeephycho/nasnet-tensorflow +yihui-he/channel-pruning +youngjoo-epfl/gconvRNN +yrlu/reinforcement_learning +yu4u/convnet-drawer +yuvalpinter/Mimick +zalandoresearch/flair +zhangqianhui/progressive_growing_of_gans_tensorflow +zhongwen/predictron +zhreshold/mxnet-ssd +zhuzilin/NP_ML +zishansami102/CNN-from-Scratch +zjjMaiMai/Deep-Alignment-Network-A-convolutional-neural-network-for-robust-face-alignment +zjy-ucas/ChineseNER +zomux/deepy +zsdonghao/Image-Captioning +zzw922cn/Automatic_Speech_Recognition diff --git a/src/logChunk/func_repos_from_other_GH_searches.txt b/src/logChunk/func_repos_from_other_GH_searches.txt new file mode 100644 index 0000000..3df36dc --- /dev/null +++ b/src/logChunk/func_repos_from_other_GH_searches.txt @@ -0,0 +1,696 @@ +0xDECAFC0FFEE/stamp2 +3389306042/12306_qiang_piao_python +50mkw/mysite +631068264/learn-sktf +631068264/learn_science +AGKhalil/diss_quad +AIVenture0/Sentiment-Analysis-Using-Deep-Learning +ALX7/ai_project +ANTsX/ANTsPy +ASLive/webserver +ATLAS8346/knowledge-distillation-confidence-distillation-final +Adamits/kaggle-housing-regression +AdrianLangseth/NAIL-Jetson-Demo-Project +AmoghLonkar/while +AmritK10/Image-Captioner-Web-App +AndreasMerentitis/TfLambdaDemo-keras +AndreasMerentitis/TfLambdaDemo-tfraw +Antymon/baselines_tf2 +Arjunbala/DistillationExperiments +ArthurMARIANI/Articles_Classifier +AswinGnanaprakash/code_box +Ayshine/Machine-Learning +BUEC500C1/twitter-summarizer-rest-service-dabadi9 +BUPT-GAMMA/OpenHINE +Baneeishaque/Handwritten-Digit-Recognition-using-Deep-Learning +Bao-Jiarong/ResNet +BarrioRQI/MachineLearningQFT +Beacon-Platform/trellis +BenjiTheC/BERTBasedTCPM +Bernard-A/ML_B-L475E-IOT_LoRa +BojanKomazec/nvidia-tensorrt-python +Botathon-CY/Parkly +CCTV-Compression/CCTV-Compression_yolov4_deepsort +CCheng00/310-Assignment-3 +CCheng00/310-Individual +Calamari-OCR/calamari +Calvin-CS/slo-classifiers +CarlKT/summerJob2020 +CeivenLean/workspace +Chenaah/RAMCO +ChetanMadan/DoItRight +ClaireSang/FinacingPGPortfolio +CrawlScript/tf_geometric +DDMAL/hpc-trainer-component +DanielSanRocha/SnakeAI +DataScienceUB/DeepLearningMaster20192020 +Demonpw/Demonpw-microblog +Demonpw/microblogpw +DewMaple/fr +DifferentiableUniverseInitiative/DHOD +DrDongSi/Ca-Backbone-Prediction +Drblessing/codewars +EdisonLeeeee/GraphGallery +EduardoFAFernandes/CatDogBot +Endyrmion/MachineLearning +EonYang/doodle-classification-server +Esmidth/CS_GAN_MOD +FLming/CRNN.tf2 +FranzWalgenbach/ModulML +FranzWalgenbach/Thesis +Garpetun/considition +GaryDoooo/ml_codes +Gauravsahadev/Covid-19-Early-detection +GeoCode-polymtl/Deep_1D_velocity +GlinZhu/SDC_Capstone_Project +GoareguerLucas/GAN-SDPC +GoogleCloudPlatform/practical-ml-vision-book +HAOzj/tensorflow2-Demo +HarshCasper/Ling-API +Harshwin/Wide-Residual-Newtork- +HarvardAgileRoboticsLab/gym-kuka-mujoco +HenriARM/gan-art +Hisairnessag3/rl_crypto +HismaelOliveira/RBM +IrennaLumbuun/RateMe +ItsCosmas/traffic-signs-classifier +JC1DA/yolo_streamer +JCelayaRdz/Artificial-Intelligence +JChander/DeepMiR2GO +JHP4911/DeepFaceLab +JRonGitHub/SDC_Capstone_Project_submit +JackRossProjects/ReadyOrNot +Jackadsa/EMD_VAE +JacobJeppesen/RS-Net +JaeYoungKim1/practice_deep_learning +JanKalo/RelAlign +Jana-Z/reinforced_snake +Jaram2019/DeepLearning_Alphabet +Jaskaran170599/Sentimental_Extraction +JayYip/bert-multitask-learning +JoeLittell/ImageClassDash +JonnyBanana/transparent_latent_gan +KDD-OpenSource/DeepADoTS +KGFlow/KGFlow +KaiChen1008/Sim-to-Real-Virtual-Guidance-for-Robot-Navigation +Kamranbarlas/API-PROJECT +KentaKawamata/installed-tensorflow-gpu1.15 +KhliustovDmitrii/aramco +Kimyuhwanpeter/4th_paper_method +Kushagraagarwal/clothes-classifier +LARC-CMU-SMU/facial-weight-change +LajosNeto/learning-ml +Laoher/python_notes +LeeHounshell/BatBot +LeelaChessZero/lczero-training +LichengXiao2017/deep-image-compression +LongxingTan/Yolov5 +Louis5499/Parallel-Computing +Lsdefine/attention-is-all-you-need-keras +Luochenghuang/TEMANN +MGL56/Projet_6 +MLH-Fellowship/neuro-art +MLTransfer/MLEncrypt-Research +MachineCF/kerasSaveLoadModel +MansoorAliSoomro/chat-bot +ManuGar/UFOD +ManuelSzewc/ML4DT +Mario-Medvedovic/deepPhosAPI +MeMAD-project/media-memorability +Meskupie/Sonicam +MexsonFernandes/IsItFake +Minyus/Config +MiroDur/MuScat_Miro +MisterVladimir/vslearn +MoritzTaylor/maml-rl-tf2 +Mvystrcilova/songRecommender +MyRespect/AdversarialAttack +N3PDF/vegasflow +NVIDIA/TensorRT +Nagakiran1/4-simple-steps-in-Builiding-OCR +Nam-SW/Response_Generation +NapsterInBlue/napsterinblue.github.io +Natthinee/newproject +Natthinee/start +Natthinee/testing +NeuralNetworkVerification/Marabou +Nickster28/PaintingMatcher +Nikolai10/scrabble-gan +NishitHada/Sentiment-Analysis +Nuitka/Nuitka +OlafenwaMoses/FireNET +OlafenwaMoses/ImageAI +Olivolja/mlHollf +PacktPublishing/Tensorflow-2-Reinforcement-Learning-Cookbook +ParmbeerJohal/Deep-Learning-for-Computer-Vision +PassengerAI/custom-tf-pose +PatricioClark/PINNs +PaulCnth/now_u_know_I_host_1_website +Peaceout21/FAQ-Bot +PennyLaneAI/qml +PeterL1n/BackgroundMattingV2-TensorFlow +Phoenix4582/Deep-Learning-Library +Prtfw/BlackSwan_NLP_transfer_learning +PureIso/exchange.core +PythonOT/POT +R-Stalker/RSNLP +RameenAbdal/StyleFlow +Raymond-Su/TensorFlowNameRecogniser +RealHulubulu/Coronavirus_Data +ReiffSoftwareServices/deployapp +Riathoir/PASSGAN-IWGAN-Tensorflow-2 +Rich0721/SSD_tf_keras +Ritchieluk/OldBrainDump +RomainStevenne/SRGAN +RuiLiFeng/flow-gan +Runist/YOLOv4 +Runist/YOLOv4_tiny +RxstydnR/PUlearning_segmentation +SRM-Hackathon/hack- +SRM-Hackathon/satrajit2_574b +SchillingEnterprises/Gymkhana_Stampede +SeJV/ComparisonRLapproaches +SeanAvery/async-tensorrt +SensorsINI/CartPoleSimulation +Serge-Kuz/skillfactory_rds +Shall-Chee/Multi-Robot_Coverage_Control_from_Time-Varing_Density_Map +ShayneTB/Chatting_with_Yourself +Shivamagrawal2014/tf_base +SilentFalls/tog +Skere9/demo-middleware-on-aws +SoR3/lab5_web +Sohaib76/Intelexica_flask-yolov5 +Soonmok/mask-rcnn-tf-2.0 +Sounce/sounce-desktop +SpencerIsTheKey/310-Software-Engineering +StarPrecursor/hepynet +StefanCobeli/Exploring-Optimism +StefanLam99/Explaining_ABSA +Supriya-Suresh/eYSIP-2017_Vegetable-Identification-Using-Transfer-Learning +Surya97/Data-Science-projects +TAN-OpenLab/FTCF-NET +TP-BIAC/CCNet +The-Bread/Reinforcement-Learning +TimothyHelton/tensorflow_2 +TolgaSari/network_compression +TomeASilva/PPO-Vs-PPO-Agent-learning-framework-to-control-Production-Flow +TomeASilva/PPO_Production_System +UKPLab/naacl18-multimodal-frame-identification +WWRS/circlepondbot +Welthungerhilfe/cgm-ml +WilliamYuhangLee/AAD +Yangangren/experiment_driving +Yuran-Zhao/tf2-multimodal_sarcasm_detection +Z-yq/TensorflowASR +Zhou602357524/12306 +a-mcego/argh-tf +a0x8o/beam +aadi350/player-cricket-detect +aboerzel/German_License_Plate_Recognition +abrahamnunes/fitr +achandec/Driver-Attentiveness-Monitoring-System +achillesheel02/wikimedia-citation-classification-task-2 +aditya140/NLPTriples +adrianodennanni/serverless-deep-learning-deploy-example +aeronmiles/TF_FLAME +agatan/paper-reading-notes +agathauy/CoE197-EE298-DL-Mini-Proj +agrinh/procyclist_performance +agupta1897/ORAs +ai-lab-circle/deep-tutorials-tensorflow-slim +aikiyy/dl4us +ajs7270/Tensorflow +alicika/snippets +alinstein/Modify-image-by-text +alvexs/ml_course +amanda-basso/SSD_Object_Detection +analysiscenter/pydens +anandharaju/Echelon_Partition_Space +anatolyefimov/parking-space-detecting +andrebola/EUSIPCO2020 +andylolu2/Alpha-Reversi +anejad/Convolutional-Neural-Network-Champions +angelnew/biblioeater +animesh/deepmind-research +antmusco/crowd_density +apacha/MusicSymbolClassifier +apple/coremltools +apple/tensorflow_macos +armando0194/MotionTransfer +arneschmidt/cancer_classification +arturomoncadatorres/deepsurvk +ashkankzme/QAforMisinformation +aswinjose89/ai-cnn +auee028/action-intent +ausmartinez/MBARI-RetinaNet-ODM +bagustris/dimensional-ser +balrajendran/TSA_Demo +bcmclean/TherapistJen +behrouzsh/deepPhosAPI +benblackcake/SRGAN_ben_v1 +benblackcake/srgan_t1 +benhsu75/sigopt-examples +benjaminabruzzo/AlphaPose +berktepebag/self_driving_car_ROS +bhklab/uhn-radcure-challenge +bio-ontology-research-group/deepgoplus +bobbyrathoree/boggins +boczekbartek/TwitterSentimentAnalyser-NN +bojone/bert4keras +bok9504/Yolov3_DeepSORT +breitmuuufrosch/OpenCvPipeline +brentritzema/senior-project +brollb/spice-completion +bubbliiiing/ssd-tf2 +bvlj/diab +candidosales/lending_loop +cascio/Kaggle-Competitions +ccaison/flask_chat +chingleo/deeplearning +chrisjihee/WiseData +cienciaydatos/ai-challenge-mars +clint-kristopher-morris/yolo-assisted-image-scrape +cod3licious/simec +codevor/webcam-playground +codingeverybody/codingyahac +constantineg1/dlclouds-examples +ct-1908/12306 +cunningham-lab/epi +cvd-q/ML_2020_project +czy36mengfei/tensorflow2_tutorials_chinese +dadwalakshay/Tundil +dakshverma2411/super-resolution-on-astronomical-images +daniel-deleon/BlueA_infer +danielegrattarola/spektral +danielwatson6/skip-thoughts +dartrevan/ChemTextMining +darwinsubramaniam/Image-Classification-by-Keras-and-Tensorflow +dataiku/dss-plugin-deeplearning-image +datawombat/ds-kaggle-notes +davidADSP/GDL_code +dawn1004/asdasd +deaspo/Unet_MedicalImagingSegmentation +deepfakes/faceswap +deeplab-ai/deephub +deeplearningturkiye/pratik-derin-ogrenme-uygulamalari +deepmind/tree +delira-dev/delira +demid5111/smart-detector +dennisbakhuis/Tutorials +dgkutnic/tensorflow +dimitardi/deep-predict +distillpub/post--growing-ca +djkormo/k8s-AKS-primer +djovanoski/Challenge +dnmarch/mrc +dorasame/Picarist +dotd/IndistrialInteractionRecognition +douglasregio/teste +duplessisaa/Docker +dushyant7917/HerbScout +dylan9033/12306 +ecaradec/humble-yolo +edhenry/roadsoda +eiurur/NNNNN +elifesciences/sciencebeam-gym +elroyhaw/StockPricePredictionWithGANs +emayfield/AFD_Decision_Corpus +emilyolafson/3d-cnn +empirical-org/Quill-NLP-Tools-and-Datasets +eric-erki/id-card-detector +ermannomillo/tennisCourt +eungbean/knowledge-distillation-cifar10 +ex-hasuolab/deeplearning-set +ezavarygin/PConv2D_Keras +ezvk7740/RLbaxter +f9e/f9k9 +fabiofumarola/ultrayolo +fatihcankurnaz/SensorGAN +faustomorales/keras-ocr +fazaki/cycle_prediction +fchollet/deep-learning-with-python-notebooks +fengjiran/tensorflow_learning +fn199544123/download-demo-beyebe +foss-for-synopsys-dwc-arc-processors/embarc_mli +francesco-mannella/Echo-State-Networks +franckalain/pneumonia-detection-midterm +franneck94/TensorCross +ftramer/ensemble-adv-training +furrypython/PConv-TensorFlow2 +fusionby2030/CNN_Fashion_MNIST +gabrielxzc/moc1-aset-project +gaunthan/ground-segmentation +gavincaoyuji/seq_2_seq +geblanco/squad-experiments +georgebrockman/DeepLearning +gillesirit/analogy +giovaninppc/MC030 +gnoejh/RocAlphaGo +gofynd/mildnet +goneByWind/Time-series-prediction +google-research/evoflow +google-research/google-research +google-research/recsim_ng +google-research/seed_rl +google/TensorNetwork +google/jax +grahamgower/genomatnn +grep0/necropedia +gsethi2409/tf-pose-estimation +guilhermealarcao/nemanet +hankcs/HanLP +hannutho/DataScienceSonacy +happyyuwei/Eidolon-Tensorflow +hayashikun/mygan +hellloxiaotian/BRDNet +hellojialee/Multi-Person-Pose-using-Body-Parts +henriwoodcock/microTensorflow +hieuctfe/Captcha-DL +hih0124/lecture_deep_learning +hoangthienan95/computer-vision-project +horovod/horovod +htdt/diqn +huggingface/node-question-answering +huggingface/transformers +hugokitano/DeepMoon---Lunar-Crater-Counting-Through-Deep-Learning +hyLiu1994/KnowledgeModels +hyperlex/Signature-detection-Practical-guide +hzk123/Thesis_maml +hzm2016/Curiosity-driven-Exploration +idankdev/Chess +ipazc/mtcnn +issaiass/ObjectDetection_Retinanet +issaiass/ObjectDetection_SSD_TFOD_API +its-gucci/disentanglement +jacobjohn2016/Deep-Learning-Specialization +jadeocr/jadeocr +jahongir7174/YOLOv5-tf +jaikishan26/Virtual_lab +jbi35/seg_training +jchaykow/trucksim +jcrodriguez1989/weeddream +jdhayford/jdhayford +jecrespo/Introduccion-a-Deep-Learning +jeffreire/transfer-learning-TF +jenlabossiere/SoftwareEngineeringA3 +jeongukjae/multilingual-bert-hate-speech-detection +jesse1029/SiGAN +jesus-a-martinez-v/cifar-10 +jeugregg/coronavirusModel +jeweldhali/stock-price-prediction +jiajunhua/AlphaPose +johanndejong/VaDER +johertrich/Proximal_Neural_Networks +jooh/gwp +jooh/sana +jpatel0/Train_model_classifier +jrkns/nlp-bot-project +jtchilders/atlas_dgcnn +judithliatsf/fillmore +julian-zucker/arbiter +junghwanchoi/Tensorflow-in-plactice +jzeimen/springboard-capstone +kalperenctnr/Caption-Generator +keiohta/tf2rl +kendryte/nncase +kenmaro3/tensorflow_practice +kingcheng2000/deepmind-research +kishorkuttan/Covid-19-drug-discovery-using-RNN-LSTMS-stack-augmented-RNN-and-RL-powered-by-Microsoft-Azure +kmohanku/Low-Light-Image-Video-Enhancement +knowledgedefinednetworking/demo-routenet +kobaryubi/MySiteSample +komiyakomiyakomiya/dockerfile_kaggle +korneelvdbroek/mp3net +koshian2/OctConv-TFKeras +koulanurag/deep-conformal +kpe/params-flow +krasserm/super-resolution +krishdb38/ML_Machine_Learning +krtbb/faces +kubeflow-kale/kale +kubeflow/pipelines +kurapan/EAST +kylemath/DeepEEG +lablup/backend.ai-kernels +larisaszatmari/facerec +laura8857/linebot +leejisue/study_git +leeleavitt/procPharm +legokichi/py_sandbox +leimao/Frozen-Graph-TensorFlow +lejeunel/ksptrack +lenngro/neuralstyletransfer +lenongsa/eeg_survey +leoHeidel/AtlasNet-tf +leona-ha/Skin-Screening_Web-App +leriomaggio/reproducible-learn +lfloretta/tf_ecosystem +lihuang3/self-learning +liuhuiaren0524/BERT +lixilinx/psgd_tf +liyuan9988/IVOPEwithACME +llamalle/TERL2 +lpphd/multivariate-attention-tcn +lu-my/demos +luisvalesilva/digitre +luke-grassroot/scratchgan +lululxvi/deepxde +lxd1190/test_proj +madrugado/Attention-Based-Aspect-Extraction +magenta/ddsp +mahathi26/cis1 +mahathi26/ciss +mamromer/Yolov3-to-Tensorflow +marcduby/MachineLearningPython +mathan1995/ChatBot-with-Tensorflow +maulikp11/final-project- +mchrestkha/machine_learning_examples +mesutpiskin/raspberry-beacon +mhn10/edge-analytics-dashboard +miamiww/face_generator +michael1997/landing-housekeeping +michaelliu03/BotInAction +michaelperel/putput +microsoft/Windows-Machine-Learning +microsoft/onnxruntime +misawa0616/myblogapp +mk60991/DeepLearning-with-keras +mlcommons/inference +modichirag/galference +mokpolar/kubeflow +monishramadoss/caidm_projects +monta0315/graduate_study +moop-china/dockerfiles +moritzlauff/thesis_programming +mostafarohani/info_bottle +mpoiitis/unsupervisedBotDetection +mriberodiaz/selection_ou_weights +mustafaaydn/yazbel-net +nagaoka-ai-innovationhub/basics-of-image-recognition-with-cnn +nathan-vm/MegaHackathon +naturalis/sdmdl +neha191091/IAF_Dynamics +neuronets/nobrainer +nickycheng4/Digital-Distributed-IoT-Assistant +nimelehin/HseDiary +nishantsapkota487/Quizapp +nju-websoft/AliNet +nkaeming/unsupervised-machine-learning-of-topological-phase-transitions-from-experimental-data +nng555/conf +nnirmall/Handwritten-Text-Recognition +noahchalifour/rnnt-speech-recognition +nottahagilani/tagpakistan-deploy-ml +novmad/praktikum-tasks +odnh/gnn-routing +olivierSaintCyr/Multi-Discriminator-GAN +oneplus-x/g-Security +opencv/opencv +openvinotoolkit/openvino +otonnesen/coursework +oyew707/Maze_Search +pabloswfly/genomcmcgan +pathak22/noreward-rl +patrickcgray/sif_analysis +pblin/re-marketdata-utils +pedro-morgado/spatialaudiogen +pedro-r-marques/keras-list-mapper +perevale/matches_prediction +perslev/MultiPlanarUNet +peterliht/knowledge-distillation-pytorch +peteryuX/esrgan-tf2 +phylovi/libsbn +pickandpick/12306 +piercetu/Hubbub +policratus/pupyl +ppaajjiikkaa/heroku-hello +pupal-deep-learning/PuPal-Beta +pydsgz/DeepVOG +pydsgz/MGMC +pytorch/pytorch +quarl-iclr/quarl +qubvel/efficientnet +ragesh-r/CarND-Capstone +raj713335/TENSORFLOW_PRACTICE +rajeev921/machine_learning +ralf-goettsche/CarND-Capstone +ramcn/chiron-0.4 +rashanarshad/deconv_api +rashanarshad/instancesegmentation_api +raven724/swe3011_proj2 +re-face/re_face +rembli/rocketman +rezakhosravi1/INM702City +rickyHong/Deepmind-research-alphafold-repl +rickyHong/Deepmind-research-repl2 +riju92/Image-Captioning +rishabhvarshney14/Detect-House-Item +rivas-lab/FasTag +rjsnh1522/code_analyzer_django_project +rjsnh1522/django-boilerplate +rknaebel/textgame-project +rlgraph/rlgraph +robbierobinette/longview-deep-learning +rockchip-linux/rknn-toolkit +rogerhcheng/LiteFlowNet2-TF2 +rohwid/rpg-stats-nn-multiclass +rt416/NSampler +rubiin/stanford-tensorflow-tutorials +sabrish89/sentenceCompletion +saiwaiyanyu/bi-lstm-crf-ner-tf2.0 +salaie-system/jetson +samsniderheld/SpriteGAN +samuelmat19/DDPG-tf2 +samyak989/CarDamageDetection +sandipanbasu/conversation-ai +sassoftware/sas-viya-programming +saum7800/reddit-flair-prediction +sc250024/python-vulnerability-testing +schillgc/Python +schutera/DeepLearningLecture_Schutera +scottemmons/sgm +seanj773/Bayesian-Op +seanjparker/distributed-rl +secondmind-labs/trieste +sen-sourav/transliterationLanguageDetect +senya0730/Tensorflow2_tutorial_pablic +serengil/deepface +serizba/cppflow +shammur/news_categorization_english +sharethis-github/simple_tensorflow_serving +sherzod-hakimov/knowledge-graph-embeddings +sheunaluko/cs230 +shin285/news_crawler +shivam1808/Mini_project +shopping-tang/Alpha-pose +sidneyarcidiacono/burroughs +siftr/notebook +silburt/DeepMoon +simeon-spasov/MCI +sjjdd/Deepose_DIY +smartcameras/PrivEdge +smrut1r/Deep-Neural-Networks-HealthCare +smrut1r/deep-learning +snooty7/BrawlStars +snowkylin/tensorflow-handbook +sonia-auv/proc_detection +sorryformyself/pong +sosthenee/Football-computer-vision +speechLabBcCuny/nnaAudiosetClassification +splicemachine/pysplice +spyder-ide/spyder-kernels +srihari-humbarwadi/progan-tensorflow2.x +srihari-humbarwadi/retinanet-tensorflow2.x +ssamot/socialist_planning +sshleifer/santander +staminajiro/shoten_7 +stanlee321/cars_classifier +stelios357/Facial-Detection-and-Recognition +stephen-w-bailey/fast-n-deep-faces +stevenzhou2017/AlphaPose +sub-mod/tf-mnist +sukruc/deeplearningai-c4-w3 +sumitIO/FaceDectection-using-EfficientDet +swagking0/Curiosity_placed +syedmeesamali/Python +synbioks/Text-Mining-NLP +ta9ryuWalrus/simclr +taheritajar/experts-group +tbd-ai/tbd-suite +tbd-group/twitter-sentiment-analysis +tdelubac/A3C +tdelubac/Deep_Q_Network +teelinsan/KerasDropconnect +tensorflow/io +tensorflow/model-analysis +tensorflow/model-optimization +tensorflow/privacy +tensorflow/profiler +tensorflow/quantum +tensorflow/recommenders +tensorflow/serving +tensorflow/tensorrt +tensorflow/text +tensorflow/tfjs +tf-encrypted/tf-encrypted +thandongtb/tabby_search +thanhbok26b/baseline-atari-ram +thanhtinhpas1/vispeechcore +theislab/scCODA +thomshaw92/PialNet +thumbe12856/superMarioBro +thuyhoang-hvtt/digit-recognition-app +titu1994/keras-attention-augmented-convs +titu1994/keras-coordconv +titu1994/tfdiffeq +triton-inference-server/server +tupleblog/generate-thai-lyrics +twofund/gan +upc-projects/minsa-twitter-sentiment-analysis +ural-gorets/snippets-back +vail131/Python +vanessailana/Seminar2 +vasumv/saliency_convnet_proj +veyorokon/handwriting-synthesis +victorphd/123 +vikra8pc/FinalProject_RutgersDS +villasen/ML-Sound-Classification +vincentvigon/RECHERCHE +vishakhagupta10/nlp_model_demo +vishalsinghji/Edge-Detection- +visiont3lab/covid-mask-classifier +vitorys/MusicGenreMetaClassifier +volotat/DiffMorph +vomin0107/ObjectDetection +w595shubham/rcpiml +wadimstt/dip +wandb/client +wangcongcong123/ttt +warteg-biru/exercise-pose-evaluation-machine +weely/webapp-python +wflosin/Drive-A-Car-AI +wngaw/blog +wolf1121/12306 +wqvbjhc/srez +wuyx/noreward-rl +wwydmanski/RLinWiFi +x41lakazam/Pharmacam +xianglinyang/DeepVisualInsight +xiaohengdai/popupRecognition +xju2/root_gnn +xryash/mnist_nn_api +xujenna/a2z +xxxxsars/Tw_railway +yeataro/TD-ONNX-EX +yinxusen/deepword +yinyifa/boggins +yogt1984/repo +yuenanchen/12306-master +yuyuenuli/-home-me +yxd886/pipeline +zaccharieramzi/tfkbnufft +zhuby1973/12306 +zhuchen03/transformer_federated +zhuojg/aadb_tf2 +zihuaweng/image_recognition_service_api +ztasre/dss-classifier +zuhlke-kerry-logistics-hackathon/warehouse-backend \ No newline at end of file diff --git a/src/logChunk/getGitLog.py b/src/logChunk/getGitLog.py index c47c040..ef47923 100644 --- a/src/logChunk/getGitLog.py +++ b/src/logChunk/getGitLog.py @@ -77,7 +77,7 @@ def processLog(projPath): log_file = projPath + os.sep + LOG_FILE if not os.path.isfile(log_file): - print("%s does not exist!!" % (log_file)) + print(("%s does not exist!!" % (log_file))) return ghDb = ghLogDb(log_file) @@ -111,16 +111,16 @@ def getGitLog(project, languages, config_info): continue proj_path = os.path.join(project, p) if(config_info.DEBUGLITE or config_info.DEBUG): - print proj_path + print(proj_path) dumpLog(proj_path, languages) #processLog(proj_path) def main(): - print "==== Utility to process Github logs ===" + print("==== Utility to process Github logs ===") if len(sys.argv) < 3: - print "!!! Usage: python ghProc.py project config_file" + print("!!! Usage: python ghProc.py project config_file") sys.exit() project = sys.argv[1] @@ -140,7 +140,7 @@ def main(): getGitLog(project, langs, config_info) #processLog(project) - print "Done!!" + print("Done!!") if __name__ == '__main__': diff --git a/src/logChunk/ghLogDb.py b/src/logChunk/ghLogDb.py index 86b985f..4b6a727 100644 --- a/src/logChunk/ghLogDb.py +++ b/src/logChunk/ghLogDb.py @@ -33,12 +33,12 @@ def timeout(signum, frame): def toStr(text): try: - text1 = str(text).encode('iso-8859-1') + text1 = str(text) temp_text = text1.replace("\'","\"") temp_text = temp_text.strip() return "\'" + str(temp_text) + "\'" except: - print type(text) + print(type(text)) return "\'NA\'" class Patch: @@ -142,7 +142,7 @@ def getFullTitleString(self): i = 0 if(patches != []): #Iterate until we find a patch that contains a method. - while(title == "" and i < len(patches)): + while(title == "" and i < len(patches)): title = patches[i].getFullTitleString() i += 1 @@ -203,7 +203,7 @@ def printSha(self): retStr += self.printPatches() - print retStr.encode('ascii', 'ignore') + print(retStr.encode('ascii', 'ignore')) def dumpPatches(self,dumpLogDb): for p in self.patches: @@ -229,18 +229,18 @@ def if_bug(self, text): global lmtzr global stoplist isBug = False - + text = text.lower() text = text.replace('error handl','') text = text.replace('error cod','') - + imp_words = [lmtzr.lemmatize(word) for word in text.lower().split() \ if ((word not in stoplist)) ] bug_desc = ' '.join([x for x in imp_words]) - + if "bug= " in bug_desc or "bug=none" in bug_desc: return isBug - + if re.search(ERR_STR, '\b'+bug_desc+'\b', re.IGNORECASE): isBug = True @@ -266,13 +266,13 @@ def __init__(self, logFile, c_info, password = ""): self.dbPass = password self.config_info = c_info #configuration info and options - + def __str__(self): - print self.project_name + print(self.project_name) for s in shas: - print s + print(s) def isSha(self,line): @@ -281,7 +281,7 @@ def isSha(self,line): if line.startswith("commit") and is_sha: sha = is_sha.group(0) if (self.config_info.DEBUG or self.config_info.DEBUGLITE): - print("COMMIT: " + sha) + print(("COMMIT: " + sha)) return sha def isAuthor(self,line,shaObj): @@ -290,9 +290,9 @@ def isAuthor(self,line,shaObj): is_auth = re.search(EMAIL, line, re.IGNORECASE) if line.startswith("Author:") and is_auth: author = is_auth.group(0) - shaObj.author = line.split(author)[0].split("Author:")[1] + shaObj.author = line.split(author)[0].split("Author:")[1] shaObj.author_email = author - print("Email: " + shaObj.author_email) + print(("Email: " + shaObj.author_email)) shaObj.author = shaObj.author.strip() return True @@ -380,9 +380,9 @@ def processPatch(self, line, patchObj, curLogChunk): pass elif line.startswith("@@ "): - if(self.config_info.DEBUG): - print("New @@: " + line) - print("HEADER: " + curLogChunk.header) + if(self.config_info.DEBUG): + print(("New @@: " + line)) + print(("HEADER: " + curLogChunk.header)) #Parse the previous chunk and store the results. if(curLogChunk.header != ""): #If there is an existing chunk to parse self.processLastChunk(patchObj, curLogChunk) @@ -444,7 +444,7 @@ def processLog(self, config = ""): project1 = os.path.split(self.log_file)[0] project1 = project1.rstrip(os.sep) self.project_name = os.path.basename(project1) - print("---------- %s ------------\n" % (self.project_name)) + print(("---------- %s ------------\n" % (self.project_name))) if(self.config_info.DATABASE): dl = dumpLogs(self.dbPass, self.config_info) @@ -466,8 +466,8 @@ def processLog(self, config = ""): lst = mockChunk.readKeywords(lst) keywords= [k[0] for k in lst if k[1] == INCLUDED] for keyword in keywords: - listToDict[str(keyword)+" Adds"]=0 - listToDict[str(keyword)+" Dels"]=0 + listToDict["\"" + str(keyword) + "\" adds"]=0 + listToDict["\"" + str(keyword) + "\" dels"]=0 inf2.write("project, sha, language, file_name, is_test, method_name,total_add,total_del,%s\n"%",".join(sorted(listToDict.keys()))) @@ -503,20 +503,20 @@ def processLog(self, config = ""): # if(self.config_info.DEBUGLITE): # print("Writing Sha:" + sha) - # if(self.config_info.DATABASE): + # if(self.config_info.DATABASE): # shaObj.dumpSha(dl) # elif(self.config_info.CSV): # shaObj.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary) # else: # shaObj.printSha() - + shaObj = Sha(self.project_name, sha) #if(self.config_info.DEBUGLITE): #Save for testing. self.shas.append(shaObj) #This will become very memory intensive in large git logs. - + is_diff = False log_mssg = "" - + continue elif self.isAuthor(line,shaObj): @@ -550,11 +550,11 @@ def processLog(self, config = ""): if(patchObj != None): #If there is an existing chunk to parse, process it if(curLogChunk.header != ""): - if(self.config_info.DEBUG): - print("New diff with previous version: " + line) - print("HEADER: " + curLogChunk.header) + if(self.config_info.DEBUG): + print(("New diff with previous version: " + line)) + print(("HEADER: " + curLogChunk.header)) self.processLastChunk(patchObj, curLogChunk) - + #Reset the current chunk obj if (self.config_info.DEBUG): print("Resetting.") @@ -574,9 +574,9 @@ def processLog(self, config = ""): #Finish the changes to the old patch object if(patchObj != None): if(curLogChunk.header != ""): #If there is an existing chunk - if (self.config_info.DEBUG): - print("New diff with no previous version: " + line) - print("HEADER: " + curLogChunk.header) + if (self.config_info.DEBUG): + print(("New diff with no previous version: " + line)) + print(("HEADER: " + curLogChunk.header)) self.processLastChunk(patchObj, curLogChunk) if (self.config_info.DEBUG): @@ -598,9 +598,9 @@ def processLog(self, config = ""): #Make sure to get the last patch in the file! if(curLogChunk.header != ""): #If there is an existing chunk to parse - if(self.config_info.DEBUG): - print("Last Patch: " + line) - print("HEADER: " + curLogChunk.header) + if(self.config_info.DEBUG): + print(("Last Patch: " + line)) + print(("HEADER: " + curLogChunk.header)) self.processLastChunk(patchObj, curLogChunk) #if shaObj != None: @@ -623,7 +623,7 @@ def processLog(self, config = ""): for s in self.shas: #s.printSha() if s != None: - if(self.config_info.DATABASE): + if(self.config_info.DATABASE): s.dumpSha(dl) elif(self.config_info.CSV): s.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary) @@ -640,7 +640,7 @@ def processLog(self, config = ""): if(self.config_info.DATABASE): print("Closing Time.") dl.close() - + if(self.config_info.CSV): inf1.close() inf2.close() @@ -648,7 +648,7 @@ def processLog(self, config = ""): fPtrPatchSummary.close() print("Sha's processed:") - print(len(self.shas)) + print((len(self.shas))) return parseFinish @@ -656,8 +656,8 @@ def processLog(self, config = ""): #---------test-----------# def test(): if len(sys.argv) < 2: - print "!!! Pass a log file." - print "usage ./ghLogDb.py ccv_all_log.txt" + print("!!! Pass a log file.") + print("usage ./ghLogDb.py ccv_all_log.txt") sys.exit() log_file = sys.argv[1] diff --git a/src/logChunk/ghLogDbTest.py b/src/logChunk/ghLogDbTest.py index eaa65e5..450c409 100644 --- a/src/logChunk/ghLogDbTest.py +++ b/src/logChunk/ghLogDbTest.py @@ -10,7 +10,7 @@ class ghLogDbTest(unittest.TestCase): def setUp(self): - + c_info = ConfigInfo("../util/sample_conf.ini") c_info2 = ConfigInfo("../util/javatest.ini") Util.DATABASE = 0 @@ -19,7 +19,7 @@ def setUp(self): self.testCommit3 = ghLogDb.ghLogDb("testfiles/ghLogDbTest/TestCommit3.txt",c_info) self.testCommit4 = ghLogDb.ghLogDb("testfiles/ghLogDbTest/TestCommit4.txt",c_info) self.testCommit5 = ghLogDb.ghLogDb("testfiles/ghLogDbTest/TestCommit5.txt",c_info) - + self.testCommit7 = ghLogDb.ghLogDb("testfiles/ghLogDbTest/TestCommit7.txt",c_info) #self.testCommit8 = ghLogDb.ghLogDb("testfiles/ghLogDbTest/TestCommit8.txt",c_info) #Broken, but I don't think its fixable. Fault of git logger self.testCommit9 = ghLogDb.ghLogDb("testfiles/ghLogDbTest/TestCommit9.txt",c_info) @@ -44,7 +44,7 @@ def test_Commit1(self): for patch in patches: self.assertTrue(patch.language == "c") self.assertTrue(patch.is_test == False) - print(patch.file_name) + print((patch.file_name)) self.assertTrue(patches[0].file_name == "bin/cuda/cwc-verify.c") methods = patches[0].methods @@ -53,13 +53,13 @@ def test_Commit1(self): self.assertTrue(methods[0].method == "main") self.assertTrue(methods[0].total_add == 1) self.assertTrue(methods[0].total_del == 1) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) self.assertTrue(methods[1].method == "GITCPROC_NON_FUNCTION") self.assertTrue(methods[1].total_add == 1) self.assertTrue(methods[1].total_del == 1) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[1].keywordDictionary) self.assertTrue(patches[1].file_name == "bin/image-net.c") @@ -68,7 +68,7 @@ def test_Commit1(self): self.assertTrue(methods[0].method == "main") self.assertTrue(methods[0].total_add == 8) self.assertTrue(methods[0].total_del == 8) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) @@ -78,7 +78,7 @@ def test_Commit1(self): self.assertTrue(methods[1].method == "ccv_convnet_verify") self.assertTrue(methods[1].total_add == 6) self.assertTrue(methods[1].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[1].keywordDictionary) self.assertTrue(patches[3].file_name == "lib/ccv_icf.c") methods = patches[3].methods @@ -94,7 +94,7 @@ def test_Commit2(self): for patch in patches: self.assertTrue(patch.language == "c") self.assertTrue(patch.is_test == False) - print(patch.file_name) + print((patch.file_name)) self.assertTrue(patches[0].file_name == "bin/cuda/cwc-bench.c") methods = patches[0].methods @@ -102,7 +102,7 @@ def test_Commit2(self): self.assertTrue(methods[0].method == "main") self.assertTrue(methods[0].total_add == 43) self.assertTrue(methods[0].total_del == 5) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) @@ -114,13 +114,13 @@ def test_Commit2(self): self.assertTrue(methods[0].method == "main") self.assertTrue(methods[0].total_add == 314) self.assertTrue(methods[0].total_del == 0) - testDict = {'assert Adds':1, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':1, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) self.assertTrue(methods[1].method == "GITCPROC_NON_FUNCTION") self.assertTrue(methods[1].total_add == 6) self.assertTrue(methods[1].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[1].keywordDictionary) def test_Commit3(self): @@ -133,7 +133,7 @@ def test_Commit3(self): for patch in patches: self.assertTrue(patch.language == "c") self.assertTrue(patch.is_test == False) - print(patch.file_name) + print((patch.file_name)) self.assertTrue(patches[0].file_name == "lib/ccv_convnet.c") methods = patches[0].methods @@ -143,14 +143,14 @@ def test_Commit3(self): self.assertTrue(methods[0].method == "_ccv_convnet_convolutional_forward_propagate_sse2") self.assertTrue(methods[0].total_add == 1) self.assertTrue(methods[0].total_del == 1) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) self.assertTrue(methods[1].method == "_ccv_convnet_convolutional_forward_propagate_neon") self.assertTrue(methods[1].total_add == 62) self.assertTrue(methods[1].total_del == 40) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[1].keywordDictionary) def test_Commit4(self): @@ -163,7 +163,7 @@ def test_Commit4(self): for patch in patches: self.assertTrue(patch.language == "c") self.assertTrue(patch.is_test == False) - print(patch.file_name) + print((patch.file_name)) methods = patches[0].methods self.assertTrue(len(methods) == 1) @@ -171,7 +171,7 @@ def test_Commit4(self): self.assertTrue(methods[0].method == "ccv_gemm") self.assertTrue(methods[0].total_add == 2) self.assertTrue(methods[0].total_del == 0) - testDict = {'assert Adds':1, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':1, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) methods = patches[1].methods @@ -179,25 +179,25 @@ def test_Commit4(self): self.assertTrue(methods[0].method == "_ccv_convnet_compute_softmax") self.assertTrue(methods[0].total_add == 0) self.assertTrue(methods[0].total_del == 18) - testDict = {'assert Adds':0, 'assert Dels': 1, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 1, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) self.assertTrue(methods[1].method == "GITCPROC_NON_FUNCTION") self.assertTrue(methods[1].total_add == 0) self.assertTrue(methods[1].total_del == 2) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[1].keywordDictionary) self.assertTrue(methods[2].method == "_ccv_convnet_compute_softmax") self.assertTrue(methods[2].total_add == 18) self.assertTrue(methods[2].total_del == 0) - testDict = {'assert Adds':1, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':1, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[2].keywordDictionary) self.assertTrue(methods[3].method == "GITCPROC_NON_FUNCTION") self.assertTrue(methods[3].total_add == 2) self.assertTrue(methods[3].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[3].keywordDictionary) def test_Commit5(self): @@ -211,14 +211,14 @@ def test_Commit5(self): for patch in patches: self.assertTrue(patch.language == "cpp") self.assertTrue(patch.is_test == False) - print(patch.file_name) + print((patch.file_name)) methods = patches[0].methods self.assertTrue(len(methods) == 8) self.assertTrue(methods[3].method == "*dlopen") self.assertTrue(methods[3].total_add == 66) self.assertTrue(methods[3].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} # self.assertEqual(testDict,methods[3].keywordDictionary) methods = patches[1].methods @@ -238,7 +238,7 @@ def test_Commit7(self): #print(len(methods)) #print(methods) #2 non-functions b/c we get 1 for each @@ where there are changes outside a function - self.assertTrue(len(methods) == 11) + self.assertTrue(len(methods) == 11) # def test_commit8(self): # self.testCommit8.processLog() @@ -253,19 +253,19 @@ def test_Commit7(self): # self.assertTrue(len(methods) == 4) # # self.assertTrue(methods[0].method == "innobase_next_autoinc") -# testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_a Adds':9, 'ut_a Dels': 4, 'ut_ad Adds':0, 'ut_ad Dels': 0} +# testDict = {'assert adds':0, 'assert dels': 0, 'ut_a adds':9, 'ut_a dels': 4, 'ut_ad adds':0, 'ut_ad dels': 0} # self.assertEqual(testDict,methods[0].keywordDictionary) # # self.assertTrue(methods[1].method == "ha_innobase::innobase_initialize_autoinc") -# testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} +# testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} # self.assertEqual(testDict,methods[1].keywordDictionary) # # self.assertTrue(methods[2].method == "ha_innobase::update_row") -# testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} +# testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} # self.assertEqual(testDict,methods[2].keywordDictionary) # # self.assertTrue(methods[3].method == "ha_innobase::get_auto_increment") -# testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} +# testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} # self.assertEqual(testDict,methods[3].keywordDictionary) def test_commit9(self): @@ -282,7 +282,7 @@ def test_commit9(self): self.assertTrue(len(methods) == 13) self.assertTrue(methods[8].method == "mysql_stmt_close") - testDict = {'assert Adds':1, 'assert Dels': 1, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[8].keywordDictionary) def test_commit10(self): @@ -298,7 +298,7 @@ def test_commit10(self): self.assertTrue(len(methods) == 8) #One non function self.assertTrue(methods[0].method == "_ccv_read_rgb_raw") - testDict = {'assert Adds':2, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':2, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) methods = patches[4].methods @@ -323,7 +323,7 @@ def test_commmit14(self): self.assertTrue(len(methods) == 1) self.assertTrue(methods[0].method == "GITCPROC_NON_FUNCTION") - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,methods[0].keywordDictionary) def test_commmit15(self): @@ -340,7 +340,7 @@ def test_Commitb1(self): for patch in patches: self.assertTrue(patch.language == "java") self.assertTrue(patch.is_test == False) - print(patch.file_name) + print((patch.file_name)) self.assertTrue(patches[0].file_name == "app/src/main/java/com/github/mobile/accounts/AccountUtils.java") methods = patches[0].methods @@ -349,7 +349,7 @@ def test_Commitb1(self): self.assertTrue(methods[0].method == "getAccounts") self.assertTrue(methods[0].total_add == 1) self.assertTrue(methods[0].total_del == 2) - dict= {'throw Adds':0, 'catch Dels': 0, 'throw Dels': 0, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + dict= {'throw adds':0, 'catch dels': 0, 'throw dels': 0, 'try adds': 0, 'try dels': 0, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(dict,methods[0].keywordDictionary) if __name__=="__main__": diff --git a/src/logChunk/ghLogDbTestPython.py b/src/logChunk/ghLogDbTestPython.py index 593810e..ab87e93 100644 --- a/src/logChunk/ghLogDbTestPython.py +++ b/src/logChunk/ghLogDbTestPython.py @@ -10,11 +10,11 @@ class ghLogDbTestPython(unittest.TestCase): def setUp(self): - + c_info = ConfigInfo("../util/pytest.ini") self.testCommit1 = ghLogDb.ghLogDb("testfiles/ghLogDbTestPython/TestCommit1.txt",c_info) self.testCommit2 = ghLogDb.ghLogDb("testfiles/ghLogDbTestPython/TestCommit2.txt",c_info) - + def test_Commit1(self): self.testCommit1.processLog() shas = self.testCommit1.shas @@ -33,10 +33,10 @@ def test_Commit1(self): self.assertTrue(methods[0].total_add == 1) self.assertTrue(methods[0].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 1, 'if Adds': 1} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 1, 'if adds': 1} self.assertEqual(testDict,methods[0].keywordDictionary) - def test_Commit2(self): + def test_Commit2(self): self.testCommit2.processLog() shas = self.testCommit2.shas self.assertTrue(shas[0].author == "David Chan") @@ -54,7 +54,7 @@ def test_Commit2(self): self.assertTrue(methods[0].total_add == 1) self.assertTrue(methods[0].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 1, 'if Adds': 1} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 1, 'if adds': 1} self.assertEqual(testDict,methods[0].keywordDictionary) if __name__=="__main__": diff --git a/src/logChunk/ghProc.py b/src/logChunk/ghProc.py index 5e62af9..e3cae59 100644 --- a/src/logChunk/ghProc.py +++ b/src/logChunk/ghProc.py @@ -25,13 +25,13 @@ def dumpLog(projPath): log_file = projPath + os.sep + LOG_FILE if os.path.isfile(log_file): - print("%s exists!!" % (log_file)) + print(("%s exists!!" % (log_file))) return with Util.cd(projPath): logCmd = "git log --date=short -U1 -- \*.java > all_log.txt" - print logCmd + print(logCmd) os.system(logCmd) def processLog(projPath, c_info, password = ""): @@ -40,10 +40,10 @@ def processLog(projPath, c_info, password = ""): log_file = projPath + os.sep + LOG_FILE if not os.path.isfile(log_file): - print("!! %s does not exist" % (log_file)) + print(("!! %s does not exist" % (log_file))) return False else: - print("Going to process %s " % (log_file)) + print(("Going to process %s " % (log_file))) ghDb = ghLogDb(log_file, c_info, password) return ghDb.processLog() @@ -51,7 +51,7 @@ def processLog(projPath, c_info, password = ""): def checkProj(project): if not os.path.isdir(project): - print("!! %s does not exist" % (project)) + print(("!! %s does not exist" % (project))) return False ''' @@ -67,10 +67,10 @@ def checkProj(project): '''main() funciton checks whether the arguments used while running the script are proper or not.''' def main(): - print "Utility to process github logs" + print("Utility to process github logs") if len(sys.argv) < 3: - print "!!! Usage: python ghProc.py project config_file [password]" + print("!!! Usage: python ghProc.py project config_file [password]") sys.exit() project = str(sys.argv[1]) @@ -98,17 +98,17 @@ def main(): else: parseFinish = processLog(project, config_info) - print "!! Done" + print("!! Done") if(config_info.LOGTIME): end = datetime.datetime.now() - print("Project: " + project) - print("Start time: " + str(start)) - print("Parse Finish time:" + str(parseFinish)) - print("End time: " + str(end)) - print("Parse time: " + str(parseFinish - start)) - print("Write time: " + str(end - parseFinish)) - print("Total time: " + str(end-start)) + print(("Project: " + project)) + print(("Start time: " + str(start))) + print(("Parse Finish time:" + str(parseFinish))) + print(("End time: " + str(end))) + print(("Parse time: " + str(parseFinish - start))) + print(("Write time: " + str(end - parseFinish))) + print(("Total time: " + str(end-start))) if __name__ == '__main__': diff --git a/src/logChunk/gitcproc.py b/src/logChunk/gitcproc.py index 8c38427..ab3ba0b 100644 --- a/src/logChunk/gitcproc.py +++ b/src/logChunk/gitcproc.py @@ -22,15 +22,18 @@ parser.add_argument("-d","--download", action="store_true", help = "Flag that indicates you want to run the step to download the projects.") parser.add_argument("-wl","--write_log", action="store_true", help = "Flag that indicates you want to run the step to write the logs.") parser.add_argument("-pl","--parse_log", action="store_true", help = "Flag that indicates you want to run the step to parse the logs.") -#parser.add_argument("-p","--password", type = str, default = None, help = "If you are outputting to the database, you must enter your password.") +parser.add_argument("-p","--password", type = str, default = None, help = "If you are outputting to the database, you must enter your password.") args = parser.parse_args() config_file = args.config_file -config_info = ConfigInfo(config_file) +config_info = ConfigInfo(config_file) if(config_info.DATABASE and args.parse_log): #If we have database output selected and are performing the parse-log step. - password = getpass.getpass(prompt="Database option selected, enter your password:") + if args.password is not None: + password = args.password + else: + password = getpass.getpass(prompt="Database option selected, enter your password:") else: password = "" @@ -41,7 +44,7 @@ #Check that the repo list file exists if(not os.path.isfile(repo_config['repo_url_file'])): - print(repo_config['repo_url_file'] + ", the file containing the list of projects to download,") + print((repo_config['repo_url_file'] + ", the file containing the list of projects to download,")) print("cannot be found. Make sure your path and name are correct.") #Create the output directory if it doesn't exist yet. @@ -53,12 +56,12 @@ if(args.write_log): #Also should include logging for time... - subprocess.call(["python", "getGitLog.py", repo_config['repo_locations'], config_file]) + subprocess.call(["python3", "getGitLog.py", repo_config['repo_locations'], config_file]) if(args.parse_log): project_set = set() if(not os.path.isfile(repo_config['repo_url_file'])): - print(repo_config['repo_url_file'] + ", the file containing the list of projects to download,") + print((repo_config['repo_url_file'] + ", the file containing the list of projects to download,")) print("cannot be found. Make sure your path and name are correct.") else: try: @@ -74,11 +77,11 @@ for next_project, name in dirs_and_names: if(len(project_set) != 0 and name not in project_set): continue - #subprocess.call(["python", "ghProc.py", next_project, config_file, password]) - subprocess.call(["nohup", "sh", "run.sh", next_project, name, config_file, password]) + #subprocess.call(["python3", "ghProc.py", next_project, config_file, password]) + subprocess.call(["nohup", "sh", "run.sh", next_project, name, config_file, password]) #Parellel Version: -#p = subprocess.Popen([sys.executable, '/path/to/script.py'], -# stdout=subprocess.PIPE, +#p = subprocess.Popen([sys.executable, '/path/to/script.py'], +# stdout=subprocess.PIPE, # stderr=subprocess.STDOUT) diff --git a/src/logChunk/installDependencies.py b/src/logChunk/installDependencies.py index 643eef4..ddb0c08 100644 --- a/src/logChunk/installDependencies.py +++ b/src/logChunk/installDependencies.py @@ -1,7 +1,7 @@ import os -os.system("pip install GitPython") -os.system("pip install PyYAML") -os.system("pip install nltk") -os.system("python -m nltk.downloader stopwords") -os.system("python -m nltk.downloader wordnet") -os.system("pip install psycopg2") +os.system("pip3 install GitPython") +os.system("pip3 install PyYAML") +os.system("pip3 install nltk") +os.system("python3 -m nltk.downloader stopwords") +os.system("python3 -m nltk.downloader wordnet") +os.system("pip3 install psycopg2-binary") diff --git a/src/logChunk/logChunk.py b/src/logChunk/logChunk.py index 535030c..fc5ed71 100644 --- a/src/logChunk/logChunk.py +++ b/src/logChunk/logChunk.py @@ -47,7 +47,7 @@ def __init__(self, text = "", language = "C", c_info = ConfigInfo("../util/sampl #Returns true if the string conforms to the pattern ,[included/excluded],[single,block] #and false otherwise def keywordValidityCheck(self, line): - toCheck = [w.strip().lower() for w in line] + toCheck = [w.strip() for w in line] if(len(toCheck) != 3): return False elif(toCheck[1] != INCLUDED and toCheck[1] != EXCLUDED): @@ -66,18 +66,18 @@ def outputKeyword(self, kw): #Read in a file with the following format: - #Keyword, Inc/Exc, Single/Block + #Keyword, Inc/Exc, Single/Block #And store them as a list of triples def readKeywords(self, lst): with open(self.KeyWordFile) as f: reader = csv.reader(f, delimiter=',', quotechar="\'") for l in reader: - l = [w.lower() for w in l] + l = [w for w in l] if(self.keywordValidityCheck(l)): next = l lst.append(next) else: - print("Invalid line in keyword config file: " + str(l)) + print(("Invalid line in keyword config file: " + str(l))) print("Skipping...") return lst @@ -90,30 +90,30 @@ def getEmptyKeywordDict(self): if(self.keyWordList == []): self.keyWordList = self.readKeywords([]) - singleKeyWordList = filter(lambda w : w[2] == SINGLE, self.keyWordList) - blockKeyWordList = filter(lambda w: w[2] == BLOCK, self.keyWordList) + singleKeyWordList = [w for w in self.keyWordList if w[2] == SINGLE] + blockKeyWordList = [w for w in self.keyWordList if w[2] == BLOCK] for keyword in singleKeyWordList: if(keyword[1] != EXCLUDED): - emptyDict[self.outputKeyword(keyword) + " Adds"]=0 - emptyDict[self.outputKeyword(keyword) + " Dels"]=0 + emptyDict[self.outputKeyword(keyword) + " adds"]=0 + emptyDict[self.outputKeyword(keyword) + " dels"]=0 for keyword in blockKeyWordList: if(keyword[1] != EXCLUDED): - emptyDict[self.outputKeyword(keyword) + " Adds"]=0 - emptyDict[self.outputKeyword(keyword) + " Dels"]=0 + emptyDict[self.outputKeyword(keyword) + " adds"]=0 + emptyDict[self.outputKeyword(keyword) + " dels"]=0 return emptyDict def printLogChunk(self): print("===========================================") - print("Total add: " + str(self.total_add)) - print("Total del: " + str(self.total_del)) - print("Header: " + str(self.header)) + print(("Total add: " + str(self.total_add))) + print(("Total del: " + str(self.total_del))) + print(("Header: " + str(self.header))) print("Functions:") for func in self.functions: func.printPatch() print("===========================================") - + def setLang(self, language = "C"): self.langSwitch = LanguageSwitcherFactory.LanguageSwitcherFactory.createLS(language) self.sT = ScopeTrackerFactory.ScopeTrackerFactory.createST(self.langSwitch, self.config_info) @@ -155,14 +155,14 @@ def sumLinesForRealFunc(self): #This sum of the real function contents shouldn't be greater than #the total lines added and deleted in the diff. if(self.config_info.DEBUG): - print("SUM of func adds: " + str(output[0])) - print("SUM of func dels: " + str(output[1])) - print("Total adds: " + str(self.total_add)) - print("Total dels: " + str(self.total_del)) + print(("SUM of func adds: " + str(output[0]))) + print(("SUM of func dels: " + str(output[1]))) + print(("Total adds: " + str(self.total_add))) + print(("Total dels: " + str(self.total_del))) #if(output[0] > self.total_add or output[1] > self.total_del): #raise CountException("Miscount between counts outside the function and the total within the functions.") - + #Just set the total count to sum of function count. if(output[0] > self.total_add): self.total_add = output[0] @@ -182,7 +182,7 @@ def getLineCountOutsideFunc(self): #Create an additional MOCK function to summarize all changes outside of functions. #If no such changes, return None - def createOutsideFuncSummary(self, keywordDictionary = {}): + def createOutsideFuncSummary(self, keywordDictionary = {}): if(keywordDictionary == {}): keywordDictionary = self.getEmptyKeywordDict() (added, deleted) = self.getLineCountOutsideFunc() @@ -195,9 +195,9 @@ def functionCount(self): if(self.initialized == False): self.parseText() return len(self.functions) - + #string -> -- - #store the next line of the chunk in the text + #store the next line of the chunk in the text def addToText(self, line): if(line.endswith("\n")): self.text += line @@ -207,19 +207,18 @@ def addToText(self, line): #Remove any parts of the line that have structures marked as excluded def removeExcludedKeywords(self, line, keywords): excludedKeywords = [k for k in keywords if k[1] == EXCLUDED] - line = line.lower() #Make this case insensitive for eK in excludedKeywords: # print(eK) line = line.replace(eK[0], "") return line - + #Determines if a line of text contains any keyword #Precondition - tested line has had all comments and strings removed. def containsKeyword(self, line, keywords): line = self.removeExcludedKeywords(line, keywords) includedKeywords = [k for k in keywords if k[1] == INCLUDED] for keyword in includedKeywords: - if(keyword in line.lower()): + if(keyword in line): return True return False @@ -243,9 +242,9 @@ def incrementBlockContext(self, keywordDict, lineType, includedKeywords, blockCo assert(False) if(lineType == ADD): - incrementDict(str(b) + " Adds", keywordDict, 1) + incrementDict(str(b) + " adds", keywordDict, 1) elif(lineType == REMOVE): - incrementDict(str(b) + " Dels", keywordDict, 1) + incrementDict(str(b) + " dels", keywordDict, 1) return keywordDict @@ -254,13 +253,15 @@ def incrementBlockContext(self, keywordDict, lineType, includedKeywords, blockCo #Otherwise use basic "in" def keywordMatch(self, keyword, line): if(keyword.startswith('\"') and keyword.endswith('\"')): - exactMatch = "(^|\W+)" + keyword[1:-1] + "(\W+|$)" - return (keyword[1:-1],re.search(exactMatch, line) != None) + slicedKeyword = keyword[1:-1] + escapedKeyword = re.escape(slicedKeyword) + exactMatch = "(^|\W*)" + escapedKeyword + "(\W*|$)" + return (slicedKeyword,re.search(exactMatch, line) != None) else: - return (keyword, keyword in line.lower()) + return (keyword, keyword in line) #String, String, list of Strings, dictionary, String -> dictionary - #Modify the keyword dictionary for this line. + #Modify the keyword dictionary for this line. def parseLineForKeywords(self, line, lineType, keywords, keywordDict, blockContext = []): assert(lineType == ADD or lineType == REMOVE) #How do we handle block statements where only internal part modified? line = self.removeExcludedKeywords(line, keywords) @@ -269,9 +270,9 @@ def parseLineForKeywords(self, line, lineType, keywords, keywordDict, blockConte keywords = sorted(keywords, key=lambda tup: -len(tup[0])) if(self.config_info.DEBUG): try: - print("LINE TO PARSE FOR KEYWORD:" + line) + print(("LINE TO PARSE FOR KEYWORD:" + line)) except: - print("LINE TO PARSE FOR KEYWORD:" + unicode(line, 'utf-8', errors='ignore')) + print(("LINE TO PARSE FOR KEYWORD:" + str(line, 'utf-8', errors='ignore'))) includedKeywords = [k for k in keywords if k[1] == INCLUDED] tmp = line @@ -281,9 +282,9 @@ def parseLineForKeywords(self, line, lineType, keywords, keywordDict, blockConte if(matched): tmp = tmp.replace(k, "") #Then remove so we don't double count if(lineType == ADD): - incrementDict(str(k) + " Adds", keywordDict, 1) + incrementDict(str(k) + " adds", keywordDict, 1) elif(lineType == REMOVE): - incrementDict(str(k) + " Dels", keywordDict, 1) + incrementDict(str(k) + " dels", keywordDict, 1) else: #I don't this case has been handled correctly for blocks. print("Unmodified") assert(0) @@ -291,7 +292,7 @@ def parseLineForKeywords(self, line, lineType, keywords, keywordDict, blockConte elif(len(blockContext) != 0): #Sum over all block keywords if(self.config_info.DEBUG): - print("Updating Block Dictionaries:" + str(blockContext)) + print(("Updating Block Dictionaries:" + str(blockContext))) keywordDict = self.incrementBlockContext(keywordDict, lineType, includedKeywords, blockContext) return keywordDict @@ -351,11 +352,11 @@ def isConstructorOrDestructor(self, line, classContext): constructPatt = self.langSwitch.getConstructorOrDestructorRegex(classContext) if(self.config_info.DEBUG): - print("Class context: " + classContext) + print(("Class context: " + classContext)) try: - print("Checking if a constructor/destructor: " + temp) + print(("Checking if a constructor/destructor: " + temp)) except: - print("Checking if a constructor/destructor: " + unicode(temp, 'utf-8', errors='ignore')) + print(("Checking if a constructor/destructor: " + str(temp, 'utf-8', errors='ignore'))) return re.search(constructPatt, temp,flags=re.IGNORECASE) @@ -372,44 +373,44 @@ def getBlockPattern(self,line,keywords): #To start, lets use a regex expression with " (<0+ parameters>) {" #Also, we should handle template methods like: "template >(<0+ parameters>) {"" #Returns a string matching the function pattern or "" if no pattern match found. - def getFunctionPattern(self, line): + def getFunctionPattern(self, line): #Remove potentially problematic structures temp = self.langSwitch.cleanFunctionLine(line) if(self.config_info.DEBUG): try: - print("Checking if function: \'" + temp + "\'") + print(("Checking if function: \'" + temp + "\'")) except: - print("Checking if function: \'" + unicode(temp, 'utf-8', errors='ignore') + "\'") + print(("Checking if function: \'" + str(temp, 'utf-8', errors='ignore') + "\'")) #Select patterns for our language and check against them funcPatterns = self.langSwitch.getFunctionRegexes() if(self.config_info.DEBUG): - print("Checking " + str(len(funcPatterns)) + " patterns.") + print(("Checking " + str(len(funcPatterns)) + " patterns.")) for p in funcPatterns: result = re.search(p, temp) if(result != None): if(self.config_info.DEBUG): - print("Found match with pattern: " + p) + print(("Found match with pattern: " + p)) return result.group(0) return "" - + def isFunction(self, line): return (self.getFunctionPattern(line) != "") #Determine if the given line is an assignment block using the { def isAssignment(self, line): - return re.search(assignPattern, line) - + return re.search(assignPattern, line) + #String -> String #Given a line of code from a diff statement, return the line with any #string literals removed. def removeStrings(self, line): return self.langSwitch.removeStrings(line) - + #String, Boolean, String, String, String -> (String, String, Boolean, String, String) #Given a line of code from a diff statement, a marker if prior lines were a multiblock #comment, the marker for the type of line, a marker for the type of comment, and @@ -564,7 +565,7 @@ def markLine(self, line): return [META, line] else: return [OTHER, line] - + #A Check to see if our regexes match class name def checkForClassName(self, searchString, classContext): if(self.langSwitch.isObjectOrientedLanguage()): @@ -572,9 +573,9 @@ def checkForClassName(self, searchString, classContext): if(className != ""): if(self.config_info.DEBUG): try: - print("Class:" + className) + print(("Class:" + className)) except: - print("Class:" + unicode(className, 'utf-8', errors='ignore')) + print(("Class:" + str(className, 'utf-8', errors='ignore'))) classContext.append(self.extractClassName(className)) #Push onto the class list return classContext @@ -603,7 +604,7 @@ def checkForFunctionName(self, phase, line, lineType, lineNum, functionName, cla shortFunctionName = self.getFunctionPattern(functionName) if(self.config_info.DEBUG): - print("Pattern: " + shortFunctionName) + print(("Pattern: " + shortFunctionName)) if(shortFunctionName != ""): isFunction = True @@ -617,9 +618,9 @@ def checkForFunctionName(self, phase, line, lineType, lineNum, functionName, cla if(isFunction): #Skip things are aren't functions if(self.config_info.DEBUG): try: - print("Function: " + shortFunctionName) + print(("Function: " + shortFunctionName)) except: - print("Function: " + unicode(shortFunctionName, 'utf-8', errors='ignore')) + print(("Function: " + str(shortFunctionName, 'utf-8', errors='ignore'))) #Must update to deal with potential changes. self.sT.increaseScope(self.sT.grabScopeLine(shortFunctionName, line, lineType), line, lineType, scopeTracker.FUNC) @@ -628,7 +629,7 @@ def checkForFunctionName(self, phase, line, lineType, lineNum, functionName, cla phase = LOOKFOREND #Count this line as an addition or deletion #this means either a { will be counted or part - #of the function name. + #of the function name. if(lineType == REMOVE): ftotal_del = 1 startFlag=1 @@ -636,18 +637,18 @@ def checkForFunctionName(self, phase, line, lineType, lineNum, functionName, cla ftotal_add = 1 startFlag=1 - #Remove the last of the function + #Remove the last of the function line = self.langSwitch.clearFunctionRemnants(line) else: #There was a non-function scope increase. if(self.config_info.DEBUG): print("Non function scope increase while searching for function name.") - + #I think this will be handled by update scope and keywords, so we don't need to handle it here. #self.sT.increaseScope(line, line, lineType, scopeTracker.GENERIC) #Check for class context last here. if(not self.sT.changeScopeFirst()): - classContext = self.checkForClassName(functionName, classContext) + classContext = self.checkForClassName(functionName, classContext) functionName = self.langSwitch.resetFunctionName(line) #Reset name and find next @@ -667,12 +668,12 @@ def checkForFunctionEnd(self, lineType, lineNum, phase, funcStart, funcEnd, func if(self.config_info.DEBUG): print("OLD") - print(self.sT.oldVerStack) + print((self.sT.oldVerStack)) print("NEW") - print(self.sT.newVerStack) - print("Line Type: " + str(lineType)) - print("Function Name: " + str(shortFunctionName)) - print(str(funcStart) + " : " + str(funcEnd)) + print((self.sT.newVerStack)) + print(("Line Type: " + str(lineType))) + print(("Function Name: " + str(shortFunctionName))) + print((str(funcStart) + " : " + str(funcEnd))) #Add this function to our list and reset the trackers. #We use shortFunctionName, which is the string that matched our expected @@ -683,7 +684,7 @@ def checkForFunctionEnd(self, lineType, lineNum, phase, funcStart, funcEnd, func raise ValueError("Function Name Parse Error") #Add assertions from current function self.functions.append(funcToAdd) - + #Reset asserts to current function functionName = "" shortFunctionName = "" @@ -697,15 +698,15 @@ def checkForFunctionEnd(self, lineType, lineNum, phase, funcStart, funcEnd, func backTrack = False for keyword in singleKeyWordList: if(keyword[1] != EXCLUDED): - keywordDictionary[self.outputKeyword(keyword) + " Adds"]=0 - keywordDictionary[self.outputKeyword(keyword) + " Dels"]=0 + keywordDictionary[self.outputKeyword(keyword) + " adds"]=0 + keywordDictionary[self.outputKeyword(keyword) + " dels"]=0 for keyword in blockKeyWordList: #Hack to make run with the 'tryDependedCatch' keyword if(not isinstance(keyword, list) or len(keyword) != KEYLISTSIZE): continue elif(keyword[1] != EXCLUDED): - keywordDictionary[self.outputKeyword(keyword) + " Adds"]=0 - keywordDictionary[self.outputKeyword(keyword) + " Dels"]=0 + keywordDictionary[self.outputKeyword(keyword) + " adds"]=0 + keywordDictionary[self.outputKeyword(keyword) + " dels"]=0 return (lineType, lineNum, phase, funcStart, funcEnd, functionName, shortFunctionName, ftotal_add, ftotal_del, foundBlock, singleKeyWordList, blockKeyWordList, keywordDictionary, backTrack) @@ -745,7 +746,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, if(foundBlock != None): #If we have an existing keyword match from before if(blockKeywordLine == lineNum - 1): if(self.config_info.DEBUG): - print("Block start found (0): " + foundBlock) + print(("Block start found (0): " + foundBlock)) sT.increaseScope(foundBlock, line, blockKeywordType, scopeTracker.SBLOCK, 1) else: #Ignore scope increases too far away from the block keyword sT.increaseScope(line, line, blockKeywordType, scopeTracker.GENERIC) @@ -757,7 +758,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, foundBlock=self.getBlockPattern(line,blockKeyWordList) if(foundBlock!=None): if(self.config_info.DEBUG): - print("Block start found (1): " + foundBlock) + print(("Block start found (1): " + foundBlock)) sT.increaseScope(foundBlock, line, lineType, scopeTracker.SBLOCK, 0) #This shouldn't set the block yet. #Reset block after this line. reset = True @@ -787,7 +788,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, #Then we ignore further block single line keywords.... if(self.config_info.DEBUG): print("Back tracking!!!") - print("BT Function Name:" + shortFunctionName) + print(("BT Function Name:" + shortFunctionName)) return (foundBlock, blockKeywordLine, blockKeywordType, shortFunctionName, keywordDictionary, sT, True) @@ -811,7 +812,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, if(sT.getBlockContext(lineType) != [] and lineType!=OTHER): if(self.config_info.DEBUG): - print("Current block context: " + str(sT.getBlockContext(lineType))) + print(("Current block context: " + str(sT.getBlockContext(lineType)))) keywordDictionary = self.parseLineForKeywords(line, lineType, blockKeyWordList, keywordDictionary, sT.getBlockContext(lineType)) @@ -819,7 +820,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, if(not sT.changeScopeFirst()): sT.decreaseScope(line, lineType) if(self.config_info.DEBUG): - print("Removed!!!!" + str(sT.getBlockContext(lineType))) + print(("Removed!!!!" + str(sT.getBlockContext(lineType)))) else: #A SIMUL change!!! if(sT.changeScopeFirst()): #Shouldn't be possible in bracket based languages. assert(sT.isScopeIncrease(line, lineType) == scopeTracker.S_SIMUL or sT.isScopeDecrease(line, lineType) == scopeTracker.S_SIMUL) @@ -830,7 +831,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, if(foundBlock != None): #If we have an existing keyword match from before if(blockKeywordLine == lineNum - 1): if(self.config_info.DEBUG): - print("Block start found (0): " + foundBlock) + print(("Block start found (0): " + foundBlock)) sT.increaseScope(foundBlock, line, blockKeywordType, scopeTracker.SBLOCK, 1, True) else: #Ignore scope increases too far away from the block keyword sT.increaseScope(line, line, blockKeywordType, scopeTracker.GENERIC, True) @@ -842,7 +843,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, foundBlock=self.getBlockPattern(line,blockKeyWordList) if(foundBlock!=None): if(self.config_info.DEBUG): - print("Block start found (SIMUL): " + foundBlock) + print(("Block start found (SIMUL): " + foundBlock)) sT.increaseScope(foundBlock, line, lineType, scopeTracker.SBLOCK, 0, True) #This shouldn't set the block yet. blockKeywordType = lineType blockKeywordLine = lineNum @@ -864,7 +865,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, if(sT.getBlockContext(lineType) != [] and lineType!=OTHER): if(self.config_info.DEBUG): - print("Current block context: " + str(sT.getBlockContext(lineType))) + print(("Current block context: " + str(sT.getBlockContext(lineType)))) keywordDictionary = self.parseLineForKeywords(line, lineType, blockKeyWordList, keywordDictionary, sT.getBlockContext(lineType)) else: # Still want to check for a block context opening @@ -885,20 +886,20 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, temp = line if(scopeChanges != [DECREASE] and scopeChanges != [INCREASE, DECREASE] and scopeChanges != [scopeTracker.S_SIMUL]): keywordDictionary = self.parseLineForKeywords(temp, lineType, singleKeyWordList, keywordDictionary) - + if(sT.getBlockContext(lineType) != [] or foundBlock != None): bC = sT.getBlockContext(lineType) if(self.config_info.DEBUG): - print("Current block context: " + str(bC)) - if(foundBlock != None): + print(("Current block context: " + str(bC))) + if(foundBlock != None): if(self.config_info.DEBUG): print("No scope increase yet for block keyword. Adding to the list.") #This means we have found block keyword, but not yet seen the scope increase #This will always happen in python, but can happen in { languages if the { #for the block is not on the same line as the keyword. bC.append(foundBlock) - - #This line is double counting on a deacrease + + #This line is double counting on a deacrease keywordDictionary = self.parseLineForKeywords(temp, lineType, blockKeyWordList, keywordDictionary, bC) if(sT.changeScopeFirst() and reset): @@ -906,7 +907,7 @@ def updateScopeAndKeywords(self, phase, line, lineType, lineNum, sT, foundBlock, blockKeywordType = "" foundBlock = None - return (foundBlock, blockKeywordLine, blockKeywordType, shortFunctionName, keywordDictionary, sT, False) + return (foundBlock, blockKeywordLine, blockKeywordType, shortFunctionName, keywordDictionary, sT, False) #Main function to parse out the contents loaded into logChunk @@ -920,8 +921,8 @@ def parseText(self): #New keyword list for both single and block keywords. This is a list of triples. if(self.keyWordList == []): self.keyWordList = self.readKeywords(keyWordList) - singleKeyWordList = filter(lambda w : w[2] == SINGLE, self.keyWordList) - blockKeyWordList = filter(lambda w: w[2] == BLOCK, self.keyWordList) + singleKeyWordList = [w for w in self.keyWordList if w[2] == SINGLE] + blockKeyWordList = [w for w in self.keyWordList if w[2] == BLOCK] foundBlock = None #This marks a line that matches a block keyword blockKeywordLine = -1 #If the keyword line doesn't have a scope increase, we need to record its line. @@ -948,20 +949,20 @@ def parseText(self): #Initialize keywords (This is repeated three times -> make into a subfunction) for keyword in singleKeyWordList: if(keyword[1] != EXCLUDED): - keywordDictionary[self.outputKeyword(keyword)+ " Adds"]=0 - keywordDictionary[self.outputKeyword(keyword)+ " Dels"]=0 - outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " Adds"]=0 - outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " Dels"]=0 + keywordDictionary[self.outputKeyword(keyword)+ " adds"]=0 + keywordDictionary[self.outputKeyword(keyword)+ " dels"]=0 + outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " adds"]=0 + outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " dels"]=0 for keyword in blockKeyWordList: #Hack to make run with the 'tryDependentCatch' keyword if(not isinstance(keyword, list) or len(keyword) != KEYLISTSIZE): continue elif(keyword[1] != EXCLUDED): - keywordDictionary[self.outputKeyword(keyword) + " Adds"]=0 - keywordDictionary[self.outputKeyword(keyword) + " Dels"]=0 - outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " Adds"]=0 - outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " Dels"]=0 + keywordDictionary[self.outputKeyword(keyword) + " adds"]=0 + keywordDictionary[self.outputKeyword(keyword) + " dels"]=0 + outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " adds"]=0 + outsideFuncKeywordDictionary[self.outputKeyword(keyword) + " dels"]=0 #----------------------------------Initialization----------------------------------# @@ -969,18 +970,18 @@ def parseText(self): for line in self.text.split("\n"): startFlag=0 lineNum += 1 - + if(self.config_info.DEBUG): try: - print("The real line: " + line) + print(("The real line: " + line)) except: - print("The real line: " + unicode(line, 'utf-8', errors='ignore')) + print(("The real line: " + str(line, 'utf-8', errors='ignore'))) + - (lineType, line)= self.markLine(line) if(lineType == META): continue - + #Remove all strings from the line. (Get rid of weird cases of brackets #or comment values being excluded from the line. line = self.removeStrings(line) @@ -989,7 +990,7 @@ def parseText(self): line = line.rstrip() #Remove whitespace at the end - + #Remove all comments from the line fChange = UNMARKED (line, lineType, commentFlag, commentType, functionName, fChange) = self.removeComments(line, commentFlag, lineType, commentType, functionName, phase) @@ -1005,7 +1006,7 @@ def parseText(self): try: priorStatus = self.sT.getContinuationFlag() if(self.config_info.DEBUG): - print("Prior Status: " + str(priorStatus)) + print(("Prior Status: " + str(priorStatus))) newStatus = self.langSwitch.isContinuationLine(line, priorStatus) if(self.config_info.DEBUG): if(newStatus == languageSwitcher.CONTINUATION): @@ -1014,7 +1015,7 @@ def parseText(self): print("CONTINUATION LINE END") elif(newStatus == languageSwitcher.CONTINUATION_START): print("CONTINUATION LINE START") - + self.sT.setContinuationFlag(newStatus) except InvalidCodeException: #continue #If the code seems invalid, just skip the line. @@ -1027,9 +1028,9 @@ def parseText(self): if(phase == LOOKFORNAME): if(self.config_info.DEBUG): try: - print("Current Name Search: " + functionName) + print(("Current Name Search: " + functionName)) except: - print("Current Name Search: " + unicode(functionName, 'utf-8', errors='ignore')) + print(("Current Name Search: " + str(functionName, 'utf-8', errors='ignore'))) #What if we've hit a function definition? if(self.langSwitch.checkForFunctionReset(functionName)): @@ -1037,9 +1038,9 @@ def parseText(self): if(self.config_info.DEBUG): try: - print("Line: \"" + line + "\"") + print(("Line: \"" + line + "\"")) except: - print("Line: \"" + unicode(line, 'utf-8', errors='ignore') + "\"") + print(("Line: \"" + str(line, 'utf-8', errors='ignore') + "\"")) #if(self.sT.isFunctionalScopeChange(line,lineType)): try: @@ -1049,7 +1050,7 @@ def parseText(self): continue except AssertionError: return self.markChunkAsError() - + if(sResult == scopeTracker.S_YES): #Problem, in python we can see a function on a line with a scope decrease try: (phase, line, lineType, lineNum, functionName, classContext, funcStart, startFlag, ftotal_add, ftotal_del) = self.checkForFunctionName(phase, line, lineType, lineNum, functionName, classContext, funcStart, startFlag, ftotal_add, ftotal_del) @@ -1154,11 +1155,11 @@ def parseText(self): self.functions.append(funcToAdd) #Remove any unmodified functions - self.functions = filter(lambda(x) : x.total_add != 0 or x.total_del != 0 , self.functions) + self.functions = [x for x in self.functions if x.total_add != 0 or x.total_del != 0] #Clear out the scope. self.sT.clearScope() - + #Create a mock function for any changes lines and keyword (single or block) #that occured outside the functions in the block. outsideFunction = None @@ -1166,19 +1167,19 @@ def parseText(self): outsideFunction = self.createOutsideFuncSummary(outsideFuncKeywordDictionary) else: outsideFunction = self.createOutsideFuncSummary() - + if(outsideFunction != None): self.functions.append(outsideFunction) if(self.config_info.DEBUG): print("Chunk End.") - + return self def markChunkAsError(self): if(self.config_info.DEBUG or self.config_info.DEBUGLITE): print("Parse Error in this chunk.") - + #We don't trust the results of parsing this chunk, so return a general error statement. self.total_add = 0 self.total_del = 0 diff --git a/src/logChunk/logChunkTestC.py b/src/logChunk/logChunkTestC.py index 37241b7..1a740b9 100644 --- a/src/logChunk/logChunkTestC.py +++ b/src/logChunk/logChunkTestC.py @@ -22,20 +22,20 @@ def readHelper(self,filename): def debugFunctions(self, funcList): print("===========================================") for func in funcList: - print(func.method) - print(func.start) - print(func.end) - print(func.total_add) - print(func.total_del) - print(func.keywordDictionary) + print((func.method)) + print((func.start)) + print((func.end)) + print((func.total_add)) + print((func.total_del)) + print((func.keywordDictionary)) print("===========================================") def setUp(self): self.keyword1 = ["wtc/Assertions.h","excluded","single"] self.keyword2 = ["ut_ad","included","single"] - self.keyword3 = ["try","Included","BLOCK"] - self.keyword4 = ["for","excludeD","block"] + self.keyword3 = ["try","included","block"] + self.keyword4 = ["for","excluded","block"] self.keyword5 = ["ut_ad included single"] self.keyword6 = ["printf(format","str)","included","single"] self.keyword7 = ["printf(format, str)","included","single"] @@ -101,7 +101,7 @@ def setUp(self): #self.chunk18 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk18.txt")) #Nope #self.chunk19 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk19.txt")) #Nope #self.chunk20 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk20.txt")) #Nope - self.chunk21 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk21.txt"), "C++",c_info) #Check C++ + self.chunk21 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk21.txt"), "C++",c_info) #Check C++ self.chunk22 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk22.txt"), "C++",c_info) #Check C++ self.chunk23 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk23.txt"), "C++",c_info) #Check C++ self.chunk24 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk24.txt"), "C++",c_info) #Check C++ @@ -125,7 +125,7 @@ def setUp(self): self.chunk42 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk42.txt"), "C++",c_info) # C++ self.chunk43 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk43.txt"), "C",c_info) #Check C self.chunk44 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk44.txt"), "C++",c_info) # C++ - self.chunk45 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk45.txt"), "C++",c_info) # C++ + self.chunk45 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk45.txt"), "C++",c_info) # C++ self.chunk46 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk46.txt"), "C++",c_info) # C++ self.chunk47 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk47.txt"), "C++",c_info) # C++ self.chunk48 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk48.txt"), "C++",c_info) # C++ @@ -313,12 +313,12 @@ def test_parseText_Single1(self): self.chunk1.parseText() funcList = self.chunk1.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 4) + self.assertTrue(len(funcList) == 4) self.assertTrue(funcList[0].method=="NdbBlob::getBlobEventName") self.assertTrue(funcList[0].total_add == 10) self.assertTrue(funcList[0].total_del == 0) - testDict = {'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = {'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) @@ -327,7 +327,7 @@ def test_parseText_Single1(self): self.assertTrue(funcList[1].end==22) self.assertTrue(funcList[1].total_add == 4) self.assertTrue(funcList[1].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) @@ -336,7 +336,7 @@ def test_parseText_Single1(self): self.assertTrue(funcList[2].end==60) self.assertTrue(funcList[2].total_add == 35) self.assertTrue(funcList[2].total_del == 0) - testDict = {'assert Adds':2, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':2, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[2].keywordDictionary) self.assertTrue(funcList[3].method==NON_FUNC) @@ -353,7 +353,7 @@ def test_parseText_Single2(self): self.assertTrue(funcList[0].total_add == 0) self.assertTrue(funcList[0].total_del == 15) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 1, 'ut_a Adds':0, 'ut_a Dels': 1} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 1, 'ut_a adds':0, 'ut_a dels': 1} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method==NON_FUNC) @@ -372,7 +372,7 @@ def test_parseText_Single3(self): self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 0) - testDict = {'assert Adds':1, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':1, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method=="safely_trapped_errors") @@ -382,7 +382,7 @@ def test_parseText_Single3(self): self.assertTrue(funcList[1].total_add == 16) self.assertTrue(funcList[1].total_del == 0) - testDict = {'assert Adds':1, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) @@ -392,7 +392,7 @@ def test_parseText_Single3(self): self.assertTrue(funcList[2].total_add == 11) self.assertTrue(funcList[2].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[2].keywordDictionary) @@ -409,7 +409,7 @@ def test_parseText_Single6(self): self.assertTrue(funcList[0].end == 10) self.assertTrue(funcList[0].total_add == 3) self.assertTrue(funcList[0].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) @@ -418,7 +418,7 @@ def test_parseText_Single6(self): self.assertTrue(funcList[1].end == 14) self.assertTrue(funcList[1].total_add == 3) self.assertTrue(funcList[1].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) @@ -427,7 +427,7 @@ def test_parseText_Single6(self): self.assertTrue(funcList[2].end == 19) self.assertTrue(funcList[2].total_add == 4) self.assertTrue(funcList[2].total_del == 0) - testDict = {'assert Adds':1, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':1, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[2].keywordDictionary) self.assertTrue(funcList[3].method == NON_FUNC) @@ -450,7 +450,7 @@ def test_parseText_Single8(self): self.assertTrue(funcList[0].end == 77) self.assertTrue(funcList[0].total_add == 16) self.assertTrue(funcList[0].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == "StackHelper") @@ -458,7 +458,7 @@ def test_parseText_Single8(self): self.assertTrue(funcList[1].end == 113) self.assertTrue(funcList[1].total_add == 1) self.assertTrue(funcList[1].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[2].method == "~StackHelper") @@ -466,13 +466,13 @@ def test_parseText_Single8(self): self.assertTrue(funcList[2].end == 117) self.assertTrue(funcList[2].total_add == 4) self.assertTrue(funcList[2].total_del == 0) - testDict = {'assert Adds':0, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[2].keywordDictionary) self.assertTrue(funcList[3].method == NON_FUNC) self.assertTrue(funcList[3].start == 0) self.assertTrue(funcList[3].end == 0) - testDict = {'assert Adds':32, 'assert Dels': 0, 'ut_ad Adds':0, 'ut_ad Dels': 0, 'ut_a Adds':0, 'ut_a Dels': 0} + testDict = {'assert adds':0, 'assert dels': 0, 'ut_ad adds':0, 'ut_ad dels': 0, 'ut_a adds':0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[3].keywordDictionary) def test_parseText_Single9(self): @@ -488,7 +488,7 @@ def test_parseText_Single9(self): self.assertTrue(funcList[6].end == 121) self.assertTrue(funcList[6].total_add == 3) self.assertTrue(funcList[6].total_del == 0) - testDict = { 'ut_ad Adds': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Dels': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert dels': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[6].keywordDictionary) def test_parseText_Single10(self): @@ -502,7 +502,7 @@ def test_parseText_Single10(self): self.assertTrue(funcList[3].end == 246) self.assertTrue(funcList[3].total_add == 44) self.assertTrue(funcList[3].total_del == 0) - testDict = { 'ut_ad Adds': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Dels': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert dels': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[3].keywordDictionary) self.assertTrue(funcList[6].method == NON_FUNC) @@ -518,7 +518,7 @@ def test_parseText_Single11(self): self.assertTrue(funcList[4].end == 199) self.assertTrue(funcList[4].total_add == 118) self.assertTrue(funcList[4].total_del == 0) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 1, 'ut_a dels': 0} self.assertEqual(testDict,funcList[4].keywordDictionary) self.assertTrue(funcList[5].method == NON_FUNC) @@ -535,7 +535,7 @@ def test_parseText_Single12(self): self.assertTrue(funcList[1].end == 184) self.assertTrue(funcList[1].total_add == 12) self.assertTrue(funcList[1].total_del == 0) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[3].method == NON_FUNC) @@ -572,7 +572,7 @@ def test_parseText_Single14(self): self.assertTrue(funcList[0].start == 8) self.assertTrue(funcList[0].end == 241) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 15, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 14, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == NON_FUNC) @@ -602,23 +602,23 @@ def test_parseText_Single22(self): self.assertTrue(len(funcList) == 7) #Can't get the last one b/c constructor out of context self.assertTrue(funcList[0].method == "MDL_map::init") self.assertTrue(funcList[1].method == "MDL_map::destroy") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[2].method == "MDL_map::find_or_insert") self.assertTrue(funcList[3].method == "MDL_map::find") self.assertTrue(funcList[4].method == "MDL_map::move_from_hash_to_lock_mutex") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 2, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 2, 'ut_a dels': 0} self.assertTrue(testDict,funcList[4].keywordDictionary) self.assertTrue(funcList[5].method == "MDL_map::remove") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 1, 'ut_a dels': 0} self.assertEqual(testDict,funcList[5].keywordDictionary) self.assertTrue(funcList[6].method == NON_FUNC) - + def test_parseText_Single23(self): self.chunk23.parseText() funcList = self.chunk23.functions @@ -626,10 +626,10 @@ def test_parseText_Single23(self): self.assertTrue(len(funcList) == 3) self.assertTrue(funcList[0].method == "MDL_ticket::has_pending_conflicting_lock_impl") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 2, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 1, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == "MDL_ticket::has_pending_conflicting_lock") #Name not in + or - - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[2].method == NON_FUNC) @@ -640,7 +640,7 @@ def test_parseText_Single24(self): self.assertTrue(len(funcList) == 25) self.assertTrue(funcList[16].method == "*get_date_time_format_str") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[16].keywordDictionary) def test_parseText_Single25(self): @@ -650,7 +650,7 @@ def test_parseText_Single25(self): self.assertTrue(len(funcList) == 5) self.assertTrue(funcList[2].method == "row_upd_index_replace_new_col_vals_index_pos") - testDict = { 'ut_ad Adds': 1, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 1, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[2].keywordDictionary) @@ -667,38 +667,38 @@ def test_parseText_Single29(self): self.assertTrue(len(funcList) == 16) self.assertTrue(funcList[0].method == "idxof") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == "idxof") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[7].method == "*sfmt_get_idstring") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[7].keywordDictionary) self.assertTrue(funcList[11].method == "sfmt_fill_array32") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 3, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 3, 'ut_a dels': 0} self.assertEqual(testDict,funcList[11].keywordDictionary) self.assertTrue(funcList[12].method == "sfmt_fill_array64") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 3, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 3, 'ut_a dels': 0} self.assertEqual(testDict,funcList[12].keywordDictionary) def test_parseText_Single31(self): - self.chunk31.parseText() + self.chunk31.parseText() funcList = self.chunk31.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "smp_callin") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 1, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 1, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) def test_parseText_Single32(self): @@ -708,16 +708,16 @@ def test_parseText_Single32(self): self.assertTrue(len(funcList) == 11) self.assertTrue(funcList[1].method == "h264_er_decode_mb") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 1, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[7].method == "alloc_picture") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 2, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 2, 'ut_a dels': 0} self.assertEqual(testDict,funcList[7].keywordDictionary) self.assertTrue(funcList[10].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[10].keywordDictionary) @@ -738,23 +738,23 @@ def test_parseText_Single35(self): self.debugFunctions(funcList) self.assertTrue(funcList[-1].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[-1].keywordDictionary) def test_parseText_Single36(self): - self.chunk36.parseText() + self.chunk36.parseText() funcList = self.chunk36.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 3) # 2 + 1 Mock self.assertTrue(funcList[1].method == "Patch") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 5, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[2].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict,funcList[2].keywordDictionary) def test_parseText_Single37(self): @@ -766,7 +766,7 @@ def test_parseText_Single37(self): self.assertTrue(funcList[0].method == "NamespaceDetails::_alloc") self.assertTrue(funcList[0].total_add == 6) self.assertTrue(funcList[0].total_del == 3) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 1, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) def test_parseText_Single38(self): @@ -783,7 +783,7 @@ def test_parseText_Single38(self): self.assertTrue(funcList[2].method == "mysql_stmt_close") self.assertTrue(funcList[2].total_add == 3) self.assertTrue(funcList[2].total_del == 3) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[2].keywordDictionary) def test_parseText_Single40(self): @@ -794,7 +794,7 @@ def test_parseText_Single40(self): self.assertTrue(funcList[0].method == "jl_unbox_##j_type") self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 2) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 2, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 2, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 2, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 2, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) def test_parseText_Single41(self): @@ -805,7 +805,7 @@ def test_parseText_Single41(self): self.assertTrue(funcList[0].method == "Mutex") self.assertTrue(funcList[4].method == "Mutex") self.assertTrue(funcList[9].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[9].keywordDictionary) def test_parseText_Single42(self): @@ -814,10 +814,10 @@ def test_parseText_Single42(self): #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "Benchmark::saveMatlabGraph") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 1, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) def test_parseText_Single43(self): @@ -826,10 +826,10 @@ def test_parseText_Single43(self): #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "incr_flush_list_size_in_bytes") - testDict = { 'ut_ad Adds': 2, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 2, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) def test_parseText_Single44(self): @@ -838,93 +838,93 @@ def test_parseText_Single44(self): #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "auto_copying_data_provider_t::get_data_into_buffers") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 4, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 4, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) def test_parseText_Single45(self): #Not Sure how I want to handle this self.chunk45.parseText() - funcList = self.chunk45.functions + funcList = self.chunk45.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "Int32BinopInputShapeTester") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 1, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) def test_parseText_Single46(self): #Not Sure how I want to handle this self.chunk46.parseText() - funcList = self.chunk46.functions + funcList = self.chunk46.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 3) self.assertTrue(funcList[0].method == "QuatF::mulP") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == "QuatF::mul") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) self.assertTrue(funcList[2].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[2].keywordDictionary) def test_parseText_Single47(self): #Not Sure how I want to handle this self.chunk47.parseText() - funcList = self.chunk47.functions + funcList = self.chunk47.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 1) self.assertTrue(funcList[0].method == "CCAnimate::initWithAnimation") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 1, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 1, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) def test_parseText_Single48(self): #Not Sure how I want to handle this self.chunk48.parseText() - funcList = self.chunk48.functions + funcList = self.chunk48.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 5) self.assertTrue(funcList[0].method == "&=") - testDict = { 'ut_ad Adds': 0, 'assert Dels': 1, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[4].method == NON_FUNC) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[4].keywordDictionary) def test_parseText_Single49(self): #Not Sure how I want to handle this self.chunk49.parseText() - funcList = self.chunk49.functions + funcList = self.chunk49.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "ClientInfo::newRequest") self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 1) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) self.assertTrue(funcList[1].method == "ClientInfo::create") self.assertTrue(funcList[1].total_add == 1) self.assertTrue(funcList[1].total_del == 1) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) def test_parseText_Single50(self): #Not Sure how I want to handle this self.chunk50.parseText() - funcList = self.chunk50.functions + funcList = self.chunk50.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 1) self.assertTrue(funcList[0].method == "xfs_buf_get") self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 7) - testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} self.assertEqual(testDict, funcList[0].keywordDictionary) @@ -934,7 +934,7 @@ def test_parseText_Single51(self): #I don't have time to fix this yet, so I'm adding a test case to make sure it doesn't crash #but instead returns a error parse statement self.chunk51.parseText() - funcList = self.chunk51.functions + funcList = self.chunk51.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 1) @@ -942,26 +942,26 @@ def test_parseText_Single51(self): def test_parseText_Single52(self): #Not Sure how I want to handle this, this case is part of a larger failing chunk. self.chunk52.parseText() - funcList = self.chunk52.functions + funcList = self.chunk52.functions #self.debugFunctions(funcList) - def test_parseText_Single53(self): + def test_parseText_Single53(self): #This is an old style K&R C function declaration. #http://stackoverflow.com/questions/3092006/function-declaration-kr-vs-ansi #The syntax is deprecated but still legal. - #I think trying to capture these may accidently lead to capturing other types of non-functions, + #I think trying to capture these may accidently lead to capturing other types of non-functions, #due to the presence of ';'. I'll throw an issue up for it. self.chunk53.parseText() - funcList = self.chunk53.functions + funcList = self.chunk53.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 1) #Re-enable this later.... #self.assertTrue(funcList[0].method == "set_offsets_for_label") - def test_parseText_Single54(self): + def test_parseText_Single54(self): self.chunk54.parseText() - funcList = self.chunk54.functions + funcList = self.chunk54.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 6) self.assertTrue(funcList[0].method == "Reset") @@ -973,7 +973,7 @@ def test_parseText_Single54(self): def test_parseText_Single55(self): # Testing our ability to find struct constructors. self.chunk55.parseText() - funcList = self.chunk55.functions + funcList = self.chunk55.functions #self.debugFunctions(funcList) self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method == "boss_jaraxxusAI") @@ -1061,12 +1061,12 @@ def test_parseText_Single55(self): # Testing our ability to find struct construc #self.assertTrue(len(funcList) == 2) #self.assertTrue(funcList[0].method == "_ccv_rgb_to_yuv") - #testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 0, 'ut_a Dels': 0} + #testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 0, 'ut_a dels': 0} #self.assertTrue(testDict,funcList[0].keywordDictionary) #self.assertTrue(len(funcList[0].assertionList) == 0) #self.assertTrue(funcList[1].method == "ccv_color_transform") - #testDict = { 'ut_ad Adds': 0, 'assert Dels': 0, 'ut_ad Dels': 0, 'ut_a Adds': 0, 'assert Adds': 2, 'ut_a Dels': 0} + #testDict = { 'ut_ad adds': 0, 'assert dels': 0, 'ut_ad dels': 0, 'ut_a adds': 0, 'assert adds': 2, 'ut_a dels': 0} #self.assertTrue(testDict,funcList[1].keywordDictionary) #self.assertTrue(len(funcList[1].assertionList) == 2) diff --git a/src/logChunk/logChunkTestJAVA.py b/src/logChunk/logChunkTestJAVA.py index 091a5dc..1464036 100644 --- a/src/logChunk/logChunkTestJAVA.py +++ b/src/logChunk/logChunkTestJAVA.py @@ -20,12 +20,12 @@ def readHelper(self,filename): def debugFunctions(self, funcList): print("===========================================") for func in funcList: - print(func.method) - print(func.start) - print(func.end) - print(func.total_add) - print(func.total_del) - print(func.keywordDictionary) + print((func.method)) + print((func.start)) + print((func.end)) + print((func.total_add)) + print((func.total_del)) + print((func.keywordDictionary)) print("===========================================") def setUp(self): @@ -90,14 +90,14 @@ def test_parseText_Block1(self): self.assertTrue(funcList[0].method=="foo") self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 1) - testDict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 0, 'try Dels': 1, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 1, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testDict= {'throw adds': 0, 'catch dels': 0, 'try adds': 0, 'try dels': 1, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 1, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method=="foo00022") self.assertTrue(funcList[1].total_add == 4) self.assertTrue(funcList[1].total_del == 2) - testDict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 1, 'try Dels': 1, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 1, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testDict= {'throw adds': 0, 'catch dels': 0, 'try adds': 1, 'try dels': 1, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 1, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testDict, funcList[1].keywordDictionary) @@ -111,13 +111,13 @@ def test_parseText_Block2(self): #ISSUE: the current cannot assign values to mul self.assertTrue(funcList[0].method=="getAccounts") self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 2) - testdict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 0, 'try adds': 0, 'try dels': 0, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method=="getAccount") self.assertTrue(funcList[1].total_add == 6) self.assertTrue(funcList[1].total_del == 2) - testdict={'throw Adds': 1, 'catch Dels': 0, 'try Adds': 3, 'try Dels': 2, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 4, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 2, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 2,'while Dels': 2} + testdict={'throw adds': 1, 'catch dels': 0, 'try adds': 3, 'try dels': 2, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 4, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 2, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 2,'while dels': 2} self.assertEqual(testdict,funcList[1].keywordDictionary) def test_parseText_Block3(self): @@ -131,7 +131,7 @@ def test_parseText_Block3(self): self.assertTrue(funcList[0].method=="ReflectiveProperty") self.assertTrue(funcList[0].total_add == 8) self.assertTrue(funcList[0].total_del == 2) - testdict= {'throw Adds': 0, 'catch Dels': 1, 'try Adds': 8, 'try Dels': 2, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 4, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 1, 'try adds': 8, 'try dels': 2, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 4, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) @@ -145,7 +145,7 @@ def test_parseText_Block4(self): self.assertTrue(funcList[0].method=="setHandle") self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 1) - testdict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 0, 'try adds': 0, 'try dels': 0, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) def test_parseText_Block5(self): @@ -160,12 +160,12 @@ def test_parseText_Block5(self): self.assertTrue(funcList[0].total_add == 19) self.assertTrue(funcList[0].total_del == 5) - testdict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 1, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 1, 'catch dels': 0, 'try adds': 0, 'try dels': 0, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 1, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method==NON_FUNC) #The add del count here is a bit off due to the way we change code that has been uncommented - testdict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 0, 'try adds': 0, 'try dels': 0, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[1].keywordDictionary) @@ -181,7 +181,7 @@ def test_parseText_Block6(self): self.assertTrue(funcList[0].method=="init") self.assertTrue(funcList[0].total_add == 0) self.assertTrue(funcList[0].total_del == 1) - testdict= {'throw Adds': 0, 'catch Dels': 1, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 1, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 1, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 1, 'try adds': 0, 'try dels': 0, 'Exception dels': 1, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 1, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) @@ -195,7 +195,7 @@ def test_parseText_Block7(self): #Need to update expected result (Question, we s self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 7) - testdict= {'throw Adds': 0, 'catch Dels': 4, 'try Adds': 0, 'try Dels': 2, 'exception Dels': 1, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 1, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 1, 'catch dels': 4, 'try adds': 0, 'try dels': 2, 'Exception dels': 1, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 1, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) @@ -210,7 +210,7 @@ def test_parseText_Block8(self): #Need to update expected result (Question, we s self.assertTrue(funcList[0].method=="getAuthToken") self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 2) - testdict= {'throw Adds': 1, 'catch Dels': 1, 'try Adds': 1, 'try Dels': 1, 'exception Dels': 1, 'raise Adds': 0, 'catch Adds': 2, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 2, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 1, 'catch dels': 1, 'try adds': 1, 'try dels': 1, 'Exception dels': 1, 'raise adds': 0, 'catch adds': 2, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 2, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) @@ -225,7 +225,7 @@ def test_parseText_Block9(self): self.assertTrue(funcList[0].method=="getAuthToken") self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 2) - testdict= {'throw Adds': 1, 'catch Dels': 1, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 1, 'raise Adds': 0, 'catch Adds': 2, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 2, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 1, 'catch dels': 1, 'try adds': 0, 'try dels': 0, 'Exception dels': 1, 'raise adds': 0, 'catch adds': 2, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 2, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) @@ -241,7 +241,7 @@ def test_parseText_Block10(self): self.assertTrue(funcList[0].method=="getToken") self.assertTrue(funcList[0].total_add == 8) self.assertTrue(funcList[0].total_del == 5) - testdict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 0, 'try Dels': 0, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 4,'for Dels': 5,'while Adds': 4,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 0, 'try adds': 0, 'try dels': 0, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 4,'for dels': 5,'while adds': 4,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) @@ -256,7 +256,7 @@ def test_parseText_Block11(self): self.assertTrue(funcList[0].method=="blockTest") self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 1) - testdict= {'throw Adds': 0, 'catch Dels': 0, 'try Adds': 1, 'try Dels': 1, 'exception Dels': 0, 'raise Adds': 0, 'catch Adds': 0, 'finally Dels': 0, 'finally Adds': 0, 'throw Dels': 0, 'exception Adds': 0, 'raise Dels': 0, 'for Adds': 0,'for Dels': 0,'while Adds': 0,'while Dels': 0} + testdict= {'throw adds': 0, 'catch dels': 0, 'try adds': 1, 'try dels': 1, 'Exception dels': 0, 'raise adds': 0, 'catch adds': 0, 'finally dels': 0, 'finally adds': 0, 'throw dels': 0, 'Exception adds': 0, 'raise dels': 0, 'for adds': 0,'for dels': 0,'while adds': 0,'while dels': 0} self.assertEqual(testdict,funcList[0].keywordDictionary) diff --git a/src/logChunk/logChunkTestPython.py b/src/logChunk/logChunkTestPython.py index c27a4fe..fff8d4f 100644 --- a/src/logChunk/logChunkTestPython.py +++ b/src/logChunk/logChunkTestPython.py @@ -22,12 +22,12 @@ def readHelper(self,filename): def debugFunctions(self, funcList): print("===========================================") for func in funcList: - print(func.method) - print(func.start) - print(func.end) - print(func.total_add) - print(func.total_del) - print(func.keywordDictionary) + print((func.method)) + print((func.start)) + print((func.end)) + print((func.total_add)) + print((func.total_del)) + print((func.keywordDictionary)) print("===========================================") @@ -69,7 +69,7 @@ def setUp(self): def test_isFunction(self): self.assertTrue(self.testChunk.isFunction(self.method1)) - self.assertTrue(self.testChunk.isFunction(self.method2)) + self.assertTrue(self.testChunk.isFunction(self.method2)) self.assertTrue(self.testChunk.isFunction(self.method3)) self.assertTrue(self.testChunk.isFunction(self.method4)) self.assertFalse(self.testChunk.isFunction(self.method5)) @@ -90,7 +90,7 @@ def test_continuationLines(self): self.assertTrue(self.testChunk.langSwitch.isContinuationLine(" /", NOT_CONTINUATION) == NOT_CONTINUATION) def test_keyword_parse(self): - print(self.testChunk.keywordMatch("\"print\"", "afo.write(data.print()")) + print((self.testChunk.keywordMatch("\"print\"", "afo.write(data.print()"))) self.assertTrue(self.testChunk.keywordMatch("\"print\"", "afo.write(data.print()") == ("print", True)) @@ -98,12 +98,12 @@ def test_parseText1(self): self.chunk1.parseText() funcList = self.chunk1.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 1) + self.assertTrue(len(funcList) == 1) self.assertTrue(funcList[0].method=="_ask_default") self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 1) - testDict = {'print Adds': 1, 'print Dels': 1, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 1, 'print dels': 1, 'if dels': 0, 'if adds': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) @@ -112,12 +112,12 @@ def test_parseText2(self): self.chunk2.parseText() funcList = self.chunk2.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 1) + self.assertTrue(len(funcList) == 1) self.assertTrue(funcList[0].method=="url") self.assertTrue(funcList[0].total_add == 4) self.assertTrue(funcList[0].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 2} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 2} self.assertEqual(testDict,funcList[0].keywordDictionary) @@ -125,18 +125,18 @@ def test_parseText3(self): self.chunk3.parseText() funcList = self.chunk3.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 2) + self.assertTrue(len(funcList) == 2) self.assertTrue(funcList[0].method=="testFunc") - - testDict = {'print Adds': 2, 'print Dels': 2, 'if Dels': 2, 'if Adds': 2} + + testDict = {'print adds': 2, 'print dels': 2, 'if dels': 2, 'if adds': 2} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method==NON_FUNC) #The exact count here is a bit off, but going to ignore... - #self.assertTrue(funcList[1].total_add == 1) + #self.assertTrue(funcList[1].total_add == 1) #self.assertTrue(funcList[1].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertTrue(testDict,funcList[1].keywordDictionary) @@ -144,14 +144,14 @@ def test_parseText4(self): self.chunk4.parseText() funcList = self.chunk4.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 1) + self.assertTrue(len(funcList) == 1) self.assertTrue(funcList[0].method=="_get_queryset") #self.assertTrue(funcList[0].total_add == 7) #self.assertTrue(funcList[0].total_del == 18) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 5, 'if Adds': 2} #Should be 4 but hacking to make test suite pass at the moment - #testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 4, 'if Adds': 2} - #testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 1, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 7, 'if adds': 2} #Should be 4 but hacking to make test suite pass at the moment + #testDict = {'print adds': 0, 'print dels': 0, 'if dels': 4, 'if adds': 2} + #testDict = {'print adds': 0, 'print dels': 0, 'if dels': 1, 'if adds': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) def test_parseText5(self): @@ -163,50 +163,50 @@ def test_parseText5(self): self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 0) - testDict = {'print Adds': 1, 'print Dels': 0, 'if Dels': 0, 'if Adds': 2} + testDict = {'print adds': 1, 'print dels': 0, 'if dels': 0, 'if adds': 2} self.assertTrue(funcList[1].method == NON_FUNC) self.assertTrue(funcList[1].total_add == 1) self.assertTrue(funcList[1].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertTrue(testDict,funcList[1].keywordDictionary) def test_parseText6(self): self.chunk6.parseText() funcList = self.chunk6.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 4) + self.assertTrue(len(funcList) == 4) self.assertTrue(funcList[0].method=="__init__") self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 0) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertTrue(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method=="from_crawler") self.assertTrue(funcList[1].total_add == 3) self.assertTrue(funcList[1].total_del == 1) - - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertTrue(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[2].method=="open") self.assertTrue(funcList[2].total_add == 1) self.assertTrue(funcList[2].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertTrue(testDict,funcList[2].keywordDictionary) self.assertTrue(funcList[3].method==NON_FUNC) self.assertTrue(funcList[3].total_add == 0) self.assertTrue(funcList[3].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertTrue(testDict,funcList[3].keywordDictionary) - + def test_parseText7(self): self.chunk7.parseText() funcList = self.chunk7.functions @@ -231,31 +231,31 @@ def test_parseText8(self): self.chunk8.parseText() funcList = self.chunk8.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 1) + self.assertTrue(len(funcList) == 1) self.assertTrue(funcList[0].method=="confusedFunction") self.assertTrue(funcList[0].total_add == 3) self.assertTrue(funcList[0].total_del == 3) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) def test_parseText9(self): self.chunk9.parseText() funcList = self.chunk9.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 3) + self.assertTrue(len(funcList) == 3) self.assertTrue(funcList[0].method=="__init__") self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 0) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) self.assertTrue(funcList[1].method=="open") self.assertTrue(funcList[1].total_add == 1) self.assertTrue(funcList[1].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertEqual(testDict,funcList[1].keywordDictionary) self.assertTrue(funcList[2].method==NON_FUNC) @@ -266,7 +266,7 @@ def test_parseText10(self): self.chunk10.parseText() funcList = self.chunk10.functions #self.debugFunctions(funcList) - self.assertTrue(len(funcList) == 3) + self.assertTrue(len(funcList) == 3) self.assertTrue(funcList[0].method=="__init__") #This is reporting 3, which is wrong by perceived set up, but not #conceptually wrong... (We would say 1 add under current definitions) @@ -290,7 +290,7 @@ def test_parseText11(self): self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 3) - testDict = {'print Adds': 1, 'print Dels': 1, 'if Dels': 0, 'if Adds': 1} + testDict = {'print adds': 1, 'print dels': 1, 'if dels': 0, 'if adds': 1} self.assertEqual(testDict,funcList[0].keywordDictionary) def test_parseText12(self): @@ -302,7 +302,7 @@ def test_parseText12(self): self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 1) - testDict = {'print Adds': 0, 'print Dels': 0, 'if Dels': 1, 'if Adds': 1} + testDict = {'print adds': 0, 'print dels': 0, 'if dels': 1, 'if adds': 1} self.assertEqual(testDict,funcList[0].keywordDictionary) @@ -315,7 +315,7 @@ def test_parseText13(self): self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 1) - testDict = {'print Adds': 1, 'print Dels': 0, 'if Dels': 1, 'if Adds': 1} + testDict = {'print adds': 1, 'print dels': 0, 'if dels': 1, 'if adds': 1} self.assertEqual(testDict,funcList[0].keywordDictionary) def test_parseText14(self): @@ -327,7 +327,7 @@ def test_parseText14(self): self.assertTrue(funcList[0].total_add == 2) self.assertTrue(funcList[0].total_del == 1) - testDict = {'print Adds': 1, 'print Dels': 0, 'if Dels': 1, 'if Adds': 1} + testDict = {'print adds': 1, 'print dels': 0, 'if dels': 1, 'if adds': 1} self.assertEqual(testDict,funcList[0].keywordDictionary) def test_parseText15(self): @@ -365,7 +365,7 @@ def test_parseText19(self): #Error with exact matching self.assertTrue(funcList[0].total_add == 1) self.assertTrue(funcList[0].total_del == 1) - testDict = {'print Adds': 1, 'print Dels': 0, 'if Dels': 0, 'if Adds': 0} + testDict = {'print adds': 1, 'print dels': 0, 'if dels': 0, 'if adds': 0} self.assertEqual(testDict,funcList[0].keywordDictionary) diff --git a/src/logChunk/requirements.txt b/src/logChunk/requirements.txt new file mode 100644 index 0000000..6a16d8c --- /dev/null +++ b/src/logChunk/requirements.txt @@ -0,0 +1,4 @@ +nltk +gitpython +pyyaml +psycopg2-binary diff --git a/src/logChunk/reset.sh b/src/logChunk/reset.sh new file mode 100755 index 0000000..c5206fd --- /dev/null +++ b/src/logChunk/reset.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -ex + +HOST="database-research.cwfckxyjfuva.us-east-1.rds.amazonaws.com" +USER="khatchad" +DB="func" +REPO_DIR="FuncProjects" + +psql -h "$HOST" -U "$USER" -d "$DB" -c "drop table if exists change_summary" +psql -h "$HOST" -U "$USER" -d "$DB" -c "drop table if exists method_change_detail" + +rm -rf "../../evaluation/repos/$REPO_DIR" diff --git a/src/logChunk/run.cmd b/src/logChunk/run.cmd new file mode 100644 index 0000000..2ac4aa0 --- /dev/null +++ b/src/logChunk/run.cmd @@ -0,0 +1 @@ +./reset.sh;python3 gitcproc.py -d -pl -wl -p funcstudy ../util/func_conf.ini diff --git a/src/logChunk/run.sh b/src/logChunk/run.sh index 7c8d7f5..7536965 100755 --- a/src/logChunk/run.sh +++ b/src/logChunk/run.sh @@ -1,2 +1,2 @@ -echo "========== nohup python ghProc.py $1 >> $2.out 2>> $2.err ============" -nohup python ghProc.py $1 $3 $4 >> ../../evaluation/log_files/$2.out 2>> ../../evaluation/log_files/$2.err +echo "========== nohup python3 ghProc.py $1 >> $2.out 2>> $2.err ============" +nohup python3 ghProc.py $1 $3 $4 >> ../../evaluation/log_files/$2.out 2>> ../../evaluation/log_files/$2.err diff --git a/src/logChunk/runAll.py b/src/logChunk/runAll.py index 8ec209d..ff932f2 100644 --- a/src/logChunk/runAll.py +++ b/src/logChunk/runAll.py @@ -13,7 +13,7 @@ def processProject(projPath): """thread worker function""" - print 'processProject : %s\n' % projPath + print('processProject : %s\n' % projPath) #print threading.current_thread().name ghProc.processLog(projPath) @@ -48,17 +48,17 @@ def main(argv): try: opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="]) except getopt.GetoptError: - print 'runAll.py -i -o ' + print('runAll.py -i -o ') sys.exit(2) if len(opts) == 0: #no argument is passed - print 'runAll.py -i -o ' + print('runAll.py -i -o ') sys.exit() for opt, arg in opts: if opt == '-h': - print 'runAll.py -i -o ' + print('runAll.py -i -o ') sys.exit() elif opt in ("-i", "--ifile"): @@ -66,8 +66,8 @@ def main(argv): elif opt in ("-o", "--ofile"): outputfile = arg - print 'Input file is :', inputfile - print 'Output file is :', outputfile + print('Input file is :', inputfile) + print('Output file is :', outputfile) #populate arrays with c and c++ projects #cpp_projects = findAll(CPP_PATH) @@ -95,13 +95,13 @@ def main(argv): # project_path = '' sys.stdout = orig_stdout sys.stderr = orig_stderr - print project_path + print(project_path) #project_paths.append(project_path) project_name = ntpath.basename(project_path) sys.stdout = open(project_name + '.out', 'w') sys.stderr = open(project_name + '.err', 'w') - print project_path + print(project_path) ghProc.processLog(project_path) f.close() diff --git a/src/logChunk/scopeTracker.py b/src/logChunk/scopeTracker.py index d83a12a..ba6822c 100644 --- a/src/logChunk/scopeTracker.py +++ b/src/logChunk/scopeTracker.py @@ -177,17 +177,17 @@ def adjustFunctionBorders(self, start, end, adds, deletes): def printScope(self): print("------------------------------------") print("Language:") - print(self.language) + print((self.language)) print("Old Stack:") - print(self.oldVerStack) + print((self.oldVerStack)) print("Old Func Cache:") - print(self.lastOldFuncContext) + print((self.lastOldFuncContext)) print("Old Block Cache:") - print(self.lastOldBlockContext) + print((self.lastOldBlockContext)) print("New Stack:") - print(self.newVerStack) + print((self.newVerStack)) print("New Func Cache:") - print(self.lastNewFuncContext) + print((self.lastNewFuncContext)) print("New Block Cache:") - print(self.lastNewBlockContext) + print((self.lastNewBlockContext)) print("------------------------------------") diff --git a/src/logChunk/stream_api_keywords.txt b/src/logChunk/stream_api_keywords.txt new file mode 100644 index 0000000..bf7c4f6 --- /dev/null +++ b/src/logChunk/stream_api_keywords.txt @@ -0,0 +1,128 @@ +"BaseStream",included,single +"Collector",included,single +"Characteristics",included,single +"Collectors",included,single +"DoubleStream",included,single +"range(",included,single +"rangeClosed(",included,single +"IntStream",included,single +"LongStream",included,single +"Spliterator",included,single +"OfDouble",included,single +"OfInt",included,single +"OfLong",included,single +"OfPrimitive",included,single +"Stream",included,single +"StreamSupport",included,single +"accumulator()",included,single +"allMatch(",included,single +"allProcesses()",included,single +"anyMatch(",included,single +"asDoubleStream()",included,single +"asIntStream()",included,single +"asLongStream()",included,single +"average()",included,single +"averagingDouble(",included,single +"averagingInt(",included,single +"averagingLong(",included,single +"boxed()",included,single +"build()",included,single +"builder()",included,single +"catalogs()",included,single +"characteristics()",included,single +"chars()",included,single +"children()",included,single +"codePoints()",included,single +"collect(",included,single +"collectingAndThen(",included,single +"combiner()",included,single +"concat(",included,single +"count()",included,single +"counting()",included,single +"datesUntil(",included,single +"descendants()",included,single +"distinct()",included,single +"doubleStream(",included,single +"doubles(",included,single +"doubles()",included,single +"drivers()",included,single +"dropWhile(",included,single +"elementsAsStream()",included,single +"estimateSize()",included,single +"filter(",included,single +"filtering(",included,single +"findAny()",included,single +"findFirst()",included,single +"finisher()",included,single +"flatMap(",included,single +"flatMapToDouble(",included,single +"flatMapToInt(",included,single +"flatMapping(",included,single +"forEach(",included,single +"forEachOrdered(",included,single +"forEachRemaining(",included,single +"generate(",included,single +"getComparator()",included,single +"getExactSizeIfKnown()",included,single +"groupingBy(",included,single +"groupingByConcurrent(",included,single +"hasCharacteristics(",included,single +"inetAddresses()",included,single +"intStream(",included,single +"ints()",included,single +"isParallel()",included,single +"joining()",included,single +"limit(",included,single +"lines()",included,single +"longStream(",included,single +"longs(",included,single +"longs()",included,single +"map(",included,single +"mapToInt(",included,single +"mapToLong(",included,single +"mapToObj(",included,single +"mapping(",included,single +"max()",included,single +"maxBy(",included,single +"min()",included,single +"minBy(",included,single +"networkInterfaces()",included,single +"noneMatch(",included,single +"parallel()",included,single +"parallelStream()",included,single +"partitioningBy(",included,single +"peek(",included,single +"reduce(",included,single +"reducing(",included,single +"results()",included,single +"sequential()",included,single +"skip(",included,single +"sorted()",included,single +"splitAsStream(",included,single +"spliterator()",included,single +"stream()",included,single +"subInterfaces()",included,single +"sum()",included,single +"summarizingDouble(",included,single +"summarizingInt(",included,single +"summarizingLong(",included,single +"summaryStatistics()",included,single +"summingDouble(",included,single +"summingInt(",included,single +"summingLong(",included,single +"supplier()",included,single +"takeWhile(",included,single +"toArray()",included,single +"toCollection(",included,single +"toConcurrentMap(",included,single +"toList()",included,single +"toMap(",included,single +"toSet()",included,single +"toUnmodifiableList()",included,single +"toUnmodifiableMap(",included,single +"toUnmodifiableSet()",included,single +"tokens()",included,single +"tryAdvance(",included,single +"trySplit()",included,single +"unordered()",included,single +"versionedStream()",included,single diff --git a/src/logChunk/stream_evolution_keywords.txt b/src/logChunk/stream_evolution_keywords.txt new file mode 100644 index 0000000..159cb11 --- /dev/null +++ b/src/logChunk/stream_evolution_keywords.txt @@ -0,0 +1,3 @@ +"parallel()",included,single +"parallelStream()",included,single +"sequential()",included,single diff --git a/src/logChunk/stream_repos.txt b/src/logChunk/stream_repos.txt new file mode 100644 index 0000000..681fb05 --- /dev/null +++ b/src/logChunk/stream_repos.txt @@ -0,0 +1 @@ +stanfordnlp/CoreNLP diff --git a/src/util/Config.py b/src/util/Config.py index cff2327..7914dfa 100644 --- a/src/util/Config.py +++ b/src/util/Config.py @@ -1,4 +1,4 @@ -from ConfigParser import SafeConfigParser +from configparser import SafeConfigParser import logging import os diff --git a/src/util/DatabaseCon.py b/src/util/DatabaseCon.py index 4b8e162..dd0c9bd 100644 --- a/src/util/DatabaseCon.py +++ b/src/util/DatabaseCon.py @@ -6,7 +6,7 @@ class DatabaseCon: def __init__(self, db, dbUser, dbHost, dbPort, passwrd): - print "Going to connect to database %s in server %s, for user %s" % (db, dbHost, dbUser) + print("Going to connect to database %s in server %s, for user %s" % (db, dbHost, dbUser)) #passwrd = raw_input("Please enter password..\n") self.conn = psycopg2.connect(database=db, user=dbUser, host=dbHost, port=dbPort,password=passwrd) @@ -23,12 +23,12 @@ def insert(self, sql_command): try: cur.execute(sql_command) except psycopg2.IntegrityError: - print "Copy already exists." + print("Copy already exists.") print(sql_command) self.conn.rollback() return except: - print "!!! Error" + print("!!! Error") print(sql_command) # #raise return @@ -42,7 +42,7 @@ def create(self, sql_command): try: cur.execute(sql_command) except: - print "Table Creation Error" + print("Table Creation Error") print(sql_command) raise @@ -61,7 +61,7 @@ def execute(self, sql_command): def test(config_file): cfg = Config(config_file) db_config = cfg.ConfigSectionMap("Database") - print "Database configuration = %r\n", db_config + print("Database configuration = %r\n", db_config) dbCon = DatabaseCon(db_config['database'], db_config['user'], db_config['host'], db_config['port'], db_config['password']) @@ -71,13 +71,13 @@ def test(config_file): rows = dbCon.execute(sql_command) for r in rows: - print r + print(r) if __name__ == '__main__': if len(sys.argv) < 2: - print "!! please give a confg file" + print("!! please give a confg file") sys.exit() test(sys.argv[1]) diff --git a/src/util/Util.py b/src/util/Util.py index fb47d58..1efeb80 100644 --- a/src/util/Util.py +++ b/src/util/Util.py @@ -49,7 +49,7 @@ def __exit__(self, etype, value, traceback): def create_dir(path): try: - print path + print(path) os.makedirs(path) except OSError as exception: @@ -69,7 +69,7 @@ def copy_dir(src, dst): def cleanup(path): if os.path.isdir(path): - print "!!! Cleaning up " , path + print("!!! Cleaning up " , path) shutil.rmtree(path) # var = raw_input("Path %s exists; do you want to delete it?" % (path)) @@ -79,7 +79,7 @@ def cleanup(path): # shutil.rmtree(path) elif os.path.isfile(path): - print "!!! Removing " , path + print("!!! Removing " , path) os.remove(path) all_extension = ['.c', '.cc', '.cpp', '.c++', '.cp', '.cxx', '.h', '.ic', \ diff --git a/src/util/func_conf.ini b/src/util/func_conf.ini new file mode 100644 index 0000000..5f87ba2 --- /dev/null +++ b/src/util/func_conf.ini @@ -0,0 +1,26 @@ +[Database] +database: func +user: khatchad +host: database-research.cwfckxyjfuva.us-east-1.rds.amazonaws.com +port: 5432 +schema: public +table_method_detail: method_change_detail +table_change_summary: change_summary + +[Repos] +repo_url_file: func_repos.txt +repo_locations: ../../evaluation/repos/FuncProjects/ + +[Keywords] +file : func_keywords.txt + +[Log] +languages: Python + +[Flags] +SEP: __ +DEBUG: 0 +DEBUGLITE: 0 +DATABASE: 1 +CSV: 0 +LOGTIME: 1 diff --git a/src/util/stream_bugs_conf.ini b/src/util/stream_bugs_conf.ini new file mode 100644 index 0000000..91e9a34 --- /dev/null +++ b/src/util/stream_bugs_conf.ini @@ -0,0 +1,26 @@ +[Database] +database: streams +user: khatchad +host: localhost +port: 5432 +schema: public +table_method_detail: method_change_detail +table_change_summary: change_summary + +[Repos] +repo_url_file: stream_repos.txt +repo_locations: ../../evaluation/repos/StreamProjects/ + +[Keywords] +file : stream_api_keywords.txt + +[Log] +languages: Java + +[Flags] +SEP: __ +DEBUG: 0 +DEBUGLITE: 0 +DATABASE: 1 +CSV: 0 +LOGTIME: 1 diff --git a/src/util/stream_evolution_conf.ini b/src/util/stream_evolution_conf.ini new file mode 100644 index 0000000..2529410 --- /dev/null +++ b/src/util/stream_evolution_conf.ini @@ -0,0 +1,26 @@ +[Database] +database: streams +user: khatchad +host: localhost +port: 5432 +schema: public +table_method_detail: method_change_detail +table_change_summary: change_summary + +[Repos] +repo_url_file: stream_repos.txt +repo_locations: ../../evaluation/repos/StreamProjects/ + +[Keywords] +file : stream_evolution_keywords.txt + +[Log] +languages: Java + +[Flags] +SEP: __ +DEBUG: 0 +DEBUGLITE: 0 +DATABASE: 1 +CSV: 0 +LOGTIME: 1 diff --git a/src/util/unicodeMagic.py b/src/util/unicodeMagic.py index 5bf9256..50595d7 100644 --- a/src/util/unicodeMagic.py +++ b/src/util/unicodeMagic.py @@ -17,7 +17,7 @@ """Based on http://stackoverflow.com/questions/3085263/create-an-utf-8-csv-file-in-python""" -import csv, codecs, cStringIO +import csv, codecs, io class UTF8Recoder: """ @@ -29,7 +29,7 @@ def __init__(self, f, encoding): def __iter__(self): return self - def next(self): + def __next__(self): return self.reader.next().encode("utf-8") class UnicodeReader: @@ -42,9 +42,9 @@ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): f = UTF8Recoder(f, encoding) self.reader = csv.reader(f, delimiter=';', dialect=dialect, **kwds) - def next(self): - row = self.reader.next() - return [unicode(s, "utf-8") for s in row] + def __next__(self): + row = next(self.reader) + return [str(s, "utf-8") for s in row] def __iter__(self): return self @@ -57,7 +57,7 @@ class UnicodeWriter: def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue - self.queue = cStringIO.StringIO() + self.queue = io.StringIO() self.writer = csv.writer(self.queue, delimiter=';', dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)()