From 9489ce13c87ab2169869af0b46e3abe6627df7bf Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Thu, 9 Jul 2015 09:26:37 -0400 Subject: [PATCH 01/12] use tox to run tests against Python2.7 and 3.4 + travis.yml --- .gitignore | 1 + .travis.yml | 10 +++++----- setup.cfg | 1 - tox.ini | 6 ++++++ travis_clouseau.sh | 0 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 tox.ini mode change 100644 => 100755 travis_clouseau.sh diff --git a/.gitignore b/.gitignore index de2d313..0d91e95 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.tox/ *.clouseau *.swn *.egg-info diff --git a/.travis.yml b/.travis.yml index d9b1cb2..300fc29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: python -python: - - "2.7" install: - - pip install -r requirements.txt + - pip install tox script: - - pip freeze - - nosetests -d \ No newline at end of file + - tox +env: + - TOXENV=py27 + - TOXENV=py34 diff --git a/setup.cfg b/setup.cfg index 4865668..088ce57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,3 @@ [nosetests] verbosity=2 nocapture=1 -with-progressive=1 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..14aa51e --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist=py27,py34 + +[testenv] +deps=-rrequirements.txt +commands=nosetests diff --git a/travis_clouseau.sh b/travis_clouseau.sh old mode 100644 new mode 100755 From 073201f2004e26da65f20a3eab37c39976bc6c49 Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Thu, 9 Jul 2015 09:34:30 -0400 Subject: [PATCH 02/12] Nose-progressive? never heard of it --- requirements.txt | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 51d2ef6..e3b4cdb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ jinja2 nose -nose-progressive diff --git a/setup.py b/setup.py index f96e153..2774acc 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ 'download_url': 'http://tbd.com', 'author_email': 'bill@if.io', 'version': '0.2.0', - 'install_requires': ['jinja2','nose','nose-progressive'], + 'install_requires': ['jinja2','nose'], 'packages': ['clouseau','tests'], 'package_data': {'clouseau': ['clients/*.py', 'patterns/*.txt', 'templates/*.html']}, 'py_modules': [], From b28cb4844eb7c225ed4b0149e2e6776a1208250b Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Mon, 13 Jul 2015 09:07:37 -0400 Subject: [PATCH 03/12] Make Python3 testing optional when running tox Tests can still be run against Python3 with: `tox -e py34` --- .travis.yml | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 300fc29..3f80983 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,3 @@ script: - tox env: - TOXENV=py27 - - TOXENV=py34 diff --git a/tox.ini b/tox.ini index 14aa51e..534f09b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py34 +envlist=py27 [testenv] deps=-rrequirements.txt From 699de2ebdcb9d18e89bcf524257c3ef2f64d8b38 Mon Sep 17 00:00:00 2001 From: Seth King Date: Sat, 1 Aug 2015 23:28:20 -0400 Subject: [PATCH 04/12] Python 3 conversion --- clouseau/clients/abstract.py | 6 +++--- clouseau/clients/console.py | 5 +++-- clouseau/clients/console_thin.py | 6 ++++-- clouseau/clouseau.py | 28 ++++++++++++++++------------ clouseau/clouseau_model.py | 1 + clouseau/commit_parser.py | 11 +++++++---- clouseau/parser.py | 16 +++++++++------- clouseau/terms_collector.py | 3 ++- tests/clouseau_tests.py | 2 +- tests/color_tests.py | 15 ++++++++------- tests/integration_tests.py | 3 ++- 11 files changed, 56 insertions(+), 40 deletions(-) diff --git a/clouseau/clients/abstract.py b/clouseau/clients/abstract.py index 6da5d96..82d4eb2 100644 --- a/clouseau/clients/abstract.py +++ b/clouseau/clients/abstract.py @@ -1,11 +1,11 @@ +from builtins import object from abc import ABCMeta, abstractmethod +from future.utils import with_metaclass -class AbstractClient: - __metaclass__ = ABCMeta - +class AbstractClient(with_metaclass(ABCMeta, object)): @abstractmethod def render(self,data): """ diff --git a/clouseau/clients/console.py b/clouseau/clients/console.py index ab59789..6da5610 100644 --- a/clouseau/clients/console.py +++ b/clouseau/clients/console.py @@ -1,6 +1,7 @@ -from abstract import AbstractClient +from __future__ import absolute_import +from .abstract import AbstractClient from jinja2 import Template, Environment, PackageLoader -from colors import * +from .colors import * import re import subprocess import sys diff --git a/clouseau/clients/console_thin.py b/clouseau/clients/console_thin.py index 0a5d857..1f4806a 100644 --- a/clouseau/clients/console_thin.py +++ b/clouseau/clients/console_thin.py @@ -1,4 +1,6 @@ -from abstract import AbstractClient +from __future__ import print_function +from __future__ import absolute_import +from .abstract import AbstractClient from jinja2 import Template, Environment, PackageLoader class ConsoleThinClient(AbstractClient): @@ -19,4 +21,4 @@ def render(self, terms, data): template = env.get_template('console_thin.html') data_to_render = template.render(data=data, has_matches=len(matches) > 0) - print data_to_render + print(data_to_render) diff --git a/clouseau/clouseau.py b/clouseau/clouseau.py index 13204f3..a276744 100755 --- a/clouseau/clouseau.py +++ b/clouseau/clouseau.py @@ -1,3 +1,6 @@ +from __future__ import print_function +from __future__ import absolute_import +from builtins import object #! /usr/bin/env python # -*- coding: utf-8 -*- # @@ -9,17 +12,17 @@ import pprint import sys import subprocess -from clients import * -from clients.colors import * -from parser import Parser -from commit_parser import CommitParser -from terms_collector import TermsCollector -from clouseau_model import ClouseauModel +from .clients import * +from .clients.colors import * +from .parser import Parser +from .commit_parser import CommitParser +from .terms_collector import TermsCollector +from .clouseau_model import ClouseauModel VERSION='0.2.0' -class Clouseau: +class Clouseau(object): """ Wrap and delegate """ @@ -41,7 +44,7 @@ def main(self , _args, client): if(not args['skip']): self.clone_repo( args['url'], args['repo_dir'] ) else: - print blue( 'Skipping git-clone or git-pull as --skip was found on the command line.' ) + print(blue( 'Skipping git-clone or git-pull as --skip was found on the command line.' )) if args['revlist'] != None and args['revlist'] != 'all': parser = CommitParser() @@ -61,14 +64,14 @@ def clone_repo(self, url, destination): _out = subprocess.check_output(['git', 'clone', url, destination]) except subprocess.CalledProcessError: - print blue( "Directory, %s, exits. Trying git-pull instead of clone." % destination ) + print(blue( "Directory, %s, exits. Trying git-pull instead of clone." % destination )) _out = subprocess.check_output(['git', '--git-dir=%s/.git' % destination, 'pull']) - print smoke( "Git says: %s" % _out ) + print(smoke( "Git says: %s" % _out )) return _out except : e = sys.exc_info()[0] - print red( 'Problem writing to destination: %s' % destination ) + print(red( 'Problem writing to destination: %s' % destination )) raise return _out @@ -83,7 +86,8 @@ def parse_args( self, arguments ): _pattern_path = os.path.join( _dir, _default_pattern_file ) _temp = os.path.join( _dir, "../temp") - p = arse.ArgumentParser (prog="clouseau", description=" Clouseau: A silly git inspector", version=VERSION) + p = arse.ArgumentParser (prog="clouseau", description=" Clouseau: A silly git inspector") + p.add_argument('--version', action='version', version=VERSION) p.add_argument('--url', '-u', required=True, action="store", dest="url", help="The fully qualified git URL (http://www.kernel.org/pub/software/scm/git/docs/git-clone.html)") p.add_argument('--term', '-t', required=False, action="store", dest="term", diff --git a/clouseau/clouseau_model.py b/clouseau/clouseau_model.py index 734b221..bb191e1 100644 --- a/clouseau/clouseau_model.py +++ b/clouseau/clouseau_model.py @@ -1,3 +1,4 @@ +from builtins import object #! /usr/bin/env python # -*- coding: utf-8 -*- # diff --git a/clouseau/commit_parser.py b/clouseau/commit_parser.py index e427bfc..9901d16 100644 --- a/clouseau/commit_parser.py +++ b/clouseau/commit_parser.py @@ -1,14 +1,17 @@ +from __future__ import print_function +from __future__ import absolute_import +from builtins import object import os import sys import re import subprocess import pprint -from clouseau_model import ClouseauModel +from .clouseau_model import ClouseauModel # ----------------------------------------------------------------------------------------------- -class CommitParser: +class CommitParser(object): """ Converts git-show's stdout to Python dictionary for any commit messages and file changes that match the terms in the patterns files """ @@ -25,7 +28,7 @@ def parse( self, terms, repo, revlist, clouseau_model, **kwargs ): for rev in revlist.strip().split(' '): output = self.get_commit(git_dir, rev) if output.strip() == '': - print "WARNING: No output was returned from git for commit [%s]. Ensure the commit exists" % rev + print("WARNING: No output was returned from git for commit [%s]. Ensure the commit exists" % rev) else: self.parse_commit( terms, output, clouseau_model ) @@ -38,7 +41,7 @@ def get_commit(self, git_dir, commit): git_show = subprocess.Popen(git_show_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) # , cwd=git_dir (out,err) = git_show.communicate() if err: - print "ERROR running git command [%s]: %s" % (git_show_cmd, err) + print("ERROR running git command [%s]: %s" % (git_show_cmd, err)) return out diff --git a/clouseau/parser.py b/clouseau/parser.py index 96d827f..ddc1397 100644 --- a/clouseau/parser.py +++ b/clouseau/parser.py @@ -1,3 +1,5 @@ +from builtins import str +from builtins import object import os import sys @@ -5,7 +7,7 @@ import subprocess # ----------------------------------------------------------------------------------------------- -class Parser: +class Parser(object): """ Converts git-grep's stdout to Python dictionary """ @@ -60,18 +62,18 @@ def search( self, git_dir, terms, revlist, clouseau ): stdout=subprocess.PIPE) (out,err) = git_grep.communicate() - + clouseau.update( {term: {}} ) - for line in out.split('\n'): + for line in out.split(b'\n'): # We don't know how a lot of the data is encoded, so make sure it's utf-8 before # processing if line == '': continue try: - line = unicode( line, 'utf-8' ) + line = str( line, 'utf-8' ) except UnicodeDecodeError: - line = unicode( line, 'latin-1' ) + line = str( line, 'latin-1' ) if file_name_heading.match( line ): @@ -79,13 +81,13 @@ def search( self, git_dir, terms, revlist, clouseau ): title = line.replace('/','_') title = title.replace('.','_').strip() _src = line.strip().encode('utf-8') - _srca = _src.split(':', 1) + _srca = _src.split(b':', 1) clouseau[term][title] = {'src' : _srca[1] } clouseau[term][title]['refspec'] = _srca[0] git_log_cmd = subprocess.Popen( ['git', '--git-dir', git_dir, 'log', _srca[0] , '-1'],\ stderr=subprocess.PIPE, stdout=subprocess.PIPE ) git_log = git_log_cmd.communicate()[0] - clouseau[term][title]['git_log'] = [ x.strip() for x in git_log.split('\n') if x != '' ] + clouseau[term][title]['git_log'] = [ x.strip() for x in git_log.split(b'\n') if x != '' ] #clouseau[term][title] = {'ref' : _srca[0] } clouseau[term][title]['matched_lines'] = [] continue diff --git a/clouseau/terms_collector.py b/clouseau/terms_collector.py index d3e4ada..4181b34 100644 --- a/clouseau/terms_collector.py +++ b/clouseau/terms_collector.py @@ -1,9 +1,10 @@ +from builtins import object import os import sys # ----------------------------------------------------------------------------------------------- -class TermsCollector: +class TermsCollector(object): """ Collects all search terms from the patterns files """ diff --git a/tests/clouseau_tests.py b/tests/clouseau_tests.py index 16edfe0..435699d 100644 --- a/tests/clouseau_tests.py +++ b/tests/clouseau_tests.py @@ -54,7 +54,7 @@ def commit_parser_merge_only_test(): terms = TermsCollector().collect_terms('clouseau/patterns/default.txt', None) model = ClouseauModel('https://github.com/cfpb/clouseau', terms) parser.parse_commit(terms, commit_output.read(), model) - exec_in_commit = model.model['exec'].keys()[0] + exec_in_commit = list(model.model['exec'].keys())[0] eq_(1, model.model['exec'][exec_in_commit]['matched_lines'][0][0]) eq_('Commit Message', model.model['exec'][exec_in_commit]['src']) diff --git a/tests/color_tests.py b/tests/color_tests.py index b31d876..c1024ba 100644 --- a/tests/color_tests.py +++ b/tests/color_tests.py @@ -1,27 +1,28 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function from clouseau.clients import colors -print dir( colors ) +print(dir( colors )) for c in colors.codes: - print colors.color( c, c ) + print(colors.color( c, c )) -print colors.ok() -print colors.fail() +print(colors.ok()) +print(colors.fail()) -print colors.ok( 'OK with text' ) -print colors.fail( 'Fail with text' ) +print(colors.ok( 'OK with text' )) +print(colors.fail( 'Fail with text' )) print ('-------------------') -print colors.gray('Gray Text') +print(colors.gray('Gray Text')) diff --git a/tests/integration_tests.py b/tests/integration_tests.py index e2cfa8b..89ddae4 100644 --- a/tests/integration_tests.py +++ b/tests/integration_tests.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os from nose.tools import * from clouseau.clouseau import Clouseau @@ -15,7 +16,7 @@ def console_client_test(): terms = ['password'] args = ['-u', 'https://github.com/virtix/cato.git'] parsed = clouseau.parse_args( args ) - print parsed + print(parsed) ids = parser.parse( terms=terms, repo=parsed['repo_dir'], revlist=parsed['revlist'], before=parsed['before'], after=parsed['after'], author=parsed['author'], github_url=parsed['github_url']) From cb97803adffbfbbc780d056aa1161ac922f6c829 Mon Sep 17 00:00:00 2001 From: Seth King Date: Sun, 2 Aug 2015 00:01:34 -0400 Subject: [PATCH 05/12] Python 3 Readme update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ed86078..0a272d8 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ See the **Get Involved** section at the end of this readme to see the current st - Unix-based OS, such as Mac or Linux (Windows support is unclear at this time.) - [git](http://git-scm.com/) - [Python 2.7](https://www.python.org/download/releases/2.7/) + or [Python 3.4](https://www.python.org/download/releases/3.4/) - [virtualenv](https://virtualenv.pypa.io/en/latest/) See the [requirements.txt](requirements.txt) file for additional dependencies to be installed in the quick setup. From cbbb4e7b0a4ff9e80c9a2d987933e99f3dc33bb6 Mon Sep 17 00:00:00 2001 From: Seth King Date: Sun, 2 Aug 2015 00:05:11 -0400 Subject: [PATCH 06/12] Python 3 Readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a272d8..ebd8a34 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ See the **Get Involved** section at the end of this readme to see the current st - Unix-based OS, such as Mac or Linux (Windows support is unclear at this time.) - [git](http://git-scm.com/) - [Python 2.7](https://www.python.org/download/releases/2.7/) - or [Python 3.4](https://www.python.org/download/releases/3.4/) + or [Python 3.4](https://www.python.org/downloads/release/python-343/) - [virtualenv](https://virtualenv.pypa.io/en/latest/) See the [requirements.txt](requirements.txt) file for additional dependencies to be installed in the quick setup. From b524804988c98525944f8ea26b2fb31c58e7ce45 Mon Sep 17 00:00:00 2001 From: Seth King Date: Mon, 3 Aug 2015 18:59:36 -0400 Subject: [PATCH 07/12] Python 3 requirements update --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 51d2ef6..d923634 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ jinja2 nose nose-progressive +future From 2c3b18194d31d5ed8d90e57bb717c58013afcf09 Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Fri, 19 Oct 2018 17:39:22 -0400 Subject: [PATCH 08/12] cleanup subprocess handling, add py3 to tox --- clouseau/clients/console.py | 20 ++++++++++---------- tox.ini | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clouseau/clients/console.py b/clouseau/clients/console.py index 6da5610..22d2f1e 100644 --- a/clouseau/clients/console.py +++ b/clouseau/clients/console.py @@ -2,10 +2,12 @@ from .abstract import AbstractClient from jinja2 import Template, Environment, PackageLoader from .colors import * +import codecs import re import subprocess import sys import os +import tempfile class ConsoleClient(AbstractClient): """ @@ -63,20 +65,18 @@ def render(self, terms, data): #m[1] = m[1].replace(term, orange_bg(term) ) - - - data_to_render = template.render(data=data) + scratchfile_handle, scratchfile_path = tempfile.mkstemp() + rendered = template.render(data=data) + + with codecs.open(scratchfile_path, 'w', 'utf-8') as scratchfile: + scratchfile.write(rendered) try: - pager = subprocess.Popen(['less', '-F', '-R', '-S', '-X', '-K'], stdin=subprocess.PIPE, stdout=sys.stdout) - lines = data_to_render.split('\n') - for line in lines: - pager.stdin.write( line.encode('utf-8') + '\n' ) - pager.stdin.close() - pager.wait() + subprocess.call(['less', '-F', '-R', '-S', '-X', '-K'], stdin=scratchfile_handle) except KeyboardInterrupt: pass - + finally: + os.remove(scratchfile_path) diff --git a/tox.ini b/tox.ini index 534f09b..325eec6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27 +envlist=py27,py36,py37 [testenv] deps=-rrequirements.txt From 32db508eebeacc22fb87f20e428a701480cbf5e3 Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Fri, 19 Oct 2018 17:49:35 -0400 Subject: [PATCH 09/12] try testing python 2 and 3 from Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3f80983..24628f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,4 +4,4 @@ install: script: - tox env: - - TOXENV=py27 + - TOXENV=py27,py36,py37 From f0ccc14bbaa0c0b009c6cb6ebf5a42c24a3b6e3e Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Fri, 19 Oct 2018 17:53:09 -0400 Subject: [PATCH 10/12] install python 2.7,3.6,and 3.7 in Travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 24628f8..bf17174 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,8 @@ language: python +python: + - 2.7 + - 3.6 + - 3.7 install: - pip install tox script: From 9f4f49fa56c44393664a7099ddcb046fe0171ae6 Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Fri, 19 Oct 2018 17:57:04 -0400 Subject: [PATCH 11/12] Stop trying to combine tox and travis --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf17174..039736c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,6 @@ python: - 3.6 - 3.7 install: - - pip install tox + - pip install -r requirements.txt script: - - tox -env: - - TOXENV=py27,py36,py37 + - nosetests From e9a79140c68a0552b8dbbdbb29df757a342d30e1 Mon Sep 17 00:00:00 2001 From: Ross Karchner Date: Fri, 19 Oct 2018 17:58:54 -0400 Subject: [PATCH 12/12] remove 3.7 from Travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 039736c..f98ff17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - 2.7 - 3.6 - - 3.7 install: - pip install -r requirements.txt script: