From 04691b9ce126c9fbed6e6b1f04b94024a201cd34 Mon Sep 17 00:00:00 2001 From: Zephyr Pellerin Date: Mon, 3 Aug 2020 11:10:40 -0700 Subject: [PATCH 1/4] Add `dos2unix` conversion option to `create_scanner_exec` --- src/microengine_utils/scanner.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/microengine_utils/scanner.py b/src/microengine_utils/scanner.py index b906bbf..8c12ffb 100644 --- a/src/microengine_utils/scanner.py +++ b/src/microengine_utils/scanner.py @@ -26,25 +26,31 @@ async def create_scanner_exec( - *cmd: 'str', + *cmd: str, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - stdin=asyncio.subprocess.DEVNULL, - check: 'bool' = False, + stdin=None, + check: bool = False, + dos2unix: bool = True ): - """Run an engine filescan `cmd`, timing out after `timeout` seconds""" + """Run an engine filescan `cmd`, timing out after `timeout` seconds + + You can use `None` in place of `asyncio.subprocess.DEVNULL` for any stream. + + ``check`` controls if an error should be thrown if the return code is non-zero + ``dos2unix`` controls if the output shoudl have CRLF converted to LF + """ + as_stream = lambda s: asyncio.subprocess.DEVNULL if s is None else s try: - proc = await asyncio.subprocess.create_subprocess_exec( - *cmd, - stdout=stdout, - stderr=stderr, - stdin=stdin, - ) + proc = await asyncio.subprocess.create_subprocess_exec(*cmd, stdout=as_stream(stdout), stderr=as_stream(stderr), stdin=as_stream(stdin)) streams = await proc.communicate() if check and proc.returncode != 0: raise CalledProcessScanError(cmd, f'Non-zero return code: {proc.returncode}') - sout, serr = (s.decode(errors='ignore') for s in streams) - return proc.returncode, sout, serr + streams = map(lambda s: s.decode(errors='ignore'), streams) + if dos2unix: + # replace all CRLF with LF, as expected + streams = map(lambda s: s.replace('\r\n'), streams) + return proc.returncode, *streams except (FileNotFoundError, BrokenPipeError, ConnectionResetError) as e: # noqa proc.kill() raise CalledProcessScanError(cmd, str(type(e))) From 91d62563e38013671e55f42e94fb2f1440b996f6 Mon Sep 17 00:00:00 2001 From: Zephyr Pellerin Date: Mon, 3 Aug 2020 11:21:26 -0700 Subject: [PATCH 2/4] Fix typing of `create_scanner_exec` --- src/microengine_utils/scanner.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/microengine_utils/scanner.py b/src/microengine_utils/scanner.py index 8c12ffb..f5798c4 100644 --- a/src/microengine_utils/scanner.py +++ b/src/microengine_utils/scanner.py @@ -5,13 +5,13 @@ import os import re from time import perf_counter -from typing import Callable, List, Mapping, Optional, Sequence, cast +from typing import Callable, List, Mapping, Optional, Sequence, Tuple, cast import datadog from polyswarmartifact import ArtifactType from polyswarmartifact.schema.verdict import Verdict -from polyswarmclient.abstractscanner import AbstractScanner, ScanResult +from polyswarmclient.abstractscanner import ScanResult from .config import EngineInfo from .constants import ( @@ -32,7 +32,7 @@ async def create_scanner_exec( stdin=None, check: bool = False, dos2unix: bool = True -): +) -> 'Tuple[int, str, str]': """Run an engine filescan `cmd`, timing out after `timeout` seconds You can use `None` in place of `asyncio.subprocess.DEVNULL` for any stream. @@ -42,15 +42,17 @@ async def create_scanner_exec( """ as_stream = lambda s: asyncio.subprocess.DEVNULL if s is None else s try: - proc = await asyncio.subprocess.create_subprocess_exec(*cmd, stdout=as_stream(stdout), stderr=as_stream(stderr), stdin=as_stream(stdin)) + proc = await asyncio.subprocess.create_subprocess_exec( + *cmd, stdout=as_stream(stdout), stderr=as_stream(stderr), stdin=as_stream(stdin) + ) streams = await proc.communicate() if check and proc.returncode != 0: raise CalledProcessScanError(cmd, f'Non-zero return code: {proc.returncode}') - streams = map(lambda s: s.decode(errors='ignore'), streams) if dos2unix: # replace all CRLF with LF, as expected - streams = map(lambda s: s.replace('\r\n'), streams) - return proc.returncode, *streams + streams = cast('Tuple[bytes, bytes]', tuple(map(lambda s: s.replace(b'\r\n', b'\n'), streams))) + sout, serr = map(lambda s: s.decode(errors='ignore'), streams) + return (proc.returncode, sout, serr) except (FileNotFoundError, BrokenPipeError, ConnectionResetError) as e: # noqa proc.kill() raise CalledProcessScanError(cmd, str(type(e))) From 6ac85cd361f3e49cb8e0f5c2448fe4ca7ed30cff Mon Sep 17 00:00:00 2001 From: Zephyr Pellerin Date: Mon, 3 Aug 2020 11:27:33 -0700 Subject: [PATCH 3/4] Load `AbstractScanner` only if typing.TYPE_CHECKING --- src/microengine_utils/scanner.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/microengine_utils/scanner.py b/src/microengine_utils/scanner.py index f5798c4..1fc30d6 100644 --- a/src/microengine_utils/scanner.py +++ b/src/microengine_utils/scanner.py @@ -5,7 +5,7 @@ import os import re from time import perf_counter -from typing import Callable, List, Mapping, Optional, Sequence, Tuple, cast +from typing import Callable, List, Mapping, Optional, Sequence, Tuple, cast, TYPE_CHECKING import datadog @@ -13,6 +13,9 @@ from polyswarmartifact.schema.verdict import Verdict from polyswarmclient.abstractscanner import ScanResult +if TYPE_CHECKING: + from polyswarmclient.abstractscanner import AbstractScanner + from .config import EngineInfo from .constants import ( SCAN_FAIL, @@ -48,11 +51,11 @@ async def create_scanner_exec( streams = await proc.communicate() if check and proc.returncode != 0: raise CalledProcessScanError(cmd, f'Non-zero return code: {proc.returncode}') - if dos2unix: - # replace all CRLF with LF, as expected - streams = cast('Tuple[bytes, bytes]', tuple(map(lambda s: s.replace(b'\r\n', b'\n'), streams))) sout, serr = map(lambda s: s.decode(errors='ignore'), streams) - return (proc.returncode, sout, serr) + if dos2unix: + # replace all CRLF with LF + sout, serr = map(lambda s: s.replace('\r\n', '\n'), (sout, serr)) + return proc.returncode, sout, serr except (FileNotFoundError, BrokenPipeError, ConnectionResetError) as e: # noqa proc.kill() raise CalledProcessScanError(cmd, str(type(e))) From ba59e34729423f1ba20bfa506f9bba78580cb9f7 Mon Sep 17 00:00:00 2001 From: Zephyr Pellerin Date: Mon, 3 Aug 2020 12:06:43 -0700 Subject: [PATCH 4/4] =?UTF-8?q?Bump=20version:=201.3.0=20=E2=86=92=201.3.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a43ce9a..0f2cb5e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.0 +current_version = 1.3.1 commit = True tag = False sign_tags = True diff --git a/setup.py b/setup.py index 1abdd99..6dfa2cc 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='microengine-utils', - version='1.3.0', + version='1.3.1', description='Library for Polyswarm Microengine Utility Package', long_description=long_description, long_description_content_type='text/markdown',