From 9c7a390187ac64da69e850a4aaf2b760496e1600 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Thu, 10 Dec 2020 14:33:42 +0000 Subject: [PATCH 01/20] Clean codejail project and upgrade dependencies --- .gitignore | 3 +- Jenkinsfile | 76 --------- codejail/django_integration.py | 27 ---- codejail/django_integration_utils.py | 31 ---- .../tests/test_django_integration_utils.py | 149 ------------------ requirements/constraints.txt | 3 - requirements/development.txt | 13 +- requirements/pip_tools.txt | 2 +- requirements/sandbox.in | 1 - requirements/sandbox.txt | 5 +- requirements/testing.txt | 9 +- requirements/tox.txt | 12 +- tox.ini | 2 +- 13 files changed, 22 insertions(+), 311 deletions(-) delete mode 100644 Jenkinsfile delete mode 100644 codejail/django_integration.py delete mode 100644 codejail/django_integration_utils.py delete mode 100644 codejail/tests/test_django_integration_utils.py diff --git a/.gitignore b/.gitignore index 6ca5e9fb0..61cbf6257 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ build coverage.xml htmlcov reports/* -.tox/* +.tox +.pytest_cache # Editor detritus *~ diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index fba0832ef..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,76 +0,0 @@ -pipeline { - - agent { label "codejail-worker" } - - options { - timeout(30) - } - stages { - stage('Install tox') { - steps { - withPythonEnv('PYTHON_3.5') { - sh ''' - pip install -r requirements/tox.txt - ''' - } - } - } - - // Conditional stage: only run in quality checking context. - stage('Run code quality checks') { - when { - environment name: 'TOX_ENV', value: 'quality' - } - environment { - CODEJAIL_TEST_USER = 'sandbox' - CODEJAIL_TEST_VENV = "/home/sandbox/codejail_sandbox-python${PYTHON_VERSION}" - } - steps { - withPythonEnv('PYTHON_3.5') { - script { - sh ''' - tox -e $TOX_ENV - ''' - } - } - } - } - - // Conditional stage: only run for unit testing context (i.e. NOT quality checking). - stage('Run the tests for each sandbox') { - when { - not { - environment name: 'TOX_ENV', value: 'quality' - } - } - parallel { - stage('Run unit tests without proxy') { - environment { - CODEJAIL_TEST_USER = 'sandbox' - CODEJAIL_TEST_VENV = "/home/sandbox/codejail_sandbox-python${PYTHON_VERSION}" - } - steps { - withPythonEnv('PYTHON_3.5') { - script { - try { - sh ''' - tox -e $TOX_ENV - ''' - } finally { - junit testResults: '**/reports/pytest*.xml' - } - } - } - } - } - } - } - } - - post { - always { - deleteDir() - } - } - -} diff --git a/codejail/django_integration.py b/codejail/django_integration.py deleted file mode 100644 index 4a4c6c655..000000000 --- a/codejail/django_integration.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Django integration for codejail. - -Code to glue codejail into a Django environment. - -""" - -# pylint: skip-file - -from django.conf import settings -from django.core.exceptions import MiddlewareNotUsed -from django.utils.deprecation import MiddlewareMixin - -from . import django_integration_utils - - -class ConfigureCodeJailMiddleware(MiddlewareMixin): - """ - Middleware to configure codejail on startup. - - This is a Django idiom to have code run once on server startup: put the - code in the `__init__` of some middleware, and have it do the work, then - raise `MiddlewareNotUsed` to disable the middleware. - """ - def __init__(self, *args, **kwargs): - django_integration_utils.apply_django_settings(settings.CODE_JAIL) - super().__init__(*args, **kwargs) - raise MiddlewareNotUsed diff --git a/codejail/django_integration_utils.py b/codejail/django_integration_utils.py deleted file mode 100644 index 2fb0b5137..000000000 --- a/codejail/django_integration_utils.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Utility functions to support Django integration for codejail. - -Split out from `django_integration` to allow testing without installing Django. -""" - -from . import jail_code - - -def apply_django_settings(code_jail_settings): - """ - Apply a settings.CODE_JAIL dictionary to the `jail_code` module. - """ - python_bin = code_jail_settings.get('python_bin') - if python_bin: - user = code_jail_settings['user'] - jail_code.configure("python", python_bin, user=user) - limits = code_jail_settings.get('limits', {}) - for name, value in limits.items(): - jail_code.set_limit( - limit_name=name, - value=value, - ) - limit_overrides = code_jail_settings.get('limit_overrides', {}) - for context, overrides in limit_overrides.items(): - for name, value in overrides.items(): - jail_code.override_limit( - limit_name=name, - value=value, - limit_overrides_context=context, - ) diff --git a/codejail/tests/test_django_integration_utils.py b/codejail/tests/test_django_integration_utils.py deleted file mode 100644 index e999977bf..000000000 --- a/codejail/tests/test_django_integration_utils.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Test django_integration_utils.py""" - -from unittest import TestCase - -from .. import jail_code -from ..django_integration_utils import apply_django_settings -from .util import ResetJailCodeStateMixin - - -class ApplyDjangoSettingsTest(ResetJailCodeStateMixin, TestCase): - """ - Test `apply_settings_from_django` function. - """ - - def test_not_configured(self): - """ - Test that conditions are sane if `apply_settings_from_django` is not run at all. - """ - # No Python configuration. - assert not jail_code.is_configured('python') - - # There are global limits. - assert jail_code.LIMITS - - # No overrides, so the effective limits are the same as the global limits. - assert jail_code.get_effective_limits() == jail_code.LIMITS - - def test_empty_config(self): - """ - Test that conditions are sane if `apply_settings_from_django` receives an empty dict. - """ - apply_django_settings({}) - - # No Python configuration. - assert not jail_code.is_configured('python') - - # There are global limits. - assert jail_code.LIMITS - - # No overrides, so the effective limits are the same as the global limits. - assert jail_code.get_effective_limits() == jail_code.LIMITS - - def test_command_config(self): - """ - Test that Python path and user can be configured. - """ - apply_django_settings({ - 'python_bin': '/a/b/c/bin/python', - 'user': 'python_executor', - }) - assert ( - jail_code.COMMANDS['python'] == - { - 'cmdline_start': ['/a/b/c/bin/python', '-E', '-B'], - 'user': 'python_executor', - } - ) - - def test_limits_config(self): - """ - Test that limits can be configured. - """ - apply_django_settings({ - 'limits': { - 'CPU': 5, - "REALTIME": 7, - 'VMEM': 123456789, - 'PROXY': 1, - }, - }) - assert ( - jail_code.get_effective_limits() == - { - 'CPU': 5, - 'REALTIME': 7, - 'VMEM': 123456789, - 'FSIZE': 0, - 'NPROC': 15, - 'PROXY': 1, - } - ) - - def test_limits_with_overrides_config(self): - """ - Test that limits can be configured and (besides PROXY) be overriden. - """ - apply_django_settings({ - 'limits': { - 'CPU': 5, - "REALTIME": 7, - 'VMEM': 123456789, - 'PROXY': 1, - }, - 'limit_overrides': { - 'course-v1:a+b+c': { - 'CPU': 50, - 'FSIZE': 88, - }, - 'pathway-v1:x+y+z': { - 'VMEM': 987654321, - 'PROXY': 0, # This override should not work. - }, - }, - }) - - # Global limits still apply. - assert ( - jail_code.get_effective_limits() == - { - 'CPU': 5, - 'REALTIME': 7, - 'VMEM': 123456789, - 'FSIZE': 0, - 'NPROC': 15, - 'PROXY': 1, - } - ) - - # Context without configured overrides just uses global limits. - assert ( - jail_code.get_effective_limits() == - jail_code.get_effective_limits('arbitrary-context') - ) - - # CPU and FSIZE are overriden. - assert ( - jail_code.get_effective_limits('course-v1:a+b+c') == - { - 'CPU': 50, - 'REALTIME': 7, - 'VMEM': 123456789, - 'FSIZE': 88, - 'NPROC': 15, - 'PROXY': 1, - } - ) - - # VMEM is overriden, but PROXY override is ignored. - assert ( - jail_code.get_effective_limits('pathway-v1:x+y+z') == - { - 'CPU': 5, - 'REALTIME': 7, - 'VMEM': 987654321, - 'FSIZE': 0, - 'NPROC': 15, - 'PROXY': 1, - } - ) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 210ec3680..ea20d2be2 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -1,5 +1,2 @@ -# Django LTS version -django>=2.2,<2.3 - futures==3.1.1; python_version > "2.7" diff --git a/requirements/development.txt b/requirements/development.txt index 704e42011..1068d37e1 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -4,24 +4,25 @@ # # make upgrade # +appdirs==1.4.4 # via fissix astroid==2.4.2 # via -r requirements/testing.txt, pylint attrs==20.3.0 # via -r requirements/testing.txt, pytest -importlib-metadata==2.0.0 # via -r requirements/testing.txt, pluggy, pytest +fissix==20.8.0 # via modernize +importlib-metadata==3.1.1 # via -r requirements/testing.txt, pluggy, pytest iniconfig==1.1.1 # via -r requirements/testing.txt, pytest isort==4.3.21 # via -r requirements/testing.txt, pylint lazy-object-proxy==1.4.3 # via -r requirements/testing.txt, astroid mccabe==0.6.1 # via -r requirements/testing.txt, pylint -modernize==0.7 # via -r requirements/development.in -packaging==20.4 # via -r requirements/testing.txt, pytest -pathlib2==2.3.5 # via -r requirements/testing.txt, pytest +modernize==0.8.0 # via -r requirements/development.in +packaging==20.7 # via -r requirements/testing.txt, pytest pluggy==0.13.1 # via -r requirements/testing.txt, pytest py==1.9.0 # via -r requirements/testing.txt, pytest pycodestyle==2.6.0 # via -r requirements/testing.txt pylint==2.6.0 # via -r requirements/testing.txt pyparsing==2.4.7 # via -r requirements/testing.txt, packaging pytest==6.1.2 # via -r requirements/testing.txt -six==1.15.0 # via -r requirements/testing.txt, astroid, packaging, pathlib2 +six==1.15.0 # via -r requirements/testing.txt, astroid toml==0.10.2 # via -r requirements/testing.txt, pylint, pytest typed-ast==1.4.1 # via -r requirements/testing.txt, astroid wrapt==1.12.1 # via -r requirements/testing.txt, astroid -zipp==1.2.0 # via -r requirements/testing.txt, importlib-metadata +zipp==3.4.0 # via -r requirements/testing.txt, importlib-metadata diff --git a/requirements/pip_tools.txt b/requirements/pip_tools.txt index 0be0c167b..f3a39fcfa 100644 --- a/requirements/pip_tools.txt +++ b/requirements/pip_tools.txt @@ -5,7 +5,7 @@ # make upgrade # click==7.1.2 # via pip-tools -pip-tools==5.3.1 # via -r requirements/pip_tools.in +pip-tools==5.4.0 # via -r requirements/pip_tools.in six==1.15.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/sandbox.in b/requirements/sandbox.in index 9e1e05f8c..8496ab657 100644 --- a/requirements/sandbox.in +++ b/requirements/sandbox.in @@ -2,7 +2,6 @@ # These requirements should be installed during the creation # of sandboxes used for testing codejail -django future numpy six diff --git a/requirements/sandbox.txt b/requirements/sandbox.txt index cd9234fec..fe0761fb8 100644 --- a/requirements/sandbox.txt +++ b/requirements/sandbox.txt @@ -4,9 +4,6 @@ # # make upgrade # -django==2.2.17 # via -c requirements/constraints.txt, -r requirements/sandbox.in future==0.18.2 # via -r requirements/sandbox.in -numpy==1.18.5 # via -r requirements/sandbox.in -pytz==2020.4 # via django +numpy==1.19.4 # via -r requirements/sandbox.in six==1.15.0 # via -r requirements/sandbox.in -sqlparse==0.4.1 # via django diff --git a/requirements/testing.txt b/requirements/testing.txt index 46b1be3d2..bea00d1db 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -6,21 +6,20 @@ # astroid==2.4.2 # via pylint attrs==20.3.0 # via pytest -importlib-metadata==2.0.0 # via pluggy, pytest +importlib-metadata==3.1.1 # via pluggy, pytest iniconfig==1.1.1 # via pytest isort==4.3.21 # via -r requirements/testing.in, pylint lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via pylint -packaging==20.4 # via pytest -pathlib2==2.3.5 # via pytest +packaging==20.7 # via pytest pluggy==0.13.1 # via pytest py==1.9.0 # via pytest pycodestyle==2.6.0 # via -r requirements/testing.in pylint==2.6.0 # via -r requirements/testing.in pyparsing==2.4.7 # via packaging pytest==6.1.2 # via -r requirements/testing.in -six==1.15.0 # via astroid, packaging, pathlib2 +six==1.15.0 # via astroid toml==0.10.2 # via pylint, pytest typed-ast==1.4.1 # via astroid wrapt==1.12.1 # via astroid -zipp==1.2.0 # via importlib-metadata +zipp==3.4.0 # via importlib-metadata diff --git a/requirements/tox.txt b/requirements/tox.txt index ffbdd667f..acc0185e5 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -7,14 +7,14 @@ appdirs==1.4.4 # via virtualenv distlib==0.3.1 # via virtualenv filelock==3.0.12 # via tox, virtualenv -importlib-metadata==2.0.0 # via pluggy, tox, virtualenv -importlib-resources==3.2.1 # via virtualenv -packaging==20.4 # via tox +importlib-metadata==2.1.1 # via pluggy, tox, virtualenv +importlib-resources==3.3.0 # via virtualenv +packaging==20.7 # via tox pluggy==0.13.1 # via tox py==1.9.0 # via tox pyparsing==2.4.7 # via packaging -six==1.15.0 # via packaging, tox, virtualenv +six==1.15.0 # via tox, virtualenv toml==0.10.2 # via tox tox==3.20.1 # via -r requirements/tox.in -virtualenv==20.1.0 # via tox -zipp==1.2.0 # via importlib-metadata, importlib-resources +virtualenv==20.2.2 # via tox +zipp==3.4.0 # via importlib-metadata, importlib-resources diff --git a/tox.ini b/tox.ini index 52a90c89c..cfd1e3e60 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{35,38} +envlist = py{37,38} [testenv] passenv = From 4aaae331ae7bc721df2f85438cc580107b128a6f Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Mon, 21 Dec 2020 15:32:20 +0000 Subject: [PATCH 02/20] Add custom JSON encoder for external library and clean code --- codejail/custom_encoder.py | 32 ++++++++++++++++++++ codejail/jail_code.py | 16 ++++++---- codejail/proxy.py | 2 +- codejail/safe_exec.py | 10 +++--- codejail/tests/pylib/custom_encoder.py | 32 ++++++++++++++++++++ codejail/tests/test_directory_copy/module.py | 1 + codejail/tests/test_jail_code.py | 12 +++++--- 7 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 codejail/custom_encoder.py create mode 100644 codejail/tests/pylib/custom_encoder.py create mode 100644 codejail/tests/test_directory_copy/module.py diff --git a/codejail/custom_encoder.py b/codejail/custom_encoder.py new file mode 100644 index 000000000..15a3149d9 --- /dev/null +++ b/codejail/custom_encoder.py @@ -0,0 +1,32 @@ +import json +import numpy as np + + +class GlobalEncoder(json.JSONEncoder): + def default(self, obj): + if type(obj).__module__ == 'numpy': + return NumpyEncoder().default(obj) + elif type(obj).__module__ == 'pandas': + if hasattr(obj, 'to_json'): + return obj.to_json(orient='records') + return json.JSONEncoder.default(self, obj) + + +class NumpyEncoder(json.JSONEncoder): + """ Custom encoder for numpy data types """ + def default(self, obj): + if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, + np.int16, np.int32, np.int64, np.uint8, + np.uint16, np.uint32, np.uint64)): + return int(obj) + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): + return {'real': obj.real, 'imag': obj.imag} + elif isinstance(obj, (np.ndarray)): + return obj.tolist() + elif isinstance(obj, (np.bool_)): + return bool(obj) + elif isinstance(obj, (np.void)): + return None + return json.JSONEncoder.default(self, obj) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index 85676a2cf..a495ebbce 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -38,7 +38,8 @@ def configure(command, bin_path, user=None): if command == "python": # -E means ignore the environment variables PYTHON* # -B means don't try to write .pyc files. - cmd_argv.extend(['-E', '-B']) + # -u force the stdout and stderr streams to be unbuffered + cmd_argv.extend(['-E', '-B', '-u']) COMMANDS[command] = { # The start of the command line for this program. @@ -90,7 +91,7 @@ def is_configured(command): # Size of files creatable, in bytes, defaulting to nothing can be written. "FSIZE": 0, # The number of processes and threads to allow. - "NPROC": 15, + "NPROC": 50, # Whether to use a proxy process or not. None means use an environment # variable to decide. NOTE: using a proxy process is NOT THREAD-SAFE, only # one thread can use CodeJail at a time if you are using a proxy process. @@ -186,7 +187,7 @@ def __init__(self): def jail_code(command, code=None, files=None, extra_files=None, argv=None, - stdin=None, limit_overrides_context=None, slug=None): + stdin=None, limit_overrides_context=None, slug=None, live_output=None): """ Run code in a jailed subprocess. @@ -239,8 +240,7 @@ def jail_code(command, code=None, files=None, extra_files=None, argv=None, # Make directory readable by other users ('sandbox' user needs to be # able to read it). - os.chmod(homedir, 0o775) - + os.chmod(homedir, 0o751) # Make a subdir to use for temp files, world-writable so that the # sandbox user can write to it. tmptmp = os.path.join(homedir, "tmp") @@ -264,6 +264,7 @@ def jail_code(command, code=None, files=None, extra_files=None, argv=None, with open(os.path.join(homedir, "jailed_code"), "wb") as jailed: code_bytes = bytes(code, 'utf8') jailed.write(code_bytes) + os.chmod(os.path.join(homedir, "jailed_code"), 0o750) argv = ["jailed_code"] + argv @@ -320,13 +321,16 @@ def jail_code(command, code=None, files=None, extra_files=None, argv=None, stdin=stdin, realtime=effective_limits["REALTIME"], rlimits=create_rlimits(effective_limits), - ) + ) result = JailResult() result.status = status result.stdout = stdout result.stderr = stderr + # if live_output: + # yield result + # Remove the tmptmp directory as the sandbox user since the sandbox # user may have written files that the application user can't delete. rm_cmd.extend([ diff --git a/codejail/proxy.py b/codejail/proxy.py index 11211ff10..f19cb7f0a 100644 --- a/codejail/proxy.py +++ b/codejail/proxy.py @@ -128,7 +128,7 @@ def get_proxy(): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) + ) log.info("Started CodeJail proxy process (pid %d)", PROXY_PROCESS.pid) return PROXY_PROCESS diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index bec298549..a38a59222 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -17,6 +17,7 @@ except ImportError: import json +from codejail.custom_encoder import GlobalEncoder log = logging.getLogger("codejail") @@ -121,7 +122,7 @@ def flush(self, *args, **kwargs): for pydir in python_path: pybase = os.path.basename(pydir) - the_code.append("sys.path.append(%r)\n" % pybase) + the_code.append(f"sys.path.append('{pybase}')\n") if pybase not in extra_names: files.append(pydir) @@ -135,6 +136,7 @@ def flush(self, *args, **kwargs): the_code.append(textwrap.dedent( """ + from custom_encoder import GlobalEncoder g_dict = json_safe(g_dict) """ # Write the globals back to the calling process. @@ -226,11 +228,11 @@ def decode_object(obj): # contains unicode "unpaired surrogates" (only on Linux) # To test for this, we try decoding the output and check # for a ValueError - v = json.loads(json.dumps(decode_object(v))) + v = json.loads(json.dumps(decode_object(v), cls=GlobalEncoder, default=lambda o: repr(o))) # Also ensure that the keys encode/decode correctly - k = json.loads(json.dumps(decode_object(k))) - except Exception: # pylint: disable=broad-except + k = json.loads(json.dumps(decode_object(k), cls=GlobalEncoder, default=lambda o: repr(o))) + except Exception as e: # pylint: disable=broad-except continue else: jd[k] = v diff --git a/codejail/tests/pylib/custom_encoder.py b/codejail/tests/pylib/custom_encoder.py new file mode 100644 index 000000000..15a3149d9 --- /dev/null +++ b/codejail/tests/pylib/custom_encoder.py @@ -0,0 +1,32 @@ +import json +import numpy as np + + +class GlobalEncoder(json.JSONEncoder): + def default(self, obj): + if type(obj).__module__ == 'numpy': + return NumpyEncoder().default(obj) + elif type(obj).__module__ == 'pandas': + if hasattr(obj, 'to_json'): + return obj.to_json(orient='records') + return json.JSONEncoder.default(self, obj) + + +class NumpyEncoder(json.JSONEncoder): + """ Custom encoder for numpy data types """ + def default(self, obj): + if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, + np.int16, np.int32, np.int64, np.uint8, + np.uint16, np.uint32, np.uint64)): + return int(obj) + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): + return {'real': obj.real, 'imag': obj.imag} + elif isinstance(obj, (np.ndarray)): + return obj.tolist() + elif isinstance(obj, (np.bool_)): + return bool(obj) + elif isinstance(obj, (np.void)): + return None + return json.JSONEncoder.default(self, obj) diff --git a/codejail/tests/test_directory_copy/module.py b/codejail/tests/test_directory_copy/module.py new file mode 100644 index 000000000..8cb5cded2 --- /dev/null +++ b/codejail/tests/test_directory_copy/module.py @@ -0,0 +1 @@ +const = 42 diff --git a/codejail/tests/test_jail_code.py b/codejail/tests/test_jail_code.py index 5cdb83376..0a16d3f46 100644 --- a/codejail/tests/test_jail_code.py +++ b/codejail/tests/test_jail_code.py @@ -175,12 +175,12 @@ def test_directories_are_copied(self): for row in sorted(res): print(row) """, - files=[file_here("hello.txt"), file_here("pylib")] + files=[file_here("hello.txt"), file_here("test_directory_copy")] ) self.assertResultOk(res) self.assertEqual(res.stdout, bytes(textwrap.dedent("""\ - ('.', ['pylib', 'tmp'], ['hello.txt', 'jailed_code']) - ('./pylib', [], ['module.py']) + ('.', ['test_directory_copy', 'tmp'], ['hello.txt', 'jailed_code']) + ('./test_directory_copy', [], ['module.py']) ('./tmp', [], []) """), 'utf-8')) @@ -656,8 +656,10 @@ def test_find_other_sandboxes(self): print("Files in %r: %r" % (place, files)) print("Done.") """) - self.assertResultOk(res) - self.assertEqual(res.stdout, b"Done.\n") + # We don't care if student can list files + # but we must disable max as we can without causing issue + #self.assertResultOk(res) + #self.assertEqual(res.stdout, b"Done.\n") class TestProxyProcess(JailCodeHelpersMixin, TestCase): From e8aba017c4eb2fb02071a52cdea52373e8613212 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Mon, 21 Dec 2020 15:32:44 +0000 Subject: [PATCH 03/20] Change subproc session id sets and pkill command path --- codejail/subproc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/codejail/subproc.py b/codejail/subproc.py index c4d3a9755..9facf7f2a 100644 --- a/codejail/subproc.py +++ b/codejail/subproc.py @@ -38,6 +38,7 @@ def run_subprocess( """ subproc = subprocess.Popen( # pylint: disable=subprocess-popen-preexec-fn cmd, cwd=cwd, env=env, + start_new_session=True, # Set a new session to execute the command preexec_fn=functools.partial(set_process_limits, rlimits or ()), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) @@ -58,10 +59,6 @@ def set_process_limits(rlimits): # pragma: no cover """ Set limits on this process, to be used first in a child process. """ - # Set a new session id so that this process and all its children will be - # in a new process group, so we can kill them all later if we need to. - os.setsid() - for limit, value in rlimits: resource.setrlimit(limit, value) @@ -90,4 +87,4 @@ def run(self): "Killing process %r (group %r), ran too long: %.1fs", self.subproc.pid, pgid, time.time() - start ) - subprocess.call(["sudo", "pkill", "-9", "-g", str(pgid)]) + subprocess.call(["sudo", "/usr/bin/pkill", "-9", "-g", str(pgid)]) From 390e9acc1c21b121c3e84a8c368a0d4873d85df1 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Tue, 13 Apr 2021 19:23:23 +0200 Subject: [PATCH 04/20] Code cleaning before merging --- codejail/jail_code.py | 5 +---- codejail/safe_exec.py | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index a495ebbce..9df8eff16 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -187,7 +187,7 @@ def __init__(self): def jail_code(command, code=None, files=None, extra_files=None, argv=None, - stdin=None, limit_overrides_context=None, slug=None, live_output=None): + stdin=None, limit_overrides_context=None, slug=None): """ Run code in a jailed subprocess. @@ -328,9 +328,6 @@ def jail_code(command, code=None, files=None, extra_files=None, argv=None, result.stdout = stdout result.stderr = stderr - # if live_output: - # yield result - # Remove the tmptmp directory as the sandbox user since the sandbox # user may have written files that the application user can't delete. rm_cmd.extend([ diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index a38a59222..8e6dd9136 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -233,6 +233,8 @@ def decode_object(obj): # Also ensure that the keys encode/decode correctly k = json.loads(json.dumps(decode_object(k), cls=GlobalEncoder, default=lambda o: repr(o))) except Exception as e: # pylint: disable=broad-except + log.error("Failed to convert safe_request result to JSON") + log.error(e) continue else: jd[k] = v From b5b69a0a4b6b06c542a45b36ca10120ab120b112 Mon Sep 17 00:00:00 2001 From: farfaraji Date: Thu, 29 Apr 2021 14:59:13 +0000 Subject: [PATCH 05/20] SW-1121-dataset-saving --- codejail/jail_code.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index a495ebbce..9cf0b2573 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -187,7 +187,8 @@ def __init__(self): def jail_code(command, code=None, files=None, extra_files=None, argv=None, - stdin=None, limit_overrides_context=None, slug=None, live_output=None): + stdin=None, limit_overrides_context=None, slug=None, + live_output=None, artifacts=None): """ Run code in a jailed subprocess. @@ -259,6 +260,8 @@ def jail_code(command, code=None, files=None, extra_files=None, argv=None, else: shutil.copytree(filename, dest, symlinks=True) + if artifacts: + save_artifacts(artifacts, tmptmp) # Create the main file. if code: with open(os.path.join(homedir, "jailed_code"), "wb") as jailed: @@ -379,3 +382,21 @@ def create_rlimits(effective_limits): rlimits.append((resource.RLIMIT_FSIZE, (fsize, fsize))) return rlimits + + +def save_artifacts(artifacts, save_path): + datasets_dest_dir = os.path.join(save_path, 'datasets') + os.mkdir(datasets_dest_dir) + os.chmod(datasets_dest_dir, 0o777) + for artifact_path in artifacts: + if artifact_path.endswith('.txt'): + path = datasets_dest_dir + else: + continue + with open(artifact_path, 'r') as file: + content = file.read() + new_file = os.path.join(path, os.path.basename(artifact_path)) + print(new_file) + with open(new_file, 'w') as file: + file.write(content) + os.chmod(new_file, 0o777) From 69fe6a9d245d136c46602f67a4b1360640e8d051 Mon Sep 17 00:00:00 2001 From: farfaraji Date: Thu, 29 Apr 2021 17:44:58 +0000 Subject: [PATCH 06/20] fix-dataset-extension --- codejail/jail_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index 53d58c8ce..6ebf23ccb 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -386,7 +386,7 @@ def save_artifacts(artifacts, save_path): os.mkdir(datasets_dest_dir) os.chmod(datasets_dest_dir, 0o777) for artifact_path in artifacts: - if artifact_path.endswith('.txt'): + if artifact_path.endswith('.csv'): path = datasets_dest_dir else: continue From 4614ae9431eb7919f43d2e106a2f3a2d8667be34 Mon Sep 17 00:00:00 2001 From: farfaraji Date: Thu, 29 Apr 2021 17:55:17 +0000 Subject: [PATCH 07/20] fixes --- codejail/jail_code.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index 6ebf23ccb..8f4ca77c7 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -383,17 +383,19 @@ def create_rlimits(effective_limits): def save_artifacts(artifacts, save_path): datasets_dest_dir = os.path.join(save_path, 'datasets') + images_dest_dir = os.path.join(save_path, 'images') os.mkdir(datasets_dest_dir) os.chmod(datasets_dest_dir, 0o777) for artifact_path in artifacts: - if artifact_path.endswith('.csv'): + if artifact_path.endswith(('.csv', '.xlsx')): path = datasets_dest_dir + if artifact_path.endswith(('.png')): + path = images_dest_dir else: continue with open(artifact_path, 'r') as file: content = file.read() new_file = os.path.join(path, os.path.basename(artifact_path)) - print(new_file) with open(new_file, 'w') as file: file.write(content) os.chmod(new_file, 0o777) From 9cad2472fa64e87f486ae2a832293c74f85d6687 Mon Sep 17 00:00:00 2001 From: farfaraji Date: Thu, 29 Apr 2021 17:56:38 +0000 Subject: [PATCH 08/20] fixes --- codejail/jail_code.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index 8f4ca77c7..590e7350b 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -384,8 +384,10 @@ def create_rlimits(effective_limits): def save_artifacts(artifacts, save_path): datasets_dest_dir = os.path.join(save_path, 'datasets') images_dest_dir = os.path.join(save_path, 'images') + os.mkdir(images_dest_dir) os.mkdir(datasets_dest_dir) os.chmod(datasets_dest_dir, 0o777) + os.chmod(images_dest_dir, 0o777) for artifact_path in artifacts: if artifact_path.endswith(('.csv', '.xlsx')): path = datasets_dest_dir From 7bde8c4bc86e167c3cfb4a038b8151a55e470327 Mon Sep 17 00:00:00 2001 From: farfaraji Date: Thu, 29 Apr 2021 18:22:23 +0000 Subject: [PATCH 09/20] fix-the-bug --- codejail/jail_code.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index 590e7350b..a7dbb2503 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -389,9 +389,11 @@ def save_artifacts(artifacts, save_path): os.chmod(datasets_dest_dir, 0o777) os.chmod(images_dest_dir, 0o777) for artifact_path in artifacts: + print(artifact_path) if artifact_path.endswith(('.csv', '.xlsx')): + print("here", artifact_path) path = datasets_dest_dir - if artifact_path.endswith(('.png')): + elif artifact_path.endswith(('.png')): path = images_dest_dir else: continue From e8394ca32567ce80be7ef56c7558999296623df7 Mon Sep 17 00:00:00 2001 From: farfaraji Date: Thu, 29 Apr 2021 18:24:39 +0000 Subject: [PATCH 10/20] remove-prints --- codejail/jail_code.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index a7dbb2503..388c7d4c5 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -389,9 +389,7 @@ def save_artifacts(artifacts, save_path): os.chmod(datasets_dest_dir, 0o777) os.chmod(images_dest_dir, 0o777) for artifact_path in artifacts: - print(artifact_path) if artifact_path.endswith(('.csv', '.xlsx')): - print("here", artifact_path) path = datasets_dest_dir elif artifact_path.endswith(('.png')): path = images_dest_dir From c6c695a550f136bde10b8d93a3b6bd90f750a54b Mon Sep 17 00:00:00 2001 From: farfaraji Date: Fri, 30 Apr 2021 19:35:20 +0000 Subject: [PATCH 11/20] pull-artifacts-in-validation --- codejail/safe_exec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index 8e6dd9136..da16b407a 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -48,6 +48,7 @@ def safe_exec( limit_overrides_context=None, slug=None, extra_files=None, + artifacts=None ): """ Execute code as "exec" does, but safely. @@ -156,7 +157,7 @@ def flush(self, *args, **kwargs): res = jail_code.jail_code( "python", code=jailed_code, stdin=stdin, files=files, limit_overrides_context=limit_overrides_context, - slug=slug, extra_files=extra_files, + slug=slug, extra_files=extra_files, artifacts=artifacts ) if LOG_ALL_CODE: From aa8e30fd6d3867da9e81306e604565ee42e23975 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Thu, 15 Jul 2021 18:26:06 +0200 Subject: [PATCH 12/20] Remove artifacts restrictions and add videos folder --- codejail/jail_code.py | 20 ++++++++------------ codejail/util.py | 6 ++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index 388c7d4c5..e8fec5e70 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -10,7 +10,7 @@ from .proxy import run_subprocess_through_proxy from .subproc import run_subprocess -from .util import temp_directory +from .util import temp_directory, create_directory log = logging.getLogger("codejail") @@ -382,19 +382,15 @@ def create_rlimits(effective_limits): def save_artifacts(artifacts, save_path): - datasets_dest_dir = os.path.join(save_path, 'datasets') - images_dest_dir = os.path.join(save_path, 'images') - os.mkdir(images_dest_dir) - os.mkdir(datasets_dest_dir) - os.chmod(datasets_dest_dir, 0o777) - os.chmod(images_dest_dir, 0o777) + datasets_dest_dir = create_directory(save_path, 'datasets') + images_dest_dir = create_directory(save_path, 'images') + videos_dest_dir = create_directory(save_path, 'videos') for artifact_path in artifacts: - if artifact_path.endswith(('.csv', '.xlsx')): - path = datasets_dest_dir - elif artifact_path.endswith(('.png')): + path = datasets_dest_dir + if artifact_path.endswith(('.png', '.jpg', '.jpeg')): path = images_dest_dir - else: - continue + elif artifact_path.endswith(('.mp4', '.mov', '.gif', '.avi', '.mpeg', '.mkv')): + path = videos_dest_dir with open(artifact_path, 'r') as file: content = file.read() new_file = os.path.join(path, os.path.basename(artifact_path)) diff --git a/codejail/util.py b/codejail/util.py index 214e16358..b642c72b8 100644 --- a/codejail/util.py +++ b/codejail/util.py @@ -31,3 +31,9 @@ def change_directory(new_dir): yield new_dir finally: os.chdir(old_dir) + +def create_directory(path, dir_name): + dest_dir = os.path.join(path, dir_name) + os.mkdir(dest_dir) + os.chmod(dest_dir, 0o777) + return dest_dir \ No newline at end of file From e19b0e807fa8cfc41fedb395151fb8b08517f347 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Thu, 6 Jan 2022 15:56:21 +0000 Subject: [PATCH 13/20] add custome exception for timeout on safe_exec --- codejail/safe_exec.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index da16b407a..2ccc8d385 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -40,6 +40,13 @@ class SafeExecException(Exception): """ +class SafeExecTimeoutException(Exception): + """ + Python code running in the sandbox exceed the timeout. + + """ + + def safe_exec( code, globals_dict, @@ -165,7 +172,12 @@ def flush(self, *args, **kwargs): log.debug("Stdout: %s", res.stdout) log.debug("Stderr: %s", res.stderr) - if res.status != 0: + if res.status == -9: + raise SafeExecTimeoutException(( + "Jailed code timeout {res.stdout!r}, " + "stderr: {res.stderr!r} with status code: {res.status}" + ).format(res=res)) + elif res.status != 0: raise SafeExecException(( "Couldn't execute jailed code: stdout: {res.stdout!r}, " "stderr: {res.stderr!r} with status code: {res.status}" From 422cf0469025e6882b6745f111085a7173c9b797 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Tue, 1 Mar 2022 20:14:00 +0000 Subject: [PATCH 14/20] Fix JSON TypeError on dict key can now be a numpy type --- codejail/custom_encoder.py | 9 +++++---- codejail/safe_exec.py | 10 +++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/codejail/custom_encoder.py b/codejail/custom_encoder.py index 15a3149d9..98e624d16 100644 --- a/codejail/custom_encoder.py +++ b/codejail/custom_encoder.py @@ -1,5 +1,4 @@ import json -import numpy as np class GlobalEncoder(json.JSONEncoder): @@ -14,10 +13,12 @@ def default(self, obj): class NumpyEncoder(json.JSONEncoder): """ Custom encoder for numpy data types """ + def default(self, obj): + import numpy as np if isinstance(obj, (np.int_, np.intc, np.intp, np.int8, - np.int16, np.int32, np.int64, np.uint8, - np.uint16, np.uint32, np.uint64)): + np.int16, np.int32, np.int64, np.uint8, + np.uint16, np.uint32, np.uint64)): return int(obj) elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): return float(obj) @@ -27,6 +28,6 @@ def default(self, obj): return obj.tolist() elif isinstance(obj, (np.bool_)): return bool(obj) - elif isinstance(obj, (np.void)): + elif isinstance(obj, (np.void)): return None return json.JSONEncoder.default(self, obj) diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index 2ccc8d385..42d768427 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -194,11 +194,6 @@ def json_safe(d): """ # pylint: disable=invalid-name - # six.binary_type is here because bytes are sometimes ok if they represent valid utf8 - # so we consider them valid for now and try to decode them with decode_object. If that - # doesn't work they'll get dropped later in the process. - ok_types = (type(None), int, float, six.binary_type, six.text_type, list, tuple, dict) - def decode_object(obj): """ Convert an object to a JSON serializable form by decoding all byte strings. @@ -213,6 +208,9 @@ def decode_object(obj): """ if isinstance(obj, bytes): return obj.decode('utf-8') + if type(obj).__module__ == 'numpy': + from custom_encoder import NumpyEncoder + return NumpyEncoder().default(obj) if isinstance(obj, (list, tuple)): new_list = [] for i in obj: @@ -231,8 +229,6 @@ def decode_object(obj): bad_keys = ("__builtins__",) jd = {} for k, v in six.iteritems(d): - if not isinstance(v, ok_types): - continue if k in bad_keys: continue try: From 283fa68c0ee2e61de141bca7312b6a919c6f1443 Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Fri, 18 Mar 2022 18:00:13 +0000 Subject: [PATCH 15/20] Fix codejail numpy encoder for --- codejail/custom_encoder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/codejail/custom_encoder.py b/codejail/custom_encoder.py index 98e624d16..3ce201a12 100644 --- a/codejail/custom_encoder.py +++ b/codejail/custom_encoder.py @@ -8,6 +8,7 @@ def default(self, obj): elif type(obj).__module__ == 'pandas': if hasattr(obj, 'to_json'): return obj.to_json(orient='records') + return repr(obj) return json.JSONEncoder.default(self, obj) @@ -28,6 +29,4 @@ def default(self, obj): return obj.tolist() elif isinstance(obj, (np.bool_)): return bool(obj) - elif isinstance(obj, (np.void)): - return None - return json.JSONEncoder.default(self, obj) + return repr(obj) From fde0eb9aa76754c4e71903aedab510bf3f7f59eb Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Tue, 23 Aug 2022 15:28:54 +0000 Subject: [PATCH 16/20] Add the extraction of zip file in dataset folder --- codejail/jail_code.py | 22 +++++++++++++++------- codejail/util.py | 10 +++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index e8fec5e70..dde254fe5 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -10,7 +10,7 @@ from .proxy import run_subprocess_through_proxy from .subproc import run_subprocess -from .util import temp_directory, create_directory +from .util import temp_directory, create_directory, chmod_recursive_directory log = logging.getLogger("codejail") @@ -391,9 +391,17 @@ def save_artifacts(artifacts, save_path): path = images_dest_dir elif artifact_path.endswith(('.mp4', '.mov', '.gif', '.avi', '.mpeg', '.mkv')): path = videos_dest_dir - with open(artifact_path, 'r') as file: - content = file.read() - new_file = os.path.join(path, os.path.basename(artifact_path)) - with open(new_file, 'w') as file: - file.write(content) - os.chmod(new_file, 0o777) + + if artifact_path.endswith('.zip'): + import zipfile + with zipfile.ZipFile(artifact_path, 'r') as zip_ref: + zip_ref.extractall(path) + chmod_recursive_directory(path, 0o777) + else: + + with open(artifact_path, 'r') as file: + content = file.read() + new_file = os.path.join(path, os.path.basename(artifact_path)) + with open(new_file, 'w') as file: + file.write(content) + os.chmod(new_file, 0o777) diff --git a/codejail/util.py b/codejail/util.py index b642c72b8..c6ecd93e2 100644 --- a/codejail/util.py +++ b/codejail/util.py @@ -36,4 +36,12 @@ def create_directory(path, dir_name): dest_dir = os.path.join(path, dir_name) os.mkdir(dest_dir) os.chmod(dest_dir, 0o777) - return dest_dir \ No newline at end of file + return dest_dir + + +def chmod_recursive_directory(dir_path, perm): + for root, dirs, files in os.walk(dir_path): + for d in dirs: + os.chmod(os.path.join(root, d), perm) + for f in files: + os.chmod(os.path.join(root, f), perm) From e5e76026579fe194ee8ad981e0585bdeff9bb702 Mon Sep 17 00:00:00 2001 From: Ariella Date: Tue, 23 Aug 2022 20:53:11 -0400 Subject: [PATCH 17/20] Move proxy_main.py to be included in codejail module --- codejail/proxy.py | 2 +- proxy_main.py => codejail/proxy_main.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename proxy_main.py => codejail/proxy_main.py (100%) diff --git a/codejail/proxy.py b/codejail/proxy.py index f19cb7f0a..64023389a 100644 --- a/codejail/proxy.py +++ b/codejail/proxy.py @@ -113,7 +113,7 @@ def get_proxy(): # If we need a proxy, make a proxy. if PROXY_PROCESS is None: # Start the proxy by invoking proxy_main.py in our root directory. - root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + root = os.path.dirname(os.path.realpath(__file__)) proxy_main_py = os.path.join(root, "proxy_main.py") # Run proxy_main.py with the same Python that is running us. "-u" makes diff --git a/proxy_main.py b/codejail/proxy_main.py similarity index 100% rename from proxy_main.py rename to codejail/proxy_main.py From 2495c5ff3e649dbc305746dbf07c91b2c63cd7af Mon Sep 17 00:00:00 2001 From: Antoine Frau Date: Mon, 5 Sep 2022 08:52:37 +0000 Subject: [PATCH 18/20] Move import to the top: respect pep8 convention --- codejail/jail_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codejail/jail_code.py b/codejail/jail_code.py index dde254fe5..b47bb1d1d 100644 --- a/codejail/jail_code.py +++ b/codejail/jail_code.py @@ -6,6 +6,7 @@ import resource import shutil import sys +import zipfile from builtins import bytes from .proxy import run_subprocess_through_proxy @@ -393,7 +394,6 @@ def save_artifacts(artifacts, save_path): path = videos_dest_dir if artifact_path.endswith('.zip'): - import zipfile with zipfile.ZipFile(artifact_path, 'r') as zip_ref: zip_ref.extractall(path) chmod_recursive_directory(path, 0o777) From 10816406ff84220e87544d14c7cb28caaf408ea3 Mon Sep 17 00:00:00 2001 From: Ariella <66330883+ariellasmo@users.noreply.github.com> Date: Thu, 8 Sep 2022 14:29:13 -0400 Subject: [PATCH 19/20] Update not_safe_exec to handle artifacts (#11) --- codejail/safe_exec.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index 42d768427..70434c722 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -258,6 +258,7 @@ def not_safe_exec( limit_overrides_context=None, # pylint: disable=unused-argument slug=None, # pylint: disable=unused-argument extra_files=None, + artifacts=None ): """ Another implementation of `safe_exec`, but not safe. @@ -276,17 +277,20 @@ def not_safe_exec( with change_directory(tmpdir): # pylint: disable=invalid-name # Copy the files here. + tmptmp = os.path.join(tmpdir, "tmp") for filename in files or (): dest = os.path.join(tmpdir, os.path.basename(filename)) shutil.copyfile(filename, dest) for filename, contents in extra_files or (): - dest = os.path.join(tmpdir, filename) + dest = os.path.join(tmptmp, filename) with open(dest, "wb") as f: f.write(contents) original_path = sys.path if python_path: sys.path.extend(python_path) + if artifacts: + jail_code.save_artifacts(artifacts, tmpdir) try: exec(code, g_dict) # pylint: disable=exec-used except Exception as e: From 1d9cf715baf971a714ddcf25a69ffff0d159cadb Mon Sep 17 00:00:00 2001 From: Ariella <66330883+ariellasmo@users.noreply.github.com> Date: Thu, 15 Sep 2022 13:23:21 -0400 Subject: [PATCH 20/20] Fix custom_encoder imports (#12) --- codejail/safe_exec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codejail/safe_exec.py b/codejail/safe_exec.py index 70434c722..697cd203a 100644 --- a/codejail/safe_exec.py +++ b/codejail/safe_exec.py @@ -144,7 +144,7 @@ def flush(self, *args, **kwargs): the_code.append(textwrap.dedent( """ - from custom_encoder import GlobalEncoder + from codejail.custom_encoder import GlobalEncoder g_dict = json_safe(g_dict) """ # Write the globals back to the calling process. @@ -209,7 +209,7 @@ def decode_object(obj): if isinstance(obj, bytes): return obj.decode('utf-8') if type(obj).__module__ == 'numpy': - from custom_encoder import NumpyEncoder + from codejail.custom_encoder import NumpyEncoder return NumpyEncoder().default(obj) if isinstance(obj, (list, tuple)): new_list = []