From a2bd0f2ca12a14f05e94eef47315ce246e399dd9 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 5 Dec 2017 11:46:52 +0000 Subject: [PATCH 01/35] add google cloud signed url support --- ckanext/cloudstorage/storage.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 93686ad..3839da7 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -131,6 +131,25 @@ def can_use_advanced_aws(self): return False + @property + def can_use_advanced_google_cloud(self): + """ + `True` if the `google-cloud` module is installed and ckanext-cloudstorage has + been configured to use Google Cloud Storage, otherwise `False`. + """ + # Are we even using google cloud? + if 'GOOGLE_STORAGE' in self.driver_name: + try: + # Yes? is the google-cloud-storage package available? + from google.cloud import storage + # shut the linter up. + assert storage + return True + except ImportError: + pass + + return False + @property def guess_mimetype(self): """ @@ -311,6 +330,20 @@ def get_url_from_filename(self, rid, filename): key=path ) + + elif self.can_use_advanced_google_cloud and self.use_secure_urls: + from google.cloud import storage + + client = storage.client.Client() + + bucket = client.get_bucket(self.container_name) + + blob = bucket.get_object(path) + return generate_signed_url( + expiration=60*60, + method='GET' + ) + # Find the object for the given key. obj = self.container.get_object(path) if obj is None: From 66a8433fa58acd33cfa1923859bb40e09a69dc48 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 5 Dec 2017 12:33:54 +0000 Subject: [PATCH 02/35] allow for both explicit sa credentials and implicit authentication --- ckanext/cloudstorage/storage.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 3839da7..489bcea 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -334,7 +334,14 @@ def get_url_from_filename(self, rid, filename): elif self.can_use_advanced_google_cloud and self.use_secure_urls: from google.cloud import storage - client = storage.client.Client() + # Read service account JSON credentials file if provided + if self.driver_options['service_account_json']: + client = storage.client.Client.from_service_account_json( + self.driver_options['service_account_json']) + # else rely on implicit credentials + # (see https://googlecloudplatform.github.io/google-cloud-python/latest/core/auth.html) + else: + client = storage.client.Client() bucket = client.get_bucket(self.container_name) From b55b52865d15b8bcb32d7c51b6e1238edcba0178 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Sat, 13 Jan 2018 07:07:56 +0300 Subject: [PATCH 03/35] call generate signed url on blob --- ckanext/cloudstorage/storage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 489bcea..80ceee5 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -344,9 +344,8 @@ def get_url_from_filename(self, rid, filename): client = storage.client.Client() bucket = client.get_bucket(self.container_name) - blob = bucket.get_object(path) - return generate_signed_url( + return blob.generate_signed_url( expiration=60*60, method='GET' ) From 9fe3dc9ad431033b7ddb74029fa0cf4db94ead6e Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Mon, 15 Jan 2018 19:08:48 +0000 Subject: [PATCH 04/35] add docker setup w/ local mounted volume --- .env | 44 ++++++++++++++ Dockerfile | 9 +++ .../cloudstorage/tests/test_cloudstorage.py | 0 docker-compose.yml | 58 +++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 .env create mode 100644 Dockerfile create mode 100644 ckanext/cloudstorage/tests/test_cloudstorage.py create mode 100644 docker-compose.yml diff --git a/.env b/.env new file mode 100644 index 0000000..636cb76 --- /dev/null +++ b/.env @@ -0,0 +1,44 @@ +# Variables in this file will be substituted into docker-compose.yml +# Save a copy of this file as .env and insert your own values. +# Verify correct substitution with "docker-compose config" +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: +# docker-compose down +# docker rmi -f docker_ckan docker_db +# docker rmi $(docker images -f dangling=true -q) +# docker-compose build +# docker-compose up -d +# docker-compose restart ckan # give the db service time to initialize the db cluster on first run + +# Image: ckan +CKAN_SITE_ID=default +# +# On AWS, your CKAN_SITE_URL is the output of: +# curl -s http://169.254.169.254/latest/meta-data/public-hostname +# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com +# When running locally, CKAN_SITE_URL must contain the port +CKAN_SITE_URL=http://localhost:5000 +# +# CKAN_PORT must be available on the host: sudo netstat -na +# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan +CKAN_PORT=5000 +# +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +# +# Image: db +POSTGRES_PASSWORD=ckan +# +# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432 +# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build +POSTGRES_PORT=5432 +# +# The datastore database will be created in the db container as docs +# Readwrite user/pass will be ckan:POSTGRES_PASSWORD +# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD +DATASTORE_READONLY_PASSWORD=datastore + +CKAN_SITE_TITLE='Testing subject' diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8c41c7a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM ckan:2.7.2 + +# install cloudstorage plugin +WORKDIR $CKAN_VENV/src +RUN echo $CKAN_VENV/src +RUN git clone https://github.com/TkTech/ckanext-cloudstorage +WORKDIR $CKAN_VENV/src/ckanext-cloudstorage +RUN $CKAN_VENV/bin/python setup.py develop + diff --git a/ckanext/cloudstorage/tests/test_cloudstorage.py b/ckanext/cloudstorage/tests/test_cloudstorage.py new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..81b7785 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,58 @@ +# docker-compose build && docker-compose up -d +# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times. +version: "3" + +volumes: + ckan_storage: + pg_data: + +services: + ckan: + container_name: ckan + build: . + links: + - db + - solr + - redis + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + environment: + # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore + - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 + - CKAN_DATAPUSHER_URL=http://datapusher:8800 + - CKAN_SITE_URL=${CKAN_SITE_URL} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - CKAN_PLUGINS=datastore cloudstorage + + volumes: + - ckan_storage:/var/lib/ckan + - .:/usr/lib/ckan/venv/src/ckanext-cloudstorage + restart: on-failure + + datapusher: + container_name: datapusher + image: clementmouchet/datapusher + ports: + - "8800:8800" + + db: + container_name: db + image: thedataplace/ckan-db:2.7.2 + environment: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - pg_data:/var/lib/postgresql/data + + solr: + container_name: solr + image: thedataplace/ckan-solr:2.7.2 + + redis: + container_name: redis + image: redis:latest From 42acc13f9b684de501d61377e140477c8c57ee36 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 17 Jan 2018 20:14:51 +0000 Subject: [PATCH 05/35] tests run, prob still misconfigured --- Dockerfile | 21 ++++++++++++---- .../cloudstorage/tests/test_cloudstorage.py | 0 docker-compose.yml | 8 ++++--- postgresql/30_setup_test_dbs.sql | 3 +++ postgresql/Dockerfile | 4 ++++ solr/Dockerfile | 4 ++++ test_entrypoint.sh | 24 +++++++++++++++++++ 7 files changed, 57 insertions(+), 7 deletions(-) delete mode 100644 ckanext/cloudstorage/tests/test_cloudstorage.py create mode 100644 postgresql/30_setup_test_dbs.sql create mode 100644 postgresql/Dockerfile create mode 100644 solr/Dockerfile create mode 100644 test_entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 8c41c7a..ccd58f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,22 @@ -FROM ckan:2.7.2 +FROM thedataplace/ckan:2.7.2 + +USER root -# install cloudstorage plugin WORKDIR $CKAN_VENV/src -RUN echo $CKAN_VENV/src +RUN ckan-pip install -r ckan/dev-requirements.txt && \ + ckan-pip install ckanapi + +# install cloudstorage plugin RUN git clone https://github.com/TkTech/ckanext-cloudstorage WORKDIR $CKAN_VENV/src/ckanext-cloudstorage -RUN $CKAN_VENV/bin/python setup.py develop +RUN sh $CKAN_VENV/bin/activate && $CKAN_VENV/bin/python setup.py develop + +COPY test_entrypoint.sh $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh +RUN cp -v $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh /test_entrypoint.sh && \ + chmod +x /test_entrypoint.sh + +ENTRYPOINT ["/test_entrypoint.sh"] + +USER ckan +CMD ["ckan-paster", "serve", "/etc/ckan/ckan.ini"] diff --git a/ckanext/cloudstorage/tests/test_cloudstorage.py b/ckanext/cloudstorage/tests/test_cloudstorage.py deleted file mode 100644 index e69de29..0000000 diff --git a/docker-compose.yml b/docker-compose.yml index 81b7785..691a589 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - CKAN_SITE_URL=${CKAN_SITE_URL} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - CKAN_PLUGINS=datastore cloudstorage + - CKAN_PLUGINS=datastore volumes: - ckan_storage:/var/lib/ckan @@ -42,7 +42,7 @@ services: db: container_name: db - image: thedataplace/ckan-db:2.7.2 + build: postgresql/ environment: - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -51,7 +51,9 @@ services: solr: container_name: solr - image: thedataplace/ckan-solr:2.7.2 + build: solr/ + ports: + - 8983:8983 redis: container_name: redis diff --git a/postgresql/30_setup_test_dbs.sql b/postgresql/30_setup_test_dbs.sql new file mode 100644 index 0000000..3104f4d --- /dev/null +++ b/postgresql/30_setup_test_dbs.sql @@ -0,0 +1,3 @@ +CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8'; +CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8'; + diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile new file mode 100644 index 0000000..0242bc8 --- /dev/null +++ b/postgresql/Dockerfile @@ -0,0 +1,4 @@ +FROM thedataplace/ckan-db:2.7.2 + +COPY 30_setup_test_dbs.sql /docker-entrypoint-initdb.d/30_setup_test_dbs.sql + diff --git a/solr/Dockerfile b/solr/Dockerfile new file mode 100644 index 0000000..63f6c00 --- /dev/null +++ b/solr/Dockerfile @@ -0,0 +1,4 @@ +FROM thedataplace/ckan-solr:2.7.2 + +RUN mkdir -p /etc/solr +RUN cp -r /opt/solr/server/solr/ckan /etc/solr/ckan diff --git a/test_entrypoint.sh b/test_entrypoint.sh new file mode 100644 index 0000000..e497a50 --- /dev/null +++ b/test_entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +abort () { + echo "$@" >&2 + exit 1 +} + +echo "test_entrypoint" +echo "$@" + +# execute stuff before ckan entrypoint +cd $CKAN_VENV/src/ckan +ckan-paster datastore set-permissions -c test-core.ini | PGPASSWORD=ckan psql -h db + +echo "configure solr" +curl 'http://solr:8983/solr/admin/cores?action=CREATE&name=ckan&instanceDir=/etc/solr/ckan' + +echo "exiting test_entrypoint.sh" +# hand over control to ckan-entrypoint, including CMD args +sh /ckan-entrypoint.sh "$@" + +exec "$@" + From 1a0b6470093fa3430cbbee92160c4e1461abb895 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Thu, 18 Jan 2018 16:03:47 +0000 Subject: [PATCH 06/35] adding s3filestore for testing purps --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ccd58f0..c0df2c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,17 @@ USER root WORKDIR $CKAN_VENV/src RUN ckan-pip install -r ckan/dev-requirements.txt && \ - ckan-pip install ckanapi + ckan-pip install ckanapi boto moto # install cloudstorage plugin RUN git clone https://github.com/TkTech/ckanext-cloudstorage +RUN git clone https://github.com/okfn/ckanext-s3filestore WORKDIR $CKAN_VENV/src/ckanext-cloudstorage RUN sh $CKAN_VENV/bin/activate && $CKAN_VENV/bin/python setup.py develop +WORKDIR $CKAN_VENV/src/ckanext-s3filestore +RUN ckan-pip install -r requirements.txt -r dev-requirements.txt + COPY test_entrypoint.sh $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh RUN cp -v $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh /test_entrypoint.sh && \ chmod +x /test_entrypoint.sh From b3da887c5b65aaaed116d99c09b89b867d61c501 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Thu, 18 Jan 2018 16:04:23 +0000 Subject: [PATCH 07/35] add first test borrowed from s3filestore --- ckanext/cloudstorage/tests/data.csv | 2 + ckanext/cloudstorage/tests/test_controller.py | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 ckanext/cloudstorage/tests/data.csv create mode 100644 ckanext/cloudstorage/tests/test_controller.py diff --git a/ckanext/cloudstorage/tests/data.csv b/ckanext/cloudstorage/tests/data.csv new file mode 100644 index 0000000..0f55c8b --- /dev/null +++ b/ckanext/cloudstorage/tests/data.csv @@ -0,0 +1,2 @@ +date,price +1950-01-01,34.730 diff --git a/ckanext/cloudstorage/tests/test_controller.py b/ckanext/cloudstorage/tests/test_controller.py new file mode 100644 index 0000000..c2deed0 --- /dev/null +++ b/ckanext/cloudstorage/tests/test_controller.py @@ -0,0 +1,41 @@ +import os + +from nose.tools import assert_equal, assert_true + +import ckan.tests.helpers as helpers +import ckan.tests.factories as factories +import ckanapi + +import boto +from moto import mock_s3 + +class TestStorageController(helpers.FunctionalTestBase): + + def _upload_resource(self): + factories.Sysadmin(apikey='my-test-key') + + app = self._get_test_app() + demo = ckanapi.TestAppCKAN(app, apikey='my-test-key') + factories.Dataset(name='my-dataset') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + resource = demo.action.resource_create(package_id='my-dataset', + upload=open(file_path), + url='file.txt') + return resource, demo, app + + @mock_s3 + @helpers.change_config('ckan.site_url', 'http://mytest.ckan.net') + def test_resource_show_url(self): + """The resource_show url is expected for uploaded resource file.""" + + resource_demo, _ = self._upload_resource() + + # does resource_show have the expected resource file url? + resource_show = demo.action.resource_show(id=resource['id']) + + expected_url = 'http://mytest.ckan.net/dataset/{0}/resource/{1}/download/data.csv' \ + .format(resource['package_id'], resource['id']) + + assert_equal(resource_show['url'], expected_url) + From b3b2692790746979571eba9e0e910b2b019c4326 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Fri, 19 Jan 2018 10:46:19 +0000 Subject: [PATCH 08/35] fix typo in unpacking --- ckanext/cloudstorage/tests/test_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/cloudstorage/tests/test_controller.py b/ckanext/cloudstorage/tests/test_controller.py index c2deed0..1cb486e 100644 --- a/ckanext/cloudstorage/tests/test_controller.py +++ b/ckanext/cloudstorage/tests/test_controller.py @@ -29,7 +29,7 @@ def _upload_resource(self): def test_resource_show_url(self): """The resource_show url is expected for uploaded resource file.""" - resource_demo, _ = self._upload_resource() + resource, demo, _ = self._upload_resource() # does resource_show have the expected resource file url? resource_show = demo.action.resource_show(id=resource['id']) From 726929f5b348cf71e4488c0b7cc3a161ed010e7c Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Fri, 19 Jan 2018 11:37:11 +0000 Subject: [PATCH 09/35] fix test dependencies and add more tests from s3filestore --- Dockerfile | 11 ++-- ckanext/cloudstorage/tests/test_controller.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c0df2c5..6a12bf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,7 @@ FROM thedataplace/ckan:2.7.2 USER root WORKDIR $CKAN_VENV/src -RUN ckan-pip install -r ckan/dev-requirements.txt && \ - ckan-pip install ckanapi boto moto +RUN ckan-pip install -r ckan/dev-requirements.txt # install cloudstorage plugin RUN git clone https://github.com/TkTech/ckanext-cloudstorage @@ -12,8 +11,12 @@ RUN git clone https://github.com/okfn/ckanext-s3filestore WORKDIR $CKAN_VENV/src/ckanext-cloudstorage RUN sh $CKAN_VENV/bin/activate && $CKAN_VENV/bin/python setup.py develop -WORKDIR $CKAN_VENV/src/ckanext-s3filestore -RUN ckan-pip install -r requirements.txt -r dev-requirements.txt +# install dev dependencies +RUN ckan-pip install --upgrade \ + enum34 \ + boto==2.38.0 \ + moto==0.4.4 \ + ckanapi==3.5 COPY test_entrypoint.sh $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh RUN cp -v $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh /test_entrypoint.sh && \ diff --git a/ckanext/cloudstorage/tests/test_controller.py b/ckanext/cloudstorage/tests/test_controller.py index 1cb486e..3852ac4 100644 --- a/ckanext/cloudstorage/tests/test_controller.py +++ b/ckanext/cloudstorage/tests/test_controller.py @@ -39,3 +39,53 @@ def test_resource_show_url(self): assert_equal(resource_show['url'], expected_url) + @mock_s3 + def test_resource_download_s3(self): + """A resource uploaded to S3 ckan be downloaded.""" + + resource, demo, app = self._upload_resource() + resource_show = demo.action.resource_show(id=resource['id']) + resource_file_url = resource_show['url'] + + file_response = app.get(resource_file_url) + + assert_equal(file_response.content_type, 'text/csv') + assert_true('date,price' in file_response.body) + + @mock_s3 + def test_resource_download_s3_no_filename(self): + """A resource uploaded can be downloaded when no filename in url.""" + resource, demo, app = self._upload_resource() + + resource_file_url = '/dataset/{0}/resource/{1}/download' \ + .format(resource['package_id'], resource['id']) + + file_response = app.get(resource_file_url) + + assert_equal(file_response.content_type, 'text/csv') + assert_true('date,price' in file_response.body) + + @mock_s3 + def test_resource_download_url_link(self): + """A resource with a url (not a file) is redirected correctly.""" + factories.Sysadmin(apikey='my-test-apikey') + + app = self._get_test_app() + demo = ckanapi.TestAppCKAN(app, apikey='my-test-apikey') + dataset = factories.Dataset() + + resource = demo.action.resource_create(package_id=dataset['id'], + url='http://example') + resource_show = demo.action.resource_show(id=resource['id']) + resource_file_url = '/dataset/{0}/resource/{1}/download' \ + .format(resource['package_id'], resource['id']) + assert_equal(resource_show['url'], 'http://example') + + conn = boto.connect_s3() + bucket = conn.get_bucket('my-bucket') + assert_equal(bucket.get_all_keys(), []) + + # attempt redirect to linked url + r = app.get(resource_file_url, status=[302, 301]) + assert_equal(r.location, 'http://example') + From d173b642084a92c716973a4a48fc73732d5dc932 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Sat, 20 Jan 2018 13:44:07 +0000 Subject: [PATCH 10/35] pin working dependencies --- Dockerfile | 3 ++- solr/Dockerfile | 2 +- test_entrypoint.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a12bf9..004bd67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ RUN ckan-pip install --upgrade \ enum34 \ boto==2.38.0 \ moto==0.4.4 \ - ckanapi==3.5 + httpretty==0.6.2 \ + ckanapi==3.5 COPY test_entrypoint.sh $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh RUN cp -v $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh /test_entrypoint.sh && \ diff --git a/solr/Dockerfile b/solr/Dockerfile index 63f6c00..a17bc16 100644 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -1,4 +1,4 @@ FROM thedataplace/ckan-solr:2.7.2 RUN mkdir -p /etc/solr -RUN cp -r /opt/solr/server/solr/ckan /etc/solr/ckan +RUN cp -r /opt/solr/server/solr/ckan /etc/solr/ckan-test diff --git a/test_entrypoint.sh b/test_entrypoint.sh index e497a50..e0a4869 100644 --- a/test_entrypoint.sh +++ b/test_entrypoint.sh @@ -14,7 +14,7 @@ cd $CKAN_VENV/src/ckan ckan-paster datastore set-permissions -c test-core.ini | PGPASSWORD=ckan psql -h db echo "configure solr" -curl 'http://solr:8983/solr/admin/cores?action=CREATE&name=ckan&instanceDir=/etc/solr/ckan' +curl 'http://solr:8983/solr/admin/cores?action=CREATE&name=ckan-test&instanceDir=/etc/solr/ckan-test' echo "exiting test_entrypoint.sh" # hand over control to ckan-entrypoint, including CMD args From ae7ba0e3b5fb8de7876281cac1fb4131108e3a20 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 24 Jan 2018 10:55:20 +0000 Subject: [PATCH 11/35] add tests for storage.py --- ckanext/cloudstorage/tests/test_storage.py | 123 +++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 ckanext/cloudstorage/tests/test_storage.py diff --git a/ckanext/cloudstorage/tests/test_storage.py b/ckanext/cloudstorage/tests/test_storage.py new file mode 100644 index 0000000..82ec916 --- /dev/null +++ b/ckanext/cloudstorage/tests/test_storage.py @@ -0,0 +1,123 @@ +import os +from nose.tools import assert_equal, assert_true, assert_false +from mock import create_autospec, patch, MagicMock +import ckanapi +from webtest import Upload + +from ckan.tests import helpers, factories +from ckan.plugins import toolkit +from ckanext.cloudstorage.storage import ResourceCloudStorage + +from pylons import config + +from libcloud.storage.types import Provider +from libcloud.storage.providers import get_driver + +google_driver = get_driver(Provider.GOOGLE_STORAGE) +mock_driver = create_autospec(google_driver) + + +class Uploader(Upload): + """Extends webtest's Upload class a bit more so it actually stores file data. + """ + + def __init__(self, *args, **kwargs): + self.file = kwargs.pop('file') + super(Uplaoder, self).__init__(*args, **kwargs) + + +class TestS3Uploader(helpers.FunctionalTestBase): + + @patch('ckanext.cloudstorage.storage.get_driver') + def test_resource_upload(self, get_driver): + """Test a basic resource file upload.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) + factories.Sysadmin(apikey='my-test-apikey') + + app = self._get_test_app() + demo = ckanapi.TestAppCKAN(app, apikey='my-test-apikey') + factories.Dataset(name='my-dataset') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + resource = demo.action.resource_create( + package_id='my-dataset', + upload=open(file_path), + url='file.txt' + ) + + key = 'resources/{0}/data.csv' \ + .format(resource['id']) + + args, kwargs = container.upload_object_via_stream.call_args + + assert_equal(kwargs['object_name'], key) + print('driver method calls', mock_driver.method_calls) + print('container method calls', container.method_calls) + + @patch('ckanext.cloudstorage.storage.get_driver') + def test_resource_upload_then_clear(self, get_driver): + """Test that clearing on upload removes the storage key.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) + + sysadmin = factories.Sysadmin(apikey="my-test-key") + + app = self._get_test_app() + demo = ckanapi.TestAppCKAN(app, apikey="my-test-key") + dataset = factories.Dataset(name='my-dataset') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + resource = demo.action.resource_create( + package_id='my-dataset', + upload=open(file_path), + url='file.txt' + ) + + key = 'resources/{0}/data.csv'.format(resource['id']) + + args, kwargs = container.upload_object_via_stream.call_args + assert_equal(kwargs['object_name'], key) + + container.get_object.return_value = 'object' + + url = toolkit.url_for( + controller='package', action='resource_edit', id=dataset['id'], resource_id=resource['id']) + env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} + app.post(url, {'clear_upload': True, 'url': 'http://asdf', 'save': 'save'}, extra_environ=env) + + args, _ = container.get_object.call_args + path = args[0] + assert_equal(path, key) + args, _ = container.delete_object.call_args + assert_equal(args[0], 'object') + + @patch('ckanext.cloudstorage.storage.get_driver') + def test_path_from_filename(self, get_driver): + """path_from_filename returns as expected.""" + dataset = factories.Dataset() + resource = factories.Resource(package_id=dataset['id']) + + uploader = ResourceCloudStorage(resource) + returned_path = uploader.path_from_filename(resource['id'], 'myfile.txt') + assert_equal(returned_path, 'resources/{0}/myfile.txt'.format(resource['id'])) + + @patch('ckanext.cloudstorage.storage.get_driver') + def test_resource_upload_with_url_and_clear(self, get_driver): + """Test that clearing an upload and using a URL does not crash.""" + + sysadmin = factories.Sysadmin(apikey='my-test-key') + + app = self._get_test_app() + dataset = factories.Dataset(name='my-dataset') + + url = toolkit.url_for(controller='package', action='new_resource', id=dataset['id']) + env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} + + app.post(url, {'clear_uplaod': True, 'id': '', # empty id from the form + 'url': 'http://asdf', 'save': 'save'}, extra_environ=env) + From ab9c2ec9f1b601ca00beca28d2cf9aa397559a5d Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Mon, 29 Jan 2018 17:29:00 +0000 Subject: [PATCH 12/35] add first tests, inspired by s3filestore --- .env | 2 +- .gitignore | 1 + ckanext/cloudstorage/tests/test_controller.py | 80 +++++++++++++------ ckanext/cloudstorage/tests/test_storage.py | 3 +- dev-requirements.txt | 1 + 5 files changed, 61 insertions(+), 26 deletions(-) create mode 100644 dev-requirements.txt diff --git a/.env b/.env index 636cb76..0329a0f 100644 --- a/.env +++ b/.env @@ -39,6 +39,6 @@ POSTGRES_PORT=5432 # The datastore database will be created in the db container as docs # Readwrite user/pass will be ckan:POSTGRES_PASSWORD # Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD -DATASTORE_READONLY_PASSWORD=datastore +DATASTORE_READONLY_PASSWORD=datastore cloudstorage CKAN_SITE_TITLE='Testing subject' diff --git a/.gitignore b/.gitignore index 979398f..a53b96d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ fl_notes.txt *.ini .noseids *~ +.coverage diff --git a/ckanext/cloudstorage/tests/test_controller.py b/ckanext/cloudstorage/tests/test_controller.py index 3852ac4..a060e6e 100644 --- a/ckanext/cloudstorage/tests/test_controller.py +++ b/ckanext/cloudstorage/tests/test_controller.py @@ -1,16 +1,34 @@ import os from nose.tools import assert_equal, assert_true +from mock import patch, create_autospec, MagicMock +import ckan.plugins import ckan.tests.helpers as helpers import ckan.tests.factories as factories +from webtest import Upload + +from ckan.common import config import ckanapi +from libcloud.storage.types import Provider +from libcloud.storage.providers import get_driver + +from ckanext.cloudstorage.controller import StorageController + +google_driver = get_driver(Provider.GOOGLE_STORAGE) + + +class Uploader(Upload): + """Extends webtest's Upload class a bit more so it actually stores file data. + """ + + def __init__(self, *args, **kwargs): + self.file = kwargs.pop('file') + super(Uplaoder, self).__init__(*args, **kwargs) -import boto -from moto import mock_s3 -class TestStorageController(helpers.FunctionalTestBase): +class TestStorageController(helpers.FunctionalTestBase): def _upload_resource(self): factories.Sysadmin(apikey='my-test-key') @@ -24,10 +42,14 @@ def _upload_resource(self): url='file.txt') return resource, demo, app - @mock_s3 + @patch('ckanext.cloudstorage.storage.get_driver') @helpers.change_config('ckan.site_url', 'http://mytest.ckan.net') - def test_resource_show_url(self): + def test_resource_show_url(self, get_driver): """The resource_show url is expected for uploaded resource file.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) resource, demo, _ = self._upload_resource() @@ -39,35 +61,51 @@ def test_resource_show_url(self): assert_equal(resource_show['url'], expected_url) - @mock_s3 - def test_resource_download_s3(self): + @patch('ckanext.cloudstorage.storage.get_driver') + def test_resource_download_s3(self, get_driver): """A resource uploaded to S3 ckan be downloaded.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) resource, demo, app = self._upload_resource() resource_show = demo.action.resource_show(id=resource['id']) resource_file_url = resource_show['url'] - file_response = app.get(resource_file_url) - - assert_equal(file_response.content_type, 'text/csv') - assert_true('date,price' in file_response.body) + assert_equal(resource_file_url, '{2}/dataset/{0}/resource/{1}/download/data.csv' \ + .format(resource['package_id'], resource['id'], 'http://ckan:5000')) - @mock_s3 - def test_resource_download_s3_no_filename(self): + @patch('ckanext.cloudstorage.storage.get_driver') + @patch('ckanext.cloudstorage.controller.h') + def test_resource_download_s3_no_filename(self, h, get_driver): """A resource uploaded can be downloaded when no filename in url.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) + resource, demo, app = self._upload_resource() resource_file_url = '/dataset/{0}/resource/{1}/download' \ .format(resource['package_id'], resource['id']) + mock_driver.get_object_cdn_url.return_value = resource_file_url + file_response = app.get(resource_file_url) - assert_equal(file_response.content_type, 'text/csv') - assert_true('date,price' in file_response.body) + h.redirect_to.assert_called_with(resource_file_url) - @mock_s3 - def test_resource_download_url_link(self): + @patch('ckanext.cloudstorage.storage.get_driver') + @patch('ckanext.cloudstorage.controller.h') + def test_resource_download_url_link(self, h, get_driver): """A resource with a url (not a file) is redirected correctly.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) + mock_driver.get_object_cdn_url.return_value = 'http://example' + factories.Sysadmin(apikey='my-test-apikey') app = self._get_test_app() @@ -81,11 +119,7 @@ def test_resource_download_url_link(self): .format(resource['package_id'], resource['id']) assert_equal(resource_show['url'], 'http://example') - conn = boto.connect_s3() - bucket = conn.get_bucket('my-bucket') - assert_equal(bucket.get_all_keys(), []) - # attempt redirect to linked url - r = app.get(resource_file_url, status=[302, 301]) - assert_equal(r.location, 'http://example') + r = app.get(resource_file_url) + h.redirect_to.assert_called_with('http://example') diff --git a/ckanext/cloudstorage/tests/test_storage.py b/ckanext/cloudstorage/tests/test_storage.py index 82ec916..ef314c5 100644 --- a/ckanext/cloudstorage/tests/test_storage.py +++ b/ckanext/cloudstorage/tests/test_storage.py @@ -1,5 +1,5 @@ import os -from nose.tools import assert_equal, assert_true, assert_false +from nose.tools import assert_equal from mock import create_autospec, patch, MagicMock import ckanapi from webtest import Upload @@ -14,7 +14,6 @@ from libcloud.storage.providers import get_driver google_driver = get_driver(Provider.GOOGLE_STORAGE) -mock_driver = create_autospec(google_driver) class Uploader(Upload): diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..7fa4cf7 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1 @@ +ckanapi==4.1 From 59ac7fdaec08db8d63dedcd90cb36e1d0bdc7fc7 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Mon, 29 Jan 2018 21:30:25 +0000 Subject: [PATCH 13/35] start work on uploader --- ckanext/cloudstorage/controller.py | 10 ++ ckanext/cloudstorage/plugin.py | 6 +- ckanext/cloudstorage/storage.py | 127 ++++++++++++++++++++- ckanext/cloudstorage/tests/test_storage.py | 89 ++++++++++++++- 4 files changed, 225 insertions(+), 7 deletions(-) diff --git a/ckanext/cloudstorage/controller.py b/ckanext/cloudstorage/controller.py index 73574c6..b7531a9 100644 --- a/ckanext/cloudstorage/controller.py +++ b/ckanext/cloudstorage/controller.py @@ -52,3 +52,13 @@ def resource_download(self, id, resource_id, filename=None): base.abort(404, _('No download is available')) h.redirect_to(uploaded_url) + print('method calls', container.method_calls) + + def uploaded_file_redirect(self, upload_to, filename): + '''Redirect static file requests to their location on S3.''' + storage_path = S3Uploader.get_storage_path(upload_to) + filepath = os.path.join(storage_path, filename) + redirect_url = 'https://{bucket_name}.s3.amazonaws.com/{filepath}' \ + .format(bucket_name=config.get('ckanext.s3filestore.aws_bucket_name'), + filepath=filepath) + redirect(redirect_url) diff --git a/ckanext/cloudstorage/plugin.py b/ckanext/cloudstorage/plugin.py index 5d7a939..9bc136b 100644 --- a/ckanext/cloudstorage/plugin.py +++ b/ckanext/cloudstorage/plugin.py @@ -53,9 +53,9 @@ def get_resource_uploader(self, data_dict): return storage.ResourceCloudStorage(data_dict) def get_uploader(self, upload_to, old_filename=None): - # We don't provide misc-file storage (group images for example) - # Returning None here will use the default Uploader. - return None + # Custom uploader for generic file uploads + print('get uploader') + return storage.FileCloudStorage(upload_to, old_filename) def before_map(self, map): sm = SubMapper( diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 93686ad..1e6447e 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -5,8 +5,8 @@ import os.path import urlparse from ast import literal_eval -from datetime import datetime, timedelta - +from datetime import timedelta +import datetime from pylons import config from ckan import model from ckan.lib import munge @@ -337,3 +337,126 @@ def get_url_from_filename(self, rid, filename): @property def package(self): return model.Package.get(self.resource['package_id']) + + +class FileCloudStorage(CloudStorage): + """ + Support upload of general files to cloudstorage. + """ + def __init__(self, upload_to, old_filename=None): + super(CloudStorage, self).__init__() + + self.filename = None + self.filepath = None + self.old_filename = old_filename + if self.old_filename: + self.old_filepath = os.path.join('storage', 'uplaods', old_filename) + + def path_from_filename(self, filename): + """ + Returns a bucket path for the given filename. + + :param: filename: The unmunged filename. + """ + return os.path.join( + 'storage', + 'uploads', + munge.munge_filename(filename) + ) + + def update_data_dict(self, data_dict, url_field, file_field, clear_field): + """ + Manipulate data from the data_dict. THis needs to be called before it + reaches any validators. + + :param url_field: Name of the field where the upload is going to be + :param file_field: Name of the key where the FieldStorage is kept (i.e. + the field where the file data actually is). + :param clear_field: Name of a boolean field which requests the upload + to be deleted + """ + + self.url = data_dict.get(url_field, '') + self.clear = data_dict.pop(clear_field, None) + self.file_field = file_field + self.upload_field_storage = data_dict.pop(file_field, None) + + if hasattr(self.upload_field_storage, 'filename'): + self.filename = self.upload_field_storage.filename + self.filename = str(datetime.datetime.utcnow()) + self.filename + self.filename = munge.munge_filename_legacy(self.filename) + self.filepath = os.path.join('storage', 'uplaods', self.filename) + data_dict[url_field] = self.filename + self.uplaod_file = self.upload_field_storage.file + # keep the file if there has been no change + elif self.old_filename and not self.old_filename.startswith('http'): + if not self.clear: + data_dict[url_field] = self.old_filename + if self.clear and self.url == self.old_filename: + data_dict[url_field] = '' + + def upload(self, max_size=2): + """ + Complete the fileupload, or clear an existing upload. + + This should happen just before a commit but after the data has + been validated and flushed to the db. This is so we do not store + anything unless the request is actually good. + :param max_size: maximum file size in MB + """ + # If a filename has been provided (a file is being uplaoded) write the + # file to the appropriate key in the container + if self.filename: + if self.can_use_advanced_azure: + from azure.storage import blob as azure_blob + from azure.storage.blob.models import ContentSettings + + blob_service = azure_blob.BlockBlobService( + self.driver_options['key'], + self.driver_options['secret'] + ) + content_settings = None + if self.guess_mimetype: + content_type, _ = mimetypes.guess_type(self.filename) + if content_type: + content_settings = ContentSettings( + content_type=content_type + ) + + return blob_service.create_blob_from_stream( + container_name=self.container_name, + blob_name=self.path_from_filename( + self.filename + ), + stream=self.file_upload, + content_settings=content_settings + ) + else: + print('uploading to', self.path_from_filename(self.filename)) + self.container.upload_object_via_stream( + self.file_upload, + object_name=self.path_from_filename( + self.filename + ) + ) + # self.upload_to_key(self.filepath, self.upload_file, + # make_public=True) + self.clear = True + if (self.clear and self.old_filename + and not self.old_filename.startwith('http')): + try: + self.container.delete_object( + self.container.get_object( + self.path_from_filename( + self.old_filename + ) + ) + ) + except ObjectDoesNotExistError: + # It's possible for the object to have already been deleted, or + # for it to not yet exist in a committed state due to an + # outstanding lease. + return + + + diff --git a/ckanext/cloudstorage/tests/test_storage.py b/ckanext/cloudstorage/tests/test_storage.py index ef314c5..e319a43 100644 --- a/ckanext/cloudstorage/tests/test_storage.py +++ b/ckanext/cloudstorage/tests/test_storage.py @@ -1,6 +1,8 @@ import os from nose.tools import assert_equal from mock import create_autospec, patch, MagicMock +import datetime + import ckanapi from webtest import Upload @@ -22,10 +24,10 @@ class Uploader(Upload): def __init__(self, *args, **kwargs): self.file = kwargs.pop('file') - super(Uplaoder, self).__init__(*args, **kwargs) + super(Uploader, self).__init__(*args, **kwargs) -class TestS3Uploader(helpers.FunctionalTestBase): +class TestResourceUploader(helpers.FunctionalTestBase): @patch('ckanext.cloudstorage.storage.get_driver') def test_resource_upload(self, get_driver): @@ -120,3 +122,86 @@ def test_resource_upload_with_url_and_clear(self, get_driver): app.post(url, {'clear_uplaod': True, 'id': '', # empty id from the form 'url': 'http://asdf', 'save': 'save'}, extra_environ=env) + +class TestFileCloudStorage(helpers.FunctionalTestBase): + + # @patch('ckanext.cloudstorage.storage.get_driver') + def test_group_image_uplaod(self): + """Test a group image file uplaod.""" + # mock_driver = MagicMock(spec=google_driver, name='driver') + # container = MagicMock(name='container') + # mock_driver.get_container.return_value = container + # mock_driver.get_object_cdn_url.return_value = 'http://cdn.url' + # get_driver.return_value = MagicMock(return_value=mock_driver) + + sysadmin = factories.Sysadmin(apikey='my-test-key') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + filename = 'image.png' + + img_uploader = Uploader(filename, file=open(file_path)) + + with patch('ckanext.cloudstorage.storage.datetime') as mock_date: + mock_date.datetime.utcnow.return_value = \ + datetime.datetime(2001, 1, 29) + context = {'user': sysadmin['name']} + helpers.call_action('group_create', context=context, + name='my-group', + image_uplaods=img_uploader, + image_url=filename, + save='save') + + key = "storage/uploads/group/2001-01-29-000000{0}" \ + .format(filename) + + group = helpers.call_action('group_show', id='my-group') + print('group', group) + + # args, kwargs = container.upload_object_via_datastream.call_args + # assert_equal(kwargs['object_name'], key) + + app = self._get_test_app() + image_file_url = '/uploads/group/{0}'.format(filename) + r = app.get(image_file_url) + assert(r.location, 'http://cdn.url') + + # @patch('ckanext.cloudstorage.storage.get_driver') + # @patch('ckanext.cloudstorage.storage.FileCloudStorage.upload') + def test_group_image_upload_then_clear(self): + """Test that clearing an upload calls delete_object""" + # mock_driver = MagicMock(spec=google_driver, name='driver') + # container = MagicMock(name='container') + # mock_driver.get_container.return_value = container + # get_driver.return_value = MagicMock(return_value=mock_driver) + + sysadmin = factories.Sysadmin(apikey='my-test-apikey') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + file_name = 'image.png' + + img_uploader = Uploader(file_name, file=open(file_path)) + + with patch('ckanext.cloudstorage.storage.datetime') as mock_date: + mock_date.datetime.utcnow.return_value = \ + datetime.datetime(2001, 1, 29) + context = {'user': sysadmin['name']} + helpers.call_action('group_create', context=context, + name='my-group', + image_uplaod=img_uploader, + image_url=file_name) + + key = 'storage/uploads/group/2001-01-29-000000{0}' \ + .format(file_name) + + # assert uplaod was called + # upload.assert_called() + + helpers.call_action('group_update', context=context, + id='my-group', name='my-group', + image_url='http://example', clear_update=True) + # assert delete object is called + # container.delete_object.assert_called() + + + + From f0c737a24279056122805435215c8425086a79ed Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 30 Jan 2018 15:20:50 +0000 Subject: [PATCH 14/35] =?UTF-8?q?get=20uploading=20&=20viewing=20generic?= =?UTF-8?q?=20files=20working=20(warning=20code=20is=20=E2=98=A2))?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ckanext/cloudstorage/controller.py | 11 +- ckanext/cloudstorage/plugin.py | 6 + ckanext/cloudstorage/storage.py | 99 ++++++++++++- ckanext/cloudstorage/tests/test_storage.py | 160 +++++++++++---------- requirements.txt | 1 + 5 files changed, 190 insertions(+), 87 deletions(-) create mode 100644 requirements.txt diff --git a/ckanext/cloudstorage/controller.py b/ckanext/cloudstorage/controller.py index b7531a9..377564e 100644 --- a/ckanext/cloudstorage/controller.py +++ b/ckanext/cloudstorage/controller.py @@ -55,10 +55,7 @@ def resource_download(self, id, resource_id, filename=None): print('method calls', container.method_calls) def uploaded_file_redirect(self, upload_to, filename): - '''Redirect static file requests to their location on S3.''' - storage_path = S3Uploader.get_storage_path(upload_to) - filepath = os.path.join(storage_path, filename) - redirect_url = 'https://{bucket_name}.s3.amazonaws.com/{filepath}' \ - .format(bucket_name=config.get('ckanext.s3filestore.aws_bucket_name'), - filepath=filepath) - redirect(redirect_url) + '''Redirect static file requests to their location on cloudstorage.''' + upload = uploader.get_uploader('notused') + uploaded_url = upload.get_object_public_url(filename) + h.redirect_to(uploaded_url) diff --git a/ckanext/cloudstorage/plugin.py b/ckanext/cloudstorage/plugin.py index 9bc136b..97b22a4 100644 --- a/ckanext/cloudstorage/plugin.py +++ b/ckanext/cloudstorage/plugin.py @@ -77,6 +77,12 @@ def before_map(self, map): action='resource_download' ) + sm.connect( + 'uploaded_file', + '/uploads/{upload_to}/{filename}', + action='uploaded_file_redirect' + ) + return map # IActions diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 1e6447e..61b1dbb 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -18,6 +18,7 @@ class CloudStorage(object): def __init__(self): + print('cloudstorage init') self.driver = get_driver( getattr( Provider, @@ -141,6 +142,22 @@ def guess_mimetype(self): config.get('ckanext.cloudstorage.guess_mimetype', False) ) + def get_object_public_url(self, filename): + """ + Returns the public url of an object. + Raises `NotImplementedError` for drivers yet unsupported, or when + `use_secure_urls` is set to `True`. + + Assumes container is made public. + """ + if self.driver_name == 'GOOGLE_STORAGE': + if self.use_secure_urls: + raise NotImplementedError("Should be pretty easy though!") + return "https://storage.googleapis.com/{0}/{1}" \ + .format(self.container_name, self.path_from_filename(filename)) + else: + raise NotImplementedError("This method hasn't been implemented yet for this driver.") + class ResourceCloudStorage(CloudStorage): def __init__(self, resource): @@ -344,7 +361,7 @@ class FileCloudStorage(CloudStorage): Support upload of general files to cloudstorage. """ def __init__(self, upload_to, old_filename=None): - super(CloudStorage, self).__init__() + super(FileCloudStorage, self).__init__() self.filename = None self.filepath = None @@ -387,7 +404,7 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): self.filename = munge.munge_filename_legacy(self.filename) self.filepath = os.path.join('storage', 'uplaods', self.filename) data_dict[url_field] = self.filename - self.uplaod_file = self.upload_field_storage.file + self.file_upload = self.upload_field_storage.file # keep the file if there has been no change elif self.old_filename and not self.old_filename.startswith('http'): if not self.clear: @@ -432,7 +449,6 @@ def upload(self, max_size=2): content_settings=content_settings ) else: - print('uploading to', self.path_from_filename(self.filename)) self.container.upload_object_via_stream( self.file_upload, object_name=self.path_from_filename( @@ -443,7 +459,7 @@ def upload(self, max_size=2): # make_public=True) self.clear = True if (self.clear and self.old_filename - and not self.old_filename.startwith('http')): + and not self.old_filename.startswith('http')): try: self.container.delete_object( self.container.get_object( @@ -458,5 +474,80 @@ def upload(self, max_size=2): # outstanding lease. return + def get_url_from_filename(self, filename): + """ + Retrieve a publically accessible URL for the given filename. + + .. note:: + + Works for Azure and any libcloud driver that implements + support for get_object_cdn_url (ex: AWS S3). + + :param filename: The resource filename. + + :returns: Externally accessible URL or None. + """ + # Find the key the file *should* be stored at. + path = self.path_from_filename(filename) + + # If advanced azure features are enabled, generate a temporary + # shared access link instead of simply redirecting to the file. + if self.can_use_advanced_azure and self.use_secure_urls: + from azure.storage import blob as azure_blob + + blob_service = azure_blob.BlockBlobService( + self.driver_options['key'], + self.driver_options['secret'] + ) + + return blob_service.make_blob_url( + container_name=self.container_name, + blob_name=path, + sas_token=blob_service.generate_blob_shared_access_signature( + container_name=self.container_name, + blob_name=path, + expiry=datetime.utcnow() + timedelta(hours=1), + permission=azure_blob.BlobPermissions.READ + ) + ) + elif self.can_use_advanced_aws and self.use_secure_urls: + from boto.s3.connection import S3Connection + s3_connection = S3Connection( + self.driver_options['key'], + self.driver_options['secret'] + ) + return s3_connection.generate_url( + expires_in=60 * 60, + method='GET', + bucket=self.container_name, + query_auth=True, + key=path + ) + + # Find the object for the given key. + obj = self.container.get_object(path) + if obj is None: + return + + # Not supported by all providers! + try: + return self.driver.get_object_cdn_url(obj) + except NotImplementedError: + if 'S3' in self.driver_name: + return urlparse.urljoin( + 'https://' + self.driver.connection.host, + '{container}/{path}'.format( + container=self.container_name, + path=path + ) + ) + # This extra 'url' property isn't documented anywhere, sadly. + # See azure_blobs.py:_xml_to_object for more. + elif 'url' in obj.extra: + return obj.extra['url'] + raise + + + diff --git a/ckanext/cloudstorage/tests/test_storage.py b/ckanext/cloudstorage/tests/test_storage.py index e319a43..ae79a96 100644 --- a/ckanext/cloudstorage/tests/test_storage.py +++ b/ckanext/cloudstorage/tests/test_storage.py @@ -8,7 +8,7 @@ from ckan.tests import helpers, factories from ckan.plugins import toolkit -from ckanext.cloudstorage.storage import ResourceCloudStorage +from ckanext.cloudstorage.storage import ResourceCloudStorage, FileCloudStorage from pylons import config @@ -125,82 +125,90 @@ def test_resource_upload_with_url_and_clear(self, get_driver): class TestFileCloudStorage(helpers.FunctionalTestBase): - # @patch('ckanext.cloudstorage.storage.get_driver') - def test_group_image_uplaod(self): - """Test a group image file uplaod.""" - # mock_driver = MagicMock(spec=google_driver, name='driver') - # container = MagicMock(name='container') - # mock_driver.get_container.return_value = container - # mock_driver.get_object_cdn_url.return_value = 'http://cdn.url' - # get_driver.return_value = MagicMock(return_value=mock_driver) - - sysadmin = factories.Sysadmin(apikey='my-test-key') - - file_path = os.path.join(os.path.dirname(__file__), 'data.csv') - filename = 'image.png' - - img_uploader = Uploader(filename, file=open(file_path)) - - with patch('ckanext.cloudstorage.storage.datetime') as mock_date: - mock_date.datetime.utcnow.return_value = \ - datetime.datetime(2001, 1, 29) - context = {'user': sysadmin['name']} - helpers.call_action('group_create', context=context, - name='my-group', - image_uplaods=img_uploader, - image_url=filename, - save='save') - - key = "storage/uploads/group/2001-01-29-000000{0}" \ - .format(filename) - - group = helpers.call_action('group_show', id='my-group') - print('group', group) - - # args, kwargs = container.upload_object_via_datastream.call_args - # assert_equal(kwargs['object_name'], key) - - app = self._get_test_app() - image_file_url = '/uploads/group/{0}'.format(filename) - r = app.get(image_file_url) - assert(r.location, 'http://cdn.url') - - # @patch('ckanext.cloudstorage.storage.get_driver') - # @patch('ckanext.cloudstorage.storage.FileCloudStorage.upload') - def test_group_image_upload_then_clear(self): - """Test that clearing an upload calls delete_object""" - # mock_driver = MagicMock(spec=google_driver, name='driver') - # container = MagicMock(name='container') - # mock_driver.get_container.return_value = container - # get_driver.return_value = MagicMock(return_value=mock_driver) - - sysadmin = factories.Sysadmin(apikey='my-test-apikey') +# # @patch('ckanext.cloudstorage.storage.get_driver') +# def test_group_image_uplaod(self): +# """Test a group image file uplaod.""" +# # mock_driver = MagicMock(spec=google_driver, name='driver') +# # container = MagicMock(name='container') +# # mock_driver.get_container.return_value = container +# # mock_driver.get_object_cdn_url.return_value = 'http://cdn.url' +# # get_driver.return_value = MagicMock(return_value=mock_driver) + +# sysadmin = factories.Sysadmin(apikey='my-test-key') + +# file_path = os.path.join(os.path.dirname(__file__), 'data.csv') +# filename = 'image.png' + +# img_uploader = Uploader(filename, file=open(file_path)) + +# with patch('ckanext.cloudstorage.storage.datetime') as mock_date: +# mock_date.datetime.utcnow.return_value = \ +# datetime.datetime(2001, 1, 29) +# context = {'user': sysadmin['name']} +# helpers.call_action('group_create', context=context, +# name='my-group', +# image_uplaods=img_uploader, +# image_url=filename, +# save='save') + +# key = "storage/uploads/group/2001-01-29-000000{0}" \ +# .format(filename) + +# group = helpers.call_action('group_show', id='my-group') +# print('group', group) + +# # args, kwargs = container.upload_object_via_datastream.call_args +# # assert_equal(kwargs['object_name'], key) + +# app = self._get_test_app() +# image_file_url = '/uploads/group/{0}'.format(filename) +# r = app.get(image_file_url) + +# # @patch('ckanext.cloudstorage.storage.get_driver') +# # @patch('ckanext.cloudstorage.storage.FileCloudStorage.upload') +# def test_group_image_upload_then_clear(self): +# """Test that clearing an upload calls delete_object""" +# # mock_driver = MagicMock(spec=google_driver, name='driver') +# # container = MagicMock(name='container') +# # mock_driver.get_container.return_value = container +# # get_driver.return_value = MagicMock(return_value=mock_driver) + +# sysadmin = factories.Sysadmin(apikey='my-test-apikey') + +# file_path = os.path.join(os.path.dirname(__file__), 'data.csv') +# file_name = 'image.png' + +# img_uploader = Uploader(file_name, file=open(file_path)) + +# with patch('ckanext.cloudstorage.storage.datetime') as mock_date: +# mock_date.datetime.utcnow.return_value = \ +# datetime.datetime(2001, 1, 29) +# context = {'user': sysadmin['name']} +# helpers.call_action('group_create', context=context, +# name='my-group', +# image_uplaod=img_uploader, +# image_url=file_name) + +# key = 'storage/uploads/group/2001-01-29-000000{0}' \ +# .format(file_name) + +# # assert uplaod was called +# # upload.assert_called() + +# helpers.call_action('group_update', context=context, +# id='my-group', name='my-group', +# image_url='http://example', clear_update=True) +# # assert delete object is called +# # container.delete_object.assert_called() + @patch('ckanext.cloudstorage.storage.get_driver') + def test_get_object_public_url(self, get_driver): + """ + Test get_object_public_url returns expected string + """ + uploader = FileCloudStorage('notused') + url = uploader.get_object_public_url('file.png') + assert_equal(url, 'https://storage.googleapis.com/test/storage/uploads/file.png') - file_path = os.path.join(os.path.dirname(__file__), 'data.csv') - file_name = 'image.png' - - img_uploader = Uploader(file_name, file=open(file_path)) - - with patch('ckanext.cloudstorage.storage.datetime') as mock_date: - mock_date.datetime.utcnow.return_value = \ - datetime.datetime(2001, 1, 29) - context = {'user': sysadmin['name']} - helpers.call_action('group_create', context=context, - name='my-group', - image_uplaod=img_uploader, - image_url=file_name) - - key = 'storage/uploads/group/2001-01-29-000000{0}' \ - .format(file_name) - - # assert uplaod was called - # upload.assert_called() - - helpers.call_action('group_update', context=context, - id='my-group', name='my-group', - image_url='http://example', clear_update=True) - # assert delete object is called - # container.delete_object.assert_called() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..904545b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pycrypto From 2a56bf1c34da7d387048f8f9332d0c311ebd957d Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 30 Jan 2018 18:15:31 +0000 Subject: [PATCH 15/35] refactor upload methods --- ckanext/cloudstorage/controller.py | 1 - ckanext/cloudstorage/storage.py | 176 ++++++++++++----------------- 2 files changed, 70 insertions(+), 107 deletions(-) diff --git a/ckanext/cloudstorage/controller.py b/ckanext/cloudstorage/controller.py index 377564e..7f75ade 100644 --- a/ckanext/cloudstorage/controller.py +++ b/ckanext/cloudstorage/controller.py @@ -52,7 +52,6 @@ def resource_download(self, id, resource_id, filename=None): base.abort(404, _('No download is available')) h.redirect_to(uploaded_url) - print('method calls', container.method_calls) def uploaded_file_redirect(self, upload_to, filename): '''Redirect static file requests to their location on cloudstorage.''' diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 61b1dbb..3b3ce90 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -158,6 +158,59 @@ def get_object_public_url(self, filename): else: raise NotImplementedError("This method hasn't been implemented yet for this driver.") + def upload_to_path(self, file_path): + """ + Upload to storage bucket + + :param file_path: File path in storage bucket + :param old_file_path: File path of old file in storage bucket. + """ + + if self.can_use_advanced_azure: + from azure.storage import blob as azure_blob + from azure.storage.blob.models import ContentSettings + + blob_service = azure_blob.BlockBlobService( + self.driver_options['key'], + self.driver_options['secret'] + ) + content_settings = None + if self.guess_mimetype: + content_type, _ = mimetypes.guess_type(file_path) + if content_type: + content_settings = ContentSettings( + content_type=content_type + ) + + return blob_service.create_blob_from_stream( + container_name=self.container_name, + blob_name=file_path, + stream=self.file_upload, + content_settings=content_settings + ) + else: + self.container.upload_object_via_stream( + self.file_upload, + object_name=file_path + ) + + def delete_object_from_path(self, file_path): + """ + Delete object from cloudstorage at `file_path` + :param file_path: Path of file to be deletedd + """ + try: + self.container.delete_object( + self.container.get_object( + old_file_path + ) + ) + except ObjectDoesNotExistError: + # It's possible for the object to have already been deleted, or + # for it to not yet exist in a committed state due to an + # outstanding lease. + return + class ResourceCloudStorage(CloudStorage): def __init__(self, resource): @@ -223,58 +276,15 @@ def upload(self, id, max_size=10): :param id: The resource_id. :param max_size: Ignored. """ + # If a filename has been provided (a file is being uplaoded) write the + # file to the appropriate key in the container if self.filename: - if self.can_use_advanced_azure: - from azure.storage import blob as azure_blob - from azure.storage.blob.models import ContentSettings - - blob_service = azure_blob.BlockBlobService( - self.driver_options['key'], - self.driver_options['secret'] - ) - content_settings = None - if self.guess_mimetype: - content_type, _ = mimetypes.guess_type(self.filename) - if content_type: - content_settings = ContentSettings( - content_type=content_type - ) - - return blob_service.create_blob_from_stream( - container_name=self.container_name, - blob_name=self.path_from_filename( - id, - self.filename - ), - stream=self.file_upload, - content_settings=content_settings - ) - else: - self.container.upload_object_via_stream( - self.file_upload, - object_name=self.path_from_filename( - id, - self.filename - ) - ) + file_path = self.path_from_filename(id, self.filename) + self.upload_to_path(file_path) + if self._clear and self.old_filename and not self.leave_files: + old_file_path = self.path_from_filename(id, self.old_filename) + self.delete_object_from_path(old_file_path) - elif self._clear and self.old_filename and not self.leave_files: - # This is only set when a previously-uploaded file is replace - # by a link. We want to delete the previously-uploaded file. - try: - self.container.delete_object( - self.container.get_object( - self.path_from_filename( - id, - self.old_filename - ) - ) - ) - except ObjectDoesNotExistError: - # It's possible for the object to have already been deleted, or - # for it to not yet exist in a committed state due to an - # outstanding lease. - return def get_url_from_filename(self, rid, filename): """ @@ -367,7 +377,7 @@ def __init__(self, upload_to, old_filename=None): self.filepath = None self.old_filename = old_filename if self.old_filename: - self.old_filepath = os.path.join('storage', 'uplaods', old_filename) + self.old_filepath = os.path.join('storage', 'uploads', old_filename) def path_from_filename(self, filename): """ @@ -394,7 +404,7 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): """ self.url = data_dict.get(url_field, '') - self.clear = data_dict.pop(clear_field, None) + self._clear = data_dict.pop(clear_field, None) self.file_field = file_field self.upload_field_storage = data_dict.pop(file_field, None) @@ -407,9 +417,9 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): self.file_upload = self.upload_field_storage.file # keep the file if there has been no change elif self.old_filename and not self.old_filename.startswith('http'): - if not self.clear: + if not self._clear: data_dict[url_field] = self.old_filename - if self.clear and self.url == self.old_filename: + if self._clear and self.url == self.old_filename: data_dict[url_field] = '' def upload(self, max_size=2): @@ -419,60 +429,14 @@ def upload(self, max_size=2): This should happen just before a commit but after the data has been validated and flushed to the db. This is so we do not store anything unless the request is actually good. - :param max_size: maximum file size in MB + :param max_size: ignored """ - # If a filename has been provided (a file is being uplaoded) write the - # file to the appropriate key in the container if self.filename: - if self.can_use_advanced_azure: - from azure.storage import blob as azure_blob - from azure.storage.blob.models import ContentSettings - - blob_service = azure_blob.BlockBlobService( - self.driver_options['key'], - self.driver_options['secret'] - ) - content_settings = None - if self.guess_mimetype: - content_type, _ = mimetypes.guess_type(self.filename) - if content_type: - content_settings = ContentSettings( - content_type=content_type - ) - - return blob_service.create_blob_from_stream( - container_name=self.container_name, - blob_name=self.path_from_filename( - self.filename - ), - stream=self.file_upload, - content_settings=content_settings - ) - else: - self.container.upload_object_via_stream( - self.file_upload, - object_name=self.path_from_filename( - self.filename - ) - ) - # self.upload_to_key(self.filepath, self.upload_file, - # make_public=True) - self.clear = True - if (self.clear and self.old_filename - and not self.old_filename.startswith('http')): - try: - self.container.delete_object( - self.container.get_object( - self.path_from_filename( - self.old_filename - ) - ) - ) - except ObjectDoesNotExistError: - # It's possible for the object to have already been deleted, or - # for it to not yet exist in a committed state due to an - # outstanding lease. - return + file_path = self.path_from_filename(self.filename) + return self.upload_to_path(file_path) + if self._clear and self.old_filename and not self.leave_files: + old_file_path = self.path_from_filename(self.old_filename) + self.delete_object_from_path(old_file_path) def get_url_from_filename(self, filename): """ From 0545a928acbc4c2dba73fbf5647962c43ffb30e9 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 30 Jan 2018 18:53:21 +0000 Subject: [PATCH 16/35] refactor get_url_from_filename to get_url_from_path --- ckanext/cloudstorage/controller.py | 6 +- ckanext/cloudstorage/storage.py | 259 +++++++++++------------------ 2 files changed, 99 insertions(+), 166 deletions(-) diff --git a/ckanext/cloudstorage/controller.py b/ckanext/cloudstorage/controller.py index 7f75ade..e9e0a20 100644 --- a/ckanext/cloudstorage/controller.py +++ b/ckanext/cloudstorage/controller.py @@ -44,7 +44,8 @@ def resource_download(self, id, resource_id, filename=None): filename = os.path.basename(resource['url']) upload = uploader.get_resource_uploader(resource) - uploaded_url = upload.get_url_from_filename(resource['id'], filename) + file_path = upload.path_from_filename(resource['id'], filename) + uploaded_url = upload.get_url_from_path(file_path) # The uploaded file is missing for some reason, such as the # provider being down. @@ -56,5 +57,6 @@ def resource_download(self, id, resource_id, filename=None): def uploaded_file_redirect(self, upload_to, filename): '''Redirect static file requests to their location on cloudstorage.''' upload = uploader.get_uploader('notused') - uploaded_url = upload.get_object_public_url(filename) + file_path = upload.path_from_filename(filename) + uploaded_url = upload.get_url_from_path(file_path) h.redirect_to(uploaded_url) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 97ed376..0cdab4c 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -230,6 +230,92 @@ def delete_object_from_path(self, file_path): # outstanding lease. return + def get_url_from_path(self, path): + """ + Retrieve a publically accessible URL for the given path + + .. note:: + + Works for Azure and any libcloud driver that implements + support for get_object_cdn_url (ex: AWS S3, Google Storage). + + :param path: The resource path. + + :returns: Externally accessible URL or None. + """ + # If advanced azure features are enabled, generate a temporary + # shared access link instead of simply redirecting to the file. + if self.can_use_advanced_azure and self.use_secure_urls: + from azure.storage import blob as azure_blob + + blob_service = azure_blob.BlockBlobService( + self.driver_options['key'], + self.driver_options['secret'] + ) + + return blob_service.make_blob_url( + container_name=self.container_name, + blob_name=path, + sas_token=blob_service.generate_blob_shared_access_signature( + container_name=self.container_name, + blob_name=path, + expiry=datetime.utcnow() + timedelta(hours=1), + permission=azure_blob.BlobPermissions.READ + ) + ) + elif self.can_use_advanced_aws and self.use_secure_urls: + from boto.s3.connection import S3Connection + s3_connection = S3Connection( + self.driver_options['key'], + self.driver_options['secret'] + ) + return s3_connection.generate_url( + expires_in=60 * 60, + method='GET', + bucket=self.container_name, + query_auth=True, + key=path + ) + + + elif self.can_use_advanced_google_cloud and self.use_secure_urls: + from google.cloud import storage + + client = storage.client.Client.from_service_account_json( + self.driver_options['secret'] + ) + + bucket = client.get_bucket(self.container_name) + blob = bucket.get_object(path) + return blob.generate_signed_url( + expiration=60*60, + method='GET' + ) + + # Find the object for the given key. + obj = self.container.get_object(path) + if obj is None: + return + + # Not supported by all providers! + try: + return self.driver.get_object_cdn_url(obj) + except NotImplementedError: + if 'S3' in self.driver_name or 'GOOGLE_STORAGE' in self.driver_name: + return urlparse.urljoin( + 'https://' + self.driver.connection.host, + '{container}/{path}'.format( + container=self.container_name, + path=path + ) + ) + # This extra 'url' property isn't documented anywhere, sadly. + # See azure_blobs.py:_xml_to_object for more. + elif 'url' in obj.extra: + return obj.extra['url'] + raise + + class ResourceCloudStorage(CloudStorage): def __init__(self, resource): @@ -304,101 +390,15 @@ def upload(self, id, max_size=10): old_file_path = self.path_from_filename(id, self.old_filename) self.delete_object_from_path(old_file_path) - - def get_url_from_filename(self, rid, filename): + def get_url_from_filename(self, id, filename): """ - Retrieve a publically accessible URL for the given resource_id - and filename. - - .. note:: - - Works for Azure and any libcloud driver that implements - support for get_object_cdn_url (ex: AWS S3). - - :param rid: The resource ID. - :param filename: The resource filename. - - :returns: Externally accessible URL or None. + Generate public URL from resource id and filename + :param id: The resource ID + :param filename: The resource filename """ - # Find the key the file *should* be stored at. - path = self.path_from_filename(rid, filename) - - # If advanced azure features are enabled, generate a temporary - # shared access link instead of simply redirecting to the file. - if self.can_use_advanced_azure and self.use_secure_urls: - from azure.storage import blob as azure_blob - - blob_service = azure_blob.BlockBlobService( - self.driver_options['key'], - self.driver_options['secret'] - ) - - return blob_service.make_blob_url( - container_name=self.container_name, - blob_name=path, - sas_token=blob_service.generate_blob_shared_access_signature( - container_name=self.container_name, - blob_name=path, - expiry=datetime.utcnow() + timedelta(hours=1), - permission=azure_blob.BlobPermissions.READ - ) - ) - elif self.can_use_advanced_aws and self.use_secure_urls: - from boto.s3.connection import S3Connection - s3_connection = S3Connection( - self.driver_options['key'], - self.driver_options['secret'] - ) - return s3_connection.generate_url( - expires_in=60 * 60, - method='GET', - bucket=self.container_name, - query_auth=True, - key=path - ) - - - elif self.can_use_advanced_google_cloud and self.use_secure_urls: - from google.cloud import storage - - # Read service account JSON credentials file if provided - if self.driver_options['service_account_json']: - client = storage.client.Client.from_service_account_json( - self.driver_options['service_account_json']) - # else rely on implicit credentials - # (see https://googlecloudplatform.github.io/google-cloud-python/latest/core/auth.html) - else: - client = storage.client.Client() - - bucket = client.get_bucket(self.container_name) - blob = bucket.get_object(path) - return blob.generate_signed_url( - expiration=60*60, - method='GET' - ) - - # Find the object for the given key. - obj = self.container.get_object(path) - if obj is None: - return + path = self.path_from_filename(id, filename) + return self.get_url_from_path(path) - # Not supported by all providers! - try: - return self.driver.get_object_cdn_url(obj) - except NotImplementedError: - if 'S3' in self.driver_name: - return urlparse.urljoin( - 'https://' + self.driver.connection.host, - '{container}/{path}'.format( - container=self.container_name, - path=path - ) - ) - # This extra 'url' property isn't documented anywhere, sadly. - # See azure_blobs.py:_xml_to_object for more. - elif 'url' in obj.extra: - return obj.extra['url'] - raise @property def package(self): @@ -479,78 +479,9 @@ def upload(self, max_size=2): def get_url_from_filename(self, filename): """ - Retrieve a publically accessible URL for the given filename. - - .. note:: - - Works for Azure and any libcloud driver that implements - support for get_object_cdn_url (ex: AWS S3). - - :param filename: The resource filename. - - :returns: Externally accessible URL or None. + Get public url from filename + :param filename: name of file """ - # Find the key the file *should* be stored at. path = self.path_from_filename(filename) - - # If advanced azure features are enabled, generate a temporary - # shared access link instead of simply redirecting to the file. - if self.can_use_advanced_azure and self.use_secure_urls: - from azure.storage import blob as azure_blob - - blob_service = azure_blob.BlockBlobService( - self.driver_options['key'], - self.driver_options['secret'] - ) - - return blob_service.make_blob_url( - container_name=self.container_name, - blob_name=path, - sas_token=blob_service.generate_blob_shared_access_signature( - container_name=self.container_name, - blob_name=path, - expiry=datetime.utcnow() + timedelta(hours=1), - permission=azure_blob.BlobPermissions.READ - ) - ) - elif self.can_use_advanced_aws and self.use_secure_urls: - from boto.s3.connection import S3Connection - s3_connection = S3Connection( - self.driver_options['key'], - self.driver_options['secret'] - ) - return s3_connection.generate_url( - expires_in=60 * 60, - method='GET', - bucket=self.container_name, - query_auth=True, - key=path - ) - - # Find the object for the given key. - obj = self.container.get_object(path) - if obj is None: - return - - # Not supported by all providers! - try: - return self.driver.get_object_cdn_url(obj) - except NotImplementedError: - if 'S3' in self.driver_name: - return urlparse.urljoin( - 'https://' + self.driver.connection.host, - '{container}/{path}'.format( - container=self.container_name, - path=path - ) - ) - # This extra 'url' property isn't documented anywhere, sadly. - # See azure_blobs.py:_xml_to_object for more. - elif 'url' in obj.extra: - return obj.extra['url'] - raise - - - - + return self.get_url_from_path(path) From 30d4986efa6fe40077ca18b0f988816346904722 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 30 Jan 2018 18:53:52 +0000 Subject: [PATCH 17/35] add plugin tests file --- ckanext/cloudstorage/tests/test_plugin.py | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ckanext/cloudstorage/tests/test_plugin.py diff --git a/ckanext/cloudstorage/tests/test_plugin.py b/ckanext/cloudstorage/tests/test_plugin.py new file mode 100644 index 0000000..7f92ea1 --- /dev/null +++ b/ckanext/cloudstorage/tests/test_plugin.py @@ -0,0 +1,35 @@ +import os +from nose.tools import assert_equal, assert_raises +from mock import patch, MagicMock + +from ckan.tests import helpers, factories +from ckan.lib import helpers as h + +import ckanapi + +from ckanext.cloudstorage.controller import StorageController +class TestPlugin(helpers.FunctionalTestBase): + + @patch('ckanext.cloudstorage.storage.get_driver') + @patch('ckanext.cloudstorage.controller.StorageController', spec=StorageController) + def test_resource_download_calls_ext_method(self, resource_download, get_driver): + """ + Test `ckanext.cloudstorage.controller.StorageController.resource_download` is called for `resource_download` action. + """ + app = self._get_test_app() + demo = ckanapi.TestAppCKAN(app, apikey='my-test-apikey') + factories.Sysadmin(apikey='my-test-apikey') + + factories.Dataset(name='my-dataset') + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + resource = demo.action.resource_create( + package_id='my-dataset', + upload=open(file_path), + url='file.txt' + ) + + # proves it's calling the right code, right? + with assert_raises(TypeError) as exc: + r = app.get(resource['url']) + assert_equal(exc.exception.message, "'MagicMock' object is not iterable") + resource_download.assert_called_once() From 352f597b3546f3e80fad9f8d8434138d37a19338 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 30 Jan 2018 19:11:28 +0000 Subject: [PATCH 18/35] replace filepath joins with method --- ckanext/cloudstorage/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 0cdab4c..8f164d6 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -416,7 +416,7 @@ def __init__(self, upload_to, old_filename=None): self.filepath = None self.old_filename = old_filename if self.old_filename: - self.old_filepath = os.path.join('storage', 'uploads', old_filename) + self.old_filepath = self.path_from_filename(old_filename) def path_from_filename(self, filename): """ @@ -451,7 +451,7 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): self.filename = self.upload_field_storage.filename self.filename = str(datetime.datetime.utcnow()) + self.filename self.filename = munge.munge_filename_legacy(self.filename) - self.filepath = os.path.join('storage', 'uplaods', self.filename) + self.filepath = self.path_from_filename(self.filename) data_dict[url_field] = self.filename self.file_upload = self.upload_field_storage.file # keep the file if there has been no change From f976efdf5e0e6bd50ef4686eae4505ed92358e89 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Tue, 30 Jan 2018 20:45:22 +0000 Subject: [PATCH 19/35] fix tests --- ckanext/cloudstorage/storage.py | 3 +-- ckanext/cloudstorage/tests/test_controller.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index 8f164d6..ae28246 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -18,7 +18,6 @@ class CloudStorage(object): def __init__(self): - print('cloudstorage init') self.driver = get_driver( getattr( Provider, @@ -221,7 +220,7 @@ def delete_object_from_path(self, file_path): try: self.container.delete_object( self.container.get_object( - old_file_path + file_path ) ) except ObjectDoesNotExistError: diff --git a/ckanext/cloudstorage/tests/test_controller.py b/ckanext/cloudstorage/tests/test_controller.py index a060e6e..3e3f346 100644 --- a/ckanext/cloudstorage/tests/test_controller.py +++ b/ckanext/cloudstorage/tests/test_controller.py @@ -73,8 +73,8 @@ def test_resource_download_s3(self, get_driver): resource_show = demo.action.resource_show(id=resource['id']) resource_file_url = resource_show['url'] - assert_equal(resource_file_url, '{2}/dataset/{0}/resource/{1}/download/data.csv' \ - .format(resource['package_id'], resource['id'], 'http://ckan:5000')) + assert_equal(resource_file_url, u'{2}/dataset/{0}/resource/{1}/download/data.csv' \ + .format(resource['package_id'], resource['id'], 'http://localhost:5000')) @patch('ckanext.cloudstorage.storage.get_driver') @patch('ckanext.cloudstorage.controller.h') From 0aeed2cdad40f1143723d92e83c29f69d74a87aa Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 31 Jan 2018 13:13:11 +0000 Subject: [PATCH 20/35] fix google secure url, refactor to raise exception if no secure url available --- ckanext/cloudstorage/controller.py | 3 +- ckanext/cloudstorage/storage.py | 80 +++++++++++++++--------------- dev-requirements.txt | 2 + 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/ckanext/cloudstorage/controller.py b/ckanext/cloudstorage/controller.py index e9e0a20..0b19163 100644 --- a/ckanext/cloudstorage/controller.py +++ b/ckanext/cloudstorage/controller.py @@ -44,8 +44,7 @@ def resource_download(self, id, resource_id, filename=None): filename = os.path.basename(resource['url']) upload = uploader.get_resource_uploader(resource) - file_path = upload.path_from_filename(resource['id'], filename) - uploaded_url = upload.get_url_from_path(file_path) + uploaded_url = upload.get_url_from_filename(resource['id'], filename) # The uploaded file is missing for some reason, such as the # provider being down. diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index ae28246..e93b178 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -244,52 +244,55 @@ def get_url_from_path(self, path): """ # If advanced azure features are enabled, generate a temporary # shared access link instead of simply redirecting to the file. - if self.can_use_advanced_azure and self.use_secure_urls: - from azure.storage import blob as azure_blob + if self.use_secure_urls: + if self.can_use_advanced_azure: + from azure.storage import blob as azure_blob - blob_service = azure_blob.BlockBlobService( - self.driver_options['key'], - self.driver_options['secret'] - ) + blob_service = azure_blob.BlockBlobService( + self.driver_options['key'], + self.driver_options['secret'] + ) - return blob_service.make_blob_url( - container_name=self.container_name, - blob_name=path, - sas_token=blob_service.generate_blob_shared_access_signature( + return blob_service.make_blob_url( container_name=self.container_name, blob_name=path, - expiry=datetime.utcnow() + timedelta(hours=1), - permission=azure_blob.BlobPermissions.READ + sas_token=blob_service.generate_blob_shared_access_signature( + container_name=self.container_name, + blob_name=path, + expiry=datetime.utcnow() + timedelta(hours=1), + permission=azure_blob.BlobPermissions.READ + ) + ) + elif self.can_use_advanced_aws: + from boto.s3.connection import S3Connection + s3_connection = S3Connection( + self.driver_options['key'], + self.driver_options['secret'] + ) + return s3_connection.generate_url( + expires_in=60 * 60, + method='GET', + bucket=self.container_name, + query_auth=True, + key=path ) - ) - elif self.can_use_advanced_aws and self.use_secure_urls: - from boto.s3.connection import S3Connection - s3_connection = S3Connection( - self.driver_options['key'], - self.driver_options['secret'] - ) - return s3_connection.generate_url( - expires_in=60 * 60, - method='GET', - bucket=self.container_name, - query_auth=True, - key=path - ) - - elif self.can_use_advanced_google_cloud and self.use_secure_urls: - from google.cloud import storage + elif self.can_use_advanced_google_cloud: + from google.cloud import storage - client = storage.client.Client.from_service_account_json( - self.driver_options['secret'] - ) + client = storage.client.Client.from_service_account_json( + self.driver_options['secret'] + ) - bucket = client.get_bucket(self.container_name) - blob = bucket.get_object(path) - return blob.generate_signed_url( - expiration=60*60, - method='GET' - ) + bucket = client.get_bucket(self.container_name) + blob = bucket.get_blob(path) + return blob.generate_signed_url( + expiration=timedelta(seconds=60*60), + method='GET', + ) + else: + raise Exception('Unable to generate secure url. Is your cloud \ + provider\'s driver installed?') # Find the object for the given key. obj = self.container.get_object(path) @@ -315,7 +318,6 @@ def get_url_from_path(self, path): raise - class ResourceCloudStorage(CloudStorage): def __init__(self, resource): """ diff --git a/dev-requirements.txt b/dev-requirements.txt index 7fa4cf7..d6a809f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1 +1,3 @@ ckanapi==4.1 +google-cloud-storage==1.7.0 + From e23940cb8fa80b45fc70392632072f466555c5ad Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 31 Jan 2018 13:15:06 +0000 Subject: [PATCH 21/35] ignore all .egg-info things --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a53b96d..d1f3f7d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ syntax: glob *.swp *.swo .DS_Store -ckan.egg-info/* +*.egg-info/ sandbox/* dist From 9eab142800b39c8a473277ba6bfed81b3f9db13c Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 31 Jan 2018 13:28:47 +0000 Subject: [PATCH 22/35] remove docker setup from repo --- Dockerfile | 30 ---------------- docker-compose.yml | 60 -------------------------------- postgresql/30_setup_test_dbs.sql | 3 -- postgresql/Dockerfile | 4 --- solr/Dockerfile | 4 --- test_entrypoint.sh | 24 ------------- 6 files changed, 125 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml delete mode 100644 postgresql/30_setup_test_dbs.sql delete mode 100644 postgresql/Dockerfile delete mode 100644 solr/Dockerfile delete mode 100644 test_entrypoint.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 004bd67..0000000 --- a/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM thedataplace/ckan:2.7.2 - -USER root - -WORKDIR $CKAN_VENV/src -RUN ckan-pip install -r ckan/dev-requirements.txt - -# install cloudstorage plugin -RUN git clone https://github.com/TkTech/ckanext-cloudstorage -RUN git clone https://github.com/okfn/ckanext-s3filestore -WORKDIR $CKAN_VENV/src/ckanext-cloudstorage -RUN sh $CKAN_VENV/bin/activate && $CKAN_VENV/bin/python setup.py develop - -# install dev dependencies -RUN ckan-pip install --upgrade \ - enum34 \ - boto==2.38.0 \ - moto==0.4.4 \ - httpretty==0.6.2 \ - ckanapi==3.5 - -COPY test_entrypoint.sh $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh -RUN cp -v $CKAN_VENV/src/ckanext-cloudstorage/test_entrypoint.sh /test_entrypoint.sh && \ - chmod +x /test_entrypoint.sh - -ENTRYPOINT ["/test_entrypoint.sh"] - -USER ckan - -CMD ["ckan-paster", "serve", "/etc/ckan/ckan.ini"] diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 691a589..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,60 +0,0 @@ -# docker-compose build && docker-compose up -d -# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times. -version: "3" - -volumes: - ckan_storage: - pg_data: - -services: - ckan: - container_name: ckan - build: . - links: - - db - - solr - - redis - ports: - - "0.0.0.0:${CKAN_PORT}:5000" - environment: - # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - - CKAN_SOLR_URL=http://solr:8983/solr/ckan - - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8800 - - CKAN_SITE_URL=${CKAN_SITE_URL} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - CKAN_PLUGINS=datastore - - volumes: - - ckan_storage:/var/lib/ckan - - .:/usr/lib/ckan/venv/src/ckanext-cloudstorage - restart: on-failure - - datapusher: - container_name: datapusher - image: clementmouchet/datapusher - ports: - - "8800:8800" - - db: - container_name: db - build: postgresql/ - environment: - - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - volumes: - - pg_data:/var/lib/postgresql/data - - solr: - container_name: solr - build: solr/ - ports: - - 8983:8983 - - redis: - container_name: redis - image: redis:latest diff --git a/postgresql/30_setup_test_dbs.sql b/postgresql/30_setup_test_dbs.sql deleted file mode 100644 index 3104f4d..0000000 --- a/postgresql/30_setup_test_dbs.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8'; -CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8'; - diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile deleted file mode 100644 index 0242bc8..0000000 --- a/postgresql/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM thedataplace/ckan-db:2.7.2 - -COPY 30_setup_test_dbs.sql /docker-entrypoint-initdb.d/30_setup_test_dbs.sql - diff --git a/solr/Dockerfile b/solr/Dockerfile deleted file mode 100644 index a17bc16..0000000 --- a/solr/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM thedataplace/ckan-solr:2.7.2 - -RUN mkdir -p /etc/solr -RUN cp -r /opt/solr/server/solr/ckan /etc/solr/ckan-test diff --git a/test_entrypoint.sh b/test_entrypoint.sh deleted file mode 100644 index e0a4869..0000000 --- a/test_entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -set -e - -abort () { - echo "$@" >&2 - exit 1 -} - -echo "test_entrypoint" -echo "$@" - -# execute stuff before ckan entrypoint -cd $CKAN_VENV/src/ckan -ckan-paster datastore set-permissions -c test-core.ini | PGPASSWORD=ckan psql -h db - -echo "configure solr" -curl 'http://solr:8983/solr/admin/cores?action=CREATE&name=ckan-test&instanceDir=/etc/solr/ckan-test' - -echo "exiting test_entrypoint.sh" -# hand over control to ckan-entrypoint, including CMD args -sh /ckan-entrypoint.sh "$@" - -exec "$@" - From f3896af26d58d71bb740ebbd93005263d4225a1f Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Thu, 1 Feb 2018 10:40:41 +0000 Subject: [PATCH 23/35] remove print in plugin.py --- ckanext/cloudstorage/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckanext/cloudstorage/plugin.py b/ckanext/cloudstorage/plugin.py index 97b22a4..29e823e 100644 --- a/ckanext/cloudstorage/plugin.py +++ b/ckanext/cloudstorage/plugin.py @@ -54,7 +54,6 @@ def get_resource_uploader(self, data_dict): def get_uploader(self, upload_to, old_filename=None): # Custom uploader for generic file uploads - print('get uploader') return storage.FileCloudStorage(upload_to, old_filename) def before_map(self, map): From 6c16b68d09acd46e470b2aacfef105bf4de0a6d1 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 14 Mar 2018 16:09:53 +0000 Subject: [PATCH 24/35] fix site-url in test --- ckanext/cloudstorage/tests/test_controller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/cloudstorage/tests/test_controller.py b/ckanext/cloudstorage/tests/test_controller.py index 3e3f346..d8afebe 100644 --- a/ckanext/cloudstorage/tests/test_controller.py +++ b/ckanext/cloudstorage/tests/test_controller.py @@ -62,6 +62,7 @@ def test_resource_show_url(self, get_driver): assert_equal(resource_show['url'], expected_url) @patch('ckanext.cloudstorage.storage.get_driver') + @helpers.change_config('ckan.site_url', 'http://localhost:5000') def test_resource_download_s3(self, get_driver): """A resource uploaded to S3 ckan be downloaded.""" mock_driver = MagicMock(spec=google_driver, name='driver') @@ -73,7 +74,7 @@ def test_resource_download_s3(self, get_driver): resource_show = demo.action.resource_show(id=resource['id']) resource_file_url = resource_show['url'] - assert_equal(resource_file_url, u'{2}/dataset/{0}/resource/{1}/download/data.csv' \ + assert_equal(resource_file_url, u'{2}/dataset/{0}/resource/{1}/download/data.csv' .format(resource['package_id'], resource['id'], 'http://localhost:5000')) @patch('ckanext.cloudstorage.storage.get_driver') From b9677ccaf1fd0671a8770e9e17ad54ef0f92261b Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 14 Mar 2018 16:21:18 +0000 Subject: [PATCH 25/35] formatting fixed in storage.py --- ckanext/cloudstorage/storage.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index e93b178..e07d077 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -134,8 +134,9 @@ def can_use_advanced_aws(self): @property def can_use_advanced_google_cloud(self): """ - `True` if the `google-cloud` module is installed and ckanext-cloudstorage has - been configured to use Google Cloud Storage, otherwise `False`. + `True` if the `google-cloud` module is installed and + ckanext-cloudstorage has been configured to use Google Cloud Storage, + otherwise `False`. """ # Are we even using google cloud? if 'GOOGLE_STORAGE' in self.driver_name: @@ -172,9 +173,11 @@ def get_object_public_url(self, filename): if self.use_secure_urls: raise NotImplementedError("Should be pretty easy though!") return "https://storage.googleapis.com/{0}/{1}" \ - .format(self.container_name, self.path_from_filename(filename)) + .format(self.container_name, + self.path_from_filename(filename)) else: - raise NotImplementedError("This method hasn't been implemented yet for this driver.") + raise NotImplementedError( + "This method hasn't been implemented yet for this driver.") def upload_to_path(self, file_path): """ @@ -400,7 +403,6 @@ def get_url_from_filename(self, id, filename): path = self.path_from_filename(id, filename) return self.get_url_from_path(path) - @property def package(self): return model.Package.get(self.resource['package_id']) @@ -476,7 +478,7 @@ def upload(self, max_size=2): return self.upload_to_path(file_path) if self._clear and self.old_filename and not self.leave_files: old_file_path = self.path_from_filename(self.old_filename) - self.delete_object_from_path(old_file_path) + self.delete_object_from_path(old_file_path) def get_url_from_filename(self, filename): """ From e32b28a166a079bf0bec99b19d196e7cb1d52afc Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Wed, 14 Mar 2018 17:57:37 +0000 Subject: [PATCH 26/35] use secure_urls only for resources --- ckanext/cloudstorage/controller.py | 5 ++-- ckanext/cloudstorage/storage.py | 11 +++++---- ckanext/cloudstorage/tests/test_storage.py | 27 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/ckanext/cloudstorage/controller.py b/ckanext/cloudstorage/controller.py index 0b19163..fc4beed 100644 --- a/ckanext/cloudstorage/controller.py +++ b/ckanext/cloudstorage/controller.py @@ -3,8 +3,7 @@ import os.path from pylons import c -from pylons.i18n import _ - +from pylons.i18n import _ from ckan import logic, model from ckan.lib import base, uploader import ckan.lib.helpers as h @@ -57,5 +56,5 @@ def uploaded_file_redirect(self, upload_to, filename): '''Redirect static file requests to their location on cloudstorage.''' upload = uploader.get_uploader('notused') file_path = upload.path_from_filename(filename) - uploaded_url = upload.get_url_from_path(file_path) + uploaded_url = upload.get_url_from_path(file_path, use_secure_urls=False) h.redirect_to(uploaded_url) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index e07d077..b7fcc74 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -232,7 +232,7 @@ def delete_object_from_path(self, file_path): # outstanding lease. return - def get_url_from_path(self, path): + def get_url_from_path(self, path, use_secure_urls): """ Retrieve a publically accessible URL for the given path @@ -247,7 +247,7 @@ def get_url_from_path(self, path): """ # If advanced azure features are enabled, generate a temporary # shared access link instead of simply redirecting to the file. - if self.use_secure_urls: + if use_secure_urls: if self.can_use_advanced_azure: from azure.storage import blob as azure_blob @@ -401,7 +401,7 @@ def get_url_from_filename(self, id, filename): :param filename: The resource filename """ path = self.path_from_filename(id, filename) - return self.get_url_from_path(path) + return self.get_url_from_path(path, self.use_secure_urls) @property def package(self): @@ -486,5 +486,8 @@ def get_url_from_filename(self, filename): :param filename: name of file """ path = self.path_from_filename(filename) - return self.get_url_from_path(path) + # We don't want to use secure urls for normal file uploads. + # Doing so would cause assets caching issues such as the logo + # to be reloaded on every page load. + return self.get_url_from_path(path, use_secure_urls=False) diff --git a/ckanext/cloudstorage/tests/test_storage.py b/ckanext/cloudstorage/tests/test_storage.py index ae79a96..c0e6cd7 100644 --- a/ckanext/cloudstorage/tests/test_storage.py +++ b/ckanext/cloudstorage/tests/test_storage.py @@ -122,6 +122,33 @@ def test_resource_upload_with_url_and_clear(self, get_driver): app.post(url, {'clear_uplaod': True, 'id': '', # empty id from the form 'url': 'http://asdf', 'save': 'save'}, extra_environ=env) + @helpers.change_config('ckanext.cloudstorage.use_secure_urls', True) + @patch('ckanext.cloudstorage.storage.get_driver') + @patch('ckanext.cloudstorage.storage.CloudStorage.get_url_from_path') + def test_path_from_filename_uses_secure_url_when_config_is_set(self, get_url_from_path, get_driver): + dataset = factories.Dataset(name='my-dataset') + resource = factories.Resource( + package_id=dataset['id'], + ) + + uploader = ResourceCloudStorage(resource) + returned_path = uploader.get_url_from_filename(resource['id'], 'myfile.txt') + get_url_from_path.assert_called_once_with('resources/{}/myfile.txt' + .format(resource['id']), True) + + @helpers.change_config('ckanext.cloudstorage.use_secure_urls', False) + @patch('ckanext.cloudstorage.storage.get_driver') + @patch('ckanext.cloudstorage.storage.CloudStorage.get_url_from_path') + def test_path_from_filename_uses_secure_url_when_option_is_false(self, get_url_from_path, get_driver): + dataset = factories.Dataset(name='my-dataset') + resource = factories.Resource( + package_id=dataset['id'], + ) + + uploader = ResourceCloudStorage(resource) + returned_path = uploader.get_url_from_filename(resource['id'], 'myfile.txt') + get_url_from_path.assert_called_once_with('resources/{}/myfile.txt'.format(resource['id']), False) + class TestFileCloudStorage(helpers.FunctionalTestBase): From a52c607c9b15b1c48ef7fb10d60c410848515a3c Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Thu, 15 Mar 2018 12:35:41 +0000 Subject: [PATCH 27/35] fix disabled tests --- ckanext/cloudstorage/storage.py | 1 - ckanext/cloudstorage/tests/test_storage.py | 173 +++++++++++---------- 2 files changed, 93 insertions(+), 81 deletions(-) diff --git a/ckanext/cloudstorage/storage.py b/ckanext/cloudstorage/storage.py index b7fcc74..81d87b2 100644 --- a/ckanext/cloudstorage/storage.py +++ b/ckanext/cloudstorage/storage.py @@ -444,7 +444,6 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): :param clear_field: Name of a boolean field which requests the upload to be deleted """ - self.url = data_dict.get(url_field, '') self._clear = data_dict.pop(clear_field, None) self.file_field = file_field diff --git a/ckanext/cloudstorage/tests/test_storage.py b/ckanext/cloudstorage/tests/test_storage.py index c0e6cd7..50bbd04 100644 --- a/ckanext/cloudstorage/tests/test_storage.py +++ b/ckanext/cloudstorage/tests/test_storage.py @@ -152,81 +152,99 @@ def test_path_from_filename_uses_secure_url_when_option_is_false(self, get_url_f class TestFileCloudStorage(helpers.FunctionalTestBase): -# # @patch('ckanext.cloudstorage.storage.get_driver') -# def test_group_image_uplaod(self): -# """Test a group image file uplaod.""" -# # mock_driver = MagicMock(spec=google_driver, name='driver') -# # container = MagicMock(name='container') -# # mock_driver.get_container.return_value = container -# # mock_driver.get_object_cdn_url.return_value = 'http://cdn.url' -# # get_driver.return_value = MagicMock(return_value=mock_driver) - -# sysadmin = factories.Sysadmin(apikey='my-test-key') - -# file_path = os.path.join(os.path.dirname(__file__), 'data.csv') -# filename = 'image.png' - -# img_uploader = Uploader(filename, file=open(file_path)) - -# with patch('ckanext.cloudstorage.storage.datetime') as mock_date: -# mock_date.datetime.utcnow.return_value = \ -# datetime.datetime(2001, 1, 29) -# context = {'user': sysadmin['name']} -# helpers.call_action('group_create', context=context, -# name='my-group', -# image_uplaods=img_uploader, -# image_url=filename, -# save='save') - -# key = "storage/uploads/group/2001-01-29-000000{0}" \ -# .format(filename) - -# group = helpers.call_action('group_show', id='my-group') -# print('group', group) - -# # args, kwargs = container.upload_object_via_datastream.call_args -# # assert_equal(kwargs['object_name'], key) - -# app = self._get_test_app() -# image_file_url = '/uploads/group/{0}'.format(filename) -# r = app.get(image_file_url) - -# # @patch('ckanext.cloudstorage.storage.get_driver') -# # @patch('ckanext.cloudstorage.storage.FileCloudStorage.upload') -# def test_group_image_upload_then_clear(self): -# """Test that clearing an upload calls delete_object""" -# # mock_driver = MagicMock(spec=google_driver, name='driver') -# # container = MagicMock(name='container') -# # mock_driver.get_container.return_value = container -# # get_driver.return_value = MagicMock(return_value=mock_driver) - -# sysadmin = factories.Sysadmin(apikey='my-test-apikey') - -# file_path = os.path.join(os.path.dirname(__file__), 'data.csv') -# file_name = 'image.png' - -# img_uploader = Uploader(file_name, file=open(file_path)) - -# with patch('ckanext.cloudstorage.storage.datetime') as mock_date: -# mock_date.datetime.utcnow.return_value = \ -# datetime.datetime(2001, 1, 29) -# context = {'user': sysadmin['name']} -# helpers.call_action('group_create', context=context, -# name='my-group', -# image_uplaod=img_uploader, -# image_url=file_name) - -# key = 'storage/uploads/group/2001-01-29-000000{0}' \ -# .format(file_name) - -# # assert uplaod was called -# # upload.assert_called() - -# helpers.call_action('group_update', context=context, -# id='my-group', name='my-group', -# image_url='http://example', clear_update=True) -# # assert delete object is called -# # container.delete_object.assert_called() + @patch('ckanext.cloudstorage.storage.FileCloudStorage') + def test_file_upload_calls_FileCloudStorage(self, FileCloudStorage): + sysadmin = factories.Sysadmin(apikey='apikey') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + filename = 'image.png' + + img_uploader = Uploader(filename, file=open(file_path)) + + with patch('ckanext.cloudstorage.storage.datetime') as mock_date: + mock_date.datetime.utcnow.returl_value = datetime.datetime(2001, 1, 29) + context = {'user': sysadmin['name']} + helpers.call_action('group_create', context=context, + name='group', + image_upload=img_uploader, + image_url=filename, + save='save') + + FileCloudStorage.assert_called_once_with('group', None) + + @patch('ckanext.cloudstorage.storage.get_driver') + def test_group_image_upload(self, get_driver): + """Test a group image file uplaod.""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + mock_driver.get_object_cdn_url.return_value = 'http://cdn.url' + get_driver.return_value = MagicMock(return_value=mock_driver) + + sysadmin = factories.Sysadmin(apikey='my-test-key') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + filename = 'image.png' + + img_uploader = Uploader(filename, file=open(file_path)) + + with patch('ckanext.cloudstorage.storage.datetime') as mock_date: + mock_date.datetime.utcnow.return_value = \ + datetime.datetime(2001, 1, 29) + context = {'user': sysadmin['name']} + helpers.call_action('group_create', context=context, + name='my-group', + image_upload=img_uploader, + image_url=filename, + save='save') + + key = "storage/uploads/2001-01-29-000000{0}" \ + .format(filename) + + group = helpers.call_action('group_show', id='my-group') + print('group', group) + + args, kwargs = container.upload_object_via_stream.call_args + assert_equal(kwargs['object_name'], unicode(key)) + + # app = self._get_test_app() + # image_file_url = '/uploads/group/{0}'.format(filename) + # r = app.get(image_file_url) + + @patch('ckanext.cloudstorage.storage.get_driver') + def test_group_image_upload_then_clear(self, get_driver): + """Test that clearing an upload calls delete_object""" + mock_driver = MagicMock(spec=google_driver, name='driver') + container = MagicMock(name='container') + mock_driver.get_container.return_value = container + get_driver.return_value = MagicMock(return_value=mock_driver) + + sysadmin = factories.Sysadmin(apikey='my-test-apikey') + + file_path = os.path.join(os.path.dirname(__file__), 'data.csv') + file_name = 'image.png' + + img_uploader = Uploader(file_name, file=open(file_path)) + + with patch('ckanext.cloudstorage.storage.datetime') as mock_date: + mock_date.datetime.utcnow.return_value = \ + datetime.datetime(2001, 1, 29) + context = {'user': sysadmin['name']} + helpers.call_action('group_create', context=context, + name='my-group', + image_upload=img_uploader, + image_url=file_name) + + object_mock = MagicMock(name='object') + container.get_object.return_value = object_mock + + helpers.call_action('group_update', context=context, + id='my-group', name='my-group', + image_url='http://example', clear_upload=True) + + # assert delete object is called + container.delete_object.assert_called_with(object_mock) + @patch('ckanext.cloudstorage.storage.get_driver') def test_get_object_public_url(self, get_driver): """ @@ -235,8 +253,3 @@ def test_get_object_public_url(self, get_driver): uploader = FileCloudStorage('notused') url = uploader.get_object_public_url('file.png') assert_equal(url, 'https://storage.googleapis.com/test/storage/uploads/file.png') - - - - - From e34bbac7f1ac03dfb928f55817c3b752830c82a4 Mon Sep 17 00:00:00 2001 From: Gerben Neven Date: Thu, 15 Mar 2018 12:35:49 +0000 Subject: [PATCH 28/35] add cover report --- cover/ckanext___init___py.html | 103 ++ cover/ckanext_cloudstorage___init___py.html | 89 ++ cover/ckanext_cloudstorage_cli_py.html | 429 +++++++ cover/ckanext_cloudstorage_controller_py.html | 209 ++++ cover/ckanext_cloudstorage_helpers_py.html | 111 ++ ...kanext_cloudstorage_logic___init___py.html | 89 ++ ...cloudstorage_logic_action___init___py.html | 89 ++ ...loudstorage_logic_action_multipart_py.html | 635 ++++++++++ ...t_cloudstorage_logic_auth___init___py.html | 89 ++ ..._cloudstorage_logic_auth_multipart_py.html | 143 +++ cover/ckanext_cloudstorage_model_py.html | 237 ++++ cover/ckanext_cloudstorage_plugin_py.html | 395 ++++++ cover/ckanext_cloudstorage_storage_py.html | 1087 +++++++++++++++++ ...cloudstorage_tests_test_controller_py.html | 341 ++++++ ...ext_cloudstorage_tests_test_plugin_py.html | 159 +++ ...xt_cloudstorage_tests_test_storage_py.html | 599 +++++++++ cover/coverage_html.js | 584 +++++++++ cover/index.html | 230 ++++ cover/jquery.ba-throttle-debounce.min.js | 9 + cover/jquery.hotkeys.js | 99 ++ cover/jquery.isonscreen.js | 53 + cover/jquery.min.js | 4 + cover/jquery.tablesorter.min.js | 2 + cover/keybd_closed.png | Bin 0 -> 112 bytes cover/keybd_open.png | Bin 0 -> 112 bytes cover/status.json | 1 + cover/style.css | 375 ++++++ 27 files changed, 6161 insertions(+) create mode 100644 cover/ckanext___init___py.html create mode 100644 cover/ckanext_cloudstorage___init___py.html create mode 100644 cover/ckanext_cloudstorage_cli_py.html create mode 100644 cover/ckanext_cloudstorage_controller_py.html create mode 100644 cover/ckanext_cloudstorage_helpers_py.html create mode 100644 cover/ckanext_cloudstorage_logic___init___py.html create mode 100644 cover/ckanext_cloudstorage_logic_action___init___py.html create mode 100644 cover/ckanext_cloudstorage_logic_action_multipart_py.html create mode 100644 cover/ckanext_cloudstorage_logic_auth___init___py.html create mode 100644 cover/ckanext_cloudstorage_logic_auth_multipart_py.html create mode 100644 cover/ckanext_cloudstorage_model_py.html create mode 100644 cover/ckanext_cloudstorage_plugin_py.html create mode 100644 cover/ckanext_cloudstorage_storage_py.html create mode 100644 cover/ckanext_cloudstorage_tests_test_controller_py.html create mode 100644 cover/ckanext_cloudstorage_tests_test_plugin_py.html create mode 100644 cover/ckanext_cloudstorage_tests_test_storage_py.html create mode 100644 cover/coverage_html.js create mode 100644 cover/index.html create mode 100644 cover/jquery.ba-throttle-debounce.min.js create mode 100644 cover/jquery.hotkeys.js create mode 100644 cover/jquery.isonscreen.js create mode 100644 cover/jquery.min.js create mode 100644 cover/jquery.tablesorter.min.js create mode 100644 cover/keybd_closed.png create mode 100644 cover/keybd_open.png create mode 100644 cover/status.json create mode 100644 cover/style.css diff --git a/cover/ckanext___init___py.html b/cover/ckanext___init___py.html new file mode 100644 index 0000000..00737bb --- /dev/null +++ b/cover/ckanext___init___py.html @@ -0,0 +1,103 @@ + + + + + + + + + + + Coverage for ckanext/__init__.py: 0% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+ + 		+

# this is a namespace package 

+

try: 

+

import pkg_resources 

+

pkg_resources.declare_namespace(__name__) 

+

except ImportError: 

+

import pkgutil 

+

__path__ = pkgutil.extend_path(__path__, __name__) 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage___init___py.html b/cover/ckanext_cloudstorage___init___py.html new file mode 100644 index 0000000..c5d5e10 --- /dev/null +++ b/cover/ckanext_cloudstorage___init___py.html @@ -0,0 +1,89 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/__init__.py: 100% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+ + + +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_cli_py.html b/cover/ckanext_cloudstorage_cli_py.html new file mode 100644 index 0000000..c18a432 --- /dev/null +++ b/cover/ckanext_cloudstorage_cli_py.html @@ -0,0 +1,429 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/cli.py: 0% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+

75

+

76

+

77

+

78

+

79

+

80

+

81

+

82

+

83

+

84

+

85

+

86

+

87

+

88

+

89

+

90

+

91

+

92

+

93

+

94

+

95

+

96

+

97

+

98

+

99

+

100

+

101

+

102

+

103

+

104

+

105

+

106

+

107

+

108

+

109

+

110

+

111

+

112

+

113

+

114

+

115

+

116

+

117

+

118

+

119

+

120

+

121

+

122

+

123

+

124

+

125

+

126

+

127

+

128

+

129

+

130

+

131

+

132

+

133

+

134

+

135

+

136

+

137

+

138

+

139

+

140

+

141

+

142

+

143

+

144

+

145

+

146

+

147

+

148

+

149

+

150

+

151

+

152

+

153

+

154

+

155

+

156

+

157

+

158

+

159

+

160

+

161

+

162

+

163

+

164

+

165

+

166

+

167

+

168

+

169

+

170

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

import os 

+

import os.path 

+

import cgi 

+

import tempfile 

+

 

+

from docopt import docopt 

+

from ckan.lib.cli import CkanCommand 

+

 

+

from ckanapi import LocalCKAN 

+

from ckanext.cloudstorage.storage import ( 

+

CloudStorage, 

+

ResourceCloudStorage 

+

) 

+

from ckanext.cloudstorage.model import ( 

+

create_tables, 

+

drop_tables 

+

) 

+

from ckan.logic import NotFound 

+

 

+

USAGE = """ckanext-cloudstorage 

+

 

+

Commands: 

+

- fix-cors Update CORS rules where possible. 

+

- migrate Upload local storage to the remote. 

+

- initdb Reinitalize database tables. 

+

 

+

Usage: 

+

cloudstorage fix-cors <domains>... [--c=<config>] 

+

cloudstorage migrate <path_to_storage> [<resource_id>] [--c=<config>] 

+

cloudstorage initdb [--c=<config>] 

+

 

+

Options: 

+

-c=<config> The CKAN configuration file. 

+

""" 

+

 

+

 

+

class FakeFileStorage(cgi.FieldStorage): 

+

def __init__(self, fp, filename): 

+

self.file = fp 

+

self.filename = filename 

+

 

+

 

+

class PasterCommand(CkanCommand): 

+

summary = 'ckanext-cloudstorage maintence utilities.' 

+

usage = USAGE 

+

 

+

def command(self): 

+

self._load_config() 

+

args = docopt(USAGE, argv=self.args) 

+

 

+

if args['fix-cors']: 

+

_fix_cors(args) 

+

elif args['migrate']: 

+

_migrate(args) 

+

elif args['initdb']: 

+

_initdb() 

+

 

+

 

+

def _migrate(args): 

+

path = args['<path_to_storage>'] 

+

single_id = args['<resource_id>'] 

+

if not os.path.isdir(path): 

+

print('The storage directory cannot be found.') 

+

return 

+

 

+

lc = LocalCKAN() 

+

resources = {} 

+

failed = [] 

+

 

+

# The resource folder is stuctured like so on disk: 

+

# - storage/ 

+

# - ... 

+

# - resources/ 

+

# - <3 letter prefix> 

+

# - <3 letter prefix> 

+

# - <remaining resource_id as filename> 

+

# ... 

+

# ... 

+

# ... 

+

for root, dirs, files in os.walk(path): 

+

# Only the bottom level of the tree actually contains any files. We 

+

# don't care at all about the overall structure. 

+

if not files: 

+

continue 

+

 

+

split_root = root.split('/') 

+

resource_id = split_root[-2] + split_root[-1] 

+

 

+

for file_ in files: 

+

ckan_res_id = resource_id + file_ 

+

if single_id and ckan_res_id != single_id: 

+

continue 

+

 

+

resources[ckan_res_id] = os.path.join( 

+

root, 

+

file_ 

+

) 

+

 

+

for i, resource in enumerate(resources.iteritems(), 1): 

+

resource_id, file_path = resource 

+

print('[{i}/{count}] Working on {id}'.format( 

+

i=i, 

+

count=len(resources), 

+

id=resource_id 

+

)) 

+

 

+

try: 

+

resource = lc.action.resource_show(id=resource_id) 

+

except NotFound: 

+

print(u'\tResource not found') 

+

continue 

+

if resource['url_type'] != 'upload': 

+

print(u'\t`url_type` is not `upload`. Skip') 

+

continue 

+

 

+

with open(file_path, 'rb') as fin: 

+

resource['upload'] = FakeFileStorage( 

+

fin, 

+

resource['url'].split('/')[-1] 

+

) 

+

try: 

+

uploader = ResourceCloudStorage(resource) 

+

uploader.upload(resource['id']) 

+

except Exception as e: 

+

failed.append(resource_id) 

+

print(u'\tError of type {0} during upload: {1}'.format(type(e), e)) 

+

 

+

if failed: 

+

log_file = tempfile.NamedTemporaryFile(delete=False) 

+

log_file.file.writelines(failed) 

+

print(u'ID of all failed uploads are saved to `{0}`'.format(log_file.name)) 

+

 

+

 

+

def _fix_cors(args): 

+

cs = CloudStorage() 

+

 

+

if cs.can_use_advanced_azure: 

+

from azure.storage import blob as azure_blob 

+

from azure.storage import CorsRule 

+

 

+

blob_service = azure_blob.BlockBlobService( 

+

cs.driver_options['key'], 

+

cs.driver_options['secret'] 

+

) 

+

 

+

blob_service.set_blob_service_properties( 

+

cors=[ 

+

CorsRule( 

+

allowed_origins=args['<domains>'], 

+

allowed_methods=['GET'] 

+

) 

+

] 

+

) 

+

print('Done!') 

+

else: 

+

print( 

+

'The driver {driver_name} being used does not currently' 

+

' support updating CORS rules through' 

+

' cloudstorage.'.format( 

+

driver_name=cs.driver_name 

+

) 

+

) 

+

 

+

 

+

def _initdb(): 

+

drop_tables() 

+

create_tables() 

+

print("DB tables are reinitialized") 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_controller_py.html b/cover/ckanext_cloudstorage_controller_py.html new file mode 100644 index 0000000..462adfc --- /dev/null +++ b/cover/ckanext_cloudstorage_controller_py.html @@ -0,0 +1,209 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/controller.py: 69% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

import os.path 

+

 

+

from pylons import c 

+

from pylons.i18n import _ 

+

from ckan import logic, model 

+

from ckan.lib import base, uploader 

+

import ckan.lib.helpers as h 

+

 

+

 

+

class StorageController(base.BaseController): 

+

def resource_download(self, id, resource_id, filename=None): 

+

context = { 

+

'model': model, 

+

'session': model.Session, 

+

'user': c.user or c.author, 

+

'auth_user_obj': c.userobj 

+

} 

+

 

+

try: 

+

resource = logic.get_action('resource_show')( 

+

context, 

+

{ 

+

'id': resource_id 

+

} 

+

) 

+

except logic.NotFound: 

+

base.abort(404, _('Resource not found')) 

+

except logic.NotAuthorized: 

+

base.abort(401, _('Unauthorized to read resource {0}'.format(id))) 

+

 

+

# This isn't a file upload, so either redirect to the source 

+

# (if available) or error out. 

+

if resource.get('url_type') != 'upload': 

+

url = resource.get('url') 

+

if not url: 

+

base.abort(404, _('No download is available')) 

+

h.redirect_to(url) 

+

 

+

if filename is None: 

+

# No filename was provided so we'll try to get one from the url. 

+

filename = os.path.basename(resource['url']) 

+

 

+

upload = uploader.get_resource_uploader(resource) 

+

uploaded_url = upload.get_url_from_filename(resource['id'], filename) 

+

 

+

# The uploaded file is missing for some reason, such as the 

+

# provider being down. 

+

if uploaded_url is None: 

+

base.abort(404, _('No download is available')) 

+

 

+

h.redirect_to(uploaded_url) 

+

 

+

def uploaded_file_redirect(self, upload_to, filename): 

+

'''Redirect static file requests to their location on cloudstorage.''' 

+

upload = uploader.get_uploader('notused') 

+

file_path = upload.path_from_filename(filename) 

+

uploaded_url = upload.get_url_from_path(file_path, use_secure_urls=False) 

+

h.redirect_to(uploaded_url) 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_helpers_py.html b/cover/ckanext_cloudstorage_helpers_py.html new file mode 100644 index 0000000..c99a7f4 --- /dev/null +++ b/cover/ckanext_cloudstorage_helpers_py.html @@ -0,0 +1,111 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/helpers.py: 67% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

from ckanext.cloudstorage.storage import ResourceCloudStorage 

+

 

+

 

+

def use_secure_urls(): 

+

return all([ 

+

ResourceCloudStorage.use_secure_urls.fget(None), 

+

# Currently implemented just AWS version 

+

'S3' in ResourceCloudStorage.driver_name.fget(None) 

+

]) 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_logic___init___py.html b/cover/ckanext_cloudstorage_logic___init___py.html new file mode 100644 index 0000000..ae16c40 --- /dev/null +++ b/cover/ckanext_cloudstorage_logic___init___py.html @@ -0,0 +1,89 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/logic/__init__.py: 100% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+ + + +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_logic_action___init___py.html b/cover/ckanext_cloudstorage_logic_action___init___py.html new file mode 100644 index 0000000..20a8e00 --- /dev/null +++ b/cover/ckanext_cloudstorage_logic_action___init___py.html @@ -0,0 +1,89 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/logic/action/__init__.py: 100% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+ + + +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_logic_action_multipart_py.html b/cover/ckanext_cloudstorage_logic_action_multipart_py.html new file mode 100644 index 0000000..dfbc67a --- /dev/null +++ b/cover/ckanext_cloudstorage_logic_action_multipart_py.html @@ -0,0 +1,635 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/logic/action/multipart.py: 15% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+

75

+

76

+

77

+

78

+

79

+

80

+

81

+

82

+

83

+

84

+

85

+

86

+

87

+

88

+

89

+

90

+

91

+

92

+

93

+

94

+

95

+

96

+

97

+

98

+

99

+

100

+

101

+

102

+

103

+

104

+

105

+

106

+

107

+

108

+

109

+

110

+

111

+

112

+

113

+

114

+

115

+

116

+

117

+

118

+

119

+

120

+

121

+

122

+

123

+

124

+

125

+

126

+

127

+

128

+

129

+

130

+

131

+

132

+

133

+

134

+

135

+

136

+

137

+

138

+

139

+

140

+

141

+

142

+

143

+

144

+

145

+

146

+

147

+

148

+

149

+

150

+

151

+

152

+

153

+

154

+

155

+

156

+

157

+

158

+

159

+

160

+

161

+

162

+

163

+

164

+

165

+

166

+

167

+

168

+

169

+

170

+

171

+

172

+

173

+

174

+

175

+

176

+

177

+

178

+

179

+

180

+

181

+

182

+

183

+

184

+

185

+

186

+

187

+

188

+

189

+

190

+

191

+

192

+

193

+

194

+

195

+

196

+

197

+

198

+

199

+

200

+

201

+

202

+

203

+

204

+

205

+

206

+

207

+

208

+

209

+

210

+

211

+

212

+

213

+

214

+

215

+

216

+

217

+

218

+

219

+

220

+

221

+

222

+

223

+

224

+

225

+

226

+

227

+

228

+

229

+

230

+

231

+

232

+

233

+

234

+

235

+

236

+

237

+

238

+

239

+

240

+

241

+

242

+

243

+

244

+

245

+

246

+

247

+

248

+

249

+

250

+

251

+

252

+

253

+

254

+

255

+

256

+

257

+

258

+

259

+

260

+

261

+

262

+

263

+

264

+

265

+

266

+

267

+

268

+

269

+

270

+

271

+

272

+

273

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

import logging 

+

import datetime 

+

 

+

from pylons import config 

+

from sqlalchemy.orm.exc import NoResultFound 

+

import ckan.model as model 

+

import ckan.lib.helpers as h 

+

import ckan.plugins.toolkit as toolkit 

+

 

+

from ckanext.cloudstorage.storage import ResourceCloudStorage 

+

from ckanext.cloudstorage.model import MultipartUpload, MultipartPart 

+

 

+

log = logging.getLogger(__name__) 

+

 

+

 

+

def _get_max_multipart_lifetime(): 

+

value = float(config.get('ckanext.cloudstorage.max_multipart_lifetime', 7)) 

+

return datetime.timedelta(value) 

+

 

+

 

+

def _get_object_url(uploader, name): 

+

return '/' + uploader.container_name + '/' + name 

+

 

+

 

+

def _delete_multipart(upload, uploader): 

+

resp = uploader.driver.connection.request( 

+

_get_object_url(uploader, upload.name) + '?uploadId=' + upload.id, 

+

method='DELETE' 

+

) 

+

if not resp.success(): 

+

raise toolkit.ValidationError(resp.error) 

+

 

+

upload.delete() 

+

upload.commit() 

+

return resp 

+

 

+

 

+

def _save_part_info(n, etag, upload): 

+

try: 

+

part = model.Session.query(MultipartPart).filter( 

+

MultipartPart.n == n, 

+

MultipartPart.upload == upload).one() 

+

except NoResultFound: 

+

part = MultipartPart(n, etag, upload) 

+

else: 

+

part.etag = etag 

+

part.save() 

+

return part 

+

 

+

 

+

def check_multipart(context, data_dict): 

+

"""Check whether unfinished multipart upload already exists. 

+

 

+

:param context: 

+

:param data_dict: dict with required `id` 

+

:returns: None or dict with `upload` - existing multipart upload info 

+

:rtype: NoneType or dict 

+

 

+

""" 

+

 

+

h.check_access('cloudstorage_check_multipart', data_dict) 

+

id = toolkit.get_or_bust(data_dict, 'id') 

+

try: 

+

upload = model.Session.query(MultipartUpload).filter_by( 

+

resource_id=id).one() 

+

except NoResultFound: 

+

return 

+

upload_dict = upload.as_dict() 

+

upload_dict['parts'] = model.Session.query(MultipartPart).filter( 

+

MultipartPart.upload == upload).count() 

+

return {'upload': upload_dict} 

+

 

+

 

+

def initiate_multipart(context, data_dict): 

+

"""Initiate new Multipart Upload. 

+

 

+

:param context: 

+

:param data_dict: dict with required keys: 

+

id: resource's id 

+

name: filename 

+

size: filesize 

+

 

+

:returns: MultipartUpload info 

+

:rtype: dict 

+

 

+

""" 

+

 

+

h.check_access('cloudstorage_initiate_multipart', data_dict) 

+

id, name, size = toolkit.get_or_bust(data_dict, ['id', 'name', 'size']) 

+

user_id = None 

+

if context['auth_user_obj']: 

+

user_id = context['auth_user_obj'].id 

+

 

+

uploader = ResourceCloudStorage({'multipart_name': name}) 

+

res_name = uploader.path_from_filename(id, name) 

+

 

+

upload_object = MultipartUpload.by_name(res_name) 

+

 

+

if upload_object is not None: 

+

_delete_multipart(upload_object, uploader) 

+

upload_object = None 

+

 

+

if upload_object is None: 

+

for old_upload in model.Session.query(MultipartUpload).filter_by( 

+

resource_id=id): 

+

_delete_multipart(old_upload, uploader) 

+

 

+

_rindex = res_name.rfind('/') 

+

if ~_rindex: 

+

try: 

+

name_prefix = res_name[:_rindex] 

+

for cloud_object in uploader.container.iterate_objects(): 

+

if cloud_object.name.startswith(name_prefix): 

+

log.info('Removing cloud object: %s' % cloud_object) 

+

cloud_object.delete() 

+

except Exception as e: 

+

log.exception('[delete from cloud] %s' % e) 

+

 

+

resp = uploader.driver.connection.request( 

+

_get_object_url(uploader, res_name) + '?uploads', 

+

method='POST' 

+

) 

+

if not resp.success(): 

+

raise toolkit.ValidationError(resp.error) 

+

try: 

+

upload_id = resp.object.find( 

+

'{%s}UploadId' % resp.object.nsmap[None]).text 

+

except AttributeError: 

+

upload_id_list = filter( 

+

lambda e: e.tag.endswith('UploadId'), 

+

resp.object.getchildren() 

+

) 

+

upload_id = upload_id_list[0].text 

+

upload_object = MultipartUpload(upload_id, id, res_name, size, name, user_id) 

+

 

+

upload_object.save() 

+

return upload_object.as_dict() 

+

 

+

 

+

def upload_multipart(context, data_dict): 

+

h.check_access('cloudstorage_upload_multipart', data_dict) 

+

upload_id, part_number, part_content = toolkit.get_or_bust( 

+

data_dict, ['uploadId', 'partNumber', 'upload']) 

+

 

+

uploader = ResourceCloudStorage({}) 

+

upload = model.Session.query(MultipartUpload).get(upload_id) 

+

 

+

resp = uploader.driver.connection.request( 

+

_get_object_url( 

+

uploader, upload.name) + '?partNumber={0}&uploadId={1}'.format( 

+

part_number, upload_id), 

+

method='PUT', 

+

data=bytearray(part_content.file.read()) 

+

) 

+

if resp.status != 200: 

+

raise toolkit.ValidationError('Upload failed: part %s' % part_number) 

+

 

+

_save_part_info(part_number, resp.headers['etag'], upload) 

+

return { 

+

'partNumber': part_number, 

+

'ETag': resp.headers['etag'] 

+

} 

+

 

+

 

+

def finish_multipart(context, data_dict): 

+

"""Called after all parts had been uploaded. 

+

 

+

Triggers call to `_commit_multipart` which will convert separate uploaded 

+

parts into single file 

+

 

+

:param context: 

+

:param data_dict: dict with required key `uploadId` - id of Multipart Upload that should be finished 

+

:returns: None 

+

:rtype: NoneType 

+

 

+

""" 

+

 

+

h.check_access('cloudstorage_finish_multipart', data_dict) 

+

upload_id = toolkit.get_or_bust(data_dict, 'uploadId') 

+

save_action = data_dict.get('save_action', False) 

+

upload = model.Session.query(MultipartUpload).get(upload_id) 

+

chunks = [ 

+

(part.n, part.etag) 

+

for part in model.Session.query(MultipartPart).filter_by( 

+

upload_id=upload_id).order_by(MultipartPart.n) 

+

] 

+

uploader = ResourceCloudStorage({}) 

+

try: 

+

obj = uploader.container.get_object(upload.name) 

+

obj.delete() 

+

except Exception: 

+

pass 

+

uploader.driver._commit_multipart( 

+

_get_object_url(uploader, upload.name), 

+

upload_id, 

+

chunks) 

+

upload.delete() 

+

upload.commit() 

+

 

+

if save_action and save_action == "go-metadata": 

+

try: 

+

res_dict = toolkit.get_action('resource_show')( 

+

context.copy(), {'id': data_dict.get('id')}) 

+

pkg_dict = toolkit.get_action('package_show')( 

+

context.copy(), {'id': res_dict['package_id']}) 

+

if pkg_dict['state'] == 'draft': 

+

toolkit.get_action('package_patch')( 

+

dict(context.copy(), allow_state_change=True), 

+

dict(id=pkg_dict['id'], state='active') 

+

) 

+

except Exception as e: 

+

log.error(e) 

+

return {'commited': True} 

+

 

+

 

+

def abort_multipart(context, data_dict): 

+

h.check_access('cloudstorage_abort_multipart', data_dict) 

+

id = toolkit.get_or_bust(data_dict, ['id']) 

+

uploader = ResourceCloudStorage({}) 

+

 

+

resource_uploads = MultipartUpload.resource_uploads(id) 

+

 

+

aborted = [] 

+

for upload in resource_uploads: 

+

_delete_multipart(upload, uploader) 

+

 

+

aborted.append(upload.id) 

+

 

+

model.Session.commit() 

+

 

+

return aborted 

+

 

+

 

+

def clean_multipart(context, data_dict): 

+

"""Clean old multipart uploads. 

+

 

+

:param context: 

+

:param data_dict: 

+

:returns: dict with: 

+

removed - amount of removed uploads. 

+

total - total amount of expired uploads. 

+

errors - list of errors raised during deletion. Appears when 

+

`total` and `removed` are different. 

+

:rtype: dict 

+

 

+

""" 

+

 

+

h.check_access('cloudstorage_clean_multipart', data_dict) 

+

uploader = ResourceCloudStorage({}) 

+

delta = _get_max_multipart_lifetime() 

+

oldest_allowed = datetime.datetime.utcnow() - delta 

+

 

+

uploads_to_remove = model.Session.query(MultipartUpload).filter( 

+

MultipartUpload.initiated < oldest_allowed 

+

) 

+

 

+

result = { 

+

'removed': 0, 

+

'total': uploads_to_remove.count(), 

+

'errors': [] 

+

} 

+

 

+

for upload in uploads_to_remove: 

+

try: 

+

_delete_multipart(upload, uploader) 

+

except toolkit.ValidationError as e: 

+

result['errors'].append(e.error_summary) 

+

else: 

+

result['removed'] += 1 

+

 

+

return result 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_logic_auth___init___py.html b/cover/ckanext_cloudstorage_logic_auth___init___py.html new file mode 100644 index 0000000..4f3db8c --- /dev/null +++ b/cover/ckanext_cloudstorage_logic_auth___init___py.html @@ -0,0 +1,89 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/logic/auth/__init__.py: 100% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+ + + +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_logic_auth_multipart_py.html b/cover/ckanext_cloudstorage_logic_auth_multipart_py.html new file mode 100644 index 0000000..7b9d694 --- /dev/null +++ b/cover/ckanext_cloudstorage_logic_auth_multipart_py.html @@ -0,0 +1,143 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/logic/auth/multipart.py: 54% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

from ckan.logic import check_access 

+

 

+

 

+

def initiate_multipart(context, data_dict): 

+

return {'success': check_access('resource_create', context, data_dict)} 

+

 

+

 

+

def upload_multipart(context, data_dict): 

+

return {'success': check_access('resource_create', context, data_dict)} 

+

 

+

 

+

def finish_multipart(context, data_dict): 

+

return {'success': check_access('resource_create', context, data_dict)} 

+

 

+

 

+

def abort_multipart(context, data_dict): 

+

return {'success': check_access('resource_create', context, data_dict)} 

+

 

+

 

+

def check_multipart(context, data_dict): 

+

return {'success': check_access('resource_create', context, data_dict)} 

+

 

+

 

+

def clean_multipart(context, data_dict): 

+

return {'success': False} 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_model_py.html b/cover/ckanext_cloudstorage_model_py.html new file mode 100644 index 0000000..dca0020 --- /dev/null +++ b/cover/ckanext_cloudstorage_model_py.html @@ -0,0 +1,237 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/model.py: 69% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

from sqlalchemy.ext.declarative import declarative_base 

+

from sqlalchemy.orm import relationship, backref 

+

import ckan.model as model 

+

from sqlalchemy import ( 

+

Column, 

+

UnicodeText, 

+

DateTime, 

+

ForeignKey, 

+

Integer, 

+

Numeric 

+

) 

+

from datetime import datetime 

+

import ckan.model.meta as meta 

+

from ckan.model.domain_object import DomainObject 

+

 

+

Base = declarative_base() 

+

metadata = Base.metadata 

+

 

+

 

+

def drop_tables(): 

+

metadata.drop_all(model.meta.engine) 

+

 

+

 

+

def create_tables(): 

+

metadata.create_all(model.meta.engine) 

+

 

+

 

+

class MultipartPart(Base, DomainObject): 

+

__tablename__ = 'cloudstorage_multipart_part' 

+

 

+

def __init__(self, n, etag, upload): 

+

self.n = n 

+

self.etag = etag 

+

self.upload = upload 

+

 

+

n = Column(Integer, primary_key=True) 

+

etag = Column(UnicodeText, primary_key=True) 

+

upload_id = Column( 

+

UnicodeText, ForeignKey('cloudstorage_multipart_upload.id'), 

+

primary_key=True 

+

) 

+

upload = relationship( 

+

'MultipartUpload', 

+

backref=backref('parts', cascade='delete, delete-orphan'), 

+

single_parent=True) 

+

 

+

 

+

class MultipartUpload(Base, DomainObject): 

+

__tablename__ = 'cloudstorage_multipart_upload' 

+

 

+

def __init__(self, id, resource_id, name, size, original_name, user_id): 

+

self.id = id 

+

self.resource_id = resource_id 

+

self.name = name 

+

self.size = size 

+

self.original_name = original_name 

+

self.user_id = user_id 

+

 

+

@classmethod 

+

def resource_uploads(cls, resource_id): 

+

query = meta.Session.query(cls).filter_by( 

+

resource_id=resource_id 

+

) 

+

return query 

+

 

+

id = Column(UnicodeText, primary_key=True) 

+

resource_id = Column(UnicodeText) 

+

name = Column(UnicodeText) 

+

initiated = Column(DateTime, default=datetime.utcnow) 

+

size = Column(Numeric) 

+

original_name = Column(UnicodeText) 

+

user_id = Column(UnicodeText) 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_plugin_py.html b/cover/ckanext_cloudstorage_plugin_py.html new file mode 100644 index 0000000..31fc8f9 --- /dev/null +++ b/cover/ckanext_cloudstorage_plugin_py.html @@ -0,0 +1,395 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/plugin.py: 69% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+

75

+

76

+

77

+

78

+

79

+

80

+

81

+

82

+

83

+

84

+

85

+

86

+

87

+

88

+

89

+

90

+

91

+

92

+

93

+

94

+

95

+

96

+

97

+

98

+

99

+

100

+

101

+

102

+

103

+

104

+

105

+

106

+

107

+

108

+

109

+

110

+

111

+

112

+

113

+

114

+

115

+

116

+

117

+

118

+

119

+

120

+

121

+

122

+

123

+

124

+

125

+

126

+

127

+

128

+

129

+

130

+

131

+

132

+

133

+

134

+

135

+

136

+

137

+

138

+

139

+

140

+

141

+

142

+

143

+

144

+

145

+

146

+

147

+

148

+

149

+

150

+

151

+

152

+

153

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

from ckan import plugins 

+

from routes.mapper import SubMapper 

+

import os.path 

+

from ckanext.cloudstorage import storage 

+

from ckanext.cloudstorage import helpers 

+

import ckanext.cloudstorage.logic.action.multipart as m_action 

+

import ckanext.cloudstorage.logic.auth.multipart as m_auth 

+

 

+

 

+

class CloudStoragePlugin(plugins.SingletonPlugin): 

+

plugins.implements(plugins.IUploader) 

+

plugins.implements(plugins.IRoutes, inherit=True) 

+

plugins.implements(plugins.IConfigurable) 

+

plugins.implements(plugins.IConfigurer) 

+

plugins.implements(plugins.IActions) 

+

plugins.implements(plugins.ITemplateHelpers) 

+

plugins.implements(plugins.IAuthFunctions) 

+

plugins.implements(plugins.IResourceController, inherit=True) 

+

 

+

# IConfigurer 

+

 

+

def update_config(self, config): 

+

plugins.toolkit.add_template_directory(config, 'templates') 

+

plugins.toolkit.add_resource('fanstatic/scripts', 'cloudstorage-js') 

+

 

+

# ITemplateHelpers 

+

 

+

def get_helpers(self): 

+

return dict( 

+

cloudstorage_use_secure_urls=helpers.use_secure_urls 

+

) 

+

 

+

def configure(self, config): 

+

 

+

required_keys = ( 

+

'ckanext.cloudstorage.driver', 

+

'ckanext.cloudstorage.driver_options', 

+

'ckanext.cloudstorage.container_name' 

+

) 

+

 

+

for rk in required_keys: 

+

if config.get(rk) is None: 

+

raise RuntimeError( 

+

'Required configuration option {0} not found.'.format( 

+

rk 

+

) 

+

) 

+

 

+

def get_resource_uploader(self, data_dict): 

+

# We provide a custom Resource uploader. 

+

return storage.ResourceCloudStorage(data_dict) 

+

 

+

def get_uploader(self, upload_to, old_filename=None): 

+

# Custom uploader for generic file uploads 

+

return storage.FileCloudStorage(upload_to, old_filename) 

+

 

+

def before_map(self, map): 

+

sm = SubMapper( 

+

map, 

+

controller='ckanext.cloudstorage.controller:StorageController' 

+

) 

+

 

+

# Override the resource download controllers so we can do our 

+

# lookup with libcloud. 

+

with sm: 

+

sm.connect( 

+

'resource_download', 

+

'/dataset/{id}/resource/{resource_id}/download', 

+

action='resource_download' 

+

) 

+

sm.connect( 

+

'resource_download', 

+

'/dataset/{id}/resource/{resource_id}/download/{filename}', 

+

action='resource_download' 

+

) 

+

 

+

sm.connect( 

+

'uploaded_file', 

+

'/uploads/{upload_to}/{filename}', 

+

action='uploaded_file_redirect' 

+

) 

+

 

+

return map 

+

 

+

# IActions 

+

 

+

def get_actions(self): 

+

return { 

+

'cloudstorage_initiate_multipart': m_action.initiate_multipart, 

+

'cloudstorage_upload_multipart': m_action.upload_multipart, 

+

'cloudstorage_finish_multipart': m_action.finish_multipart, 

+

'cloudstorage_abort_multipart': m_action.abort_multipart, 

+

'cloudstorage_check_multipart': m_action.check_multipart, 

+

'cloudstorage_clean_multipart': m_action.clean_multipart, 

+

} 

+

 

+

# IAuthFunctions 

+

 

+

def get_auth_functions(self): 

+

return { 

+

'cloudstorage_initiate_multipart': m_auth.initiate_multipart, 

+

'cloudstorage_upload_multipart': m_auth.upload_multipart, 

+

'cloudstorage_finish_multipart': m_auth.finish_multipart, 

+

'cloudstorage_abort_multipart': m_auth.abort_multipart, 

+

'cloudstorage_check_multipart': m_auth.check_multipart, 

+

'cloudstorage_clean_multipart': m_auth.clean_multipart, 

+

} 

+

 

+

# IResourceController 

+

 

+

def before_delete(self, context, resource, resources): 

+

# let's get all info about our resource. It somewhere in resources 

+

# but if there is some possibility that it isn't(magic?) we have 

+

# `else` clause 

+

 

+

for res in resources: 

+

if res['id'] == resource['id']: 

+

break 

+

else: 

+

return 

+

# just ignore simple links 

+

if res['url_type'] != 'upload': 

+

return 

+

 

+

# we don't want to change original item from resources, just in case 

+

# someone will use it in another `before_delete`. So, let's copy it 

+

# and add `clear_upload` flag 

+

res_dict = dict(res.items() + [('clear_upload', True)]) 

+

 

+

uploader = self.get_resource_uploader(res_dict) 

+

 

+

# to be on the safe side, let's check existence of container 

+

container = getattr(uploader, 'container', None) 

+

if container is None: 

+

return 

+

 

+

# and now uploader removes our file. 

+

uploader.upload(resource['id']) 

+

 

+

# and all other files linked to this resource 

+

if not uploader.leave_files: 

+

upload_path = os.path.dirname( 

+

uploader.path_from_filename( 

+

resource['id'], 

+

'fake-name' 

+

) 

+

) 

+

 

+

for old_file in uploader.container.iterate_objects(): 

+

if old_file.name.startswith(upload_path): 

+

old_file.delete() 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_storage_py.html b/cover/ckanext_cloudstorage_storage_py.html new file mode 100644 index 0000000..678c655 --- /dev/null +++ b/cover/ckanext_cloudstorage_storage_py.html @@ -0,0 +1,1087 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/storage.py: 66% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+

75

+

76

+

77

+

78

+

79

+

80

+

81

+

82

+

83

+

84

+

85

+

86

+

87

+

88

+

89

+

90

+

91

+

92

+

93

+

94

+

95

+

96

+

97

+

98

+

99

+

100

+

101

+

102

+

103

+

104

+

105

+

106

+

107

+

108

+

109

+

110

+

111

+

112

+

113

+

114

+

115

+

116

+

117

+

118

+

119

+

120

+

121

+

122

+

123

+

124

+

125

+

126

+

127

+

128

+

129

+

130

+

131

+

132

+

133

+

134

+

135

+

136

+

137

+

138

+

139

+

140

+

141

+

142

+

143

+

144

+

145

+

146

+

147

+

148

+

149

+

150

+

151

+

152

+

153

+

154

+

155

+

156

+

157

+

158

+

159

+

160

+

161

+

162

+

163

+

164

+

165

+

166

+

167

+

168

+

169

+

170

+

171

+

172

+

173

+

174

+

175

+

176

+

177

+

178

+

179

+

180

+

181

+

182

+

183

+

184

+

185

+

186

+

187

+

188

+

189

+

190

+

191

+

192

+

193

+

194

+

195

+

196

+

197

+

198

+

199

+

200

+

201

+

202

+

203

+

204

+

205

+

206

+

207

+

208

+

209

+

210

+

211

+

212

+

213

+

214

+

215

+

216

+

217

+

218

+

219

+

220

+

221

+

222

+

223

+

224

+

225

+

226

+

227

+

228

+

229

+

230

+

231

+

232

+

233

+

234

+

235

+

236

+

237

+

238

+

239

+

240

+

241

+

242

+

243

+

244

+

245

+

246

+

247

+

248

+

249

+

250

+

251

+

252

+

253

+

254

+

255

+

256

+

257

+

258

+

259

+

260

+

261

+

262

+

263

+

264

+

265

+

266

+

267

+

268

+

269

+

270

+

271

+

272

+

273

+

274

+

275

+

276

+

277

+

278

+

279

+

280

+

281

+

282

+

283

+

284

+

285

+

286

+

287

+

288

+

289

+

290

+

291

+

292

+

293

+

294

+

295

+

296

+

297

+

298

+

299

+

300

+

301

+

302

+

303

+

304

+

305

+

306

+

307

+

308

+

309

+

310

+

311

+

312

+

313

+

314

+

315

+

316

+

317

+

318

+

319

+

320

+

321

+

322

+

323

+

324

+

325

+

326

+

327

+

328

+

329

+

330

+

331

+

332

+

333

+

334

+

335

+

336

+

337

+

338

+

339

+

340

+

341

+

342

+

343

+

344

+

345

+

346

+

347

+

348

+

349

+

350

+

351

+

352

+

353

+

354

+

355

+

356

+

357

+

358

+

359

+

360

+

361

+

362

+

363

+

364

+

365

+

366

+

367

+

368

+

369

+

370

+

371

+

372

+

373

+

374

+

375

+

376

+

377

+

378

+

379

+

380

+

381

+

382

+

383

+

384

+

385

+

386

+

387

+

388

+

389

+

390

+

391

+

392

+

393

+

394

+

395

+

396

+

397

+

398

+

399

+

400

+

401

+

402

+

403

+

404

+

405

+

406

+

407

+

408

+

409

+

410

+

411

+

412

+

413

+

414

+

415

+

416

+

417

+

418

+

419

+

420

+

421

+

422

+

423

+

424

+

425

+

426

+

427

+

428

+

429

+

430

+

431

+

432

+

433

+

434

+

435

+

436

+

437

+

438

+

439

+

440

+

441

+

442

+

443

+

444

+

445

+

446

+

447

+

448

+

449

+

450

+

451

+

452

+

453

+

454

+

455

+

456

+

457

+

458

+

459

+

460

+

461

+

462

+

463

+

464

+

465

+

466

+

467

+

468

+

469

+

470

+

471

+

472

+

473

+

474

+

475

+

476

+

477

+

478

+

479

+

480

+

481

+

482

+

483

+

484

+

485

+

486

+

487

+

488

+

489

+

490

+

491

+

492

+

493

+

494

+

495

+

496

+

497

+

498

+

499

+ + 		+

#!/usr/bin/env python 

+

# -*- coding: utf-8 -*- 

+

import cgi 

+

import mimetypes 

+

import os.path 

+

import urlparse 

+

from ast import literal_eval 

+

from datetime import timedelta 

+

import datetime 

+

from pylons import config 

+

from ckan import model 

+

from ckan.lib import munge 

+

import ckan.plugins as p 

+

 

+

from libcloud.storage.types import Provider, ObjectDoesNotExistError 

+

from libcloud.storage.providers import get_driver 

+

 

+

 

+

class CloudStorage(object): 

+

def __init__(self): 

+

self.driver = get_driver( 

+

getattr( 

+

Provider, 

+

self.driver_name 

+

) 

+

)(**self.driver_options) 

+

self._container = None 

+

 

+

def path_from_filename(self, rid, filename): 

+

raise NotImplemented 

+

 

+

@property 

+

def container(self): 

+

""" 

+

Return the currently configured libcloud container. 

+

""" 

+

if self._container is None: 

+

self._container = self.driver.get_container( 

+

container_name=self.container_name 

+

) 

+

 

+

return self._container 

+

 

+

@property 

+

def driver_options(self): 

+

""" 

+

A dictionary of options ckanext-cloudstorage has been configured to 

+

pass to the apache-libcloud driver. 

+

""" 

+

return literal_eval(config['ckanext.cloudstorage.driver_options']) 

+

 

+

@property 

+

def driver_name(self): 

+

""" 

+

The name of the driver (ex: AZURE_BLOBS, S3) that ckanext-cloudstorage 

+

is configured to use. 

+

 

+

 

+

.. note:: 

+

 

+

This value is used to lookup the apache-libcloud driver to use 

+

based on the Provider enum. 

+

""" 

+

return config['ckanext.cloudstorage.driver'] 

+

 

+

@property 

+

def container_name(self): 

+

""" 

+

The name of the container (also called buckets on some providers) 

+

ckanext-cloudstorage is configured to use. 

+

""" 

+

return config['ckanext.cloudstorage.container_name'] 

+

 

+

@property 

+

def use_secure_urls(self): 

+

""" 

+

`True` if ckanext-cloudstroage is configured to generate secure 

+

one-time URLs to resources, `False` otherwise. 

+

""" 

+

return p.toolkit.asbool( 

+

config.get('ckanext.cloudstorage.use_secure_urls', False) 

+

) 

+

 

+

@property 

+

def leave_files(self): 

+

""" 

+

`True` if ckanext-cloudstorage is configured to leave files on the 

+

provider instead of removing them when a resource/package is deleted, 

+

otherwise `False`. 

+

""" 

+

return p.toolkit.asbool( 

+

config.get('ckanext.cloudstorage.leave_files', False) 

+

) 

+

 

+

@property 

+

def can_use_advanced_azure(self): 

+

""" 

+

`True` if the `azure-storage` module is installed and 

+

ckanext-cloudstorage has been configured to use Azure, otherwise 

+

`False`. 

+

""" 

+

# Are we even using Azure? 

+

if self.driver_name == 'AZURE_BLOBS': 

+

try: 

+

# Yes? Is the azure-storage package available? 

+

from azure import storage 

+

# Shut the linter up. 

+

assert storage 

+

return True 

+

except ImportError: 

+

pass 

+

 

+

return False 

+

 

+

@property 

+

def can_use_advanced_aws(self): 

+

""" 

+

`True` if the `boto` module is installed and ckanext-cloudstorage has 

+

been configured to use Amazon S3, otherwise `False`. 

+

""" 

+

# Are we even using AWS? 

+

if 'S3' in self.driver_name: 

+

try: 

+

# Yes? Is the boto package available? 

+

import boto 

+

# Shut the linter up. 

+

assert boto 

+

return True 

+

except ImportError: 

+

pass 

+

 

+

return False 

+

 

+

@property 

+

def can_use_advanced_google_cloud(self): 

+

""" 

+

`True` if the `google-cloud` module is installed and 

+

ckanext-cloudstorage has been configured to use Google Cloud Storage, 

+

otherwise `False`. 

+

""" 

+

# Are we even using google cloud? 

+

if 'GOOGLE_STORAGE' in self.driver_name: 

+

try: 

+

# Yes? is the google-cloud-storage package available? 

+

from google.cloud import storage 

+

# shut the linter up. 

+

assert storage 

+

return True 

+

except ImportError: 

+

pass 

+

 

+

return False 

+

 

+

@property 

+

def guess_mimetype(self): 

+

""" 

+

`True` if ckanext-cloudstorage is configured to guess mime types, 

+

`False` otherwise. 

+

""" 

+

return p.toolkit.asbool( 

+

config.get('ckanext.cloudstorage.guess_mimetype', False) 

+

) 

+

 

+

def get_object_public_url(self, filename): 

+

""" 

+

Returns the public url of an object. 

+

Raises `NotImplementedError` for drivers yet unsupported, or when 

+

`use_secure_urls` is set to `True`. 

+

 

+

Assumes container is made public. 

+

""" 

+

if self.driver_name == 'GOOGLE_STORAGE': 

+

if self.use_secure_urls: 

+

raise NotImplementedError("Should be pretty easy though!") 

+

return "https://storage.googleapis.com/{0}/{1}" \ 

+

.format(self.container_name, 

+

self.path_from_filename(filename)) 

+

else: 

+

raise NotImplementedError( 

+

"This method hasn't been implemented yet for this driver.") 

+

 

+

def upload_to_path(self, file_path): 

+

""" 

+

Upload to storage bucket 

+

 

+

:param file_path: File path in storage bucket 

+

:param old_file_path: File path of old file in storage bucket. 

+

""" 

+

 

+

if self.can_use_advanced_azure: 

+

from azure.storage import blob as azure_blob 

+

from azure.storage.blob.models import ContentSettings 

+

 

+

blob_service = azure_blob.BlockBlobService( 

+

self.driver_options['key'], 

+

self.driver_options['secret'] 

+

) 

+

content_settings = None 

+

if self.guess_mimetype: 

+

content_type, _ = mimetypes.guess_type(file_path) 

+

if content_type: 

+

content_settings = ContentSettings( 

+

content_type=content_type 

+

) 

+

 

+

return blob_service.create_blob_from_stream( 

+

container_name=self.container_name, 

+

blob_name=file_path, 

+

stream=self.file_upload, 

+

content_settings=content_settings 

+

) 

+

else: 

+

self.container.upload_object_via_stream( 

+

self.file_upload, 

+

object_name=file_path 

+

) 

+

 

+

def delete_object_from_path(self, file_path): 

+

""" 

+

Delete object from cloudstorage at `file_path` 

+

:param file_path: Path of file to be deletedd 

+

""" 

+

try: 

+

self.container.delete_object( 

+

self.container.get_object( 

+

file_path 

+

) 

+

) 

+

except ObjectDoesNotExistError: 

+

# It's possible for the object to have already been deleted, or 

+

# for it to not yet exist in a committed state due to an 

+

# outstanding lease. 

+

return 

+

 

+

def get_url_from_path(self, path, use_secure_urls): 

+

""" 

+

Retrieve a publically accessible URL for the given path 

+

 

+

.. note:: 

+

 

+

Works for Azure and any libcloud driver that implements 

+

support for get_object_cdn_url (ex: AWS S3, Google Storage). 

+

 

+

:param path: The resource path. 

+

 

+

:returns: Externally accessible URL or None. 

+

""" 

+

# If advanced azure features are enabled, generate a temporary 

+

# shared access link instead of simply redirecting to the file. 

+

if use_secure_urls: 

+

if self.can_use_advanced_azure: 

+

from azure.storage import blob as azure_blob 

+

 

+

blob_service = azure_blob.BlockBlobService( 

+

self.driver_options['key'], 

+

self.driver_options['secret'] 

+

) 

+

 

+

return blob_service.make_blob_url( 

+

container_name=self.container_name, 

+

blob_name=path, 

+

sas_token=blob_service.generate_blob_shared_access_signature( 

+

container_name=self.container_name, 

+

blob_name=path, 

+

expiry=datetime.utcnow() + timedelta(hours=1), 

+

permission=azure_blob.BlobPermissions.READ 

+

) 

+

) 

+

elif self.can_use_advanced_aws: 

+

from boto.s3.connection import S3Connection 

+

s3_connection = S3Connection( 

+

self.driver_options['key'], 

+

self.driver_options['secret'] 

+

) 

+

return s3_connection.generate_url( 

+

expires_in=60 * 60, 

+

method='GET', 

+

bucket=self.container_name, 

+

query_auth=True, 

+

key=path 

+

) 

+

 

+

elif self.can_use_advanced_google_cloud: 

+

from google.cloud import storage 

+

 

+

client = storage.client.Client.from_service_account_json( 

+

self.driver_options['secret'] 

+

) 

+

 

+

bucket = client.get_bucket(self.container_name) 

+

blob = bucket.get_blob(path) 

+

return blob.generate_signed_url( 

+

expiration=timedelta(seconds=60*60), 

+

method='GET', 

+

) 

+

else: 

+

raise Exception('Unable to generate secure url. Is your cloud \ 

+

provider\'s driver installed?') 

+

 

+

# Find the object for the given key. 

+

obj = self.container.get_object(path) 

+

if obj is None: 

+

return 

+

 

+

# Not supported by all providers! 

+

try: 

+

return self.driver.get_object_cdn_url(obj) 

+

except NotImplementedError: 

+

if 'S3' in self.driver_name or 'GOOGLE_STORAGE' in self.driver_name: 

+

return urlparse.urljoin( 

+

'https://' + self.driver.connection.host, 

+

'{container}/{path}'.format( 

+

container=self.container_name, 

+

path=path 

+

) 

+

) 

+

# This extra 'url' property isn't documented anywhere, sadly. 

+

# See azure_blobs.py:_xml_to_object for more. 

+

elif 'url' in obj.extra: 

+

return obj.extra['url'] 

+

raise 

+

 

+

 

+

class ResourceCloudStorage(CloudStorage): 

+

def __init__(self, resource): 

+

""" 

+

Support for uploading resources to any storage provider 

+

implemented by the apache-libcloud library. 

+

 

+

:param resource: The resource dict. 

+

""" 

+

super(ResourceCloudStorage, self).__init__() 

+

 

+

self.filename = None 

+

self.old_filename = None 

+

self.file = None 

+

self.resource = resource 

+

 

+

upload_field_storage = resource.pop('upload', None) 

+

self._clear = resource.pop('clear_upload', None) 

+

multipart_name = resource.pop('multipart_name', None) 

+

 

+

# Check to see if a file has been provided 

+

if isinstance(upload_field_storage, cgi.FieldStorage): 

+

self.filename = munge.munge_filename(upload_field_storage.filename) 

+

self.file_upload = upload_field_storage.file 

+

resource['url'] = self.filename 

+

resource['url_type'] = 'upload' 

+

elif multipart_name and self.can_use_advanced_aws: 

+

# This means that file was successfully uploaded and stored 

+

# at cloud. 

+

# Currently implemented just AWS version 

+

resource['url'] = munge.munge_filename(multipart_name) 

+

resource['url_type'] = 'upload' 

+

elif self._clear and resource.get('id'): 

+

# Apparently, this is a created-but-not-commited resource whose 

+

# file upload has been canceled. We're copying the behaviour of 

+

# ckaenxt-s3filestore here. 

+

old_resource = model.Session.query( 

+

model.Resource 

+

).get( 

+

resource['id'] 

+

) 

+

 

+

self.old_filename = old_resource.url 

+

resource['url_type'] = '' 

+

 

+

def path_from_filename(self, rid, filename): 

+

""" 

+

Returns a bucket path for the given resource_id and filename. 

+

 

+

:param rid: The resource ID. 

+

:param filename: The unmunged resource filename. 

+

""" 

+

return os.path.join( 

+

'resources', 

+

rid, 

+

munge.munge_filename(filename) 

+

) 

+

 

+

def upload(self, id, max_size=10): 

+

""" 

+

Complete the file upload, or clear an existing upload. 

+

 

+

:param id: The resource_id. 

+

:param max_size: Ignored. 

+

""" 

+

# If a filename has been provided (a file is being uplaoded) write the 

+

# file to the appropriate key in the container 

+

if self.filename: 

+

file_path = self.path_from_filename(id, self.filename) 

+

self.upload_to_path(file_path) 

+

if self._clear and self.old_filename and not self.leave_files: 

+

old_file_path = self.path_from_filename(id, self.old_filename) 

+

self.delete_object_from_path(old_file_path) 

+

 

+

def get_url_from_filename(self, id, filename): 

+

""" 

+

Generate public URL from resource id and filename 

+

:param id: The resource ID 

+

:param filename: The resource filename 

+

""" 

+

path = self.path_from_filename(id, filename) 

+

return self.get_url_from_path(path, self.use_secure_urls) 

+

 

+

@property 

+

def package(self): 

+

return model.Package.get(self.resource['package_id']) 

+

 

+

 

+

class FileCloudStorage(CloudStorage): 

+

""" 

+

Support upload of general files to cloudstorage. 

+

""" 

+

def __init__(self, upload_to, old_filename=None): 

+

super(FileCloudStorage, self).__init__() 

+

 

+

self.filename = None 

+

self.filepath = None 

+

self.old_filename = old_filename 

+

if self.old_filename: 

+

self.old_filepath = self.path_from_filename(old_filename) 

+

 

+

def path_from_filename(self, filename): 

+

""" 

+

Returns a bucket path for the given filename. 

+

 

+

:param: filename: The unmunged filename. 

+

""" 

+

return os.path.join( 

+

'storage', 

+

'uploads', 

+

munge.munge_filename(filename) 

+

) 

+

 

+

def update_data_dict(self, data_dict, url_field, file_field, clear_field): 

+

""" 

+

Manipulate data from the data_dict. THis needs to be called before it 

+

reaches any validators. 

+

 

+

:param url_field: Name of the field where the upload is going to be 

+

:param file_field: Name of the key where the FieldStorage is kept (i.e. 

+

the field where the file data actually is). 

+

:param clear_field: Name of a boolean field which requests the upload 

+

to be deleted 

+

""" 

+

print('before update dict', self.filename, self.filepath) 

+

print('args', data_dict, url_field, file_field, clear_field) 

+

print('clear_field value', data_dict.get(clear_field)) 

+

self.url = data_dict.get(url_field, '') 

+

self._clear = data_dict.pop(clear_field, None) 

+

self.file_field = file_field 

+

self.upload_field_storage = data_dict.pop(file_field, None) 

+

print('upload field', self.upload_field_storage) 

+

print(self.url, self._clear, self.file_field, self.upload_field_storage) 

+

 

+

if hasattr(self.upload_field_storage, 'filename'): 

+

self.filename = self.upload_field_storage.filename 

+

self.filename = str(datetime.datetime.utcnow()) + self.filename 

+

self.filename = munge.munge_filename_legacy(self.filename) 

+

self.filepath = self.path_from_filename(self.filename) 

+

data_dict[url_field] = self.filename 

+

self.file_upload = self.upload_field_storage.file 

+

# keep the file if there has been no change 

+

elif self.old_filename and not self.old_filename.startswith('http'): 

+

if not self._clear: 

+

data_dict[url_field] = self.old_filename 

+

if self._clear and self.url == self.old_filename: 

+

data_dict[url_field] = '' 

+

print('after update dict', self.filename, self.filepath) 

+

 

+

def upload(self, max_size=2): 

+

""" 

+

Complete the fileupload, or clear an existing upload. 

+

 

+

This should happen just before a commit but after the data has 

+

been validated and flushed to the db. This is so we do not store 

+

anything unless the request is actually good. 

+

:param max_size: ignored 

+

""" 

+

print('upload', self.filename, self._clear, self.old_filename, self.leave_files) 

+

if self.filename: 

+

file_path = self.path_from_filename(self.filename) 

+

return self.upload_to_path(file_path) 

+

if self._clear and self.old_filename and not self.leave_files: 

+

old_file_path = self.path_from_filename(self.old_filename) 

+

self.delete_object_from_path(old_file_path) 

+

 

+

def get_url_from_filename(self, filename): 

+

""" 

+

Get public url from filename 

+

:param filename: name of file 

+

""" 

+

path = self.path_from_filename(filename) 

+

# We don't want to use secure urls for normal file uploads. 

+

# Doing so would cause assets caching issues such as the logo 

+

# to be reloaded on every page load. 

+

return self.get_url_from_path(path, use_secure_urls=False) 

+

 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_tests_test_controller_py.html b/cover/ckanext_cloudstorage_tests_test_controller_py.html new file mode 100644 index 0000000..c9a30a6 --- /dev/null +++ b/cover/ckanext_cloudstorage_tests_test_controller_py.html @@ -0,0 +1,341 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/tests/test_controller.py: 97% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+

75

+

76

+

77

+

78

+

79

+

80

+

81

+

82

+

83

+

84

+

85

+

86

+

87

+

88

+

89

+

90

+

91

+

92

+

93

+

94

+

95

+

96

+

97

+

98

+

99

+

100

+

101

+

102

+

103

+

104

+

105

+

106

+

107

+

108

+

109

+

110

+

111

+

112

+

113

+

114

+

115

+

116

+

117

+

118

+

119

+

120

+

121

+

122

+

123

+

124

+

125

+

126

+ + 		+

import os 

+

 

+

from nose.tools import assert_equal, assert_true 

+

from mock import patch, create_autospec, MagicMock 

+

 

+

import ckan.plugins 

+

import ckan.tests.helpers as helpers 

+

import ckan.tests.factories as factories 

+

from webtest import Upload 

+

 

+

from ckan.common import config 

+

import ckanapi 

+

from libcloud.storage.types import Provider 

+

from libcloud.storage.providers import get_driver 

+

 

+

from ckanext.cloudstorage.controller import StorageController 

+

 

+

google_driver = get_driver(Provider.GOOGLE_STORAGE) 

+

 

+

 

+

class Uploader(Upload): 

+

"""Extends webtest's Upload class a bit more so it actually stores file data. 

+

""" 

+

 

+

def __init__(self, *args, **kwargs): 

+

self.file = kwargs.pop('file') 

+

super(Uplaoder, self).__init__(*args, **kwargs) 

+

 

+

 

+

 

+

class TestStorageController(helpers.FunctionalTestBase): 

+

def _upload_resource(self): 

+

factories.Sysadmin(apikey='my-test-key') 

+

 

+

app = self._get_test_app() 

+

demo = ckanapi.TestAppCKAN(app, apikey='my-test-key') 

+

factories.Dataset(name='my-dataset') 

+

 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

resource = demo.action.resource_create(package_id='my-dataset', 

+

upload=open(file_path), 

+

url='file.txt') 

+

return resource, demo, app 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@helpers.change_config('ckan.site_url', 'http://mytest.ckan.net') 

+

def test_resource_show_url(self, get_driver): 

+

"""The resource_show url is expected for uploaded resource file.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

 

+

resource, demo, _ = self._upload_resource() 

+

 

+

# does resource_show have the expected resource file url? 

+

resource_show = demo.action.resource_show(id=resource['id']) 

+

 

+

expected_url = 'http://mytest.ckan.net/dataset/{0}/resource/{1}/download/data.csv' \ 

+

.format(resource['package_id'], resource['id']) 

+

 

+

assert_equal(resource_show['url'], expected_url) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@helpers.change_config('ckan.site_url', 'http://localhost:5000') 

+

def test_resource_download_s3(self, get_driver): 

+

"""A resource uploaded to S3 ckan be downloaded.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

 

+

resource, demo, app = self._upload_resource() 

+

resource_show = demo.action.resource_show(id=resource['id']) 

+

resource_file_url = resource_show['url'] 

+

 

+

assert_equal(resource_file_url, u'{2}/dataset/{0}/resource/{1}/download/data.csv' 

+

.format(resource['package_id'], resource['id'], 'http://localhost:5000')) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@patch('ckanext.cloudstorage.controller.h') 

+

def test_resource_download_s3_no_filename(self, h, get_driver): 

+

"""A resource uploaded can be downloaded when no filename in url.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

 

+

resource, demo, app = self._upload_resource() 

+

 

+

resource_file_url = '/dataset/{0}/resource/{1}/download' \ 

+

.format(resource['package_id'], resource['id']) 

+

 

+

mock_driver.get_object_cdn_url.return_value = resource_file_url 

+

 

+

file_response = app.get(resource_file_url) 

+

 

+

h.redirect_to.assert_called_with(resource_file_url) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@patch('ckanext.cloudstorage.controller.h') 

+

def test_resource_download_url_link(self, h, get_driver): 

+

"""A resource with a url (not a file) is redirected correctly.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

mock_driver.get_object_cdn_url.return_value = 'http://example' 

+

 

+

factories.Sysadmin(apikey='my-test-apikey') 

+

 

+

app = self._get_test_app() 

+

demo = ckanapi.TestAppCKAN(app, apikey='my-test-apikey') 

+

dataset = factories.Dataset() 

+

 

+

resource = demo.action.resource_create(package_id=dataset['id'], 

+

url='http://example') 

+

resource_show = demo.action.resource_show(id=resource['id']) 

+

resource_file_url = '/dataset/{0}/resource/{1}/download' \ 

+

.format(resource['package_id'], resource['id']) 

+

assert_equal(resource_show['url'], 'http://example') 

+

 

+

# attempt redirect to linked url 

+

r = app.get(resource_file_url) 

+

h.redirect_to.assert_called_with('http://example') 

+

 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_tests_test_plugin_py.html b/cover/ckanext_cloudstorage_tests_test_plugin_py.html new file mode 100644 index 0000000..29209ce --- /dev/null +++ b/cover/ckanext_cloudstorage_tests_test_plugin_py.html @@ -0,0 +1,159 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/tests/test_plugin.py: 100% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+ + 		+

import os 

+

from nose.tools import assert_equal, assert_raises 

+

from mock import patch, MagicMock 

+

 

+

from ckan.tests import helpers, factories 

+

from ckan.lib import helpers as h 

+

 

+

import ckanapi 

+

 

+

from ckanext.cloudstorage.controller import StorageController 

+

class TestPlugin(helpers.FunctionalTestBase): 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@patch('ckanext.cloudstorage.controller.StorageController', spec=StorageController) 

+

def test_resource_download_calls_ext_method(self, resource_download, get_driver): 

+

""" 

+

Test `ckanext.cloudstorage.controller.StorageController.resource_download` is called for `resource_download` action. 

+

""" 

+

app = self._get_test_app() 

+

demo = ckanapi.TestAppCKAN(app, apikey='my-test-apikey') 

+

factories.Sysadmin(apikey='my-test-apikey') 

+

 

+

factories.Dataset(name='my-dataset') 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

resource = demo.action.resource_create( 

+

package_id='my-dataset', 

+

upload=open(file_path), 

+

url='file.txt' 

+

) 

+

 

+

# proves it's calling the right code, right? 

+

with assert_raises(TypeError) as exc: 

+

r = app.get(resource['url']) 

+

assert_equal(exc.exception.message, "'MagicMock' object is not iterable") 

+

resource_download.assert_called_once() 

+ +
+
+ + + + + diff --git a/cover/ckanext_cloudstorage_tests_test_storage_py.html b/cover/ckanext_cloudstorage_tests_test_storage_py.html new file mode 100644 index 0000000..f60e88c --- /dev/null +++ b/cover/ckanext_cloudstorage_tests_test_storage_py.html @@ -0,0 +1,599 @@ + + + + + + + + + + + Coverage for ckanext/cloudstorage/tests/test_storage.py: 100% + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ r + m + x + p   toggle line displays +

+

+ j + k   next/prev highlighted chunk +

+

+ 0   (zero) top of page +

+

+ 1   (one) first highlighted chunk +

+
+
+ +
+ + + + + +
+

1

+

2

+

3

+

4

+

5

+

6

+

7

+

8

+

9

+

10

+

11

+

12

+

13

+

14

+

15

+

16

+

17

+

18

+

19

+

20

+

21

+

22

+

23

+

24

+

25

+

26

+

27

+

28

+

29

+

30

+

31

+

32

+

33

+

34

+

35

+

36

+

37

+

38

+

39

+

40

+

41

+

42

+

43

+

44

+

45

+

46

+

47

+

48

+

49

+

50

+

51

+

52

+

53

+

54

+

55

+

56

+

57

+

58

+

59

+

60

+

61

+

62

+

63

+

64

+

65

+

66

+

67

+

68

+

69

+

70

+

71

+

72

+

73

+

74

+

75

+

76

+

77

+

78

+

79

+

80

+

81

+

82

+

83

+

84

+

85

+

86

+

87

+

88

+

89

+

90

+

91

+

92

+

93

+

94

+

95

+

96

+

97

+

98

+

99

+

100

+

101

+

102

+

103

+

104

+

105

+

106

+

107

+

108

+

109

+

110

+

111

+

112

+

113

+

114

+

115

+

116

+

117

+

118

+

119

+

120

+

121

+

122

+

123

+

124

+

125

+

126

+

127

+

128

+

129

+

130

+

131

+

132

+

133

+

134

+

135

+

136

+

137

+

138

+

139

+

140

+

141

+

142

+

143

+

144

+

145

+

146

+

147

+

148

+

149

+

150

+

151

+

152

+

153

+

154

+

155

+

156

+

157

+

158

+

159

+

160

+

161

+

162

+

163

+

164

+

165

+

166

+

167

+

168

+

169

+

170

+

171

+

172

+

173

+

174

+

175

+

176

+

177

+

178

+

179

+

180

+

181

+

182

+

183

+

184

+

185

+

186

+

187

+

188

+

189

+

190

+

191

+

192

+

193

+

194

+

195

+

196

+

197

+

198

+

199

+

200

+

201

+

202

+

203

+

204

+

205

+

206

+

207

+

208

+

209

+

210

+

211

+

212

+

213

+

214

+

215

+

216

+

217

+

218

+

219

+

220

+

221

+

222

+

223

+

224

+

225

+

226

+

227

+

228

+

229

+

230

+

231

+

232

+

233

+

234

+

235

+

236

+

237

+

238

+

239

+

240

+

241

+

242

+

243

+

244

+

245

+

246

+

247

+

248

+

249

+

250

+

251

+

252

+

253

+

254

+

255

+ + 		+

import os 

+

from nose.tools import assert_equal 

+

from mock import create_autospec, patch, MagicMock 

+

import datetime 

+

 

+

import ckanapi 

+

from webtest import Upload 

+

 

+

from ckan.tests import helpers, factories 

+

from ckan.plugins import toolkit 

+

from ckanext.cloudstorage.storage import ResourceCloudStorage, FileCloudStorage 

+

 

+

from pylons import config 

+

 

+

from libcloud.storage.types import Provider 

+

from libcloud.storage.providers import get_driver 

+

 

+

google_driver = get_driver(Provider.GOOGLE_STORAGE) 

+

 

+

 

+

class Uploader(Upload): 

+

"""Extends webtest's Upload class a bit more so it actually stores file data. 

+

""" 

+

 

+

def __init__(self, *args, **kwargs): 

+

self.file = kwargs.pop('file') 

+

super(Uploader, self).__init__(*args, **kwargs) 

+

 

+

 

+

class TestResourceUploader(helpers.FunctionalTestBase): 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_resource_upload(self, get_driver): 

+

"""Test a basic resource file upload.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

factories.Sysadmin(apikey='my-test-apikey') 

+

 

+

app = self._get_test_app() 

+

demo = ckanapi.TestAppCKAN(app, apikey='my-test-apikey') 

+

factories.Dataset(name='my-dataset') 

+

 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

resource = demo.action.resource_create( 

+

package_id='my-dataset', 

+

upload=open(file_path), 

+

url='file.txt' 

+

) 

+

 

+

key = 'resources/{0}/data.csv' \ 

+

.format(resource['id']) 

+

 

+

args, kwargs = container.upload_object_via_stream.call_args 

+

 

+

assert_equal(kwargs['object_name'], key) 

+

print('driver method calls', mock_driver.method_calls) 

+

print('container method calls', container.method_calls) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_resource_upload_then_clear(self, get_driver): 

+

"""Test that clearing on upload removes the storage key.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

 

+

sysadmin = factories.Sysadmin(apikey="my-test-key") 

+

 

+

app = self._get_test_app() 

+

demo = ckanapi.TestAppCKAN(app, apikey="my-test-key") 

+

dataset = factories.Dataset(name='my-dataset') 

+

 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

resource = demo.action.resource_create( 

+

package_id='my-dataset', 

+

upload=open(file_path), 

+

url='file.txt' 

+

) 

+

 

+

key = 'resources/{0}/data.csv'.format(resource['id']) 

+

 

+

args, kwargs = container.upload_object_via_stream.call_args 

+

assert_equal(kwargs['object_name'], key) 

+

 

+

container.get_object.return_value = 'object' 

+

 

+

url = toolkit.url_for( 

+

controller='package', action='resource_edit', id=dataset['id'], resource_id=resource['id']) 

+

env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} 

+

app.post(url, {'clear_upload': True, 'url': 'http://asdf', 'save': 'save'}, extra_environ=env) 

+

 

+

args, _ = container.get_object.call_args 

+

path = args[0] 

+

assert_equal(path, key) 

+

args, _ = container.delete_object.call_args 

+

assert_equal(args[0], 'object') 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_path_from_filename(self, get_driver): 

+

"""path_from_filename returns as expected.""" 

+

dataset = factories.Dataset() 

+

resource = factories.Resource(package_id=dataset['id']) 

+

 

+

uploader = ResourceCloudStorage(resource) 

+

returned_path = uploader.path_from_filename(resource['id'], 'myfile.txt') 

+

assert_equal(returned_path, 'resources/{0}/myfile.txt'.format(resource['id'])) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_resource_upload_with_url_and_clear(self, get_driver): 

+

"""Test that clearing an upload and using a URL does not crash.""" 

+

 

+

sysadmin = factories.Sysadmin(apikey='my-test-key') 

+

 

+

app = self._get_test_app() 

+

dataset = factories.Dataset(name='my-dataset') 

+

 

+

url = toolkit.url_for(controller='package', action='new_resource', id=dataset['id']) 

+

env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} 

+

 

+

app.post(url, {'clear_uplaod': True, 'id': '', # empty id from the form 

+

'url': 'http://asdf', 'save': 'save'}, extra_environ=env) 

+

 

+

@helpers.change_config('ckanext.cloudstorage.use_secure_urls', True) 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@patch('ckanext.cloudstorage.storage.CloudStorage.get_url_from_path') 

+

def test_path_from_filename_uses_secure_url_when_config_is_set(self, get_url_from_path, get_driver): 

+

dataset = factories.Dataset(name='my-dataset') 

+

resource = factories.Resource( 

+

package_id=dataset['id'], 

+

) 

+

 

+

uploader = ResourceCloudStorage(resource) 

+

returned_path = uploader.get_url_from_filename(resource['id'], 'myfile.txt') 

+

get_url_from_path.assert_called_once_with('resources/{}/myfile.txt' 

+

.format(resource['id']), True) 

+

 

+

@helpers.change_config('ckanext.cloudstorage.use_secure_urls', False) 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

@patch('ckanext.cloudstorage.storage.CloudStorage.get_url_from_path') 

+

def test_path_from_filename_uses_secure_url_when_option_is_false(self, get_url_from_path, get_driver): 

+

dataset = factories.Dataset(name='my-dataset') 

+

resource = factories.Resource( 

+

package_id=dataset['id'], 

+

) 

+

 

+

uploader = ResourceCloudStorage(resource) 

+

returned_path = uploader.get_url_from_filename(resource['id'], 'myfile.txt') 

+

get_url_from_path.assert_called_once_with('resources/{}/myfile.txt'.format(resource['id']), False) 

+

 

+

 

+

class TestFileCloudStorage(helpers.FunctionalTestBase): 

+

 

+

@patch('ckanext.cloudstorage.storage.FileCloudStorage') 

+

def test_file_upload_calls_FileCloudStorage(self, FileCloudStorage): 

+

sysadmin = factories.Sysadmin(apikey='apikey') 

+

 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

filename = 'image.png' 

+

 

+

img_uploader = Uploader(filename, file=open(file_path)) 

+

 

+

with patch('ckanext.cloudstorage.storage.datetime') as mock_date: 

+

mock_date.datetime.utcnow.returl_value = datetime.datetime(2001, 1, 29) 

+

context = {'user': sysadmin['name']} 

+

helpers.call_action('group_create', context=context, 

+

name='group', 

+

image_upload=img_uploader, 

+

image_url=filename, 

+

save='save') 

+

 

+

FileCloudStorage.assert_called_once_with('group', None) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_group_image_upload(self, get_driver): 

+

"""Test a group image file uplaod.""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

mock_driver.get_object_cdn_url.return_value = 'http://cdn.url' 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

 

+

sysadmin = factories.Sysadmin(apikey='my-test-key') 

+

 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

filename = 'image.png' 

+

 

+

img_uploader = Uploader(filename, file=open(file_path)) 

+

 

+

with patch('ckanext.cloudstorage.storage.datetime') as mock_date: 

+

mock_date.datetime.utcnow.return_value = \ 

+

datetime.datetime(2001, 1, 29) 

+

context = {'user': sysadmin['name']} 

+

helpers.call_action('group_create', context=context, 

+

name='my-group', 

+

image_upload=img_uploader, 

+

image_url=filename, 

+

save='save') 

+

 

+

key = "storage/uploads/2001-01-29-000000{0}" \ 

+

.format(filename) 

+

 

+

group = helpers.call_action('group_show', id='my-group') 

+

print('group', group) 

+

 

+

args, kwargs = container.upload_object_via_stream.call_args 

+

assert_equal(kwargs['object_name'], unicode(key)) 

+

 

+

# app = self._get_test_app() 

+

# image_file_url = '/uploads/group/{0}'.format(filename) 

+

# r = app.get(image_file_url) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_group_image_upload_then_clear(self, get_driver): 

+

"""Test that clearing an upload calls delete_object""" 

+

mock_driver = MagicMock(spec=google_driver, name='driver') 

+

container = MagicMock(name='container') 

+

mock_driver.get_container.return_value = container 

+

get_driver.return_value = MagicMock(return_value=mock_driver) 

+

 

+

sysadmin = factories.Sysadmin(apikey='my-test-apikey') 

+

 

+

file_path = os.path.join(os.path.dirname(__file__), 'data.csv') 

+

file_name = 'image.png' 

+

 

+

img_uploader = Uploader(file_name, file=open(file_path)) 

+

 

+

with patch('ckanext.cloudstorage.storage.datetime') as mock_date: 

+

mock_date.datetime.utcnow.return_value = \ 

+

datetime.datetime(2001, 1, 29) 

+

context = {'user': sysadmin['name']} 

+

helpers.call_action('group_create', context=context, 

+

name='my-group', 

+

image_upload=img_uploader, 

+

image_url=file_name) 

+

 

+

object_mock = MagicMock(name='object') 

+

container.get_object.return_value = object_mock 

+

 

+

helpers.call_action('group_update', context=context, 

+

id='my-group', name='my-group', 

+

image_url='http://example', clear_upload=True) 

+

 

+

# assert delete object is called 

+

container.delete_object.assert_called_with(object_mock) 

+

 

+

@patch('ckanext.cloudstorage.storage.get_driver') 

+

def test_get_object_public_url(self, get_driver): 

+

""" 

+

Test get_object_public_url returns expected string 

+

""" 

+

uploader = FileCloudStorage('notused') 

+

url = uploader.get_object_public_url('file.png') 

+

assert_equal(url, 'https://storage.googleapis.com/test/storage/uploads/file.png') 

+ +
+
+ + + + + diff --git a/cover/coverage_html.js b/cover/coverage_html.js new file mode 100644 index 0000000..f6f5de2 --- /dev/null +++ b/cover/coverage_html.js @@ -0,0 +1,584 @@ +// Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +// For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +// Coverage.py HTML report browser code. +/*jslint browser: true, sloppy: true, vars: true, plusplus: true, maxerr: 50, indent: 4 */ +/*global coverage: true, document, window, $ */ + +coverage = {}; + +// Find all the elements with shortkey_* class, and use them to assign a shortcut key. +coverage.assign_shortkeys = function () { + $("*[class*='shortkey_']").each(function (i, e) { + $.each($(e).attr("class").split(" "), function (i, c) { + if (/^shortkey_/.test(c)) { + $(document).bind('keydown', c.substr(9), function () { + $(e).click(); + }); + } + }); + }); +}; + +// Create the events for the help panel. +coverage.wire_up_help_panel = function () { + $("#keyboard_icon").click(function () { + // Show the help panel, and position it so the keyboard icon in the + // panel is in the same place as the keyboard icon in the header. + $(".help_panel").show(); + var koff = $("#keyboard_icon").offset(); + var poff = $("#panel_icon").position(); + $(".help_panel").offset({ + top: koff.top-poff.top, + left: koff.left-poff.left + }); + }); + $("#panel_icon").click(function () { + $(".help_panel").hide(); + }); +}; + +// Create the events for the filter box. +coverage.wire_up_filter = function () { + // Cache elements. + var table = $("table.index"); + var table_rows = table.find("tbody tr"); + var table_row_names = table_rows.find("td.name a"); + var no_rows = $("#no_rows"); + + // Create a duplicate table footer that we can modify with dynamic summed values. + var table_footer = $("table.index tfoot tr"); + var table_dynamic_footer = table_footer.clone(); + table_dynamic_footer.attr('class', 'total_dynamic hidden'); + table_footer.after(table_dynamic_footer); + + // Observe filter keyevents. + $("#filter").on("keyup change", $.debounce(150, function (event) { + var filter_value = $(this).val(); + + if (filter_value === "") { + // Filter box is empty, remove all filtering. + table_rows.removeClass("hidden"); + + // Show standard footer, hide dynamic footer. + table_footer.removeClass("hidden"); + table_dynamic_footer.addClass("hidden"); + + // Hide placeholder, show table. + if (no_rows.length > 0) { + no_rows.hide(); + } + table.show(); + + } + else { + // Filter table items by value. + var hidden = 0; + var shown = 0; + + // Hide / show elements. + $.each(table_row_names, function () { + var element = $(this).parents("tr"); + + if ($(this).text().indexOf(filter_value) === -1) { + // hide + element.addClass("hidden"); + hidden++; + } + else { + // show + element.removeClass("hidden"); + shown++; + } + }); + + // Show placeholder if no rows will be displayed. + if (no_rows.length > 0) { + if (shown === 0) { + // Show placeholder, hide table. + no_rows.show(); + table.hide(); + } + else { + // Hide placeholder, show table. + no_rows.hide(); + table.show(); + } + } + + // Manage dynamic header: + if (hidden > 0) { + // Calculate new dynamic sum values based on visible rows. + for (var column = 2; column < 20; column++) { + // Calculate summed value. + var cells = table_rows.find('td:nth-child(' + column + ')'); + if (!cells.length) { + // No more columns...! + break; + } + + var sum = 0, numer = 0, denom = 0; + $.each(cells.filter(':visible'), function () { + var ratio = $(this).data("ratio"); + if (ratio) { + var splitted = ratio.split(" "); + numer += parseInt(splitted[0], 10); + denom += parseInt(splitted[1], 10); + } + else { + sum += parseInt(this.innerHTML, 10); + } + }); + + // Get footer cell element. + var footer_cell = table_dynamic_footer.find('td:nth-child(' + column + ')'); + + // Set value into dynamic footer cell element. + if (cells[0].innerHTML.indexOf('%') > -1) { + // Percentage columns use the numerator and denominator, + // and adapt to the number of decimal places. + var match = /\.([0-9]+)/.exec(cells[0].innerHTML); + var places = 0; + if (match) { + places = match[1].length; + } + var pct = numer * 100 / denom; + footer_cell.text(pct.toFixed(places) + '%'); + } + else { + footer_cell.text(sum); + } + } + + // Hide standard footer, show dynamic footer. + table_footer.addClass("hidden"); + table_dynamic_footer.removeClass("hidden"); + } + else { + // Show standard footer, hide dynamic footer. + table_footer.removeClass("hidden"); + table_dynamic_footer.addClass("hidden"); + } + } + })); + + // Trigger change event on setup, to force filter on page refresh + // (filter value may still be present). + $("#filter").trigger("change"); +}; + +// Loaded on index.html +coverage.index_ready = function ($) { + // Look for a cookie containing previous sort settings: + var sort_list = []; + var cookie_name = "COVERAGE_INDEX_SORT"; + var i; + + // This almost makes it worth installing the jQuery cookie plugin: + if (document.cookie.indexOf(cookie_name) > -1) { + var cookies = document.cookie.split(";"); + for (i = 0; i < cookies.length; i++) { + var parts = cookies[i].split("="); + + if ($.trim(parts[0]) === cookie_name && parts[1]) { + sort_list = eval("[[" + parts[1] + "]]"); + break; + } + } + } + + // Create a new widget which exists only to save and restore + // the sort order: + $.tablesorter.addWidget({ + id: "persistentSort", + + // Format is called by the widget before displaying: + format: function (table) { + if (table.config.sortList.length === 0 && sort_list.length > 0) { + // This table hasn't been sorted before - we'll use + // our stored settings: + $(table).trigger('sorton', [sort_list]); + } + else { + // This is not the first load - something has + // already defined sorting so we'll just update + // our stored value to match: + sort_list = table.config.sortList; + } + } + }); + + // Configure our tablesorter to handle the variable number of + // columns produced depending on report options: + var headers = []; + var col_count = $("table.index > thead > tr > th").length; + + headers[0] = { sorter: 'text' }; + for (i = 1; i < col_count-1; i++) { + headers[i] = { sorter: 'digit' }; + } + headers[col_count-1] = { sorter: 'percent' }; + + // Enable the table sorter: + $("table.index").tablesorter({ + widgets: ['persistentSort'], + headers: headers + }); + + coverage.assign_shortkeys(); + coverage.wire_up_help_panel(); + coverage.wire_up_filter(); + + // Watch for page unload events so we can save the final sort settings: + $(window).unload(function () { + document.cookie = cookie_name + "=" + sort_list.toString() + "; path=/"; + }); +}; + +// -- pyfile stuff -- + +coverage.pyfile_ready = function ($) { + // If we're directed to a particular line number, highlight the line. + var frag = location.hash; + if (frag.length > 2 && frag[1] === 'n') { + $(frag).addClass('highlight'); + coverage.set_sel(parseInt(frag.substr(2), 10)); + } + else { + coverage.set_sel(0); + } + + $(document) + .bind('keydown', 'j', coverage.to_next_chunk_nicely) + .bind('keydown', 'k', coverage.to_prev_chunk_nicely) + .bind('keydown', '0', coverage.to_top) + .bind('keydown', '1', coverage.to_first_chunk) + ; + + $(".button_toggle_run").click(function (evt) {coverage.toggle_lines(evt.target, "run");}); + $(".button_toggle_exc").click(function (evt) {coverage.toggle_lines(evt.target, "exc");}); + $(".button_toggle_mis").click(function (evt) {coverage.toggle_lines(evt.target, "mis");}); + $(".button_toggle_par").click(function (evt) {coverage.toggle_lines(evt.target, "par");}); + + coverage.assign_shortkeys(); + coverage.wire_up_help_panel(); + + coverage.init_scroll_markers(); + + // Rebuild scroll markers after window high changing + $(window).resize(coverage.resize_scroll_markers); +}; + +coverage.toggle_lines = function (btn, cls) { + btn = $(btn); + var hide = "hide_"+cls; + if (btn.hasClass(hide)) { + $("#source ."+cls).removeClass(hide); + btn.removeClass(hide); + } + else { + $("#source ."+cls).addClass(hide); + btn.addClass(hide); + } +}; + +// Return the nth line div. +coverage.line_elt = function (n) { + return $("#t" + n); +}; + +// Return the nth line number div. +coverage.num_elt = function (n) { + return $("#n" + n); +}; + +// Return the container of all the code. +coverage.code_container = function () { + return $(".linenos"); +}; + +// Set the selection. b and e are line numbers. +coverage.set_sel = function (b, e) { + // The first line selected. + coverage.sel_begin = b; + // The next line not selected. + coverage.sel_end = (e === undefined) ? b+1 : e; +}; + +coverage.to_top = function () { + coverage.set_sel(0, 1); + coverage.scroll_window(0); +}; + +coverage.to_first_chunk = function () { + coverage.set_sel(0, 1); + coverage.to_next_chunk(); +}; + +coverage.is_transparent = function (color) { + // Different browsers return different colors for "none". + return color === "transparent" || color === "rgba(0, 0, 0, 0)"; +}; + +coverage.to_next_chunk = function () { + var c = coverage; + + // Find the start of the next colored chunk. + var probe = c.sel_end; + var color, probe_line; + while (true) { + probe_line = c.line_elt(probe); + if (probe_line.length === 0) { + return; + } + color = probe_line.css("background-color"); + if (!c.is_transparent(color)) { + break; + } + probe++; + } + + // There's a next chunk, `probe` points to it. + var begin = probe; + + // Find the end of this chunk. + var next_color = color; + while (next_color === color) { + probe++; + probe_line = c.line_elt(probe); + next_color = probe_line.css("background-color"); + } + c.set_sel(begin, probe); + c.show_selection(); +}; + +coverage.to_prev_chunk = function () { + var c = coverage; + + // Find the end of the prev colored chunk. + var probe = c.sel_begin-1; + var probe_line = c.line_elt(probe); + if (probe_line.length === 0) { + return; + } + var color = probe_line.css("background-color"); + while (probe > 0 && c.is_transparent(color)) { + probe--; + probe_line = c.line_elt(probe); + if (probe_line.length === 0) { + return; + } + color = probe_line.css("background-color"); + } + + // There's a prev chunk, `probe` points to its last line. + var end = probe+1; + + // Find the beginning of this chunk. + var prev_color = color; + while (prev_color === color) { + probe--; + probe_line = c.line_elt(probe); + prev_color = probe_line.css("background-color"); + } + c.set_sel(probe+1, end); + c.show_selection(); +}; + +// Return the line number of the line nearest pixel position pos +coverage.line_at_pos = function (pos) { + var l1 = coverage.line_elt(1), + l2 = coverage.line_elt(2), + result; + if (l1.length && l2.length) { + var l1_top = l1.offset().top, + line_height = l2.offset().top - l1_top, + nlines = (pos - l1_top) / line_height; + if (nlines < 1) { + result = 1; + } + else { + result = Math.ceil(nlines); + } + } + else { + result = 1; + } + return result; +}; + +// Returns 0, 1, or 2: how many of the two ends of the selection are on +// the screen right now? +coverage.selection_ends_on_screen = function () { + if (coverage.sel_begin === 0) { + return 0; + } + + var top = coverage.line_elt(coverage.sel_begin); + var next = coverage.line_elt(coverage.sel_end-1); + + return ( + (top.isOnScreen() ? 1 : 0) + + (next.isOnScreen() ? 1 : 0) + ); +}; + +coverage.to_next_chunk_nicely = function () { + coverage.finish_scrolling(); + if (coverage.selection_ends_on_screen() === 0) { + // The selection is entirely off the screen: select the top line on + // the screen. + var win = $(window); + coverage.select_line_or_chunk(coverage.line_at_pos(win.scrollTop())); + } + coverage.to_next_chunk(); +}; + +coverage.to_prev_chunk_nicely = function () { + coverage.finish_scrolling(); + if (coverage.selection_ends_on_screen() === 0) { + var win = $(window); + coverage.select_line_or_chunk(coverage.line_at_pos(win.scrollTop() + win.height())); + } + coverage.to_prev_chunk(); +}; + +// Select line number lineno, or if it is in a colored chunk, select the +// entire chunk +coverage.select_line_or_chunk = function (lineno) { + var c = coverage; + var probe_line = c.line_elt(lineno); + if (probe_line.length === 0) { + return; + } + var the_color = probe_line.css("background-color"); + if (!c.is_transparent(the_color)) { + // The line is in a highlighted chunk. + // Search backward for the first line. + var probe = lineno; + var color = the_color; + while (probe > 0 && color === the_color) { + probe--; + probe_line = c.line_elt(probe); + if (probe_line.length === 0) { + break; + } + color = probe_line.css("background-color"); + } + var begin = probe + 1; + + // Search forward for the last line. + probe = lineno; + color = the_color; + while (color === the_color) { + probe++; + probe_line = c.line_elt(probe); + color = probe_line.css("background-color"); + } + + coverage.set_sel(begin, probe); + } + else { + coverage.set_sel(lineno); + } +}; + +coverage.show_selection = function () { + var c = coverage; + + // Highlight the lines in the chunk + c.code_container().find(".highlight").removeClass("highlight"); + for (var probe = c.sel_begin; probe > 0 && probe < c.sel_end; probe++) { + c.num_elt(probe).addClass("highlight"); + } + + c.scroll_to_selection(); +}; + +coverage.scroll_to_selection = function () { + // Scroll the page if the chunk isn't fully visible. + if (coverage.selection_ends_on_screen() < 2) { + // Need to move the page. The html,body trick makes it scroll in all + // browsers, got it from http://stackoverflow.com/questions/3042651 + var top = coverage.line_elt(coverage.sel_begin); + var top_pos = parseInt(top.offset().top, 10); + coverage.scroll_window(top_pos - 30); + } +}; + +coverage.scroll_window = function (to_pos) { + $("html,body").animate({scrollTop: to_pos}, 200); +}; + +coverage.finish_scrolling = function () { + $("html,body").stop(true, true); +}; + +coverage.init_scroll_markers = function () { + var c = coverage; + // Init some variables + c.lines_len = $('td.text p').length; + c.body_h = $('body').height(); + c.header_h = $('div#header').height(); + c.missed_lines = $('td.text p.mis, td.text p.par'); + + // Build html + c.resize_scroll_markers(); +}; + +coverage.resize_scroll_markers = function () { + var c = coverage, + min_line_height = 3, + max_line_height = 10, + visible_window_h = $(window).height(); + + $('#scroll_marker').remove(); + // Don't build markers if the window has no scroll bar. + if (c.body_h <= visible_window_h) { + return; + } + + $("body").append("
 
"); + var scroll_marker = $('#scroll_marker'), + marker_scale = scroll_marker.height() / c.body_h, + line_height = scroll_marker.height() / c.lines_len; + + // Line height must be between the extremes. + if (line_height > min_line_height) { + if (line_height > max_line_height) { + line_height = max_line_height; + } + } + else { + line_height = min_line_height; + } + + var previous_line = -99, + last_mark, + last_top; + + c.missed_lines.each(function () { + var line_top = Math.round($(this).offset().top * marker_scale), + id_name = $(this).attr('id'), + line_number = parseInt(id_name.substring(1, id_name.length)); + + if (line_number === previous_line + 1) { + // If this solid missed block just make previous mark higher. + last_mark.css({ + 'height': line_top + line_height - last_top + }); + } + else { + // Add colored line in scroll_marker block. + scroll_marker.append('
'); + last_mark = $('#m' + line_number); + last_mark.css({ + 'height': line_height, + 'top': line_top + }); + last_top = line_top; + } + + previous_line = line_number; + }); +}; diff --git a/cover/index.html b/cover/index.html new file mode 100644 index 0000000..e81b248 --- /dev/null +++ b/cover/index.html @@ -0,0 +1,230 @@ + + + + + + + + Coverage report + + + + + + + + + + + + + +
+ Hide keyboard shortcuts +

Hot-keys on this page

+
+

+ n + s + m + x + + c   change column sorting +

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Modulestatementsmissingexcludedcoverage
Total790313060%
ckanext/cloudstorage/__init__.py000100%
ckanext/cloudstorage/cli.py828200%
ckanext/cloudstorage/controller.py3210069%
ckanext/cloudstorage/helpers.py31067%
ckanext/cloudstorage/logic/__init__.py000100%
ckanext/cloudstorage/logic/action/__init__.py000100%
ckanext/cloudstorage/logic/action/multipart.py135115015%
ckanext/cloudstorage/logic/auth/__init__.py000100%
ckanext/cloudstorage/logic/auth/multipart.py136054%
ckanext/cloudstorage/model.py4213069%
ckanext/cloudstorage/plugin.py5918069%
ckanext/cloudstorage/storage.py19366066%
ckanext/cloudstorage/tests/test_controller.py742097%
ckanext/cloudstorage/tests/test_plugin.py2000100%
ckanext/cloudstorage/tests/test_storage.py13700100%
+ +

+ No items found using the specified filter. +

+
+ + + + + diff --git a/cover/jquery.ba-throttle-debounce.min.js b/cover/jquery.ba-throttle-debounce.min.js new file mode 100644 index 0000000..648fe5d --- /dev/null +++ b/cover/jquery.ba-throttle-debounce.min.js @@ -0,0 +1,9 @@ +/* + * jQuery throttle / debounce - v1.1 - 3/7/2010 + * http://benalman.com/projects/jquery-throttle-debounce-plugin/ + * + * Copyright (c) 2010 "Cowboy" Ben Alman + * Dual licensed under the MIT and GPL licenses. + * http://benalman.com/about/license/ + */ +(function(b,c){var $=b.jQuery||b.Cowboy||(b.Cowboy={}),a;$.throttle=a=function(e,f,j,i){var h,d=0;if(typeof f!=="boolean"){i=j;j=f;f=c}function g(){var o=this,m=+new Date()-d,n=arguments;function l(){d=+new Date();j.apply(o,n)}function k(){h=c}if(i&&!h){l()}h&&clearTimeout(h);if(i===c&&m>e){l()}else{if(f!==true){h=setTimeout(i?k:l,i===c?e-m:e)}}}if($.guid){g.guid=j.guid=j.guid||$.guid++}return g};$.debounce=function(d,e,f){return f===c?a(d,e,false):a(d,f,e!==false)}})(this); diff --git a/cover/jquery.hotkeys.js b/cover/jquery.hotkeys.js new file mode 100644 index 0000000..09b21e0 --- /dev/null +++ b/cover/jquery.hotkeys.js @@ -0,0 +1,99 @@ +/* + * jQuery Hotkeys Plugin + * Copyright 2010, John Resig + * Dual licensed under the MIT or GPL Version 2 licenses. + * + * Based upon the plugin by Tzury Bar Yochay: + * http://github.com/tzuryby/hotkeys + * + * Original idea by: + * Binny V A, http://www.openjs.com/scripts/events/keyboard_shortcuts/ +*/ + +(function(jQuery){ + + jQuery.hotkeys = { + version: "0.8", + + specialKeys: { + 8: "backspace", 9: "tab", 13: "return", 16: "shift", 17: "ctrl", 18: "alt", 19: "pause", + 20: "capslock", 27: "esc", 32: "space", 33: "pageup", 34: "pagedown", 35: "end", 36: "home", + 37: "left", 38: "up", 39: "right", 40: "down", 45: "insert", 46: "del", + 96: "0", 97: "1", 98: "2", 99: "3", 100: "4", 101: "5", 102: "6", 103: "7", + 104: "8", 105: "9", 106: "*", 107: "+", 109: "-", 110: ".", 111 : "/", + 112: "f1", 113: "f2", 114: "f3", 115: "f4", 116: "f5", 117: "f6", 118: "f7", 119: "f8", + 120: "f9", 121: "f10", 122: "f11", 123: "f12", 144: "numlock", 145: "scroll", 191: "/", 224: "meta" + }, + + shiftNums: { + "`": "~", "1": "!", "2": "@", "3": "#", "4": "$", "5": "%", "6": "^", "7": "&", + "8": "*", "9": "(", "0": ")", "-": "_", "=": "+", ";": ": ", "'": "\"", ",": "<", + ".": ">", "/": "?", "\\": "|" + } + }; + + function keyHandler( handleObj ) { + // Only care when a possible input has been specified + if ( typeof handleObj.data !== "string" ) { + return; + } + + var origHandler = handleObj.handler, + keys = handleObj.data.toLowerCase().split(" "); + + handleObj.handler = function( event ) { + // Don't fire in text-accepting inputs that we didn't directly bind to + if ( this !== event.target && (/textarea|select/i.test( event.target.nodeName ) || + event.target.type === "text") ) { + return; + } + + // Keypress represents characters, not special keys + var special = event.type !== "keypress" && jQuery.hotkeys.specialKeys[ event.which ], + character = String.fromCharCode( event.which ).toLowerCase(), + key, modif = "", possible = {}; + + // check combinations (alt|ctrl|shift+anything) + if ( event.altKey && special !== "alt" ) { + modif += "alt+"; + } + + if ( event.ctrlKey && special !== "ctrl" ) { + modif += "ctrl+"; + } + + // TODO: Need to make sure this works consistently across platforms + if ( event.metaKey && !event.ctrlKey && special !== "meta" ) { + modif += "meta+"; + } + + if ( event.shiftKey && special !== "shift" ) { + modif += "shift+"; + } + + if ( special ) { + possible[ modif + special ] = true; + + } else { + possible[ modif + character ] = true; + possible[ modif + jQuery.hotkeys.shiftNums[ character ] ] = true; + + // "$" can be triggered as "Shift+4" or "Shift+$" or just "$" + if ( modif === "shift+" ) { + possible[ jQuery.hotkeys.shiftNums[ character ] ] = true; + } + } + + for ( var i = 0, l = keys.length; i < l; i++ ) { + if ( possible[ keys[i] ] ) { + return origHandler.apply( this, arguments ); + } + } + }; + } + + jQuery.each([ "keydown", "keyup", "keypress" ], function() { + jQuery.event.special[ this ] = { add: keyHandler }; + }); + +})( jQuery ); diff --git a/cover/jquery.isonscreen.js b/cover/jquery.isonscreen.js new file mode 100644 index 0000000..0182ebd --- /dev/null +++ b/cover/jquery.isonscreen.js @@ -0,0 +1,53 @@ +/* Copyright (c) 2010 + * @author Laurence Wheway + * Dual licensed under the MIT (http://www.opensource.org/licenses/mit-license.php) + * and GPL (http://www.opensource.org/licenses/gpl-license.php) licenses. + * + * @version 1.2.0 + */ +(function($) { + jQuery.extend({ + isOnScreen: function(box, container) { + //ensure numbers come in as intgers (not strings) and remove 'px' is it's there + for(var i in box){box[i] = parseFloat(box[i])}; + for(var i in container){container[i] = parseFloat(container[i])}; + + if(!container){ + container = { + left: $(window).scrollLeft(), + top: $(window).scrollTop(), + width: $(window).width(), + height: $(window).height() + } + } + + if( box.left+box.width-container.left > 0 && + box.left < container.width+container.left && + box.top+box.height-container.top > 0 && + box.top < container.height+container.top + ) return true; + return false; + } + }) + + + jQuery.fn.isOnScreen = function (container) { + for(var i in container){container[i] = parseFloat(container[i])}; + + if(!container){ + container = { + left: $(window).scrollLeft(), + top: $(window).scrollTop(), + width: $(window).width(), + height: $(window).height() + } + } + + if( $(this).offset().left+$(this).width()-container.left > 0 && + $(this).offset().left < container.width+container.left && + $(this).offset().top+$(this).height()-container.top > 0 && + $(this).offset().top < container.height+container.top + ) return true; + return false; + } +})(jQuery); diff --git a/cover/jquery.min.js b/cover/jquery.min.js new file mode 100644 index 0000000..d1608e3 --- /dev/null +++ b/cover/jquery.min.js @@ -0,0 +1,4 @@ +/*! jQuery v1.11.1 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l="1.11.1",m=function(a,b){return new m.fn.init(a,b)},n=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,o=/^-ms-/,p=/-([\da-z])/gi,q=function(a,b){return b.toUpperCase()};m.fn=m.prototype={jquery:l,constructor:m,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=m.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return m.each(this,a,b)},map:function(a){return this.pushStack(m.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},m.extend=m.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||m.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(m.isPlainObject(c)||(b=m.isArray(c)))?(b?(b=!1,f=a&&m.isArray(a)?a:[]):f=a&&m.isPlainObject(a)?a:{},g[d]=m.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},m.extend({expando:"jQuery"+(l+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===m.type(a)},isArray:Array.isArray||function(a){return"array"===m.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return!m.isArray(a)&&a-parseFloat(a)>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==m.type(a)||a.nodeType||m.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(k.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&m.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(o,"ms-").replace(p,q)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=r(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(n,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(r(Object(a))?m.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=r(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),m.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||m.guid++,e):void 0},now:function(){return+new Date},support:k}),m.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function r(a){var b=a.length,c=m.type(a);return"function"===c||m.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var s=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+-new Date,v=a.document,w=0,x=0,y=gb(),z=gb(),A=gb(),B=function(a,b){return a===b&&(l=!0),0},C="undefined",D=1<<31,E={}.hasOwnProperty,F=[],G=F.pop,H=F.push,I=F.push,J=F.slice,K=F.indexOf||function(a){for(var b=0,c=this.length;c>b;b++)if(this[b]===a)return b;return-1},L="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",N="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",O=N.replace("w","w#"),P="\\["+M+"*("+N+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+O+"))|)"+M+"*\\]",Q=":("+N+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+P+")*)|.*)\\)|)",R=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),S=new RegExp("^"+M+"*,"+M+"*"),T=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),V=new RegExp(Q),W=new RegExp("^"+O+"$"),X={ID:new RegExp("^#("+N+")"),CLASS:new RegExp("^\\.("+N+")"),TAG:new RegExp("^("+N.replace("w","w*")+")"),ATTR:new RegExp("^"+P),PSEUDO:new RegExp("^"+Q),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+L+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)};try{I.apply(F=J.call(v.childNodes),v.childNodes),F[v.childNodes.length].nodeType}catch(eb){I={apply:F.length?function(a,b){H.apply(a,J.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function fb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],!a||"string"!=typeof a)return d;if(1!==(k=b.nodeType)&&9!==k)return[];if(p&&!e){if(f=_.exec(a))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return I.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName&&b.getElementsByClassName)return I.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=9===k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+qb(o[l]);w=ab.test(a)&&ob(b.parentNode)||b,x=o.join(",")}if(x)try{return I.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function gb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function hb(a){return a[u]=!0,a}function ib(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function jb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function kb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||D)-(~a.sourceIndex||D);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function lb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function nb(a){return hb(function(b){return b=+b,hb(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function ob(a){return a&&typeof a.getElementsByTagName!==C&&a}c=fb.support={},f=fb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=fb.setDocument=function(a){var b,e=a?a.ownerDocument||a:v,g=e.defaultView;return e!==n&&9===e.nodeType&&e.documentElement?(n=e,o=e.documentElement,p=!f(e),g&&g!==g.top&&(g.addEventListener?g.addEventListener("unload",function(){m()},!1):g.attachEvent&&g.attachEvent("onunload",function(){m()})),c.attributes=ib(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ib(function(a){return a.appendChild(e.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(e.getElementsByClassName)&&ib(function(a){return a.innerHTML="
",a.firstChild.className="i",2===a.getElementsByClassName("i").length}),c.getById=ib(function(a){return o.appendChild(a).id=u,!e.getElementsByName||!e.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if(typeof b.getElementById!==C&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c=typeof a.getAttributeNode!==C&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return typeof b.getElementsByTagName!==C?b.getElementsByTagName(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return typeof b.getElementsByClassName!==C&&p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(e.querySelectorAll))&&(ib(function(a){a.innerHTML="",a.querySelectorAll("[msallowclip^='']").length&&q.push("[*^$]="+M+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+M+"*(?:value|"+L+")"),a.querySelectorAll(":checked").length||q.push(":checked")}),ib(function(a){var b=e.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+M+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ib(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",Q)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===e||a.ownerDocument===v&&t(v,a)?-1:b===e||b.ownerDocument===v&&t(v,b)?1:k?K.call(k,a)-K.call(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,f=a.parentNode,g=b.parentNode,h=[a],i=[b];if(!f||!g)return a===e?-1:b===e?1:f?-1:g?1:k?K.call(k,a)-K.call(k,b):0;if(f===g)return kb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?kb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},e):n},fb.matches=function(a,b){return fb(a,null,null,b)},fb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return fb(b,n,null,[a]).length>0},fb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},fb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&E.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},fb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},fb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=fb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=fb.selectors={cacheLength:50,createPseudo:hb,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||fb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&fb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+M+")"+a+"("+M+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||typeof a.getAttribute!==C&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=fb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||fb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?hb(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=K.call(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:hb(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?hb(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),!c.pop()}}),has:hb(function(a){return function(b){return fb(a,b).length>0}}),contains:hb(function(a){return function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:hb(function(a){return W.test(a||"")||fb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:nb(function(){return[0]}),last:nb(function(a,b){return[b-1]}),eq:nb(function(a,b,c){return[0>c?c+b:c]}),even:nb(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:nb(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:nb(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:nb(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function rb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function sb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function tb(a,b,c){for(var d=0,e=b.length;e>d;d++)fb(a,b[d],c);return c}function ub(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function vb(a,b,c,d,e,f){return d&&!d[u]&&(d=vb(d)),e&&!e[u]&&(e=vb(e,f)),hb(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||tb(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:ub(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=ub(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?K.call(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=ub(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):I.apply(g,r)})}function wb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=rb(function(a){return a===b},h,!0),l=rb(function(a){return K.call(b,a)>-1},h,!0),m=[function(a,c,d){return!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d))}];f>i;i++)if(c=d.relative[a[i].type])m=[rb(sb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return vb(i>1&&sb(m),i>1&&qb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&wb(a.slice(i,e)),f>e&&wb(a=a.slice(e)),f>e&&qb(a))}m.push(c)}return sb(m)}function xb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=G.call(i));s=ub(s)}I.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&fb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?hb(f):f}return h=fb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=wb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,xb(e,d)),f.selector=a}return f},i=fb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&ob(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&qb(j),!a)return I.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&ob(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ib(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),ib(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||jb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ib(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||jb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),ib(function(a){return null==a.getAttribute("disabled")})||jb(L,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),fb}(a);m.find=s,m.expr=s.selectors,m.expr[":"]=m.expr.pseudos,m.unique=s.uniqueSort,m.text=s.getText,m.isXMLDoc=s.isXML,m.contains=s.contains;var t=m.expr.match.needsContext,u=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,v=/^.[^:#\[\.,]*$/;function w(a,b,c){if(m.isFunction(b))return m.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return m.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(v.test(b))return m.filter(b,a,c);b=m.filter(b,a)}return m.grep(a,function(a){return m.inArray(a,b)>=0!==c})}m.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?m.find.matchesSelector(d,a)?[d]:[]:m.find.matches(a,m.grep(b,function(a){return 1===a.nodeType}))},m.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(m(a).filter(function(){for(b=0;e>b;b++)if(m.contains(d[b],this))return!0}));for(b=0;e>b;b++)m.find(a,d[b],c);return c=this.pushStack(e>1?m.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(w(this,a||[],!1))},not:function(a){return this.pushStack(w(this,a||[],!0))},is:function(a){return!!w(this,"string"==typeof a&&t.test(a)?m(a):a||[],!1).length}});var x,y=a.document,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=m.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||x).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof m?b[0]:b,m.merge(this,m.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:y,!0)),u.test(c[1])&&m.isPlainObject(b))for(c in b)m.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=y.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return x.find(a);this.length=1,this[0]=d}return this.context=y,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):m.isFunction(a)?"undefined"!=typeof x.ready?x.ready(a):a(m):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),m.makeArray(a,this))};A.prototype=m.fn,x=m(y);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};m.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!m(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),m.fn.extend({has:function(a){var b,c=m(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(m.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=t.test(a)||"string"!=typeof a?m(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&m.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?m.unique(f):f)},index:function(a){return a?"string"==typeof a?m.inArray(this[0],m(a)):m.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(m.unique(m.merge(this.get(),m(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}m.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return m.dir(a,"parentNode")},parentsUntil:function(a,b,c){return m.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return m.dir(a,"nextSibling")},prevAll:function(a){return m.dir(a,"previousSibling")},nextUntil:function(a,b,c){return m.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return m.dir(a,"previousSibling",c)},siblings:function(a){return m.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return m.sibling(a.firstChild)},contents:function(a){return m.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:m.merge([],a.childNodes)}},function(a,b){m.fn[a]=function(c,d){var e=m.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=m.filter(d,e)),this.length>1&&(C[a]||(e=m.unique(e)),B.test(a)&&(e=e.reverse())),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return m.each(a.match(E)||[],function(a,c){b[c]=!0}),b}m.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):m.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){m.each(b,function(b,c){var d=m.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&m.each(arguments,function(a,c){var d;while((d=m.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?m.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},m.extend({Deferred:function(a){var b=[["resolve","done",m.Callbacks("once memory"),"resolved"],["reject","fail",m.Callbacks("once memory"),"rejected"],["notify","progress",m.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return m.Deferred(function(c){m.each(b,function(b,f){var g=m.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&m.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?m.extend(a,d):d}},e={};return d.pipe=d.then,m.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&m.isFunction(a.promise)?e:0,g=1===f?a:m.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&m.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;m.fn.ready=function(a){return m.ready.promise().done(a),this},m.extend({isReady:!1,readyWait:1,holdReady:function(a){a?m.readyWait++:m.ready(!0)},ready:function(a){if(a===!0?!--m.readyWait:!m.isReady){if(!y.body)return setTimeout(m.ready);m.isReady=!0,a!==!0&&--m.readyWait>0||(H.resolveWith(y,[m]),m.fn.triggerHandler&&(m(y).triggerHandler("ready"),m(y).off("ready")))}}});function I(){y.addEventListener?(y.removeEventListener("DOMContentLoaded",J,!1),a.removeEventListener("load",J,!1)):(y.detachEvent("onreadystatechange",J),a.detachEvent("onload",J))}function J(){(y.addEventListener||"load"===event.type||"complete"===y.readyState)&&(I(),m.ready())}m.ready.promise=function(b){if(!H)if(H=m.Deferred(),"complete"===y.readyState)setTimeout(m.ready);else if(y.addEventListener)y.addEventListener("DOMContentLoaded",J,!1),a.addEventListener("load",J,!1);else{y.attachEvent("onreadystatechange",J),a.attachEvent("onload",J);var c=!1;try{c=null==a.frameElement&&y.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!m.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}I(),m.ready()}}()}return H.promise(b)};var K="undefined",L;for(L in m(k))break;k.ownLast="0"!==L,k.inlineBlockNeedsLayout=!1,m(function(){var a,b,c,d;c=y.getElementsByTagName("body")[0],c&&c.style&&(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),typeof b.style.zoom!==K&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",k.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(d))}),function(){var a=y.createElement("div");if(null==k.deleteExpando){k.deleteExpando=!0;try{delete a.test}catch(b){k.deleteExpando=!1}}a=null}(),m.acceptData=function(a){var b=m.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var M=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,N=/([A-Z])/g;function O(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(N,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:M.test(c)?m.parseJSON(c):c}catch(e){}m.data(a,b,c)}else c=void 0}return c}function P(a){var b;for(b in a)if(("data"!==b||!m.isEmptyObject(a[b]))&&"toJSON"!==b)return!1;return!0}function Q(a,b,d,e){if(m.acceptData(a)){var f,g,h=m.expando,i=a.nodeType,j=i?m.cache:a,k=i?a[h]:a[h]&&h; +if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||m.guid++:h),j[k]||(j[k]=i?{}:{toJSON:m.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=m.extend(j[k],b):j[k].data=m.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[m.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[m.camelCase(b)])):f=g,f}}function R(a,b,c){if(m.acceptData(a)){var d,e,f=a.nodeType,g=f?m.cache:a,h=f?a[m.expando]:m.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){m.isArray(b)?b=b.concat(m.map(b,m.camelCase)):b in d?b=[b]:(b=m.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!P(d):!m.isEmptyObject(d))return}(c||(delete g[h].data,P(g[h])))&&(f?m.cleanData([a],!0):k.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}m.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?m.cache[a[m.expando]]:a[m.expando],!!a&&!P(a)},data:function(a,b,c){return Q(a,b,c)},removeData:function(a,b){return R(a,b)},_data:function(a,b,c){return Q(a,b,c,!0)},_removeData:function(a,b){return R(a,b,!0)}}),m.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=m.data(f),1===f.nodeType&&!m._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=m.camelCase(d.slice(5)),O(f,d,e[d])));m._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){m.data(this,a)}):arguments.length>1?this.each(function(){m.data(this,a,b)}):f?O(f,a,m.data(f,a)):void 0},removeData:function(a){return this.each(function(){m.removeData(this,a)})}}),m.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=m._data(a,b),c&&(!d||m.isArray(c)?d=m._data(a,b,m.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=m.queue(a,b),d=c.length,e=c.shift(),f=m._queueHooks(a,b),g=function(){m.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return m._data(a,c)||m._data(a,c,{empty:m.Callbacks("once memory").add(function(){m._removeData(a,b+"queue"),m._removeData(a,c)})})}}),m.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},W=/^(?:checkbox|radio)$/i;!function(){var a=y.createElement("input"),b=y.createElement("div"),c=y.createDocumentFragment();if(b.innerHTML="
a",k.leadingWhitespace=3===b.firstChild.nodeType,k.tbody=!b.getElementsByTagName("tbody").length,k.htmlSerialize=!!b.getElementsByTagName("link").length,k.html5Clone="<:nav>"!==y.createElement("nav").cloneNode(!0).outerHTML,a.type="checkbox",a.checked=!0,c.appendChild(a),k.appendChecked=a.checked,b.innerHTML="",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,c.appendChild(b),b.innerHTML="",k.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,k.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){k.noCloneEvent=!1}),b.cloneNode(!0).click()),null==k.deleteExpando){k.deleteExpando=!0;try{delete b.test}catch(d){k.deleteExpando=!1}}}(),function(){var b,c,d=y.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(k[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),k[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var X=/^(?:input|select|textarea)$/i,Y=/^key/,Z=/^(?:mouse|pointer|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=/^([^.]*)(?:\.(.+)|)$/;function ab(){return!0}function bb(){return!1}function cb(){try{return y.activeElement}catch(a){}}m.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=m.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof m===K||a&&m.event.triggered===a.type?void 0:m.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(E)||[""],h=b.length;while(h--)f=_.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=m.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=m.event.special[o]||{},l=m.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&m.expr.match.needsContext.test(e),namespace:p.join(".")},i),(n=g[o])||(n=g[o]=[],n.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?n.splice(n.delegateCount++,0,l):n.push(l),m.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m.hasData(a)&&m._data(a);if(r&&(k=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=_.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=m.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,n=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=n.length;while(f--)g=n[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(n.splice(f,1),g.selector&&n.delegateCount--,l.remove&&l.remove.call(a,g));i&&!n.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||m.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)m.event.remove(a,o+b[j],c,d,!0);m.isEmptyObject(k)&&(delete r.handle,m._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,n,o=[d||y],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||y,3!==d.nodeType&&8!==d.nodeType&&!$.test(p+m.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[m.expando]?b:new m.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:m.makeArray(c,[b]),k=m.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!m.isWindow(d)){for(i=k.delegateType||p,$.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||y)&&o.push(l.defaultView||l.parentWindow||a)}n=0;while((h=o[n++])&&!b.isPropagationStopped())b.type=n>1?i:k.bindType||p,f=(m._data(h,"events")||{})[b.type]&&m._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&m.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&m.acceptData(d)&&g&&d[p]&&!m.isWindow(d)){l=d[g],l&&(d[g]=null),m.event.triggered=p;try{d[p]()}catch(r){}m.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=m.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(m._data(this,"events")||{})[a.type]||[],k=m.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=m.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((m.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?m(c,this).index(i)>=0:m.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h]","i"),hb=/^\s+/,ib=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,jb=/<([\w:]+)/,kb=/\s*$/g,rb={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:k.htmlSerialize?[0,"",""]:[1,"X
","
"]},sb=db(y),tb=sb.appendChild(y.createElement("div"));rb.optgroup=rb.option,rb.tbody=rb.tfoot=rb.colgroup=rb.caption=rb.thead,rb.th=rb.td;function ub(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==K?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==K?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||m.nodeName(d,b)?f.push(d):m.merge(f,ub(d,b));return void 0===b||b&&m.nodeName(a,b)?m.merge([a],f):f}function vb(a){W.test(a.type)&&(a.defaultChecked=a.checked)}function wb(a,b){return m.nodeName(a,"table")&&m.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function xb(a){return a.type=(null!==m.find.attr(a,"type"))+"/"+a.type,a}function yb(a){var b=pb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function zb(a,b){for(var c,d=0;null!=(c=a[d]);d++)m._data(c,"globalEval",!b||m._data(b[d],"globalEval"))}function Ab(a,b){if(1===b.nodeType&&m.hasData(a)){var c,d,e,f=m._data(a),g=m._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)m.event.add(b,c,h[c][d])}g.data&&(g.data=m.extend({},g.data))}}function Bb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!k.noCloneEvent&&b[m.expando]){e=m._data(b);for(d in e.events)m.removeEvent(b,d,e.handle);b.removeAttribute(m.expando)}"script"===c&&b.text!==a.text?(xb(b).text=a.text,yb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),k.html5Clone&&a.innerHTML&&!m.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&W.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}m.extend({clone:function(a,b,c){var d,e,f,g,h,i=m.contains(a.ownerDocument,a);if(k.html5Clone||m.isXMLDoc(a)||!gb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(tb.innerHTML=a.outerHTML,tb.removeChild(f=tb.firstChild)),!(k.noCloneEvent&&k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||m.isXMLDoc(a)))for(d=ub(f),h=ub(a),g=0;null!=(e=h[g]);++g)d[g]&&Bb(e,d[g]);if(b)if(c)for(h=h||ub(a),d=d||ub(f),g=0;null!=(e=h[g]);g++)Ab(e,d[g]);else Ab(a,f);return d=ub(f,"script"),d.length>0&&zb(d,!i&&ub(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,l,n=a.length,o=db(b),p=[],q=0;n>q;q++)if(f=a[q],f||0===f)if("object"===m.type(f))m.merge(p,f.nodeType?[f]:f);else if(lb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(jb.exec(f)||["",""])[1].toLowerCase(),l=rb[i]||rb._default,h.innerHTML=l[1]+f.replace(ib,"<$1>")+l[2],e=l[0];while(e--)h=h.lastChild;if(!k.leadingWhitespace&&hb.test(f)&&p.push(b.createTextNode(hb.exec(f)[0])),!k.tbody){f="table"!==i||kb.test(f)?""!==l[1]||kb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)m.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}m.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),k.appendChecked||m.grep(ub(p,"input"),vb),q=0;while(f=p[q++])if((!d||-1===m.inArray(f,d))&&(g=m.contains(f.ownerDocument,f),h=ub(o.appendChild(f),"script"),g&&zb(h),c)){e=0;while(f=h[e++])ob.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=m.expando,j=m.cache,l=k.deleteExpando,n=m.event.special;null!=(d=a[h]);h++)if((b||m.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)n[e]?m.event.remove(d,e):m.removeEvent(d,e,g.handle);j[f]&&(delete j[f],l?delete d[i]:typeof d.removeAttribute!==K?d.removeAttribute(i):d[i]=null,c.push(f))}}}),m.fn.extend({text:function(a){return V(this,function(a){return void 0===a?m.text(this):this.empty().append((this[0]&&this[0].ownerDocument||y).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?m.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||m.cleanData(ub(c)),c.parentNode&&(b&&m.contains(c.ownerDocument,c)&&zb(ub(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&m.cleanData(ub(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&m.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return m.clone(this,a,b)})},html:function(a){return V(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(fb,""):void 0;if(!("string"!=typeof a||mb.test(a)||!k.htmlSerialize&&gb.test(a)||!k.leadingWhitespace&&hb.test(a)||rb[(jb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(ib,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(m.cleanData(ub(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,m.cleanData(ub(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,n=this,o=l-1,p=a[0],q=m.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&nb.test(p))return this.each(function(c){var d=n.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(i=m.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=m.map(ub(i,"script"),xb),f=g.length;l>j;j++)d=i,j!==o&&(d=m.clone(d,!0,!0),f&&m.merge(g,ub(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,m.map(g,yb),j=0;f>j;j++)d=g[j],ob.test(d.type||"")&&!m._data(d,"globalEval")&&m.contains(h,d)&&(d.src?m._evalUrl&&m._evalUrl(d.src):m.globalEval((d.text||d.textContent||d.innerHTML||"").replace(qb,"")));i=c=null}return this}}),m.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){m.fn[a]=function(a){for(var c,d=0,e=[],g=m(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),m(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Cb,Db={};function Eb(b,c){var d,e=m(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:m.css(e[0],"display");return e.detach(),f}function Fb(a){var b=y,c=Db[a];return c||(c=Eb(a,b),"none"!==c&&c||(Cb=(Cb||m("