From 81a720cf3b22380100b05cb19a2784b103fe7550 Mon Sep 17 00:00:00 2001 From: bobosss Date: Thu, 30 Oct 2014 16:21:32 +0200 Subject: [PATCH 1/4] support for string uploads and more options for s3 --- flask_store/__init__.py | 2 +- flask_store/providers/__init__.py | 13 ++++++----- flask_store/providers/s3.py | 38 +++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/flask_store/__init__.py b/flask_store/__init__.py index d537263..878e09d 100644 --- a/flask_store/__init__.py +++ b/flask_store/__init__.py @@ -127,7 +127,7 @@ def check_config(self, app): if hasattr(self.Provider, 'REQUIRED_CONFIGURATION'): for name in self.Provider.REQUIRED_CONFIGURATION: - if not app.config.get(name): + if app.config.get(name, None) == None: raise NotConfiguredError( '{0} must be configured in your flask application ' 'configuration'.format(name)) diff --git a/flask_store/providers/__init__.py b/flask_store/providers/__init__.py index d9b52bb..45a520c 100644 --- a/flask_store/providers/__init__.py +++ b/flask_store/providers/__init__.py @@ -10,6 +10,9 @@ import os import shortuuid import urlparse +import random +import string + from flask import current_app from flask_store.utils import is_path, path_to_uri @@ -50,16 +53,14 @@ def __init__(self, fp, location=None): # Save the fp - could be a FileStorage instance or a path self.fp = fp - # Get the filename if is_path(fp): self.filename = os.path.basename(fp) + + elif isinstance(fp, FileStorage): + self.filename = fp.filename else: - if not isinstance(fp, FileStorage): - raise ValueError( - 'File pointer must be an instance of a ' - 'werkzeug.datastructures.FileStorage') - self.filename = fp.filename + self.filename = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(12)) # Save location self.location = location diff --git a/flask_store/providers/s3.py b/flask_store/providers/s3.py index 91012bb..2bb3b51 100644 --- a/flask_store/providers/s3.py +++ b/flask_store/providers/s3.py @@ -22,6 +22,12 @@ class FooForm(Form): app.config['STORE_PROVIDER'] = 'flask_store.providers.s3.S3Provider' app.config['STORE_S3_ACCESS_KEY'] = 'foo' app.confog['STORE_S3_SECRET_KEY'] = 'bar' + app.confog['STORE_S3_REDUCED_REDUNDANCY'] = False + app.confog['STORE_S3_HEADERS'] = { + 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', + 'Cache-Control': 'max-age=86400', + } + store = Store(app) @@ -50,7 +56,7 @@ def upload(): import io import mimetypes import os - +from StringIO import StringIO from flask import copy_current_request_context, current_app from flask_store.exceptions import NotConfiguredError from flask_store.providers import Provider @@ -68,7 +74,10 @@ class S3Provider(Provider): 'STORE_S3_ACCESS_KEY', 'STORE_S3_SECRET_KEY', 'STORE_S3_BUCKET', - 'STORE_S3_REGION'] + 'STORE_S3_REGION', + 'STORE_S3_REDUCED_REDUNDANCY', + 'STORE_S3_HEADERS'] + policy = 'public-read' @staticmethod def app_defaults(app): @@ -98,11 +107,14 @@ def connect(self): """ if not hasattr(self, '_s3connection'): - s3connection = boto.s3.connect_to_region( - current_app.config['STORE_S3_REGION'], - aws_access_key_id=current_app.config['STORE_S3_ACCESS_KEY'], - aws_secret_access_key=current_app.config['STORE_S3_SECRET_KEY']) - setattr(self, '_s3connection', s3connection) + try: + s3connection = boto.s3.connect_to_region( + current_app.config['STORE_S3_REGION'], + aws_access_key_id=current_app.config['STORE_S3_ACCESS_KEY'], + aws_secret_access_key=current_app.config['STORE_S3_SECRET_KEY']) + setattr(self, '_s3connection', s3connection) + except S3ResponseError: + raise return getattr(self, '_s3connection') def bucket(self, s3connection): @@ -170,14 +182,20 @@ def save(self): key = bucket.new_key(path) key.set_metadata('Content-Type', mimetype) - key.set_contents_from_file(fp) - key.set_acl('public-read') + for header, value in current_app.config['STORE_S3_HEADERS'][0].iteritems(): + key.set_metadata(header, value) + if isinstance(fp,StringIO): + key.set_contents_from_string(fp.getvalue(), reduced_redundancy=current_app.config['STORE_S3_REDUCED_REDUNDANCY']) + else: + key.set_contents_from_file(fp, reduced_redundancy=current_app.config['STORE_S3_REDUCED_REDUNDANCY']) + + key.set_acl(self.policy) # Update the filename - it may have changes self.filename = filename def open(self): - """ Opens an S3 key and returns an oepn File Like object pointer. + """ Opens an S3 key and returns an open File Like object pointer. Returns ------- From 6ec8448a2cb89a68390aad2653038cacddb31c09 Mon Sep 17 00:00:00 2001 From: bobosss Date: Fri, 31 Oct 2014 14:28:31 +0200 Subject: [PATCH 2/4] also support buffer file save on local provider --- flask_store/providers/local.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/flask_store/providers/local.py b/flask_store/providers/local.py index 0caa6e9..e283e04 100644 --- a/flask_store/providers/local.py +++ b/flask_store/providers/local.py @@ -37,6 +37,7 @@ def upload(): import errno import os +from StringIO import StringIO from flask_store.providers import Provider @@ -130,9 +131,14 @@ def save(self): if not os.path.isdir(directory): raise IOError('{0} is not a directory'.format(directory)) - # Save the file - fp.save(path) - fp.close() + if isinstance(fp,StringIO): + output = open(path, "wb") + output.write(fp.getvalue()) + output.close() + else: + # Save the file + fp.save(path) + fp.close() # Update the filename - it may have changes self.filename = filename @@ -153,3 +159,4 @@ def open(self): raise IOError('File does not exist: {0}'.format(self.absolute_path)) return fp + From f58dcbca0efaabaeef31aba9819344eb61fc7be7 Mon Sep 17 00:00:00 2001 From: bobosss Date: Fri, 31 Oct 2014 15:25:20 +0200 Subject: [PATCH 3/4] make bucket a property --- flask_store/providers/s3.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/flask_store/providers/s3.py b/flask_store/providers/s3.py index 2bb3b51..ffba22b 100644 --- a/flask_store/providers/s3.py +++ b/flask_store/providers/s3.py @@ -79,6 +79,12 @@ class S3Provider(Provider): 'STORE_S3_HEADERS'] policy = 'public-read' + _bucket = None + + def __init__(self, *args, **kwargs): + super (S3Provider, self).__init__(*args, **kwargs) + self._bucket = current_app.config.get('STORE_S3_BUCKET') + @staticmethod def app_defaults(app): """ Sets sensible application configuration settings for this @@ -117,12 +123,22 @@ def connect(self): raise return getattr(self, '_s3connection') - def bucket(self, s3connection): - """ Returns an S3 bucket instance - """ - return s3connection.get_bucket( - current_app.config.get('STORE_S3_BUCKET')) + + @property + def bucket(self): + """make bucket a property so user can change the bucket on runtime""" + return self._bucket + @bucket.setter + def bucket(self, value): + self._bucket = value + + + + def get_bucket(self, s3connection): + """ Returns an S3 bucket instance""" + + return s3connection.get_bucket(self.bucket) def join(self, *parts): """ Joins paths into a url. @@ -155,7 +171,7 @@ def exists(self, filename): """ s3connection = self.connect() - bucket = self.bucket(s3connection) + bucket = self.get_bucket(s3connection) path = self.join(self.store_path, filename) key = boto.s3.key.Key(name=path, bucket=bucket) @@ -173,7 +189,7 @@ def save(self): fp = self.fp s3connection = self.connect() - bucket = self.bucket(s3connection) + bucket = self.get_bucket(s3connection) filename = self.safe_filename(self.filename) path = self.join(self.store_path, filename) mimetype, encoding = mimetypes.guess_type(filename) @@ -204,7 +220,7 @@ def open(self): """ s3connection = self.connect() - bucket = self.bucket(s3connection) + bucket = self.get_bucket(s3connection) key = bucket.get_key(self.relative_path) if not key: @@ -213,6 +229,7 @@ def open(self): return io.BytesIO(key.read()) # In memory + class S3GeventProvider(S3Provider): """ A Gevent Support for :class:`.S3Provider`. Calling :meth:`.save` here will spawn a greenlet which will handle the actual upload process. From 31c92c2b5afc9005df962320c2ac80fced166d07 Mon Sep 17 00:00:00 2001 From: Costas Date: Fri, 19 Apr 2024 23:12:20 +0300 Subject: [PATCH 4/4] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9fe7836..9327d64 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def read_requirements(filename): requirements = [] try: - with open(filename, 'rb') as f: + with open(filename, 'r') as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#') or line == '':