From 3c308dac3c9038ec479daed88a0c3d9c70224ebb Mon Sep 17 00:00:00 2001 From: James Cunningham Date: Tue, 2 Aug 2016 14:48:37 -0700 Subject: [PATCH] Move all of deploy's api functionality into bases. --- freight/api/bases/__init__.py | 0 freight/api/bases/details.py | 75 ++++++++++++ freight/api/bases/index.py | 214 ++++++++++++++++++++++++++++++++++ freight/api/bases/log.py | 78 +++++++++++++ freight/api/deploy_details.py | 69 +---------- freight/api/deploy_index.py | 206 +------------------------------- freight/api/deploy_log.py | 71 +---------- 7 files changed, 383 insertions(+), 330 deletions(-) create mode 100644 freight/api/bases/__init__.py create mode 100644 freight/api/bases/details.py create mode 100644 freight/api/bases/index.py create mode 100644 freight/api/bases/log.py diff --git a/freight/api/bases/__init__.py b/freight/api/bases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freight/api/bases/details.py b/freight/api/bases/details.py new file mode 100644 index 00000000..2570d953 --- /dev/null +++ b/freight/api/bases/details.py @@ -0,0 +1,75 @@ +from __future__ import absolute_import + +from flask_restful import reqparse + +from freight.api.base import ApiView +from freight.api.serializer import serialize +from freight.config import db, redis +from freight.models import App, Task, TaskStatus +from freight.notifiers import NotifierEvent +from freight.notifiers.utils import send_task_notifications +from freight.utils.redis import lock + + +class BaseMixin(object): + def _get_obj(self, app=None, env=None, number=None, obj_model=None, **kwargs): + obj_id = [v for k, v in kwargs.iteritems() if k.endswith('_id')] + if obj_id: + return obj_model.query.get(obj_id[0]) + try: + app = App.query.filter(App.name == app)[0] + except IndexError: + return None + try: + return obj_model.query.filter( + obj_model.app_id == app.id, + obj_model.environment == env, + obj_model.number == number, + )[0] + except IndexError: + return None + + +class BaseDetailsApiView(ApiView, BaseMixin): + def __init__(self): + raise NotImplementedError + # self.obj_model = Task + + def get(self, **kwargs): + """ + Retrive a task. + """ + kwargs['obj_model'] = self.obj_model + obj = self._get_obj(**kwargs) + if obj is None: + return self.error('Invalid obj', name='invalid_resource', status_code=404) + + return self.respond(serialize(obj)) + + put_parser = reqparse.RequestParser() + put_parser.add_argument('status', choices=('cancelled',)) + + def put(self, **kwargs): + kwargs['obj_model'] = self.obj_model + obj = self._get_obj(**kwargs) + if obj is None: + return self.error('Invalid obj', name='invalid_resource', status_code=404) + + with lock(redis, '{}:{}'.format(type(obj).__name__, obj.id), timeout=5): + # we have to refetch in order to ensure lock state changes + obj = self.obj_model.query.get(obj.id) + task = Task.query.get(obj.task_id) + args = self.put_parser.parse_args() + if args.status: + assert task.status in (TaskStatus.pending, TaskStatus.in_progress) + assert args.status == 'cancelled' + did_cancel = task.status == TaskStatus.pending + task.status = TaskStatus.cancelled + + db.session.add(task) + db.session.commit() + + if args.status and did_cancel: + send_task_notifications(task, NotifierEvent.TASK_FINISHED) + + return self.respond(serialize(obj)) diff --git a/freight/api/bases/index.py b/freight/api/bases/index.py new file mode 100644 index 00000000..6cc32ce1 --- /dev/null +++ b/freight/api/bases/index.py @@ -0,0 +1,214 @@ +from __future__ import absolute_import + +import json + +from flask_restful import reqparse, inputs + +from freight import checks, vcs +from freight.api.base import ApiView +from freight.api.serializer import serialize +from freight.config import db, redis +from freight.exceptions import CheckError, CheckPending +from freight.models import ( + App, Repository, Task, TaskStatus, User, + TaskConfig, TaskConfigType, +) +from freight.notifiers import NotifierEvent +from freight.notifiers.utils import send_task_notifications +from freight.utils.auth import get_current_user +from freight.utils.redis import lock +from freight.utils.workspace import Workspace + + +class BaseIndexApiView(ApiView): + """ + All of the helper functions that deploy and build share. + """ + def __init__(self): + # self.obj_model = Dummy + # self.sequence_model = DummySequence + + raise NotImplementedError + + def _get_internal_ref(self, app, env, ref): + # find the most recent green status for this app + if ref == ':current': + return app.get_current_sha(env) + + # the previous stable ref (before current) + if ref == ':previous': + current_sha = app.get_current_sha(env) + + if not current_sha: + return + + return app.get_previous_sha(env, current_sha=current_sha) + raise ValueError('Unknown ref: {}'.format(ref)) + + get_parser = reqparse.RequestParser() + get_parser.add_argument('app', location='args') + get_parser.add_argument('user', location='args') + get_parser.add_argument('env', location='args') + get_parser.add_argument('ref', location='args') + get_parser.add_argument('status', location='args', action='append') + + def get(self): + """ + Retrieve a list of objects. + + If any parameters are invalid the result will simply be an empty list. + """ + args = self.get_parser.parse_args() + + qs_filters = [] + + if args.app: + app = App.query.filter(App.name == args.app).first() + if not app: + return self.respond([]) + qs_filters.append(self.obj_model.app_id == app.id) + + if args.user: + user = User.query.filter(User.name == args.user).first() + if not user: + return self.respond([]) + qs_filters.append(Task.user_id == user.id) + + if args.env: + qs_filters.append(self.obj_model.environment == args.env) + + if args.ref: + qs_filters.append(Task.ref == args.ref) + + if args.status: + status_list = map(TaskStatus.label_to_id, args.status) + qs_filters.append(Task.status.in_(status_list)) + + obj_qs = self.obj_model.query.filter(*qs_filters).order_by(self.obj_model.id.desc()) + + return self.paginate(obj_qs, on_results=serialize) + + post_parser = reqparse.RequestParser() + post_parser.add_argument('app', required=True) + post_parser.add_argument('params', type=json.loads) + post_parser.add_argument('user') + post_parser.add_argument('env', default='production') + post_parser.add_argument('ref') + post_parser.add_argument('force', default=False, type=inputs.boolean) + + def post(self): + """ + Given any constraints for a task are within acceptable bounds, create + a new task and enqueue it. + """ + args = self.post_parser.parse_args() + + user = get_current_user() + if not user: + username = args.user + if not username: + return self.error('Missing required argument "user"', status_code=400) + + with lock(redis, 'user:create:{}'.format(username), timeout=5): + # TODO(dcramer): this needs to be a get_or_create pattern and + # ideally moved outside of the lock + user = User.query.filter(User.name == username).first() + if not user: + user = User(name=username) + db.session.add(user) + db.session.flush() + elif args.user: + return self.error('Cannot specify user when using session authentication.', status_code=400) + + app = App.query.filter(App.name == args.app).first() + if not app: + return self.error('Invalid app', name='invalid_resource', status_code=404) + + obj_config = TaskConfig.query.filter( + TaskConfig.app_id == app.id, + # TODO(jtcunning) Ehhhhhhhhhhh. + TaskConfig.type == getattr(TaskConfigType, self.obj_model.__name__.lower()), + ).first() + if not obj_config: + return self.error('Missing config', name='missing_conf', status_code=404) + + params = None + + repo = Repository.query.get(app.repository_id) + + workspace = Workspace( + path=repo.get_path(), + ) + + vcs_backend = vcs.get( + repo.vcs, + url=repo.url, + workspace=workspace, + ) + + with lock(redis, 'repo:update:{}'.format(repo.id)): + vcs_backend.clone_or_update() + + ref = args.ref or app.get_default_ref(args.env) + + # look for our special refs (prefixed via a colon) + # TODO(dcramer): this should be supported outside of just this endpoint + if ref.startswith(':'): + sha = self._get_internal_ref(app, args.env, ref) + if not sha: + return self.error('Invalid ref', name='invalid_ref', status_code=400) + else: + try: + sha = vcs_backend.get_sha(ref) + except vcs.UnknownRevision: + return self.error('Invalid ref', name='invalid_ref', status_code=400) + + if args.params is not None: + params = args.params + + if not args.force: + for check_config in obj_config.checks: + check = checks.get(check_config['type']) + try: + check.check(app, sha, check_config['config']) + except CheckPending: + pass + except CheckError as e: + return self.error( + message=unicode(e), + name='check_failed', + ) + + with lock(redis, '{}:create:{}'.format(self.obj_model.__name__, app.id), timeout=5): + task = Task( + app_id=app.id, + # TODO(dcramer): ref should default based on app config + ref=ref, + sha=sha, + params=params, + status=TaskStatus.pending, + user_id=user.id, + provider=obj_config.provider, + data={ + 'force': args.force, + 'provider_config': obj_config.provider_config, + 'notifiers': obj_config.notifiers, + 'checks': obj_config.checks, + }, + ) + db.session.add(task) + db.session.flush() + db.session.refresh(task) + + obj = self.obj_model( + task_id=task.id, + app_id=app.id, + environment=args.env, + number=self.sequence_model.get_clause(app.id, args.env), + ) + db.session.add(obj) + db.session.commit() + + send_task_notifications(task, NotifierEvent.TASK_QUEUED) + + return self.respond(serialize(obj), status_code=201) diff --git a/freight/api/bases/log.py b/freight/api/bases/log.py new file mode 100644 index 00000000..2d195d89 --- /dev/null +++ b/freight/api/bases/log.py @@ -0,0 +1,78 @@ +from __future__ import absolute_import + +from flask_restful import reqparse + +from freight.api.base import ApiView +from freight.api.bases.details import BaseMixin +from freight.config import db +from freight.models import LogChunk + + +class BaseLogApiView(ApiView, BaseMixin): + get_parser = reqparse.RequestParser() + get_parser.add_argument('offset', location='args', type=int, default=0) + get_parser.add_argument('limit', location='args', type=int) + + def __init__(self): + raise NotImplementedError + # self.obj_model = Task + + def get(self, **kwargs): + """ + Retrieve a log. + """ + kwargs['obj_model'] = self.obj_model + obj = self._get_obj(**kwargs) + if obj is None: + return self.error( + 'Invalid {}'.format(type(obj)), + name='invalid_resource', + status_code=404 + ) + + args = self.get_parser.parse_args() + + queryset = db.session.query( + LogChunk.text, LogChunk.offset, LogChunk.size + ).filter( + LogChunk.task_id == obj.task_id, + ).order_by(LogChunk.offset.asc()) + + if args.offset == -1: + # starting from the end so we need to know total size + tail = db.session.query(LogChunk.offset + LogChunk.size).filter( + LogChunk.task_id == obj.task_id, + ).order_by(LogChunk.offset.desc()).limit(1).scalar() + + if tail is None: + logchunks = [] + else: + if args.limit: + queryset = queryset.filter( + (LogChunk.offset + LogChunk.size) >= max(tail - args.limit + 1, 0), + ) + else: + if args.offset: + queryset = queryset.filter( + LogChunk.offset >= args.offset, + ) + if args.limit: + queryset = queryset.filter( + LogChunk.offset < args.offset + args.limit, + ) + + logchunks = list(queryset) + + if logchunks: + next_offset = logchunks[-1].offset + logchunks[-1].size + else: + next_offset = args.offset + + links = [self.build_cursor_link('next', next_offset)] + + context = { + 'text': ''.join(l.text for l in logchunks), + 'nextOffset': next_offset, + } + + return self.respond(context, links=links) diff --git a/freight/api/deploy_details.py b/freight/api/deploy_details.py index cd88bb2a..f35384f0 100644 --- a/freight/api/deploy_details.py +++ b/freight/api/deploy_details.py @@ -1,68 +1,9 @@ from __future__ import absolute_import -from flask_restful import reqparse +from freight.api.bases.details import BaseDetailsApiView +from freight.models import Deploy -from freight.api.base import ApiView -from freight.api.serializer import serialize -from freight.config import db, redis -from freight.models import App, Task, Deploy, TaskStatus -from freight.notifiers import NotifierEvent -from freight.notifiers.utils import send_task_notifications -from freight.utils.redis import lock - -class DeployMixin(object): - def _get_deploy(self, app=None, env=None, number=None, deploy_id=None): - if deploy_id: - return Deploy.query.get(deploy_id) - try: - app = App.query.filter(App.name == app)[0] - except IndexError: - return None - try: - return Deploy.query.filter( - Deploy.app_id == app.id, - Deploy.environment == env, - Deploy.number == number, - )[0] - except IndexError: - return None - - -class DeployDetailsApiView(ApiView, DeployMixin): - def get(self, **kwargs): - """ - Retrive a task. - """ - deploy = self._get_deploy(**kwargs) - if deploy is None: - return self.error('Invalid deploy', name='invalid_resource', status_code=404) - - return self.respond(serialize(deploy)) - - put_parser = reqparse.RequestParser() - put_parser.add_argument('status', choices=('cancelled',)) - - def put(self, **kwargs): - deploy = self._get_deploy(**kwargs) - if deploy is None: - return self.error('Invalid deploy', name='invalid_resource', status_code=404) - - with lock(redis, 'deploy:{}'.format(deploy.id), timeout=5): - # we have to refetch in order to ensure lock state changes - deploy = Deploy.query.get(deploy.id) - task = Task.query.get(deploy.task_id) - args = self.put_parser.parse_args() - if args.status: - assert task.status in (TaskStatus.pending, TaskStatus.in_progress) - assert args.status == 'cancelled' - did_cancel = task.status == TaskStatus.pending - task.status = TaskStatus.cancelled - - db.session.add(task) - db.session.commit() - - if args.status and did_cancel: - send_task_notifications(task, NotifierEvent.TASK_FINISHED) - - return self.respond(serialize(deploy)) +class DeployDetailsApiView(BaseDetailsApiView): + def __init__(self): + self.obj_model = Deploy diff --git a/freight/api/deploy_index.py b/freight/api/deploy_index.py index 540f76f2..f786f679 100644 --- a/freight/api/deploy_index.py +++ b/freight/api/deploy_index.py @@ -1,204 +1,10 @@ from __future__ import absolute_import -import json +from freight.api.bases.index import BaseIndexApiView +from freight.models import Deploy, DeploySequence -from flask_restful import reqparse, inputs -from freight import checks, vcs -from freight.api.base import ApiView -from freight.api.serializer import serialize -from freight.config import db, redis -from freight.exceptions import CheckError, CheckPending -from freight.models import ( - App, Repository, Task, Deploy, DeploySequence, TaskStatus, User, - TaskConfig, TaskConfigType, -) -from freight.notifiers import NotifierEvent -from freight.notifiers.utils import send_task_notifications -from freight.utils.auth import get_current_user -from freight.utils.redis import lock -from freight.utils.workspace import Workspace - - -class DeployIndexApiView(ApiView): - def _get_internal_ref(self, app, env, ref): - # find the most recent green deploy for this app - if ref == ':current': - return app.get_current_sha(env) - - # the previous stable ref (before current) - if ref == ':previous': - current_sha = app.get_current_sha(env) - - if not current_sha: - return - - return app.get_previous_sha(env, current_sha=current_sha) - raise ValueError('Unknown ref: {}'.format(ref)) - - get_parser = reqparse.RequestParser() - get_parser.add_argument('app', location='args') - get_parser.add_argument('user', location='args') - get_parser.add_argument('env', location='args') - get_parser.add_argument('ref', location='args') - get_parser.add_argument('status', location='args', action='append') - - def get(self): - """ - Retrieve a list of deploys. - - If any parameters are invalid the result will simply be an empty list. - """ - args = self.get_parser.parse_args() - - qs_filters = [] - - if args.app: - app = App.query.filter(App.name == args.app).first() - if not app: - return self.respond([]) - qs_filters.append(Deploy.app_id == app.id) - - if args.user: - user = User.query.filter(User.name == args.user).first() - if not user: - return self.respond([]) - qs_filters.append(Task.user_id == user.id) - - if args.env: - qs_filters.append(Deploy.environment == args.env) - - if args.ref: - qs_filters.append(Task.ref == args.ref) - - if args.status: - status_list = map(TaskStatus.label_to_id, args.status) - qs_filters.append(Task.status.in_(status_list)) - - deploy_qs = Deploy.query.filter(*qs_filters).order_by(Deploy.id.desc()) - - return self.paginate(deploy_qs, on_results=serialize) - - post_parser = reqparse.RequestParser() - post_parser.add_argument('app', required=True) - post_parser.add_argument('params', type=json.loads) - post_parser.add_argument('user') - post_parser.add_argument('env', default='production') - post_parser.add_argument('ref') - post_parser.add_argument('force', default=False, type=inputs.boolean) - - def post(self): - """ - Given any constraints for a task are within acceptable bounds, create - a new task and enqueue it. - """ - args = self.post_parser.parse_args() - - user = get_current_user() - if not user: - username = args.user - if not username: - return self.error('Missing required argument "user"', status_code=400) - - with lock(redis, 'user:create:{}'.format(username), timeout=5): - # TODO(dcramer): this needs to be a get_or_create pattern and - # ideally moved outside of the lock - user = User.query.filter(User.name == username).first() - if not user: - user = User(name=username) - db.session.add(user) - db.session.flush() - elif args.user: - return self.error('Cannot specify user when using session authentication.', status_code=400) - - app = App.query.filter(App.name == args.app).first() - if not app: - return self.error('Invalid app', name='invalid_resource', status_code=404) - - deploy_config = TaskConfig.query.filter( - TaskConfig.app_id == app.id, - TaskConfig.type == TaskConfigType.deploy, - ).first() - if not deploy_config: - return self.error('Missing deploy config', name='missing_conf', status_code=404) - - params = None - - repo = Repository.query.get(app.repository_id) - - workspace = Workspace( - path=repo.get_path(), - ) - - vcs_backend = vcs.get( - repo.vcs, - url=repo.url, - workspace=workspace, - ) - - with lock(redis, 'repo:update:{}'.format(repo.id)): - vcs_backend.clone_or_update() - - ref = args.ref or app.get_default_ref(args.env) - - # look for our special refs (prefixed via a colon) - # TODO(dcramer): this should be supported outside of just this endpoint - if ref.startswith(':'): - sha = self._get_internal_ref(app, args.env, ref) - if not sha: - return self.error('Invalid ref', name='invalid_ref', status_code=400) - else: - try: - sha = vcs_backend.get_sha(ref) - except vcs.UnknownRevision: - return self.error('Invalid ref', name='invalid_ref', status_code=400) - - if args.params is not None: - params = args.params - - if not args.force: - for check_config in deploy_config.checks: - check = checks.get(check_config['type']) - try: - check.check(app, sha, check_config['config']) - except CheckPending: - pass - except CheckError as e: - return self.error( - message=unicode(e), - name='check_failed', - ) - - with lock(redis, 'deploy:create:{}'.format(app.id), timeout=5): - task = Task( - app_id=app.id, - # TODO(dcramer): ref should default based on app config - ref=ref, - sha=sha, - params=params, - status=TaskStatus.pending, - user_id=user.id, - provider=deploy_config.provider, - data={ - 'force': args.force, - 'provider_config': deploy_config.provider_config, - 'notifiers': deploy_config.notifiers, - 'checks': deploy_config.checks, - }, - ) - db.session.add(task) - db.session.flush() - db.session.refresh(task) - - deploy = Deploy( - task_id=task.id, - app_id=app.id, - environment=args.env, - number=DeploySequence.get_clause(app.id, args.env), - ) - db.session.add(deploy) - db.session.commit() - - send_task_notifications(task, NotifierEvent.TASK_QUEUED) - - return self.respond(serialize(deploy), status_code=201) +class DeployIndexApiView(BaseIndexApiView): + def __init__(self): + self.obj_model = Deploy + self.sequence_model = DeploySequence diff --git a/freight/api/deploy_log.py b/freight/api/deploy_log.py index 87bf6206..e31a6634 100644 --- a/freight/api/deploy_log.py +++ b/freight/api/deploy_log.py @@ -1,70 +1,9 @@ from __future__ import absolute_import -from flask_restful import reqparse +from freight.api.bases.log import BaseLogApiView +from freight.models import Deploy -from freight.api.base import ApiView -from freight.config import db -from freight.models import LogChunk -from .deploy_details import DeployMixin - - -class DeployLogApiView(ApiView, DeployMixin): - get_parser = reqparse.RequestParser() - get_parser.add_argument('offset', location='args', type=int, default=0) - get_parser.add_argument('limit', location='args', type=int) - - def get(self, **kwargs): - """ - Retrieve deploy log. - """ - deploy = self._get_deploy(**kwargs) - if deploy is None: - return self.error('Invalid deploy', name='invalid_resource', status_code=404) - - args = self.get_parser.parse_args() - - queryset = db.session.query( - LogChunk.text, LogChunk.offset, LogChunk.size - ).filter( - LogChunk.task_id == deploy.task_id, - ).order_by(LogChunk.offset.asc()) - - if args.offset == -1: - # starting from the end so we need to know total size - tail = db.session.query(LogChunk.offset + LogChunk.size).filter( - LogChunk.task_id == deploy.task_id, - ).order_by(LogChunk.offset.desc()).limit(1).scalar() - - if tail is None: - logchunks = [] - else: - if args.limit: - queryset = queryset.filter( - (LogChunk.offset + LogChunk.size) >= max(tail - args.limit + 1, 0), - ) - else: - if args.offset: - queryset = queryset.filter( - LogChunk.offset >= args.offset, - ) - if args.limit: - queryset = queryset.filter( - LogChunk.offset < args.offset + args.limit, - ) - - logchunks = list(queryset) - - if logchunks: - next_offset = logchunks[-1].offset + logchunks[-1].size - else: - next_offset = args.offset - - links = [self.build_cursor_link('next', next_offset)] - - context = { - 'text': ''.join(l.text for l in logchunks), - 'nextOffset': next_offset, - } - - return self.respond(context, links=links) +class DeployLogApiView(BaseLogApiView): + def __init__(self): + self.obj_model = Deploy