From 2cb03448773d145c06521ca063a8fa7c3aabb5a3 Mon Sep 17 00:00:00 2001 From: Harshit Sharma Date: Thu, 20 Jun 2019 17:19:31 +0530 Subject: [PATCH 1/2] Add daily-round-robin scheduler algorithm Adds daily-round-robin algorithm for oncall scheduler. This algorithm creates events for a roster in a daily round robin fashion. --- db/schema.v0.sql | 8 +- src/oncall/api/v0/roster_user.py | 39 +- src/oncall/scheduler/daily-round-robin.py | 442 ++++++++++++++++++++++ src/oncall/ui/static/js/oncall.js | 3 +- src/oncall/ui/templates/index.html | 28 +- 5 files changed, 509 insertions(+), 11 deletions(-) create mode 100644 src/oncall/scheduler/daily-round-robin.py diff --git a/db/schema.v0.sql b/db/schema.v0.sql index b1b999eb..19c39776 100644 --- a/db/schema.v0.sql +++ b/db/schema.v0.sql @@ -312,7 +312,7 @@ CREATE TABLE IF NOT EXISTS `contact_mode` ( -- Initialize contact modes -- ----------------------------------------------------- INSERT INTO `contact_mode` (`name`) -VALUES ('email'), ('sms'), ('call'), ('slack'), ('teams_messenger'); +VALUES ('email'), ('sms'), ('call'), ('slack'); -- ----------------------------------------------------- -- Table `user_contact` @@ -464,8 +464,10 @@ VALUES ('default', 'Default scheduling algorithm'), ('round-robin', 'Round robin in roster order; does not respect vacations/conflicts'), - ('no-skip-matching', - 'Default scheduling algorithm; doesn\'t skips creating events if matching events already exist on the calendar'); + ('no-skip-matching', + 'Default scheduling algorithm; doesn\'t skips creating events if matching events already exist on the calendar'), + ('daily-round-robin', + 'Follows roster order of round-robin, but divides week long shifts into day long shifts among different users'); -- ----------------------------------------------------- -- Initialize notification types diff --git a/src/oncall/api/v0/roster_user.py b/src/oncall/api/v0/roster_user.py index dfb9cc94..7c112abc 100644 --- a/src/oncall/api/v0/roster_user.py +++ b/src/oncall/api/v0/roster_user.py @@ -8,7 +8,8 @@ from ...utils import load_json_body, unsubscribe_notifications, create_audit from ... import db from ...constants import ROSTER_USER_DELETED, ROSTER_USER_EDITED - +import logging +logger = logging.getLogger() @login_required def on_delete(req, resp, team, roster, user): @@ -44,15 +45,49 @@ def on_delete(req, resp, team, roster, user): roster_id = cursor.fetchone() if roster_id is None: raise HTTPNotFound() + + cursor.execute('''SELECT `user_id` FROM `roster_user` + WHERE `roster_id` = %s''', + roster_id) + user_ids = [r[0] for r in cursor] + cursor.execute('''SELECT `id` FROM `user` + WHERE `name` = %s''', + user) + user_id = [r[0] for r in cursor] + cursor.execute('''SELECT `id` FROM `schedule` + WHERE `roster_id` = %s AND `last_scheduled_user_id`=%s ''', + (roster_id,user_id[0])) + schedule_id = [r[0] for r in cursor] + last_scheduled_user_id=None + if(schedule_id): + cursor.execute('''SELECT `last_scheduled_user_id` FROM `schedule` + WHERE `roster_id` = %s AND `id`=%s ''', + (roster_id,schedule_id[0])) + last_scheduled_user_id = [r[0] for r in cursor] + + if(last_scheduled_user_id): + if(last_scheduled_user_id[0]==user_id[0]): + user_index=user_ids.index(user_id[0]) + last_scheduled_user=0 + + if(user_index==0): + last_scheduled_user=user_ids[len(user_ids)-1] + else: + last_scheduled_user=user_ids[user_index-1] + for ids in schedule_id: + cursor.execute('UPDATE `schedule` SET `last_scheduled_user_id` = %s WHERE `id` = %s', (last_scheduled_user, ids)) + cursor.execute('''DELETE FROM `roster_user` WHERE `roster_id`= %s AND `user_id`=(SELECT `id` FROM `user` WHERE `name`=%s)''', (roster_id, user)) + cursor.execute('''DELETE `schedule_order` FROM `schedule_order` JOIN `schedule` ON `schedule`.`id` = `schedule_order`.`schedule_id` WHERE `roster_id` = %s AND user_id = (SELECT `id` FROM `user` WHERE `name` = %s)''', (roster_id, user)) + create_audit({'roster': roster, 'user': user}, team, ROSTER_USER_DELETED, req, cursor) # Remove user from the team if needed @@ -130,4 +165,4 @@ def on_put(req, resp, team, roster, user): cursor.close() connection.close() resp.status = HTTP_200 - resp.body = '[]' + resp.body = '[]' \ No newline at end of file diff --git a/src/oncall/scheduler/daily-round-robin.py b/src/oncall/scheduler/daily-round-robin.py new file mode 100644 index 00000000..f9cac3a6 --- /dev/null +++ b/src/oncall/scheduler/daily-round-robin.py @@ -0,0 +1,442 @@ +from datetime import datetime, timedelta +from pytz import timezone, utc +from oncall.utils import gen_link_id +from falcon import HTTPBadRequest +from ujson import dumps as json_dumps +import time +import logging +import operator +import requests + +logger = logging.getLogger() + +UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=utc) +SECONDS_IN_A_DAY = 24 * 60 * 60 +SECONDS_IN_A_WEEK = SECONDS_IN_A_DAY * 7 + +columns = { + 'id': '`temp_event`.`id` as `id`', + 'start': '`temp_event`.`start` as `start`', + 'end': '`temp_event`.`end` as `end`', + 'role': '`role`.`name` as `role`', + 'team': '`team`.`name` as `team`', + 'user': '`user`.`name` as `user`', + 'full_name': '`user`.`full_name` as `full_name`', + 'schedule_id': '`temp_event`.`schedule_id`', + 'link_id': '`temp_event`.`link_id`', + 'note': '`temp_event`.`note`', +} + +constraints = { + 'start__lt': '`temp_event`.`start` < %s', + 'end__ge': '`temp_event`.`end` >= %s', + 'team__eq': '`team`.`name` = %s', +} + +all_columns = ', '.join(columns.values()) + + +class Scheduler(object): + def __init__(self): + pass + + # DB interactions + def get_role_id(self, role_name, cursor): + cursor.execute('SELECT `id` FROM `role` WHERE `name` = %s', role_name) + role_id = cursor.fetchone()['id'] + return role_id + + def get_schedule_last_event_end(self, schedule, cursor, table_name='event'): + query = 'SELECT `end` FROM `%s` WHERE `schedule_id` = %%r ORDER BY `end` DESC LIMIT 1' % table_name + cursor.execute(query, schedule['id']) + if cursor.rowcount != 0: + return cursor.fetchone()['end'] + else: + return None + + def get_schedule_last_epoch(self, schedule, cursor): + cursor.execute('SELECT `last_epoch_scheduled` FROM `schedule` WHERE `id` = %s', + schedule['id']) + if cursor.rowcount != 0: + return cursor.fetchone()['last_epoch_scheduled'] + else: + return None + + def get_roster_user_ids(self, roster_id, cursor): + cursor.execute(''' + SELECT `roster_user`.`user_id` FROM `roster_user` + JOIN `user` ON `user`.`id` = `roster_user`.`user_id` + WHERE `roster_user`.`in_rotation` = 1 AND `roster_user`.`roster_id` = %r + AND `user`.`active` = TRUE''', roster_id) + return [r['user_id'] for r in cursor] + + def get_busy_user_by_event_range(self, user_ids, team_id, events, cursor, table_name='event'): + ''' Find which users have overlapping events for the same team in this time range''' + query_params = [user_ids] + range_check = [] + for e in events: + range_check.append('(%s < `end` AND `start` < %s)') + query_params += [e['start'], e['end']] + + cursor.execute('''SELECT `subscription_id`, `role_id` + FROM `team_subscription` + WHERE `team_id` = %s''', + team_id) + subscriptions = cursor.fetchall() + team_check = ['team_id = %s'] + query_params.append(team_id) + for sub in subscriptions: + team_check.append('(team_id = %s AND role_id = %s)') + query_params += [sub['subscription_id'], sub['role_id']] + + query = ''' + SELECT DISTINCT `user_id` FROM `%s` + WHERE `user_id` in %%s AND (%s) AND (%s or `role_id`= (SELECT `id` FROM `role` WHERE `name` = 'vacation')) + ''' % (table_name, ' OR '.join(range_check), ' OR '.join(team_check)) + + cursor.execute(query, query_params) + return [r['user_id'] for r in cursor.fetchall()] + + def find_least_active_user_id_by_team(self, user_ids, team_id, start_time, role_id, cursor, table_name='event'): + ''' + Of the people who have been oncall before, finds those who haven't been oncall for the longest. Start + time refers to the start time of the event being created, so we don't accidentally look at future + events when determining who was oncall in the past. Done on a per-role basis, so we don't take manager + or vacation shifts into account + ''' + query = ''' + SELECT `user_id`, MAX(`end`) AS `last_end` FROM `%s` + WHERE `team_id` = %%s AND `user_id` IN %%s AND `end` <= %%s + AND `role_id` = %%s + GROUP BY `user_id` + ''' % table_name + + cursor.execute(query, (team_id, user_ids, start_time, role_id)) + if cursor.rowcount != 0: + # Grab user id with lowest last scheduled time + return min(cursor.fetchall(), key=operator.itemgetter('last_end'))['user_id'] + else: + return None + + def find_new_user_in_roster(self, roster_id, team_id, start_time, role_id, cursor, table_name='event'): + ''' + Return roster users who haven't been scheduled for any event on this team's calendar for this schedule's role. + Ignores events from other teams. + ''' + query = ''' + SELECT DISTINCT `user`.`id` FROM `roster_user` + JOIN `user` ON `user`.`id` = `roster_user`.`user_id` AND `roster_user`.`roster_id` = %%s + LEFT JOIN `%s` ON `%s`.`user_id` = `user`.`id` AND `%s`.`team_id` = %%s AND `%s`.`end` <= %%s + AND `%s`.`role_id` = %%s + WHERE `roster_user`.`in_rotation` = 1 AND `%s`.`id` IS NULL + ''' % (table_name, table_name, table_name, table_name, table_name, table_name) + + cursor.execute(query, (roster_id, team_id, start_time, role_id)) + if cursor.rowcount != 0: + logger.debug('Found new guy') + return {row['id'] for row in cursor} + + def create_events(self, team_id, schedule_id, user_id, events, role_id, cursor, skip_match=True, table_name='event'): + if len(events) == 0: + return + + [event] = events + t_id=team_id + s_id=schedule_id + start=event['start'] + end=event['start']+86400 + u_id=user_id + r_id=role_id + cursor.execute('''SELECT `name` FROM `team` + WHERE `id` = %s''', + t_id) + team_name = {row['name'] for row in cursor} + cursor.execute('''SELECT `roster_id` FROM `schedule` + WHERE `id` = %s''', + s_id) + roster_id = {row['roster_id'] for row in cursor} + + cursor.execute('''SELECT `user_id` FROM `roster_user` + WHERE `roster_id` = %s''', + roster_id) + user_ids = [r['user_id'] for r in cursor] + cursor.execute('''SELECT `auto_populate_threshold` FROM `schedule` + WHERE `id` = %s''', + s_id) + threshold_days=[row['auto_populate_threshold'] for row in cursor] + + cursor.execute('''SELECT `last_scheduled_user_id` FROM `schedule` + WHERE `id` = %s''', + s_id) + + last_scheduled_user = [row['last_scheduled_user_id'] for row in cursor] + if(not last_scheduled_user[0]): + last_scheduled_user=[user_ids[len(user_ids)-1]] + + def execute_query(t_id, s_id, start, end, u_id, r_id): + event_args = (t_id, s_id, start, end, u_id, r_id) + logger.debug('inserting event: %s', event_args) + query = ''' + INSERT INTO `%s` ( + `team_id`, `schedule_id`, `start`, `end`, `user_id`, `role_id` + ) VALUES ( + %%s, %%s, %%s, %%s, %%s, %%s + )''' % table_name + cursor.execute(query, event_args) + user_index=0 + + last_index=user_ids.index(last_scheduled_user[0]) + if(last_index==len(user_ids)-1): + user_index=0 + else: + user_index=last_index+1 + + new_user_ids=[] + while(len(new_user_ids) next_epoch: + epoch_events = self.generate_events(schedule, schedule['events'], next_epoch) + next_epoch += timedelta(days=7 * period) + if epoch_events: + future_events.append(epoch_events) + # Return future events and the last epoch events were scheduled for. + return future_events, self.utc_from_naive_date(next_epoch - timedelta(days=7 * period), schedule) + + def find_next_user_id(self, schedule, future_events, cursor, table_name='event'): + team_id = schedule['team_id'] + role_id = schedule['role_id'] + roster_id = schedule['roster_id'] + # find people without conflicting events + # TODO: finer grain conflict checking + user_ids = set(self.get_roster_user_ids(roster_id, cursor)) + if not user_ids: + logger.info('Empty roster, skipping') + return None + logger.debug('filtering users: %s', user_ids) + start = min([e['start'] for e in future_events]) + for uid in self.get_busy_user_by_event_range(user_ids, team_id, future_events, cursor, table_name): + user_ids.remove(uid) + if not user_ids: + logger.info('All users have conflicting events, skipping...') + return None + new_user_ids = self.find_new_user_in_roster(roster_id, team_id, start, role_id, cursor, table_name) + available_and_new = new_user_ids & user_ids + if available_and_new: + logger.info('Picking new and available user from %s', available_and_new) + return available_and_new.pop() + + logger.debug('picking user between: %s, team: %s', user_ids, team_id) + return self.find_least_active_user_id_by_team(user_ids, team_id, start, role_id, cursor, table_name) + + def schedule(self, team, schedules, dbinfo): + connection, cursor = dbinfo + events = [] + for schedule in schedules: + if schedule['auto_populate_threshold'] <= 0: + self.set_last_epoch(schedule['id'], time.time(), cursor) + continue + logger.info('\t\tschedule: %s', str(schedule['id'])) + schedule['timezone'] = team['scheduling_timezone'] + # Calculate events for schedule + future_events, last_epoch = self.calculate_future_events(schedule, cursor) + for epoch in future_events: + # Add (start_time, schedule_id, role_id, roster_id, epoch_events) to events + events.append((schedule, epoch)) + self.set_last_epoch(schedule['id'], last_epoch, cursor) + + # Create events in the db, associating a user to them + # Iterate through events in order of (start time, role) to properly assign users + for schedule, epoch in sorted(events, key=lambda x: (min(ev['start'] for ev in x[1]), x[0]['role_id'])): + user_id = self.find_next_user_id(schedule, epoch, cursor) + if not user_id: + logger.info('Failed to find available user') + continue + logger.info('Found user: %s', user_id) + self.create_events(team['id'], schedule['id'], user_id, epoch, schedule['role_id'], cursor) + connection.commit() + + def build_preview_response(self, cursor, start__lt, end__ge, team__eq, table_name='temp_event'): + # get existing events + + cols = all_columns + query = '''SELECT %s FROM `%s` + JOIN `user` ON `user`.`id` = `%s`.`user_id` + JOIN `team` ON `team`.`id` = `%s`.`team_id` + JOIN `role` ON `role`.`id` = `%s`.`role_id`''' % (cols, table_name, table_name, table_name, table_name) + where_params = [constraints['start__lt'], constraints['end__ge']] + where_vals = [start__lt, end__ge] + + # Deal with team subscriptions and team parameters + team_where = [constraints['team__eq']] + subs_vals = [team__eq] + subs_and = ' AND '.join(team_where) + cursor.execute('''SELECT `subscription_id`, `role_id` FROM `team_subscription` + JOIN `team` ON `team_id` = `team`.`id` + WHERE %s''' % subs_and, subs_vals) + if cursor.rowcount != 0: + # Build where clause based on team params and subscriptions + subs_and = '(%s OR (%s))' % (subs_and, ' OR '.join(['`team`.`id` = %s AND `role`.`id` = %s' % + (row['subscription_id'], row['role_id']) for row in cursor])) + where_params.append(subs_and) + where_vals += subs_vals + + where_query = ' AND '.join(where_params) + if where_query: + query = '%s WHERE %s' % (query, where_query) + cursor.execute(query, where_vals) + data = cursor.fetchall() + return json_dumps(data) + + def populate(self, schedule, start_time, dbinfo, table_name='event'): + connection, cursor = dbinfo + start_dt = datetime.fromtimestamp(start_time, utc) + start_epoch = self.epoch_from_datetime(start_dt) + + # Get schedule info + role_id = schedule['role_id'] + team_id = schedule['team_id'] + first_event_start = min(ev['start'] for ev in schedule['events']) + period = self.get_period_len(schedule) + handoff = start_epoch + timedelta(seconds=first_event_start) + handoff = timezone(schedule['timezone']).localize(handoff) + + # Start scheduling from the next occurrence of the hand-off time. + if start_dt > handoff: + start_epoch += timedelta(weeks=period) + handoff += timedelta(weeks=period) + if handoff < utc.localize(datetime.utcnow()): + cursor.execute("DROP TEMPORARY TABLE IF EXISTS `temp_event`") + connection.commit() + raise HTTPBadRequest('Invalid populate/preview request', 'cannot populate/preview starting in the past') + #user_id = self.find_next_user_id(schedule, start_time, cursor, table_name) + #self.create_events(team_id, schedule['id'], user_id, start_time, role_id, cursor, table_name=table_name) + + future_events, last_epoch = self.calculate_future_events(schedule, cursor, start_epoch) + self.set_last_epoch(schedule['id'], last_epoch, cursor) + + # Delete existing events from the start of the first event + future_events = [filter(lambda x: x['start'] >= start_time, evs) for evs in future_events] + future_events = filter(lambda x: x != [], future_events) + if future_events: + first_event_start = min(future_events[0], key=lambda x: x['start'])['start'] + query = 'DELETE FROM %s WHERE schedule_id = %%s AND start >= %%s' % table_name + cursor.execute(query, (schedule['id'], first_event_start)) + + # Create events in the db, associating a user to them + for epoch in future_events: + user_id = self.find_next_user_id(schedule, epoch, cursor, table_name) + if not user_id: + continue + self.create_events(team_id, schedule['id'], user_id, epoch, role_id, cursor, table_name=table_name) + break + connection.commit() \ No newline at end of file diff --git a/src/oncall/ui/static/js/oncall.js b/src/oncall/ui/static/js/oncall.js index fc129d91..6220ff16 100644 --- a/src/oncall/ui/static/js/oncall.js +++ b/src/oncall/ui/static/js/oncall.js @@ -1738,7 +1738,8 @@ var oncall = { scheduleItem: '.module-card', schedulerTemplates: { 'default': $('#default-scheduler-template').html(), - 'round-robin': $('#round-robin-scheduler-template').html() + 'round-robin': $('#round-robin-scheduler-template').html(), + 'daily-round-robin': $('#daily-round-robin-scheduler-template').html() }, schedulerTypeContainer: '.scheduler-type-container', schedulesUrl: '/api/v0/schedules/', diff --git a/src/oncall/ui/templates/index.html b/src/oncall/ui/templates/index.html index 9f12f32b..7603bf33 100644 --- a/src/oncall/ui/templates/index.html +++ b/src/oncall/ui/templates/index.html @@ -965,7 +965,7 @@

Populate Schedule

{{#unless selected_schedule.advanced_mode}}
- +
{{/unless}} @@ -1102,9 +1102,10 @@


- +
@@ -1125,7 +1126,19 @@

- + + + + From 3790a7f94d8d6f1c35c2c75a4460736160d27d55 Mon Sep 17 00:00:00 2001 From: Harshit Sharma Date: Fri, 21 Jun 2019 16:59:43 +0530 Subject: [PATCH 2/2] UI improvements daily-round-robin scheduler algo --- src/oncall/ui/static/css/oncall.css | 42 +++++++++++++++++++++++++++++ src/oncall/ui/static/js/oncall.js | 19 +++++++++++++ src/oncall/ui/templates/index.html | 17 ++++++++++-- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/src/oncall/ui/static/css/oncall.css b/src/oncall/ui/static/css/oncall.css index 718c41ff..0b6f919f 100644 --- a/src/oncall/ui/static/css/oncall.css +++ b/src/oncall/ui/static/css/oncall.css @@ -1103,6 +1103,48 @@ div[data-admin="false"] .team-info .toggle-rotation { border-bottom: 1px dashed #ccc; } +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler { + border: 1px solid #ccc; + color: #5c5b5c; + border-radius: 4px; + margin: 5px 0; + padding: 10px; +} + +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler ul { + margin-top: 5px; + max-height: 300px; + overflow-y: scroll; +} + +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler li { + cursor: move; + padding: 2px; + transition: background .3s; + user-select: none; +} + +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler svg { + display: inline-block; + font-size: inherit; + height: 1em; + overflow: visible; + vertical-align: -.125em; + width: 25px; +} + +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler li:hover { + background: #efefef; +} + +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler .dragging { + opacity: .3; +} + +.module-schedule-create[data-scheduler="daily-month-plan"] .daily-month-plan-scheduler .drag-over { + border-bottom: 1px dashed #ccc; +} + .module-schedule { width: 24%; margin: 0 .5%; diff --git a/src/oncall/ui/static/js/oncall.js b/src/oncall/ui/static/js/oncall.js index 6220ff16..f6728444 100644 --- a/src/oncall/ui/static/js/oncall.js +++ b/src/oncall/ui/static/js/oncall.js @@ -2349,6 +2349,25 @@ var oncall = { return schedule.id === parseInt(schedule_id) })[0]; + if (order !== undefined) { + order = JSON.parse(order); + } + // If order includes all roster users, use order. Otherwise, just use roster + if (order !== undefined && order.every(function(u) { return users.indexOf(u) !== -1})) { + $container.html(template(order)); + } else { + $container.html(template(users)); + } + this.drag.init($container.find('.draggable')); + } else if ( scheduler === 'daily-month-plan' ) { + // Pass in user list to template if custom order is selected + var schedule_id = $card.attr('data-edit-id'), + order = $card.attr('data-schedule-data'), + users = this.data.teamData.rosters[roster].users.map(function(user) {return user.name}); + schedule = this.data.teamData.rosters[roster].schedules.filter(function (schedule) { + return schedule.id === parseInt(schedule_id) + })[0]; + if (order !== undefined) { order = JSON.parse(order); } diff --git a/src/oncall/ui/templates/index.html b/src/oncall/ui/templates/index.html index 7603bf33..2883b286 100644 --- a/src/oncall/ui/templates/index.html +++ b/src/oncall/ui/templates/index.html @@ -1222,8 +1222,21 @@

-