From c6d152bb6206bf898fbb5a2dee7b42597e6a16e1 Mon Sep 17 00:00:00 2001 From: kojiagile Date: Tue, 21 Mar 2017 20:30:57 +1000 Subject: [PATCH 01/10] Add slack python library in requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index ced5fbe..8d5839d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -83,6 +83,7 @@ scikit-learn==0.16.1 scipy==0.15.1 simplegeneric==0.8.1 six==1.10.0 +slackclient==1.0.5 smart-open==1.3.5 subprocess32==3.2.7 tincan==0.0.5 From 8c6433d8a964b81e3f21835d6f71c9966370fa2f Mon Sep 17 00:00:00 2001 From: kojiagile Date: Tue, 21 Mar 2017 20:31:37 +1000 Subject: [PATCH 02/10] Add user id in OfflinePlatformAuthToken table --- clatoolkit_project/dataintegration/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clatoolkit_project/dataintegration/views.py b/clatoolkit_project/dataintegration/views.py index 050d543..562519d 100644 --- a/clatoolkit_project/dataintegration/views.py +++ b/clatoolkit_project/dataintegration/views.py @@ -604,7 +604,7 @@ def slack_client_auth(request): return HttpResponseServerError('

Internal Server Error (500)

More than one records were found.') else: token_storage = OfflinePlatformAuthToken( - user_smid=json_val['user_id'], token=access_token, platform=xapi_settings.PLATFORM_SLACK) + user_smid=json_val['user_id'], token=access_token, platform=xapi_settings.PLATFORM_SLACK, user = request.user) token_storage.save() # Set user ID and access token in social media ID register or update page From e9e8bc3d9693058decf4c7aeaab7e01bc4eee63a Mon Sep 17 00:00:00 2001 From: kojiagile Date: Tue, 28 Mar 2017 17:27:55 +1000 Subject: [PATCH 03/10] Fix bug: When there is no xAPI statement --- clatoolkit_project/dashboard/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clatoolkit_project/dashboard/utils.py b/clatoolkit_project/dashboard/utils.py index 995a851..e2a6ade 100644 --- a/clatoolkit_project/dashboard/utils.py +++ b/clatoolkit_project/dashboard/utils.py @@ -877,6 +877,9 @@ def sentiment_classifier(unit): getter = xapi_getter() filters.statement_id = sm_obj.statement_id stmt = getter.get_xapi_statements(sm_obj.unit_id, sm_obj.user_id, filters) + if stmt is None or len(stmt) == 0: + continue + message = stmt[0]['object']['definition']['name']['en-US'] message = message.encode('utf-8', 'replace') From 3f7bfb009ce5c8f4aeee83e90e26dd056818b89a Mon Sep 17 00:00:00 2001 From: kojiagile Date: Mon, 3 Apr 2017 17:32:59 +1000 Subject: [PATCH 04/10] Modify installation instruction(Local Installation using VirtualEnv) --- README.md | 90 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4b69ef9..df6ab7b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,17 @@ If you do not have VirtualEnv installed: $ pip install virtualenv $ pip install virtualenvwrapper $ mkdir ~/.virtualenvs +``` + +Add the following lines in .bashrc (or .bash_profile) +```bash $ export WORKON_HOME=~/.virtualenvs +$ source /usr/local/bin/virtualenvwrapper.sh +``` + +Apply the two lines +```bash +$ source .bashrc ``` **Create a virtual environment for CLAToolkit:** @@ -42,14 +52,21 @@ $ cd clatoolkit/clatoolkit_project/clatoolkit_project **Install Python and Django Requirements** - -A requirements.txt file is provided in the code repository. This will take a while especially the installation of numpy. If numpy fails you may have to find a platform specific deployment method eg using apt-get on ubuntu ($ sudo apt-get install python-numpy python-scipy python-matplotlib ipython ipython-notebook python-pandas python-sympy python-nose). - +***Run these commands below before running requirements.txt.*** +Install prerequisite libraries that are required to install libraries in requirements.txt ```bash -$ sudo pip install -r requirements.txt +$ sudo apt-get install python-dev libpq-dev libxml2-dev libxslt-dev libigraph0-dev ``` -**Install Postgres** +**Install PostgreSQL** +Install postgreSQL9.4 on Ubuntu 14.04 +PostgreSQL 9.3 is installed as default database on Ubuntu 14.04. However, the CLA toolkit uses PostgreSQL 9.4 or above. +```bash +$ sudo add-apt-repository "deb https://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" +$ wget --quiet -O - https://postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +$ sudo apt-get update +$ sudo apt-get install postgresql-9.4 postgresql-contrib +``` On a Mac install postgres.app using these instructions: http://postgresapp.com/ and add to path using: @@ -57,37 +74,86 @@ and add to path using: export PATH="/Applications/Postgres.app/Contents/Versions/9.4/bin:$PATH" ``` +**Install requirements** +A requirements.txt file is provided in the code repository. This will take a while especially the installation of numpy. If numpy fails you may have to find a platform specific deployment method eg using apt-get on ubuntu ($ sudo apt-get install python-numpy python-scipy python-matplotlib ipython ipython-notebook python-pandas python-sympy python-nose). + +```bash +$ sudo pip install -r requirements.txt +``` + +**Create PostgreSQL database instance** You can either create a new postgres database for your CLAToolkit Instance or use database with preloaded Social Media content. A preloaded database is available upon request which comes with a set of migrations. -Instructions to create a new postgres database +Create a superuser (if necessary) +```bash +$ sudo su - postgres +$ createuser -P -s -e username +``` + Follow http://killtheyak.com/use-postgresql-with-django-flask/ to create a user and database for django ```bash -$ sudo createdb -U username --locale=en_US.utf-8 -E utf-8 -O username cladjangodb -T template0 +$ sudo -u username createdb --locale=en_US.utf-8 -E utf-8 -O username cladjangodb -T template0 ``` Load an existing database by first creating a new database and then importing data from backup database ```bash -$ sudo createdb -U username --locale=en_US.utf-8 -E utf-8 -O username newdatabasename -T template0 +$ sudo -u username createdb --locale=en_US.utf-8 -E utf-8 -O username newdatabasename -T template0 $ psql newdatabasename < backedupdbname.bak ``` +**Edit .env** Edit ```.env``` and add secret key + DB details + +**Migration** If a new database was created, you will need to setup the database tables and create a superuser. ```bash $ python manage.py migrate $ python manage.py createsuperuser ``` +If you see an error saying that dotenv has no attribute 'read_dotenv' when you run migrate command, other types of dotenv are likely to conflict with django-dotenv. If so, uninstall them and (re)install django-dotenv if necessary. +Example error message: +```bash +Traceback (most recent call last): + File "manage.py", line 8, in + dotenv.read_dotenv(os.path.join(BASE_DIR, ".env")) +AttributeError: 'module' object has no attribute 'read_dotenv' +``` +```bash +$ sudo pip uninstall dotenv +$ sudo pip uninstall python-dotenv +$ sudo pip install django-dotenv==1.4.1 +``` + + +**Insert the default data into database** +Default LRS data needs to be stored in the database in advance. Run the insert SQL below. +```bash +insert into xapi_clientapp values (1, 'CLAToolkit LRS', 'Connected Learning Analytics Toolkit', '1', '1bsL1k5HiQnFOo0J', 'http', 'lrs.beyondlms.org', 43, '/xapi/OAuth/initiate','/xapi/OAuth/token', '/xapi/OAuth/authorize', '/xapi/statements', '/regclatoolkitu/'); +``` + + +**Install Bower Component** +To install bower, follow the instruction on [https://bower.io/](https://bower.io/) + +```bash +$ cd clatoolkit_project/static +$ sudo bower install --allow-root +``` + Now you can run the django webserver: ```bash $ python manage.py runserver ``` -If a new database was created go to http://localhost:8000/admin and login with superuser account -Add a unit offering with hashtags (for twitter) and group id (for facebook) -Add users with twitter id and facebook id -Login is at http://localhost:8000/ +If a new database was created go to http://localhost:8000/admin and login with superuser account. +Edit the user's profile (admin home - Users - click the user). + +Go to http://localhost:8000/ and login to the CLA toolkit. Create a unit via unit offering page (Click the username on the right top corner - Click Create Offering). Once a unit is created, it will be listed in user's dashboard. + +When a unit is created, there will be a link to the user registration page (e.g. https://localhost/clatoolkit/unitofferings/1/register/). To add other toolkit users to the unit, give them the link and let them login or create a new user. + Installation on Ubuntu using Apache - From a0a10601ec7d6a53d083dd313462ad237d1fe0dd Mon Sep 17 00:00:00 2001 From: kojiagile Date: Wed, 5 Apr 2017 14:42:11 +1000 Subject: [PATCH 05/10] Add message about LRS and fix insert SQL --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index df6ab7b..81e0ee3 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ The Connected Learning Analytics toolkit (new django architecture, superseeding Local Installation using VirtualEnv --------- -**CLAToolkit is built with Django. The installation is pretty standard but requires Postgres (for JSON document queries), Numpy and a range of Machine Learning Libraries such as Scikit Learn and Gensim** - +**CLAToolkit is built with Django. The installation is pretty standard but requires PostgreSQL, Numpy and a range of Machine Learning Libraries such as Scikit Learn and Gensim** +**CLAToolkit also requires Learning Record Store (LRS) to store/retrieve JSON data. You can see the instruction for installing LRS [here](https://github.com/zwaters/ADL_LRS)** If you do not have VirtualEnv installed: ```bash @@ -101,8 +101,12 @@ $ sudo -u username createdb --locale=en_US.utf-8 -E utf-8 -O username newdatabas $ psql newdatabasename < backedupdbname.bak ``` -**Edit .env** -Edit ```.env``` and add secret key + DB details +**Configure CLAToolkit environment file (.env) with your database credentials and social media secret ID and password** +```bash +$ cp .env.example .env +$ nano .env +``` +Then, edit ```.env``` and add secret key and DB details **Migration** @@ -130,7 +134,7 @@ $ sudo pip install django-dotenv==1.4.1 **Insert the default data into database** Default LRS data needs to be stored in the database in advance. Run the insert SQL below. ```bash -insert into xapi_clientapp values (1, 'CLAToolkit LRS', 'Connected Learning Analytics Toolkit', '1', '1bsL1k5HiQnFOo0J', 'http', 'lrs.beyondlms.org', 43, '/xapi/OAuth/initiate','/xapi/OAuth/token', '/xapi/OAuth/authorize', '/xapi/statements', '/regclatoolkitu/'); +insert into xapi_clientapp values (1, 'CLAToolkit LRS', 'Connected Learning Analytics Toolkit', '', '', 'http', 'lrs.beyondlms.org', 43, '/xapi/OAuth/initiate','/xapi/OAuth/token', '/xapi/OAuth/authorize', '/xapi/statements', '/regclatoolkitu/'); ``` From d431101ee3f2c44936fa69290ea231ad9405324e Mon Sep 17 00:00:00 2001 From: kojiagile Date: Thu, 6 Apr 2017 00:16:31 +1000 Subject: [PATCH 06/10] Modify clatoolkit apache installation instruction --- docs/apache-install.md | 48 ++++-------------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/docs/apache-install.md b/docs/apache-install.md index 62f263d..03ba98c 100644 --- a/docs/apache-install.md +++ b/docs/apache-install.md @@ -1,52 +1,12 @@ # Setting up the CLA Toolkit with Apache How to setup the CLA Toolkit on Ubuntu 14.04 using the Apache server. -**Install dependancies:** -```bash -$ sudo apt-get update -$ sudo apt-get install git python-pip python-dev apache2 libapache2-mod-wsgi postgresql postgresql-contrib python-psycopg2 libxml2-dev libxslt-dev libpq-dev -``` - -**Clone the CLAtoolkit repo locally:** -```bash -$ git clone https://github.com/kirstykitto/CLAtoolkit.git -$ cd CLAtoolkit -``` - -**Setup virtualenv:** -```bash -$ sudo pip install virtualenv -$ sudo pip install virtualenvwrapper -$ mkvirtualenv clatoolkit -``` - -**Install Python dependancies:** -```bash -$ pip install -r requirements.txt -``` - -**Setup Postgres:** -```bash -$ sudo -u postgres createuser -P clatoolkit -s -$ sudo createdb -U clatoolkit --locale=en_US.utf-8 -E utf-8 -O clatoolkit cladjangodb -T template0 -h 127.0.0.1 --username=clatoolkit -``` -When prompted for a password, use the password for the Postgres user you just created +**The CLA toolkit needs to be installed in advance. The instruction can be found [here](https://github.com/kojiagile/CLAtoolkit/blob/koji/README.md#local-installation-using-virtualenv)**  inst -**Configure clatoolkit environment with your database credentials:** +**Install Apache and dependancies** ```bash -$ cp .env.example .env -$ nano .env -``` -Make sure to change the `DEBUG` flag to 0 if this instance is being used in production - -**Initialise the Database:** -```bash -$ python clatoolkit_project/manage.py migrate -``` - -**Create a superuser:** -```bash -$ python manage.py createsuperuser +$ sudo apt-get update +$ sudo apt-get install apache2 libapache2-mod-wsgi python-psycopg2 ``` **Edit the Apache configuration:** From 556b5f99e8d74915b5dd73bf317576648c8c4649 Mon Sep 17 00:00:00 2001 From: kojiagile Date: Thu, 6 Apr 2017 10:07:35 +1000 Subject: [PATCH 07/10] Delete unnecessary message --- docs/apache-install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-install.md b/docs/apache-install.md index 03ba98c..45bfc99 100644 --- a/docs/apache-install.md +++ b/docs/apache-install.md @@ -1,7 +1,7 @@ # Setting up the CLA Toolkit with Apache How to setup the CLA Toolkit on Ubuntu 14.04 using the Apache server. -**The CLA toolkit needs to be installed in advance. The instruction can be found [here](https://github.com/kojiagile/CLAtoolkit/blob/koji/README.md#local-installation-using-virtualenv)**  inst +**The CLA toolkit needs to be installed in advance. The instruction can be found [here](https://github.com/kojiagile/CLAtoolkit/blob/koji/README.md#local-installation-using-virtualenv)**   **Install Apache and dependancies** ```bash From 06e78e9e3dc67b2b7efb89559293d29dbd01b448 Mon Sep 17 00:00:00 2001 From: kojiagile Date: Mon, 10 Apr 2017 15:28:49 +1000 Subject: [PATCH 08/10] Fix page title --- .../templates/dataintegration/dataimport_complete.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clatoolkit_project/templates/dataintegration/dataimport_complete.html b/clatoolkit_project/templates/dataintegration/dataimport_complete.html index 249a8c3..4bbdbb0 100644 --- a/clatoolkit_project/templates/dataintegration/dataimport_complete.html +++ b/clatoolkit_project/templates/dataintegration/dataimport_complete.html @@ -2,7 +2,7 @@ - GitHub data refresh + {{ platform }} data import

{{ platform }} data have been refreshed.

From f03edaaab01d660113e1deddda8e51f29ab237ea Mon Sep 17 00:00:00 2001 From: kojiagile Date: Fri, 21 Apr 2017 14:09:47 +1000 Subject: [PATCH 09/10] Add analytics API (platform/verb timeseries endpoints) --- clatoolkit_project/api/__init__.py | 0 clatoolkit_project/api/analytics/__init__.py | 0 .../api/analytics/endpoint/__init__.py | 0 .../api/analytics/endpoint/timeseries.py | 375 ++++++++++++++++++ clatoolkit_project/api/analytics/models.py | 18 + .../api/analytics/validator/__init__.py | 0 .../validator/timeseries_validator.py | 55 +++ .../api/analytics/validator/validator.py | 59 +++ clatoolkit_project/api/analytics/views.py | 67 ++++ clatoolkit_project/api/error/__init__.py | 0 .../api/error/application_error.py | 8 + clatoolkit_project/api/error/error.py | 26 ++ .../api/error/invalid_parameter_error.py | 9 + clatoolkit_project/api/models.py | 0 clatoolkit_project/api/urls.py | 15 + clatoolkit_project/clatoolkit/models.py | 47 ++- .../clatoolkit_project/settings.py | 4 +- clatoolkit_project/clatoolkit_project/urls.py | 1 + clatoolkit_project/common/util.py | 85 +++- clatoolkit_project/dashboard/views.py | 10 - .../plugins/slack/cladi_plugin.py | 9 +- .../templates/dashboard/dashboard.html | 105 +++-- .../xapi/statement/xapi_settings.py | 20 + 23 files changed, 838 insertions(+), 75 deletions(-) create mode 100644 clatoolkit_project/api/__init__.py create mode 100644 clatoolkit_project/api/analytics/__init__.py create mode 100644 clatoolkit_project/api/analytics/endpoint/__init__.py create mode 100644 clatoolkit_project/api/analytics/endpoint/timeseries.py create mode 100644 clatoolkit_project/api/analytics/models.py create mode 100644 clatoolkit_project/api/analytics/validator/__init__.py create mode 100644 clatoolkit_project/api/analytics/validator/timeseries_validator.py create mode 100644 clatoolkit_project/api/analytics/validator/validator.py create mode 100644 clatoolkit_project/api/analytics/views.py create mode 100644 clatoolkit_project/api/error/__init__.py create mode 100644 clatoolkit_project/api/error/application_error.py create mode 100644 clatoolkit_project/api/error/error.py create mode 100644 clatoolkit_project/api/error/invalid_parameter_error.py create mode 100644 clatoolkit_project/api/models.py create mode 100644 clatoolkit_project/api/urls.py diff --git a/clatoolkit_project/api/__init__.py b/clatoolkit_project/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clatoolkit_project/api/analytics/__init__.py b/clatoolkit_project/api/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clatoolkit_project/api/analytics/endpoint/__init__.py b/clatoolkit_project/api/analytics/endpoint/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clatoolkit_project/api/analytics/endpoint/timeseries.py b/clatoolkit_project/api/analytics/endpoint/timeseries.py new file mode 100644 index 0000000..04a696a --- /dev/null +++ b/clatoolkit_project/api/analytics/endpoint/timeseries.py @@ -0,0 +1,375 @@ +__author__ = 'Koji' + +import copy +from collections import OrderedDict +from django.db import connection +from ..models import TimeseriesProperty +from common.util import Utility + +from clatoolkit.models import UnitOffering +from xapi.statement.xapi_settings import xapi_settings +from api.error.application_error import ApplicationError + + + +class Timeseries(object): + STR_ORDER_BY_DATE = 'date' + + + @classmethod + def get_platform_timeseries(self, request, args, kw): + try: + ts_prop = self.get_timeseries_property(request, args, kw) + platform_list = self.get_platform_list(ts_prop.filter_string, ts_prop.unit) + platform_list.sort() + + # Subtract the month because month starts from 0 in JavaScript + start = Utility.format_date( + str(Utility.max_date(ts_prop.start_date, ts_prop.unit.start_date)), + '-', '-', True) + end = Utility.format_date( + str(Utility.min_date(ts_prop.end_date, ts_prop.unit.end_date)), + '-', '-', True) + values = OrderedDict ([ + ('platforms', ','.join(platform_list)), + ('start', start), + ('end', end), + ('activities', self.get_timeseries(ts_prop, platform_list)) + ]) + + return values + + except Exception as exp: + raise ApplicationError(exp, 'An unexpected error has occurred.') + + + @classmethod + def get_verb_timeseries(self, request, args, kw): + try: + ts_prop = self.get_timeseries_property(request, args, kw) + verb_list = self.get_verb_list(ts_prop.filter_string, ts_prop.unit, ts_prop.platform_filter_string) + platform_list = self.get_platform_list(ts_prop.platform_filter_string, ts_prop.unit) + verb_list.sort() + platform_list.sort() + + activity_list = [] + for verb in verb_list: + obj = OrderedDict([ + ('verb', verb), + ('activities', self.get_timeseries(ts_prop, platform_list, verb)) + ]) + activity_list.append(obj) + + # Modify the data structure to reduce redundant data (date element) + # + # activity_list has something like this. + # { + # "verb": "created", + # "activities": [{ + # "date": "2016-08-01", + # "Slack": 1, + # },{ + # ... + # ]},{ + # "verb": "shared", + # "activities": [{ + # "date": "2016-08-01", + # "Slack": 0, + # },{ + # ... + # + # date element exist in every activities, which is redundant. + # The following code eliminate it and changes the structure above to: + # { + # "date": "2017-00-01", + # "commented": { + # "Slack": 0, + # "Trello": 0 + # }, + # "created": { + # "Slack": 0, + # "Trello": 0 + # }, + # ... + # + new_activity_list = [] + activity = activity_list[0] + index = 0 + for i in xrange(len(activity['activities'])): + # Add date in activity_list to a variable + new_activity = OrderedDict([('date', activity['activities'][i]['date'])]) + + for j in xrange(len(activity_list)): + inner_activities = activity_list[j]['activities'] + single_activity = inner_activities[i] + # Remove date element so the other elements can be retrieve easily + single_activity.pop('date', None) + + obj = OrderedDict([]) + for key in single_activity.keys(): + obj[key] = single_activity[key] + + new_activity[activity_list[j]['verb']] = obj + + new_activity_list.append(new_activity) + + # Replace the old list with the new one + activity_list = new_activity_list + + # Subtract the month because month starts from 0 in JavaScript + start = Utility.format_date( + str(Utility.max_date(ts_prop.start_date, ts_prop.unit.start_date)), + '-', '-', True) + end = Utility.format_date( + str(Utility.min_date(ts_prop.end_date, ts_prop.unit.end_date)), + '-', '-', True) + + values = OrderedDict ([ + ('verbs', ','.join(verb_list)), + ('platforms', ','.join(platform_list)), + ('start', start), + ('end', end), + ('activities', activity_list) + ]) + + return values + + except Exception as exp: + raise ApplicationError(exp, 'An unexpected error has occurred.') + + + @classmethod + def get_platform_list(self, platform_filter_string, unit): + platform_list = [] + # When "all" is specified + if platform_filter_string: + if platform_filter_string.find('all') != -1: + platform_list = unit.get_required_platforms() + else: + platform_list = platform_filter_string.split(',') + else: + platform_list = unit.get_required_platforms() + + return platform_list + + + @classmethod + def get_verb_list(self, verb_filter_string, unit, platform_filter_string = None): + verb_list = [] + platform_list = [] + if platform_filter_string: + platform_list = platform_filter_string.split(',') + + if verb_filter_string: + if verb_filter_string.find('all') != -1: + verb_list = unit.get_required_verbs() + else: + verb_list = verb_filter_string.split(',') + else: + verb_list = unit.get_required_verbs(platform_list = platform_list) + + return verb_list + + + @classmethod + def get_timeseries_property(self, request, args, kw, month_subtract = True): + ts_prop = TimeseriesProperty() + ts_prop.unit = UnitOffering.objects.get(id = int(kw['unit_id'])) + ts_prop.user = None + ts_prop.start_date = request.GET.get('start', None) + ts_prop.end_date = request.GET.get('end', None) + ts_prop.month_subtract = month_subtract + ts_prop.order_by, ts_prop.order = self.get_orderby(request.GET.get('order', None)) + ts_prop.filter_string = request.GET.get('filter', None) + platforms = request.GET.get('platforms', None) + ts_prop.platform_filter_string = platforms if platforms is not None and platforms != 'all' else 'all' + return ts_prop + + + @classmethod + def get_timeseries(self, ts_prop, platform_list, verb = None): + sql = self.get_timeseries_sql(ts_prop, platform_list, verb) + # print sql + + cursor = connection.cursor() + cursor.execute(sql) + result = cursor.fetchall() + activities = [] + + record_count = self.get_series_record_count(ts_prop.unit, ts_prop.start_date, ts_prop.end_date) + # for row in result: + for i in xrange(record_count): + index = 0 + activity = OrderedDict([('date', '')]) # Add date element + for platform in platform_list: + row = result[i + (index * record_count) ] + curdate = row[1] + activity[platform] = int(row[2]) + index += 1 + + # In JavaScript, month starts at 0, thus subtract 1 from the month if month_subtract is True + month = curdate.month -1 if ts_prop.month_subtract else curdate.month + activity['date'] = "%s-%s-%s" % (curdate.year, '%02d' % month, '%02d' % curdate.day) + activities.append(activity) + + return activities + + + @classmethod + def get_series_record_count(self, unit, start_date, end_date): + cursor = connection.cursor() + cursor.execute(self.get_generate_series_select_sql(unit, start_date, end_date)) + result = cursor.fetchall() + return len(result) + + + @classmethod + def get_timeseries_sql(self, ts_prop, platform_list, verb = None): + with_queries = self.get_with_queries(ts_prop, platform_list, verb) + sql_list = [] + for platform in platform_list: + sql = """(select + '%s'::text as platform, + to_date(to_char(filled_dates.day, 'YYYY-MM-DD'), 'YYYY-MM-DD') date + , coalesce(daily_counts_%s.smcount, filled_dates.blank_count) as counts + from filled_dates + left outer join daily_counts_%s on daily_counts_%s.day = filled_dates.day + order by filled_dates.day asc) + """ % (platform, platform, platform, platform) + sql_list.append(sql) + + if len(platform_list) > 1: + sql = with_queries + ' union '.join(sql_list) + ' order by platform, date' + else: + sql = with_queries + ' ' + ''.join(sql_list) + + return sql + + + @classmethod + def get_with_queries(self, ts_prop, platform_list, verb = None): + with_clause_list = [] + daily_counts_list = [] + for platform in platform_list: + daily_counts_list.append(self.get_daily_counts_with_queries(ts_prop, platform, verb = verb)) + + return 'with ' + self.get_generate_series_with_queries(ts_prop, platform) \ + + ',' + ', '.join(daily_counts_list) + + + @classmethod + def get_daily_counts_with_queries(self, ts_prop, platform, verb = None): + ts_prop.platform = platform + ts_prop.verb = verb + clauses = self.get_clauses(ts_prop) + # Create WITH queries for daily counts + return """ + daily_counts_%s as ( + select + date_trunc('day', to_timestamp(substring(CAST(clatoolkit_learningrecord.datetimestamp as text) from 1 for 11), 'YYYY-MM-DD')) as day, + count(*) as smcount + FROM clatoolkit_learningrecord + WHERE clatoolkit_learningrecord.unit_id = %s + %s %s %s + group by day + order by day asc + ) + """ % (platform, ts_prop.unit.id, + clauses['user_clause'], clauses['platform_clause'], clauses['verb_clause']) + + + @classmethod + def get_generate_series_with_queries(self, ts_prop, platform): + generate_series_sql = self.get_generate_series_select_sql(ts_prop.unit, ts_prop.start_date, ts_prop.end_date) + return " filled_dates as ( %s ) " % (generate_series_sql) + + + @classmethod + def get_generate_series_select_sql(self, unit, start_date, end_date): + # more info on postgres timeseries + # http://no0p.github.io/postgresql/2014/05/08/timeseries-tips-pg.html + + # Create WITH queries for generating series data + start_clause = "(select start_date from clatoolkit_unitoffering where id = %s)" % (unit.id) + end_clause = "(select end_date from clatoolkit_unitoffering where id = %s)" % (unit.id) + + if start_date is not None: + # When start date param is larger than start date in unit profile + if Utility.compare_to(start_date, unit.start_date) == 1: + start_clause = " '%s' " % start_date + + if end_date is not None: + # When start date param is larger than start date in unit profile + if Utility.compare_to(end_date, unit.end_date) == -1: + end_clause = " '%s' " % end_date + + return "SELECT generate_series( %s, %s, interval '1 day') as day, 0 as blank_count" \ + % (start_clause, end_clause) + + + @classmethod + def get_clauses(self, ts_prop): + platformclause = "" + if ts_prop.platform: + platforms = ts_prop.platform.split(',') + if len(platforms) == 1: + if ts_prop.platform is not None and ts_prop.platform != "all": + platformclause = " and clatoolkit_learningrecord.platform = '%s' " % (ts_prop.platform) + elif len(platforms) > 1: + names = [] + for p in platforms: + names.append("'" + p + "'") + + platformclause = " and clatoolkit_learningrecord.platform in (%s) " % (', '.join(names)) + + verb_clause = "" + if ts_prop.verb is not None: + verb_clause = " and clatoolkit_learningrecord.verb = '%s' " % (ts_prop.verb) + + user_clause = "" + if ts_prop.user is not None: + user_clause = " and clatoolkit_learningrecord.user_id = %s " % (ts_prop.user.id) + + orderby_clause = ' order by filled_dates.day asc' + if ts_prop.order_by is not None: + if ts_prop.order_by == self.STR_ORDER_BY_DATE: + orderby_clause = ' order by filled_dates.day %s' % (ts_prop.order) + # If other order by clause are required... + # elif order_by != self.STR_ORDER_BY_DATE: + # pass + else: + # Default order by clause + orderby_clause = ' order by filled_dates.day %s' % (ts_prop.order) + + return { + 'platform_clause': platformclause, + 'verb_clause': verb_clause, + 'user_clause': user_clause, + } + + + @classmethod + def get_orderby(self, order_by_str): + asc = 'asc' + desc = 'desc' + + if order_by_str is None: + # Default order + return self.STR_ORDER_BY_DATE, asc + + orderby_list = order_by_str.split(',') + for orderby in orderby_list: + o = orderby.split('-') + if len(o) == 1: + if o[0] == self.STR_ORDER_BY_DATE: + return o[0], asc + else: + # Default order + return self.STR_ORDER_BY_DATE, asc + elif len(o) == 2: + if o[1] == self.STR_ORDER_BY_DATE: + return o[1], desc + else: + # Default order + return self.STR_ORDER_BY_DATE, asc + diff --git a/clatoolkit_project/api/analytics/models.py b/clatoolkit_project/api/analytics/models.py new file mode 100644 index 0000000..c9aed1a --- /dev/null +++ b/clatoolkit_project/api/analytics/models.py @@ -0,0 +1,18 @@ +__author__ = 'Koji' + +class AnalysisProperty(object): + platform = None + verb = None + user = None + unit = None + + +class TimeseriesProperty(AnalysisProperty): + start_date = None + end_date = None + order_by = None # column name to sort records + order = 'asc' + month_subtract = True + filter_string = None + platform_filter_string = None + diff --git a/clatoolkit_project/api/analytics/validator/__init__.py b/clatoolkit_project/api/analytics/validator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clatoolkit_project/api/analytics/validator/timeseries_validator.py b/clatoolkit_project/api/analytics/validator/timeseries_validator.py new file mode 100644 index 0000000..cd83895 --- /dev/null +++ b/clatoolkit_project/api/analytics/validator/timeseries_validator.py @@ -0,0 +1,55 @@ +__author__ = 'Koji' + +import re +from ..endpoint.timeseries import Timeseries +from xapi.statement.xapi_settings import xapi_settings +from clatoolkit.models import UnitOffering +from common.util import Utility +from validator import Validator + +from api.error.application_error import ApplicationError +from api.error.invalid_parameter_error import InvalidParameterError + + +class TimeseriesValidator(Validator): + # Parameter validation + @classmethod + def valid_timeseries_params(self, request, args, kw): + try: + try: + UnitOffering.objects.get(id = int(kw['unit_id'])) + except exp: + raise InvalidParameterError(exp, 'Unit ID') + + # TODO: validate user (implement it later?) + # Validate a user + # ts_prop.user = None + + if not self.valid_date(request.GET.get('start', None), '%Y-%m-%d'): + raise InvalidParameterError(None, 'Start date') + + if not self.valid_date(request.GET.get('end', None), '%Y-%m-%d'): + raise InvalidParameterError(None, 'End date') + + if request.GET.get('order', None) and \ + (request.GET.get('order', None) != Timeseries.STR_ORDER_BY_DATE \ + and request.GET.get('order', None) != '-' + Timeseries.STR_ORDER_BY_DATE): + raise InvalidParameterError(None, 'Order By') + + if kw['type'] == 'verb': + if not self.valid_verb_names(request.GET.get('filter', None), kw['unit_id']): + raise InvalidParameterError(None, 'Filter') + if not self.valid_platform_names(request.GET.get('platforms', None), kw['unit_id']): + raise InvalidParameterError(None, 'Platforms') + + elif kw['type'] == 'platform': + if not self.valid_platform_names(request.GET.get('filter', None), kw['unit_id']): + raise InvalidParameterError(None, 'Filter') + + return True + + except InvalidParameterError as ipexp: + raise ipexp + + except Exception as exp: + raise ApplicationError(exp, 'An unexpected error has occurred.') diff --git a/clatoolkit_project/api/analytics/validator/validator.py b/clatoolkit_project/api/analytics/validator/validator.py new file mode 100644 index 0000000..652bea5 --- /dev/null +++ b/clatoolkit_project/api/analytics/validator/validator.py @@ -0,0 +1,59 @@ +__author__ = 'Koji' + +import re +from ..endpoint.timeseries import Timeseries +from xapi.statement.xapi_settings import xapi_settings +from clatoolkit.models import UnitOffering +from common.util import Utility + + +class Validator(object): + @classmethod + def valid_platform_names(self, platform_filter, unit_id): + # If the parameter does not match to platform/verb name defined in the toolkit, it is invalid param. + try: + unit = UnitOffering.objects.get(id = int(unit_id)) + platform_list = unit.get_required_platforms() + # Empty string is acceptable + if platform_filter and platform_filter != '': + filter_vals = platform_filter.split(',') + # compare_list = xapi_settings.get_platform_list() + for val in filter_vals: + if not val in platform_list and val != 'all': + return False + except: + return False + + return True + + + @classmethod + def valid_verb_names(self, verb_filter, unit_id): + # If the parameter does not match to platform/verb name defined in the toolkit, it is invalid param. + try: + unit = UnitOffering.objects.get(id = int(unit_id)) + verb_list = unit.get_required_verbs() + # Empty string is acceptable + if verb_filter and verb_filter != '': + filter_vals = verb_filter.split(',') + # compare_list = xapi_settings.get_verb_list() + for val in filter_vals: + if not val in verb_list and val != 'all': + return False + except: + return False + + return True + + + @classmethod + def valid_date(self, date_str, format_string): + try: + # if date_str and re.match('\d{4}-\d{2}-\d{2}$' , date_str) is None: + if date_str and Utility.validate_date(date_str, format_string) is None: + return False + except: + return False + + return True + diff --git a/clatoolkit_project/api/analytics/views.py b/clatoolkit_project/api/analytics/views.py new file mode 100644 index 0000000..348e4b8 --- /dev/null +++ b/clatoolkit_project/api/analytics/views.py @@ -0,0 +1,67 @@ +__author__ = 'Koji' + +import json + +from collections import OrderedDict +from django.http import HttpResponse, JsonResponse +from endpoint.timeseries import Timeseries +from validator.timeseries_validator import TimeseriesValidator + +from rest_framework import authentication, permissions, viewsets, filters +from rest_framework.views import APIView +# from rest_framework.decorators import api_view +from rest_framework.response import Response +from rest_framework import status + +from ..error.application_error import ApplicationError +from ..error.invalid_parameter_error import InvalidParameterError + + +class DefaultsMixin(object): + """Default settings for view authentication, permissions, + filtering and pagination.""" + + authentication_classes = ( + authentication.SessionAuthentication, + ) + + permission_classes = ( + permissions.IsAuthenticated, + ) + paginate_by = 300 + paginate_by_param = 'page_size' + max_paginate_by = 1000 + + filter_backends = ( + filters.SearchFilter, + filters.DjangoFilterBackend, + filters.OrderingFilter + ) + + +class TimeseriesRequest(DefaultsMixin, APIView): + TIMESERIES_DATATYPE_PLATFORM = 'platform' + TIMESERIES_DATATYPE_VERB = 'verb' + + # https://docs.djangoproject.com/en/1.10/ref/class-based-views/base/#django.views.generic.base.View.as_view + def get(self, request, *args, **kw): + resp = None + try: + TimeseriesValidator.valid_timeseries_params(request, args, kw) + + if kw['type'] == self.TIMESERIES_DATATYPE_PLATFORM: + resp = Timeseries.get_platform_timeseries(request, args, kw) + + elif kw['type'] == self.TIMESERIES_DATATYPE_VERB: + resp = Timeseries.get_verb_timeseries(request, args, kw) + + except InvalidParameterError as ipexp: + ipexp.print_errorlog_message() + resp = {'status': 'error', 'message': '%s' % (ipexp.message)} + + except ApplicationError as appexp: + appexp.print_errorlog_message() + resp = {'status': 'error', 'message': '%s' % (appexp.message)} + + return JsonResponse(resp, status=status.HTTP_200_OK) + diff --git a/clatoolkit_project/api/error/__init__.py b/clatoolkit_project/api/error/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clatoolkit_project/api/error/application_error.py b/clatoolkit_project/api/error/application_error.py new file mode 100644 index 0000000..c81f38d --- /dev/null +++ b/clatoolkit_project/api/error/application_error.py @@ -0,0 +1,8 @@ +__author__ = 'Koji' + +from error import Error + +class ApplicationError(Error): + + def __init__(self, expression = None, message = None): + super(ApplicationError, self).__init__(expression, message) diff --git a/clatoolkit_project/api/error/error.py b/clatoolkit_project/api/error/error.py new file mode 100644 index 0000000..42e2713 --- /dev/null +++ b/clatoolkit_project/api/error/error.py @@ -0,0 +1,26 @@ +__author__ = 'Koji' + +from datetime import datetime + + +class Error(Exception): + # Base class for exceptions. + + def __init__(self, expression = None, message = None): + self.expression = expression + self.message = message + self.datetime = datetime.now().strftime("%d/%m/%Y %H:%M:%S.%f") + + def get_errorlog_basic_message(self, error_type = 'error'): + return '[%s] [%s] ' % (self.datetime, error_type) + + def print_errorlog_message(self): + message = self.get_errorlog_message() + if self.expression and self.expression.args: + message += ' [original error]: %s' % (self.expression.args) + + print message + + def get_errorlog_message(self, additional_msg = ''): + return '%s%s: %s' % (self.get_errorlog_basic_message('error'), type(self.expression), self.message) + diff --git a/clatoolkit_project/api/error/invalid_parameter_error.py b/clatoolkit_project/api/error/invalid_parameter_error.py new file mode 100644 index 0000000..a844f19 --- /dev/null +++ b/clatoolkit_project/api/error/invalid_parameter_error.py @@ -0,0 +1,9 @@ +__author__ = 'Koji' + +from error import Error + +class InvalidParameterError(Error): + + def __init__(self, expression = None, param_name = ''): + super(InvalidParameterError, self).__init__(expression, 'Invalid parameter: %s' % param_name) + \ No newline at end of file diff --git a/clatoolkit_project/api/models.py b/clatoolkit_project/api/models.py new file mode 100644 index 0000000..e69de29 diff --git a/clatoolkit_project/api/urls.py b/clatoolkit_project/api/urls.py new file mode 100644 index 0000000..cb946e0 --- /dev/null +++ b/clatoolkit_project/api/urls.py @@ -0,0 +1,15 @@ +__author__ = 'Koji' + +from analytics.views import TimeseriesRequest +from django.conf.urls import patterns, url +from rest_framework.routers import DefaultRouter + + +urlpatterns = patterns('', + # Analytics API + url(r'^unit/(?P[a-zA-Z0-9]+)/timeseries/platforms/$', + TimeseriesRequest.as_view(), {'type': TimeseriesRequest.TIMESERIES_DATATYPE_PLATFORM}, name='PlatformTimeseries'), + url(r'^unit/(?P[a-zA-Z0-9]+)/timeseries/verbs/$', + TimeseriesRequest.as_view(), {'type': TimeseriesRequest.TIMESERIES_DATATYPE_VERB}, name='VerbTimeseries'), +) + diff --git a/clatoolkit_project/clatoolkit/models.py b/clatoolkit_project/clatoolkit/models.py index 10b1c48..cb18c3b 100644 --- a/clatoolkit_project/clatoolkit/models.py +++ b/clatoolkit_project/clatoolkit/models.py @@ -1,5 +1,6 @@ import os from django.db import models +from django.conf import settings from django.contrib.auth.models import User from django_pgjson.fields import JsonField from django.core.exceptions import ObjectDoesNotExist @@ -78,6 +79,12 @@ class OfflinePlatformAuthToken(models.Model): platform = models.CharField(max_length=1000, blank=False) user = models.ForeignKey(User) +class UserPlatformResourceMap(models.Model): + user = models.ForeignKey(User) + unit = models.ForeignKey('UnitOffering') + resource_id = models.CharField(max_length=5000, blank=False) + platform = models.CharField(max_length=100, blank=False) + class UnitOffering(models.Model): code = models.CharField(max_length=5000, blank=False, verbose_name="Unit Code", unique=True) name = models.CharField(max_length=5000, blank=False, verbose_name="Unit Name") @@ -220,21 +227,23 @@ def get_required_platforms(self): platforms = [] if len(self.twitter_hashtags_as_list()): - platforms.append('twitter') + platforms.append(xapi_settings.PLATFORM_TWITTER) if len(self.facebook_groups_as_list()): - platforms.append('facebook') + platforms.append(xapi_settings.PLATFORM_FACEBOOK) if len(self.forum_urls_as_list()): - platforms.append('forum') + platforms.append(xapi_settings.PLATFORM_FORUM) if len(self.youtube_channelIds_as_list()): - platforms.append('youtube') + platforms.append(xapi_settings.PLATFORM_YOUTUBE) if len(self.diigo_tags_as_list()): - platforms.append('diigo') + platforms.append(xapi_settings.PLATFORM_DIIGO) if len(self.blogmember_urls_as_list()): - platforms.append('blog') - if len(self.github_urls_as_list()): - platforms.append('github') + platforms.append(xapi_settings.PLATFORM_BLOG) if len(self.trello_boards_as_list()): - platforms.append('trello') + platforms.append(xapi_settings.PLATFORM_TRELLO) + if self.github_member_count() > 0: + platforms.append(xapi_settings.PLATFORM_GITHUB) + if self.slack_member_count() > 0: + platforms.append(xapi_settings.PLATFORM_SLACK) return platforms @@ -259,6 +268,20 @@ def get_cca_dashboard_params(self): return ','.join(params) + def get_required_verbs(self, platform_list = None): + platforms = platform_list + if platforms is None or len(platforms) == 0 or 'all' in platforms: + platforms = self.get_required_platforms() + + verb_list = [] + for p in platforms: + platform_verbs = settings.DATAINTEGRATION_PLUGINS[p].xapi_verbs + platform_verbs.sort() + # verb_list[p] = platform_verbs + verb_list.extend(platform_verbs) + + return list(set(verb_list)) + class OauthFlowTemp(models.Model): googleid = models.CharField(max_length=1000, blank=False) @@ -341,12 +364,6 @@ class UserClassification(models.Model): trained = models.BooleanField(blank=False, default=False) created_at = models.DateTimeField(auto_now_add=True) -class UserPlatformResourceMap(models.Model): - user = models.ForeignKey(User) - unit = models.ForeignKey(UnitOffering) - resource_id = models.CharField(max_length=5000, blank=False) - platform = models.CharField(max_length=100, blank=False) - class ApiCredentials(models.Model): platform_uid = models.CharField(max_length=5000, blank=False) credentials_json = JsonField() diff --git a/clatoolkit_project/clatoolkit_project/settings.py b/clatoolkit_project/clatoolkit_project/settings.py index 932abfb..2f1c109 100644 --- a/clatoolkit_project/clatoolkit_project/settings.py +++ b/clatoolkit_project/clatoolkit_project/settings.py @@ -74,8 +74,8 @@ 'clatoolkit', 'dataintegration', 'dashboard', - #'common', - 'xapi' + 'xapi', + 'api', ) MIDDLEWARE_CLASSES = ( diff --git a/clatoolkit_project/clatoolkit_project/urls.py b/clatoolkit_project/clatoolkit_project/urls.py index 57bd5cc..c472b36 100644 --- a/clatoolkit_project/clatoolkit_project/urls.py +++ b/clatoolkit_project/clatoolkit_project/urls.py @@ -8,6 +8,7 @@ url(r'^$', views.userlogin, name='userlogin'), url(r'^clatoolkit/', include('clatoolkit.urls')), url(r'^api/', include(router.urls)), + url(r'^api/', include('api.urls')), url(r'^admin/', include(admin.site.urls)), url(r'^dataintegration/', include('dataintegration.urls')), url(r'^dashboard/', include('dashboard.urls')), diff --git a/clatoolkit_project/common/util.py b/clatoolkit_project/common/util.py index 7099559..62b474b 100644 --- a/clatoolkit_project/common/util.py +++ b/clatoolkit_project/common/util.py @@ -1,7 +1,7 @@ from django.http import HttpResponse from datetime import datetime -from isodate.isodatetime import parse_datetime +from isodate.isodatetime import parse_datetime, parse_date class Utility(object): @@ -11,6 +11,7 @@ def get_site_url(self, request): url = '%s://%s' % (protocol, request.get_host()) return url + @classmethod def convert_to_datetime_object(self, timestr): try: @@ -20,6 +21,7 @@ def convert_to_datetime_object(self, timestr): return date_object + @classmethod def format_date(self, date_str, splitter, connector, isMonthSubtract): ret = '' @@ -32,7 +34,88 @@ def format_date(self, date_str, splitter, connector, isMonthSubtract): dateAry = date_str.split(splitter) return dateAry[0] + connector + str(int(dateAry[1]) - month_subtract).zfill(2) + connector + dateAry[2] + @classmethod def convert_unixtime_to_datetime(self, unix_time): unix_time = float(unix_time) return datetime.fromtimestamp(unix_time) + + + @classmethod + def validate_date(self, datestr, format_string): + date_object = None + try: + date_object = self.convert_to_date_object(datestr, format_string) + except ValueError: + raise ValueError("Incorrect data format. It must be YYYY-MM-DD") + + ret = True if date_object else False + return ret + + + @classmethod + def convert_to_date_object(self, datestr, format_string): + date_object = None + try: + date_object = datetime.strptime(datestr, format_string) + except ValueError as ve: + print type(ve) + print ve.args + raise ve + + return date_object + + + @classmethod + def compare_to(self, d1, d2): + if d1 is None and d2 is None: + return 0 + elif d1 is None and d2 is not None: + return -1 + elif d1 is not None and d2 is None: + return 1 + + date1 = d1 + date2 = d2 + if not isinstance(date1, datetime): + date1 = self.convert_to_date_object(str(d1), '%Y-%m-%d') + if not isinstance(date2, datetime): + date2 = self.convert_to_date_object(str(d2), '%Y-%m-%d') + + if date1 == date2: + return 0 + elif date1 < date2: + return -1 + elif date1 > date2: + return 1 + + + @classmethod + def max_date(self, d1, d2): + if d1 is None and d2 is not None: + return d2 + elif d1 is not None and d2 is None: + return d1 + + if self.compare_to(d1, d2) == 1: + return d1 + elif self.compare_to(d1, d2) == -1: + return d2 + else: + return d1 + + + @classmethod + def min_date(self, d1, d2): + if d1 is None and d2 is not None: + return d2 + elif d1 is not None and d2 is None: + return d1 + + if self.compare_to(d1, d2) == 1: + return d2 + elif self.compare_to(d1, d2) == -1: + return d1 + else: + return d1 + diff --git a/clatoolkit_project/dashboard/views.py b/clatoolkit_project/dashboard/views.py index e5d6471..9ca32a3 100644 --- a/clatoolkit_project/dashboard/views.py +++ b/clatoolkit_project/dashboard/views.py @@ -260,7 +260,6 @@ def dashboard(request): # Activity Time line data (verbs and platform) timeline_data = get_verb_timeline_data(unit, platform, None) - platform_timeline_data = get_platform_timeline_data(unit, platform, None) # p = platform if platform != "all" else None activememberstable = get_active_members_table(unit, platform) @@ -274,15 +273,6 @@ def dashboard(request): 'posts_timeline': timeline_data['posts'], 'shares_timeline': timeline_data['shares'], 'likes_timeline': timeline_data['likes'], 'comments_timeline': timeline_data['comments'], - 'twitter_timeline': platform_timeline_data[xapi_settings.PLATFORM_TWITTER], - 'facebook_timeline': platform_timeline_data[xapi_settings.PLATFORM_FACEBOOK], - 'youtube_timeline': platform_timeline_data[xapi_settings.PLATFORM_YOUTUBE], - 'blog_timeline': platform_timeline_data[xapi_settings.PLATFORM_BLOG], - 'trello_timeline': platform_timeline_data[xapi_settings.PLATFORM_TRELLO], - 'github_timeline': platform_timeline_data[xapi_settings.PLATFORM_GITHUB], - 'slack_timeline': platform_timeline_data[xapi_settings.PLATFORM_SLACK], - 'forum_timeline': [], 'diigo_timeline':[], - 'activity_pie_series': activity_pie_series, 'platformactivity_pie_series': platformactivity_pie_series } diff --git a/clatoolkit_project/dataintegration/plugins/slack/cladi_plugin.py b/clatoolkit_project/dataintegration/plugins/slack/cladi_plugin.py index 32c0dd0..6ef3ae4 100644 --- a/clatoolkit_project/dataintegration/plugins/slack/cladi_plugin.py +++ b/clatoolkit_project/dataintegration/plugins/slack/cladi_plugin.py @@ -36,8 +36,9 @@ def __init__(self): platform_url = "https://slack.com/" xapi_verbs = [xapi_settings.VERB_CREATED, xapi_settings.VERB_COMMENTED, xapi_settings.VERB_SHARED, - xapi_settings.VERB_MENTIONED, xapi_settings.VERB_LIKED, xapi_settings.VERB_REMOVED] - xapi_objects = [xapi_settings.OBJECT_NOTE, xapi_settings.OBJECT_FILE, ] + xapi_settings.VERB_MENTIONED, xapi_settings.VERB_BOOKMARKED, xapi_settings.VERB_REMOVED, + xapi_settings.VERB_ATTACHED] + xapi_objects = [xapi_settings.OBJECT_NOTE, xapi_settings.OBJECT_FILE, xapi_settings.VERB_COMMENTED] user_api_association_name = 'Slack Username' # eg the username for a signed up user that will appear in data extracted via a social API unit_api_association_name = 'Slack Team' # eg Slack team @@ -76,8 +77,8 @@ def __init__(self): xapi_settings.OBJECT_FILE, xapi_settings.VERB_COMMENTED] xapi_verbs_to_includein_verbactivitywidget = [xapi_settings.VERB_CREATED, xapi_settings.VERB_COMMENTED, - xapi_settings.VERB_SHARED, xapi_settings.VERB_MENTIONED, - xapi_settings.VERB_LIKED, xapi_settings.VERB_REMOVED] + xapi_settings.VERB_SHARED, xapi_settings.VERB_MENTIONED, xapi_settings.VERB_BOOKMARKED, + xapi_settings.VERB_REMOVED, xapi_settings.VERB_ATTACHED] def __init__(self): diff --git a/clatoolkit_project/templates/dashboard/dashboard.html b/clatoolkit_project/templates/dashboard/dashboard.html index 4a9444b..3a3c6f5 100644 --- a/clatoolkit_project/templates/dashboard/dashboard.html +++ b/clatoolkit_project/templates/dashboard/dashboard.html @@ -148,24 +148,73 @@ {% block js_block %} {% autoescape off %}