diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 0cd16ae6..d4ffbc48 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -8,12 +8,12 @@ jobs: build: runs-on: ubuntu-latest - + services: # Label used to access the service container postgres: - image: postgres:11 - env: + image: postgres:14 + env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres options: >- @@ -22,24 +22,35 @@ jobs: --health-timeout 5s --health-retries 5 ports: - - 5432/tcp - env: - TESTMODEL_URL: http://localhost:8080/intermine-demo - TESTMODEL_PATH: intermine-demo + - 5432:5432 steps: - - uses: actions/checkout@v2 - - name: Set up python 3.7 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '11' + - name: Set up python 3.10 + uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.10' - name: Install dependencies run: | + set -euo pipefail + sudo apt-get install libxml2-dev libxslt-dev python3-dev python -m pip install --upgrade pip + pip install -U setuptools pip install -r requirements.txt - name: Install PostgreSQL client run: | + set -euo pipefail sudo apt-get update -y sudo apt-get install -y libpq-dev postgresql-client sudo service postgresql start - name: Run unit tests - run: ./config/ci/init-solr.sh && ./config/ci/init.sh && python setup.py test && python setup.py livetest + run: | + set -euo pipefail + ./config/ci/init-solr.sh ${GITHUB_WORKSPACE} + ./config/ci/init.sh ${GITHUB_WORKSPACE} http://localhost:8080/intermine-demo + python setup.py test + python setup.py livetest diff --git a/.gitignore b/.gitignore index 44023b0c..2ebe5c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,10 @@ intermine-docs-1.00.03.zip build MANIFEST .idea + +# CI + +server/ + +solr-*.tgz +solr-*/ diff --git a/config/ci/init-solr.sh b/config/ci/init-solr.sh index 1ea5a44b..1aae3488 100755 --- a/config/ci/init-solr.sh +++ b/config/ci/init-solr.sh @@ -1,7 +1,46 @@ +#!/bin/bash + # set up solr for testmine # testmine's setup script populates these empty indexes -wget http://archive.apache.org/dist/lucene/solr/8.6.2/solr-8.6.2.tgz -tar xzf solr-8.6.2.tgz && ./solr-8.6.2/bin/solr start -./solr-8.6.2/bin/solr create -c intermine-search -./solr-8.6.2/bin/solr create -c intermine-autocomplete +set -euo pipefail + +if [ "$#" != "1" ]; then + echo "Usage: $0 " + exit 2 +fi + +WORKSPACE_DIR=$1 + +cd "$WORKSPACE_DIR" + +SOLR_VERSION=8.6.2 +SOLR_PACKAGE=solr-${SOLR_VERSION}.tgz +SOLR_DIR=solr-${SOLR_VERSION} +SOLR=${SOLR_DIR}/bin/solr + +create_solr_core() { + local core_name=$1 + + local status + + status=$(curl -s "http://localhost:8983/solr/admin/cores?action=STATUS&core=${core_name}" | jq --arg core_name "${core_name}" '.status[$core_name]') + + if [ "$status" = "{}" ]; then + ${SOLR} create -c "${core_name}" + else + echo "Solr core ${core_name} already exists" + fi +} + +if [ ! -d $SOLR_DIR ]; then + if [ ! -f $SOLR_PACKAGE ]; then + wget http://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/${SOLR_PACKAGE} + fi + + tar xzf $SOLR_PACKAGE +fi + +${SOLR} restart +create_solr_core intermine-search +create_solr_core intermine-autocomplete diff --git a/config/ci/init.sh b/config/ci/init.sh index 42f1fcd7..28835f93 100755 --- a/config/ci/init.sh +++ b/config/ci/init.sh @@ -1,44 +1,61 @@ #!/bin/bash -set -e +set -euo pipefail -if [ -z $(which wget) ]; then - # use curl - GET='curl' -else - GET='wget -O -' +if [ "$#" != "2" ]; then + echo "Usage: $0 " + exit 2 fi +WORKSPACE_DIR=$1 +TESTMODEL_URL=$2 + +INTERMINE_DIR=${WORKSPACE_DIR}/server +TESTMINE_DIR=${INTERMINE_DIR}/testmine + +cd "${WORKSPACE_DIR}" + # Pull in the server code. -git clone --single-branch --depth 1 https://github.com/intermine/intermine.git server + +if [ ! -d "${INTERMINE_DIR}" ]; then + git clone --single-branch --depth 1 -b dev https://github.com/ucam-department-of-psychiatry/intermine.git "${INTERMINE_DIR}" +fi export PSQL_USER=test export PSQL_PWD=test +export PSQL_HOST=localhost +export PGPASSWORD=${PGPASSWORD:-postgres} export KEYSTORE=${PWD}/keystore.jks echo "#---> Running unit tests" -sudo -u postgres createuser test -sudo -u postgres psql -c "alter user test with encrypted password 'test';" +sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists intermine-demo +sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists userprofile-demo + +sudo -E -u postgres dropuser -h "${PSQL_HOST}" --if-exists test +sudo -E -u postgres createuser -h "${PSQL_HOST}" test +sudo -E -u postgres psql -h "${PSQL_HOST}" -c "alter user test with encrypted password 'test';" # Set up properties -PROPDIR=$HOME/.intermine -TESTMODEL_PROPS=$PROPDIR/testmodel.properties -SED_SCRIPT='s/PSQL_USER/test/' +PROPDIR=${HOME}/.intermine +TESTMODEL_PROPS=${PROPDIR}/testmodel.properties + +mkdir -p "${PROPDIR}" -mkdir -p $PROPDIR +echo "#--- creating ${TESTMODEL_PROPS}" +cp "${INTERMINE_DIR}"/config/testmodel.properties "${TESTMODEL_PROPS}" +sed -i -e "s/PSQL_HOST/${PSQL_HOST}/" "$TESTMODEL_PROPS" +sed -i -e "s/PSQL_USER/${PSQL_USER}/" "$TESTMODEL_PROPS" +sed -i -e "s/PSQL_PWD/${PSQL_PWD}/" "$TESTMODEL_PROPS" -echo "#--- creating $TESTMODEL_PROPS" -cp server/config/testmodel.properties $TESTMODEL_PROPS -sed -i -e $SED_SCRIPT $TESTMODEL_PROPS # We will need a fully operational web-application echo '#---> Building and releasing web application to test against' -(cd server/testmine && ./setup.sh) +(cd "${TESTMINE_DIR}" && ./setup.sh "${INTERMINE_DIR}") # Travis is so slow sleep 90 # let webapp startup # Warm up the keyword search by requesting results, but ignoring the results -$GET "$TESTMODEL_URL/service/search" > /dev/null +wget -O - "${TESTMODEL_URL}/service/search" > /dev/null # Start any list upgrades -$GET "$TESTMODEL_URL/service/lists?token=test-user-token" > /dev/null +wget -O - "${TESTMODEL_URL}/service/lists?token=test-user-token" > /dev/null diff --git a/intermine/results.py b/intermine/results.py index 7a49ffc2..3f8db79c 100644 --- a/intermine/results.py +++ b/intermine/results.py @@ -603,7 +603,7 @@ def __init__(self, credentials=None, token=None): self.token = token if credentials and len(credentials) == 2: encoded = '{0}:{1}'.format(*credentials).encode('utf8') - base64string = 'Basic {0}'.format(base64.encodestring(encoded)[:-1].decode('ascii')) + base64string = 'Basic {0}'.format(base64.encodebytes(encoded)[:-1].decode('ascii')) self.auth_header = base64string self.using_authentication = True elif self.token is not None: diff --git a/intermine/webservice.py b/intermine/webservice.py index bee49f4b..97b5acc5 100644 --- a/intermine/webservice.py +++ b/intermine/webservice.py @@ -5,16 +5,10 @@ import requests -try: - from urlparse import urlparse - from UserDict import DictMixin - from urllib import urlopen - from urllib import urlencode -except ImportError: - from urllib.parse import urlparse - from urllib.parse import urlencode - from collections import MutableMapping as DictMixin - from urllib.request import urlopen +from urllib.parse import urlparse +from urllib.parse import urlencode +from collections.abc import MutableMapping as DictMixin +from urllib.request import urlopen try: import simplejson as json # Prefer this as it is faster diff --git a/requirements.txt b/requirements.txt index d837d583..bda7ec4b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -certifi==2020.4.5.1 -chardet==3.0.4 -cycler==0.10.0 -Cython==0.29.19 -idna==2.9 -kiwisolver==1.2.0 -lxml==4.5.1 -matplotlib==3.2.1 -numpy==1.18.4 -pandas==1.0.3 -pyparsing==2.4.7 -python-dateutil==2.8.1 -pytz==2020.1 -requests==2.23.0 -simplejson==3.17.0 -six==1.15.0 -urllib3==1.25.9 +certifi==2024.8.30 +chardet==5.2.0 +cycler==0.12.1 +Cython==3.0.11 +idna==3.8 +kiwisolver==1.4.5 +lxml==5.3.0 +matplotlib==3.9.2 +numpy==2.1.0 +pandas==2.2.2 +pyparsing==3.1.4 +python-dateutil==2.9.0 +pytz==2024.1 +requests==2.32.4 +simplejson==3.19.3 +six==1.16.0 +urllib3==2.5.0 diff --git a/tests/test_core.py b/tests/test_core.py index d04036c4..2df631fc 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,6 +2,7 @@ import unittest import logging import sys +import xml.etree.ElementTree as ET from intermine.model import * from intermine.webservice import * @@ -50,6 +51,23 @@ def do_unpredictable_test(self, test, attempts=0, error=None): raise RuntimeError("Max error count reached - last error: " + str( error)) + def checkQueryHasConstraint(self, query, **kwargs): + xpath_attrs = [] + + for k, v in kwargs.items(): + xpath_attrs.append(f"[@{k}='{v}']") + + xpath_attrs_str = "".join(xpath_attrs) + xpath = f"./constraint{xpath_attrs_str}" + + constraint = query.find(xpath) + + ET.indent(query, space=" ", level=0) + xml = ET.tostring(query, method="xml") + self.assertIsNotNone(constraint, f"Failed to find constraint matching:\n{xpath} in:\n{xml}") + + return constraint + class TestInstantiation(WebserviceTest): # pragma: no cover def testMakeModel(self): @@ -843,10 +861,41 @@ def testXML(self): self.q.add_join("Employee.department", "outer") self.q.add_sort_order("Employee.age") self.q.set_logic("(A and B) or (A and C and D) and (E or F or G)") - expected = 'JohnPaulMary1..1030..35' - self.assertEqual(expected, self.q.to_xml()) + + xml = self.q.to_xml() + query = ET.fromstring(xml) + + self.assertEqual(query.attrib["constraintLogic"], "((A and B) or (A and C and D)) and (E or F or G)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value=10) + self.checkQueryHasConstraint(query, path="Employee.department", op="LOOKUP", value="Sales", extraValue="Wernham-Hogg") + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees.name", op="ONE OF") + + names = [e.text for e in constraint.findall("value")] + self.assertIn("John", names) + self.assertIn("Paul", names) + self.assertIn("Mary", names) + + self.checkQueryHasConstraint(query, path="Employee.department.manager", op="=", loopPath="Employee") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="some list of employees") + + constraint = self.checkQueryHasConstraint(query, path="Employee.age", op="OVERLAPS") + values = [e.text for e in constraint.findall("value")] + self.assertIn("1..10", values) + self.assertIn("30..35", values) + + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees", type="Manager") + # Clones must produce identical XML - self.assertEqual(expected, self.q.clone().to_xml()) + self.assertEqual(xml, self.q.clone().to_xml()) def testSugaryQueryConstruction(self): """Test use of operation coercion which is similar to SQLAlchemy""" @@ -855,8 +904,6 @@ def testSugaryQueryConstruction(self): Employee = model.table("Employee") Manager = model.table("Manager") - expected = 'JohnPaulMary' - # SQL style q = Employee.\ select("name", "age", "department.name").\ @@ -871,7 +918,31 @@ def testSugaryQueryConstruction(self): order_by(Employee.age).\ set_logic("(A and B) or (A and C and D) and (E or F)") - self.assertEqual(expected, q.to_xml()) + xml = q.to_xml() + query = ET.fromstring(xml) + + self.assertEqual(query.attrib["constraintLogic"], "((A and B) or (A and C and D)) and (E or F)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="10") + self.checkQueryHasConstraint(query, path="Employee.department", op="LOOKUP", value="Sales", extraValue="Wernham-Hogg") + + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees.name", op="ONE OF") + names = [e.text for e in constraint.findall("value")] + self.assertIn("John", names) + self.assertIn("Paul", names) + self.assertIn("Mary", names) + + self.checkQueryHasConstraint(query, path="Employee.department.manager", op="=", loopPath="Employee") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="my-list") + self.checkQueryHasConstraint(query, path="Employee.department.employees", type="Manager") # SQLAlchemy style q = self.service.query(Employee).\ @@ -886,34 +957,48 @@ def testSugaryQueryConstruction(self): outerjoin(Employee.department).\ order_by(Employee.age).\ set_logic("(A and B) or (A and C and D) and (E or F)") + xml = q.to_xml() + query = ET.fromstring(xml) - self.assertEqual(expected, q.to_xml()) + self.assertEqual(query.attrib["constraintLogic"], "((A and B) or (A and C and D)) and (E or F)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="10") + self.checkQueryHasConstraint(query, path="Employee.department", op="LOOKUP", value="Sales", extraValue="Wernham-Hogg") + + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees.name", op="ONE OF") + names = [e.text for e in constraint.findall("value")] + self.assertIn("John", names) + self.assertIn("Paul", names) + self.assertIn("Mary", names) + + self.checkQueryHasConstraint(query, path="Employee.department.manager", op="=", loopPath="Employee") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="my-list") + self.checkQueryHasConstraint(query, path="Employee.department.employees", type="Manager") def testKWCons(self): """Test use of constraints provided in kwargs""" model = self.q.model - expected = '' - q = model.Employee.select("name").where(age=10) + xml = q.to_xml() + query = ET.fromstring(xml) + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name") - self.assertEqual(expected, q.to_xml()) + self.checkQueryHasConstraint(query, path="Employee.age", op="=", value=10) def testLogicConstraintTrees(self): # Actually SQL-Alchemy-esque - expected = """ - - - - - - - - - """ e = self.service.model.Employee CEO = self.service.model.CEO q = self.service.query(e, e.department.name).\ @@ -927,10 +1012,23 @@ def testLogicConstraintTrees(self): outerjoin(e.department).\ order_by(e.age) - expected = re.sub(r'\s+', ' ', expected) - expected = re.sub(r'>\s+<', '><', expected) - expected = expected.strip() - self.assertEqual(expected, q.to_xml()) + xml = q.to_xml() + query = ET.fromstring(xml) + + self.assertEqual(query.attrib["constraintLogic"], "(A and B) or (C and D)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.age Employee.end Employee.fullTime Employee.id Employee.name Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="10") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="my-list") + self.checkQueryHasConstraint(query, value="David", op="LOOKUP", path="Employee.department.manager") + self.checkQueryHasConstraint(query, path="Employee.department.manager", type="CEO") class TestTemplate(TestQuery): # pragma: no cover @@ -993,34 +1091,77 @@ def testURLs(self): t.add_constraint("Employee.name", '=', "Fred") t.add_constraint("Employee.age", ">", 25) - expectedQ = ('/QUERY-PATH', { - 'query': - '', - 'start': 0 - }, 'object', ['Employee.name', 'Employee.age', 'Employee.id'], - self.model.get_class("Employee")) - self.assertEqual(expectedQ, q.results()) - self.assertEqual(list(expectedQ), q.get_results_list()) + results = q.results() + + self.assertEqual(results[0], "/QUERY-PATH") + query_xml = results[1]["query"] + + query = ET.fromstring(query_xml) + + self.assertEqual(query.attrib["constraintLogic"], "A and B") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.id") + + self.checkQueryHasConstraint(query, path="Employee.name", op="=", value="Fred") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="25") + + self.assertEqual(results[1]["start"], 0) + self.assertEqual(results[2], "object") + self.assertIn("Employee.name", results[3]) + self.assertIn("Employee.age", results[3]) + self.assertIn("Employee.id", results[3]) + self.assertEqual(results[4], self.model.get_class("Employee")) + + self.assertEqual(list(results), q.get_results_list()) + + rows = q.rows() + self.assertEqual(rows[0], "/QUERY-PATH") + query_xml = rows[1]["query"] + + query = ET.fromstring(query_xml) + + self.assertEqual(query.attrib["constraintLogic"], "A and B") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.id") + + self.checkQueryHasConstraint(query, path="Employee.name", op="=", value="Fred") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="25") + self.assertEqual(rows[1]["start"], 0) + self.assertEqual(rows[2], "rr") + self.assertIn("Employee.name", rows[3]) + self.assertIn("Employee.age", rows[3]) + self.assertIn("Employee.id", rows[3]) + self.assertEqual(rows[4], self.model.get_class("Employee")) + + self.assertEqual(list(rows), q.get_row_list()) + + results = q.results(start=10, size=200) + self.assertEqual(results[0], "/QUERY-PATH") + + query_xml = results[1]["query"] + query = ET.fromstring(query_xml) + + self.assertEqual(query.attrib["constraintLogic"], "A and B") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.id") + + self.checkQueryHasConstraint(query, path="Employee.name", op="=", value="Fred") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="25") + + self.assertEqual(results[1]["start"], 10) + self.assertEqual(results[1]["size"], 200) + self.assertEqual(results[2], "object") + self.assertIn("Employee.name", results[3]) + self.assertIn("Employee.age", results[3]) + self.assertIn("Employee.id", results[3]) + self.assertEqual(results[4], self.model.get_class("Employee")) - expectedQ = ('/QUERY-PATH', { - 'query': - '', - 'start': 0 - }, 'rr', ['Employee.name', 'Employee.age', 'Employee.id'], - self.model.get_class("Employee")) - self.assertEqual(expectedQ, q.rows()) - self.assertEqual(list(expectedQ), q.get_row_list()) - expectedQ = ('/QUERY-PATH', { - 'query': - '', - 'start': 10, - 'size': 200 - }, 'object', ['Employee.name', 'Employee.age', 'Employee.id'], - self.model.get_class("Employee")) - self.assertEqual(expectedQ, q.results(start=10, size=200)) self.assertEqual( - list(expectedQ), q.get_results_list( + list(results), q.get_results_list( start=10, size=200)) expected1 = ('/TEMPLATE-PATH', {