From 04f945df709e16f56ead5c12f101773e0e76fad4 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 15:49:57 +0100 Subject: [PATCH 01/22] Use later Postgres in GitHub test workflow --- .github/workflows/im-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 0cd16ae6..c5fe0907 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -8,12 +8,12 @@ jobs: build: runs-on: ubuntu-latest - + services: # Label used to access the service container postgres: - image: postgres:11 - env: + image: postgres:13 + env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres options: >- From aa688439243de87ffa8b44dcb8289d87b9aa164d Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 15:54:42 +0100 Subject: [PATCH 02/22] Use Python 3.10 in GitHub workflow Also use later versions of checkout and setup-python --- .github/workflows/im-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index c5fe0907..28f0dd18 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -27,11 +27,11 @@ jobs: TESTMODEL_URL: http://localhost:8080/intermine-demo TESTMODEL_PATH: intermine-demo steps: - - uses: actions/checkout@v2 - - name: Set up python 3.7 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up python 3.10 + uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.10' - name: Install dependencies run: | python -m pip install --upgrade pip From 8dc7e3c7634ab4045952209cd691607473696dd6 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 15:55:06 +0100 Subject: [PATCH 03/22] Better error handling in worflow --- .github/workflows/im-build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 28f0dd18..7d3c92b5 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -34,12 +34,16 @@ jobs: python-version: '3.10' - name: Install dependencies run: | + set -euxo pipefail python -m pip install --upgrade pip pip install -r requirements.txt - name: Install PostgreSQL client run: | + set -euxo pipefail sudo apt-get update -y sudo apt-get install -y libpq-dev postgresql-client sudo service postgresql start - name: Run unit tests - run: ./config/ci/init-solr.sh && ./config/ci/init.sh && python setup.py test && python setup.py livetest + run: | + set -euxo pipefail + ./config/ci/init-solr.sh && ./config/ci/init.sh && python setup.py test && python setup.py livetest From 771011cc0b6a616d71a580193e0120e47ce5d335 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 16:01:07 +0100 Subject: [PATCH 04/22] Install python3-lxml build dependencies in workflow --- .github/workflows/im-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 7d3c92b5..156f454d 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -35,6 +35,7 @@ jobs: - name: Install dependencies run: | set -euxo pipefail + sudo apt-get build-dep python3-lxml python -m pip install --upgrade pip pip install -r requirements.txt - name: Install PostgreSQL client From 2c3c460c1528abe3beafc17a19901e867640c1ab Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 16:07:11 +0100 Subject: [PATCH 05/22] Try another way to install lxml dependencies with apt-get build-dep we need a deb-src entry in sources.list. We can do that but I'd rather not. --- .github/workflows/im-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 156f454d..3092cf27 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | set -euxo pipefail - sudo apt-get build-dep python3-lxml + sudo apt-get install libxml2-dev libxslt-dev python-dev python -m pip install --upgrade pip pip install -r requirements.txt - name: Install PostgreSQL client From 4746a5c90074b2ca05b4fdd08d89abb88a71fe05 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 16:08:33 +0100 Subject: [PATCH 06/22] Use later setuptools --- .github/workflows/im-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 3092cf27..b36839bd 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -37,6 +37,7 @@ jobs: set -euxo pipefail sudo apt-get install libxml2-dev libxslt-dev python-dev python -m pip install --upgrade pip + pip install -U setuptools pip install -r requirements.txt - name: Install PostgreSQL client run: | From 3a538ea4009a2241cb7e684587fb6f64c721ab0f Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 16:11:28 +0100 Subject: [PATCH 07/22] python3-dev, not python-dev --- .github/workflows/im-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index b36839bd..741b8687 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | set -euxo pipefail - sudo apt-get install libxml2-dev libxslt-dev python-dev + sudo apt-get install libxml2-dev libxslt-dev python3-dev python -m pip install --upgrade pip pip install -U setuptools pip install -r requirements.txt From be7285441afbfdc55dfb6988070ccce4ba113708 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 16:25:23 +0100 Subject: [PATCH 08/22] Upgrade all of the python packages --- requirements.txt | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index d837d583..9815f6a4 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -certifi==2020.4.5.1 -chardet==3.0.4 -cycler==0.10.0 -Cython==0.29.19 -idna==2.9 -kiwisolver==1.2.0 -lxml==4.5.1 -matplotlib==3.2.1 -numpy==1.18.4 -pandas==1.0.3 -pyparsing==2.4.7 -python-dateutil==2.8.1 -pytz==2020.1 -requests==2.23.0 -simplejson==3.17.0 -six==1.15.0 -urllib3==1.25.9 +certifi==2024.8.30 +chardet==5.2.0 +cycler==0.12.1 +Cython==3.0.11 +idna==3.8 +kiwisolver==1.4.5 +lxml==5.3.0 +matplotlib==3.9.2 +numpy==2.1.0 +pandas==2.2.2 +pyparsing==3.1.4 +python-dateutil==2.9.0 +pytz==2024.1 +requests==2.32.3 +simplejson==3.19.3 +six==1.16.0 +urllib3==2.2.2 From f056fc06b62b111a24c1b4d9f8178e8b9d9f04f0 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 2 Sep 2024 16:41:54 +0100 Subject: [PATCH 09/22] Fix import errors --- intermine/webservice.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/intermine/webservice.py b/intermine/webservice.py index bee49f4b..97b5acc5 100644 --- a/intermine/webservice.py +++ b/intermine/webservice.py @@ -5,16 +5,10 @@ import requests -try: - from urlparse import urlparse - from UserDict import DictMixin - from urllib import urlopen - from urllib import urlencode -except ImportError: - from urllib.parse import urlparse - from urllib.parse import urlencode - from collections import MutableMapping as DictMixin - from urllib.request import urlopen +from urllib.parse import urlparse +from urllib.parse import urlencode +from collections.abc import MutableMapping as DictMixin +from urllib.request import urlopen try: import simplejson as json # Prefer this as it is faster From d48f766aea39eca2f1cd867151b42b2210864d45 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 09:41:57 +0000 Subject: [PATCH 10/22] Less noisy CI output --- .github/workflows/im-build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 741b8687..71fc8b8e 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -34,18 +34,21 @@ jobs: python-version: '3.10' - name: Install dependencies run: | - set -euxo pipefail + set -euo pipefail sudo apt-get install libxml2-dev libxslt-dev python3-dev python -m pip install --upgrade pip pip install -U setuptools pip install -r requirements.txt - name: Install PostgreSQL client run: | - set -euxo pipefail + set -euo pipefail sudo apt-get update -y sudo apt-get install -y libpq-dev postgresql-client sudo service postgresql start - name: Run unit tests run: | - set -euxo pipefail - ./config/ci/init-solr.sh && ./config/ci/init.sh && python setup.py test && python setup.py livetest + set -euo pipefail + ./config/ci/init-solr.sh + ./config/ci/init.sh + python setup.py test + python setup.py livetest From 3b7a5762ac81cb6a5aebf25395de82da73be2a1d Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 09:53:33 +0000 Subject: [PATCH 11/22] Make Solr initialisation runable locally --- .github/workflows/im-build.yml | 2 +- .gitignore | 5 ++++ config/ci/init-solr.sh | 47 +++++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 71fc8b8e..46cf191a 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -48,7 +48,7 @@ jobs: - name: Run unit tests run: | set -euo pipefail - ./config/ci/init-solr.sh + ./config/ci/init-solr.sh ${GITHUB_WORKSPACE} ./config/ci/init.sh python setup.py test python setup.py livetest diff --git a/.gitignore b/.gitignore index 44023b0c..c5f22652 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,8 @@ intermine-docs-1.00.03.zip build MANIFEST .idea + +# solr (CI) + +solr-*.tgz +solr-*/ diff --git a/config/ci/init-solr.sh b/config/ci/init-solr.sh index 1ea5a44b..1aae3488 100755 --- a/config/ci/init-solr.sh +++ b/config/ci/init-solr.sh @@ -1,7 +1,46 @@ +#!/bin/bash + # set up solr for testmine # testmine's setup script populates these empty indexes -wget http://archive.apache.org/dist/lucene/solr/8.6.2/solr-8.6.2.tgz -tar xzf solr-8.6.2.tgz && ./solr-8.6.2/bin/solr start -./solr-8.6.2/bin/solr create -c intermine-search -./solr-8.6.2/bin/solr create -c intermine-autocomplete +set -euo pipefail + +if [ "$#" != "1" ]; then + echo "Usage: $0 " + exit 2 +fi + +WORKSPACE_DIR=$1 + +cd "$WORKSPACE_DIR" + +SOLR_VERSION=8.6.2 +SOLR_PACKAGE=solr-${SOLR_VERSION}.tgz +SOLR_DIR=solr-${SOLR_VERSION} +SOLR=${SOLR_DIR}/bin/solr + +create_solr_core() { + local core_name=$1 + + local status + + status=$(curl -s "http://localhost:8983/solr/admin/cores?action=STATUS&core=${core_name}" | jq --arg core_name "${core_name}" '.status[$core_name]') + + if [ "$status" = "{}" ]; then + ${SOLR} create -c "${core_name}" + else + echo "Solr core ${core_name} already exists" + fi +} + +if [ ! -d $SOLR_DIR ]; then + if [ ! -f $SOLR_PACKAGE ]; then + wget http://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/${SOLR_PACKAGE} + fi + + tar xzf $SOLR_PACKAGE +fi + +${SOLR} restart +create_solr_core intermine-search +create_solr_core intermine-autocomplete From 5472ff2f740f8c1503ed3af28270911285f00382 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 10:43:04 +0000 Subject: [PATCH 12/22] Improvements to run workflow locally --- .github/workflows/im-build.yml | 7 ++--- config/ci/init.sh | 52 +++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 46cf191a..a2ef35b2 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -22,10 +22,7 @@ jobs: --health-timeout 5s --health-retries 5 ports: - - 5432/tcp - env: - TESTMODEL_URL: http://localhost:8080/intermine-demo - TESTMODEL_PATH: intermine-demo + - 5432:5432 steps: - uses: actions/checkout@v4 - name: Set up python 3.10 @@ -49,6 +46,6 @@ jobs: run: | set -euo pipefail ./config/ci/init-solr.sh ${GITHUB_WORKSPACE} - ./config/ci/init.sh + ./config/ci/init.sh ${GITHUB_WORKSPACE} http://localhost:8080/intermine-demo python setup.py test python setup.py livetest diff --git a/config/ci/init.sh b/config/ci/init.sh index 42f1fcd7..79be7d55 100755 --- a/config/ci/init.sh +++ b/config/ci/init.sh @@ -1,44 +1,58 @@ #!/bin/bash -set -e +set -euo pipefail -if [ -z $(which wget) ]; then - # use curl - GET='curl' -else - GET='wget -O -' +if [ "$#" != "2" ]; then + echo "Usage: $0 " + exit 2 fi +WORKSPACE_DIR=$1 +TESTMODEL_URL=$2 + +INTERMINE_DIR=${WORKSPACE_DIR}/server +TESTMINE_DIR=${INTERMINE_DIR}/testmine + +cd "${WORKSPACE_DIR}" + # Pull in the server code. -git clone --single-branch --depth 1 https://github.com/intermine/intermine.git server + +if [ ! -d "${INTERMINE_DIR}" ]; then + git clone --single-branch --depth 1 -b build-workflows https://github.com/ucam-department-of-psychiatry/intermine.git "${INTERMINE_DIR}" +fi export PSQL_USER=test export PSQL_PWD=test +export PSQL_HOST=localhost +export PGPASSWORD=${PGPASSWORD:-postgres} export KEYSTORE=${PWD}/keystore.jks echo "#---> Running unit tests" -sudo -u postgres createuser test -sudo -u postgres psql -c "alter user test with encrypted password 'test';" +sudo -E -u postgres dropuser -h "${PSQL_HOST}" --if-exists test +sudo -E -u postgres createuser -h "${PSQL_HOST}" test +sudo -E -u postgres psql -h "${PSQL_HOST}" -c "alter user test with encrypted password 'test';" # Set up properties -PROPDIR=$HOME/.intermine -TESTMODEL_PROPS=$PROPDIR/testmodel.properties -SED_SCRIPT='s/PSQL_USER/test/' +PROPDIR=${HOME}/.intermine +TESTMODEL_PROPS=${PROPDIR}/testmodel.properties + +mkdir -p "${PROPDIR}" -mkdir -p $PROPDIR +echo "#--- creating ${TESTMODEL_PROPS}" +cp "${INTERMINE_DIR}"/config/testmodel.properties "${TESTMODEL_PROPS}" +sed -i -e "s/PSQL_HOST/${PSQL_HOST}/" "$TESTMODEL_PROPS" +sed -i -e "s/PSQL_USER/${PSQL_USER}/" "$TESTMODEL_PROPS" +sed -i -e "s/PSQL_PWD/${PSQL_PWD}/" "$TESTMODEL_PROPS" -echo "#--- creating $TESTMODEL_PROPS" -cp server/config/testmodel.properties $TESTMODEL_PROPS -sed -i -e $SED_SCRIPT $TESTMODEL_PROPS # We will need a fully operational web-application echo '#---> Building and releasing web application to test against' -(cd server/testmine && ./setup.sh) +(cd "${TESTMINE_DIR}" && ./setup.sh "${INTERMINE_DIR}") # Travis is so slow sleep 90 # let webapp startup # Warm up the keyword search by requesting results, but ignoring the results -$GET "$TESTMODEL_URL/service/search" > /dev/null +wget -O - "${TESTMODEL_URL}/service/search" > /dev/null # Start any list upgrades -$GET "$TESTMODEL_URL/service/lists?token=test-user-token" > /dev/null +wget -O - "${TESTMODEL_URL}/service/lists?token=test-user-token" > /dev/null From 98ed65eefe47bf60676ad0dfda1ed89d172e5244 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 11:01:52 +0000 Subject: [PATCH 13/22] Set up Java in workflow --- .github/workflows/im-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index a2ef35b2..d70f2702 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -25,6 +25,10 @@ jobs: - 5432:5432 steps: - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: '11' - name: Set up python 3.10 uses: actions/setup-python@v5 with: From ec652e45b2a140d1ec0d293b35f0c7866ca2c23d Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 11:03:44 +0000 Subject: [PATCH 14/22] Ignore Intermine directory --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c5f22652..2ebe5c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,9 @@ build MANIFEST .idea -# solr (CI) +# CI + +server/ solr-*.tgz solr-*/ From e054326d2e59153ba35754b5006499da2cbaa21a Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 16:48:35 +0000 Subject: [PATCH 15/22] Fix XML comparison tests --- tests/test_core.py | 243 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 192 insertions(+), 51 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index d04036c4..2df631fc 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,6 +2,7 @@ import unittest import logging import sys +import xml.etree.ElementTree as ET from intermine.model import * from intermine.webservice import * @@ -50,6 +51,23 @@ def do_unpredictable_test(self, test, attempts=0, error=None): raise RuntimeError("Max error count reached - last error: " + str( error)) + def checkQueryHasConstraint(self, query, **kwargs): + xpath_attrs = [] + + for k, v in kwargs.items(): + xpath_attrs.append(f"[@{k}='{v}']") + + xpath_attrs_str = "".join(xpath_attrs) + xpath = f"./constraint{xpath_attrs_str}" + + constraint = query.find(xpath) + + ET.indent(query, space=" ", level=0) + xml = ET.tostring(query, method="xml") + self.assertIsNotNone(constraint, f"Failed to find constraint matching:\n{xpath} in:\n{xml}") + + return constraint + class TestInstantiation(WebserviceTest): # pragma: no cover def testMakeModel(self): @@ -843,10 +861,41 @@ def testXML(self): self.q.add_join("Employee.department", "outer") self.q.add_sort_order("Employee.age") self.q.set_logic("(A and B) or (A and C and D) and (E or F or G)") - expected = 'JohnPaulMary1..1030..35' - self.assertEqual(expected, self.q.to_xml()) + + xml = self.q.to_xml() + query = ET.fromstring(xml) + + self.assertEqual(query.attrib["constraintLogic"], "((A and B) or (A and C and D)) and (E or F or G)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value=10) + self.checkQueryHasConstraint(query, path="Employee.department", op="LOOKUP", value="Sales", extraValue="Wernham-Hogg") + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees.name", op="ONE OF") + + names = [e.text for e in constraint.findall("value")] + self.assertIn("John", names) + self.assertIn("Paul", names) + self.assertIn("Mary", names) + + self.checkQueryHasConstraint(query, path="Employee.department.manager", op="=", loopPath="Employee") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="some list of employees") + + constraint = self.checkQueryHasConstraint(query, path="Employee.age", op="OVERLAPS") + values = [e.text for e in constraint.findall("value")] + self.assertIn("1..10", values) + self.assertIn("30..35", values) + + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees", type="Manager") + # Clones must produce identical XML - self.assertEqual(expected, self.q.clone().to_xml()) + self.assertEqual(xml, self.q.clone().to_xml()) def testSugaryQueryConstruction(self): """Test use of operation coercion which is similar to SQLAlchemy""" @@ -855,8 +904,6 @@ def testSugaryQueryConstruction(self): Employee = model.table("Employee") Manager = model.table("Manager") - expected = 'JohnPaulMary' - # SQL style q = Employee.\ select("name", "age", "department.name").\ @@ -871,7 +918,31 @@ def testSugaryQueryConstruction(self): order_by(Employee.age).\ set_logic("(A and B) or (A and C and D) and (E or F)") - self.assertEqual(expected, q.to_xml()) + xml = q.to_xml() + query = ET.fromstring(xml) + + self.assertEqual(query.attrib["constraintLogic"], "((A and B) or (A and C and D)) and (E or F)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="10") + self.checkQueryHasConstraint(query, path="Employee.department", op="LOOKUP", value="Sales", extraValue="Wernham-Hogg") + + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees.name", op="ONE OF") + names = [e.text for e in constraint.findall("value")] + self.assertIn("John", names) + self.assertIn("Paul", names) + self.assertIn("Mary", names) + + self.checkQueryHasConstraint(query, path="Employee.department.manager", op="=", loopPath="Employee") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="my-list") + self.checkQueryHasConstraint(query, path="Employee.department.employees", type="Manager") # SQLAlchemy style q = self.service.query(Employee).\ @@ -886,34 +957,48 @@ def testSugaryQueryConstruction(self): outerjoin(Employee.department).\ order_by(Employee.age).\ set_logic("(A and B) or (A and C and D) and (E or F)") + xml = q.to_xml() + query = ET.fromstring(xml) - self.assertEqual(expected, q.to_xml()) + self.assertEqual(query.attrib["constraintLogic"], "((A and B) or (A and C and D)) and (E or F)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="10") + self.checkQueryHasConstraint(query, path="Employee.department", op="LOOKUP", value="Sales", extraValue="Wernham-Hogg") + + constraint = self.checkQueryHasConstraint(query, path="Employee.department.employees.name", op="ONE OF") + names = [e.text for e in constraint.findall("value")] + self.assertIn("John", names) + self.assertIn("Paul", names) + self.assertIn("Mary", names) + + self.checkQueryHasConstraint(query, path="Employee.department.manager", op="=", loopPath="Employee") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="my-list") + self.checkQueryHasConstraint(query, path="Employee.department.employees", type="Manager") def testKWCons(self): """Test use of constraints provided in kwargs""" model = self.q.model - expected = '' - q = model.Employee.select("name").where(age=10) + xml = q.to_xml() + query = ET.fromstring(xml) + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name") - self.assertEqual(expected, q.to_xml()) + self.checkQueryHasConstraint(query, path="Employee.age", op="=", value=10) def testLogicConstraintTrees(self): # Actually SQL-Alchemy-esque - expected = """ - - - - - - - - - """ e = self.service.model.Employee CEO = self.service.model.CEO q = self.service.query(e, e.department.name).\ @@ -927,10 +1012,23 @@ def testLogicConstraintTrees(self): outerjoin(e.department).\ order_by(e.age) - expected = re.sub(r'\s+', ' ', expected) - expected = re.sub(r'>\s+<', '><', expected) - expected = expected.strip() - self.assertEqual(expected, q.to_xml()) + xml = q.to_xml() + query = ET.fromstring(xml) + + self.assertEqual(query.attrib["constraintLogic"], "(A and B) or (C and D)") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.age asc") + self.assertEqual(query.attrib["view"], "Employee.age Employee.end Employee.fullTime Employee.id Employee.name Employee.department.name") + + join = query.find("join") + self.assertEqual(join.attrib["path"], "Employee.department") + self.assertEqual(join.attrib["style"], "OUTER") + + self.checkQueryHasConstraint(query, path="Employee.name", op="IS NOT NULL") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="10") + self.checkQueryHasConstraint(query, path="Employee", op="IN", value="my-list") + self.checkQueryHasConstraint(query, value="David", op="LOOKUP", path="Employee.department.manager") + self.checkQueryHasConstraint(query, path="Employee.department.manager", type="CEO") class TestTemplate(TestQuery): # pragma: no cover @@ -993,34 +1091,77 @@ def testURLs(self): t.add_constraint("Employee.name", '=', "Fred") t.add_constraint("Employee.age", ">", 25) - expectedQ = ('/QUERY-PATH', { - 'query': - '', - 'start': 0 - }, 'object', ['Employee.name', 'Employee.age', 'Employee.id'], - self.model.get_class("Employee")) - self.assertEqual(expectedQ, q.results()) - self.assertEqual(list(expectedQ), q.get_results_list()) + results = q.results() + + self.assertEqual(results[0], "/QUERY-PATH") + query_xml = results[1]["query"] + + query = ET.fromstring(query_xml) + + self.assertEqual(query.attrib["constraintLogic"], "A and B") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.id") + + self.checkQueryHasConstraint(query, path="Employee.name", op="=", value="Fred") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="25") + + self.assertEqual(results[1]["start"], 0) + self.assertEqual(results[2], "object") + self.assertIn("Employee.name", results[3]) + self.assertIn("Employee.age", results[3]) + self.assertIn("Employee.id", results[3]) + self.assertEqual(results[4], self.model.get_class("Employee")) + + self.assertEqual(list(results), q.get_results_list()) + + rows = q.rows() + self.assertEqual(rows[0], "/QUERY-PATH") + query_xml = rows[1]["query"] + + query = ET.fromstring(query_xml) + + self.assertEqual(query.attrib["constraintLogic"], "A and B") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.id") + + self.checkQueryHasConstraint(query, path="Employee.name", op="=", value="Fred") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="25") + self.assertEqual(rows[1]["start"], 0) + self.assertEqual(rows[2], "rr") + self.assertIn("Employee.name", rows[3]) + self.assertIn("Employee.age", rows[3]) + self.assertIn("Employee.id", rows[3]) + self.assertEqual(rows[4], self.model.get_class("Employee")) + + self.assertEqual(list(rows), q.get_row_list()) + + results = q.results(start=10, size=200) + self.assertEqual(results[0], "/QUERY-PATH") + + query_xml = results[1]["query"] + query = ET.fromstring(query_xml) + + self.assertEqual(query.attrib["constraintLogic"], "A and B") + self.assertEqual(query.attrib["model"], "testmodel") + self.assertEqual(query.attrib["sortOrder"], "Employee.name asc") + self.assertEqual(query.attrib["view"], "Employee.name Employee.age Employee.id") + + self.checkQueryHasConstraint(query, path="Employee.name", op="=", value="Fred") + self.checkQueryHasConstraint(query, path="Employee.age", op=">", value="25") + + self.assertEqual(results[1]["start"], 10) + self.assertEqual(results[1]["size"], 200) + self.assertEqual(results[2], "object") + self.assertIn("Employee.name", results[3]) + self.assertIn("Employee.age", results[3]) + self.assertIn("Employee.id", results[3]) + self.assertEqual(results[4], self.model.get_class("Employee")) - expectedQ = ('/QUERY-PATH', { - 'query': - '', - 'start': 0 - }, 'rr', ['Employee.name', 'Employee.age', 'Employee.id'], - self.model.get_class("Employee")) - self.assertEqual(expectedQ, q.rows()) - self.assertEqual(list(expectedQ), q.get_row_list()) - expectedQ = ('/QUERY-PATH', { - 'query': - '', - 'start': 10, - 'size': 200 - }, 'object', ['Employee.name', 'Employee.age', 'Employee.id'], - self.model.get_class("Employee")) - self.assertEqual(expectedQ, q.results(start=10, size=200)) self.assertEqual( - list(expectedQ), q.get_results_list( + list(results), q.get_results_list( start=10, size=200)) expected1 = ('/TEMPLATE-PATH', { From 72f564334da68242c6b1f6f5da18e26ccfd69e58 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 17:20:34 +0000 Subject: [PATCH 16/22] Drop test databases in CI workflow --- config/ci/init.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/ci/init.sh b/config/ci/init.sh index 79be7d55..3a23ac17 100755 --- a/config/ci/init.sh +++ b/config/ci/init.sh @@ -29,6 +29,9 @@ export KEYSTORE=${PWD}/keystore.jks echo "#---> Running unit tests" +sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists intermine-demo +sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists userprofile-demo + sudo -E -u postgres dropuser -h "${PSQL_HOST}" --if-exists test sudo -E -u postgres createuser -h "${PSQL_HOST}" test sudo -E -u postgres psql -h "${PSQL_HOST}" -c "alter user test with encrypted password 'test';" From 3bfe4fc0e9676383304b3ae6703d4549ef842851 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 17 Nov 2025 17:21:02 +0000 Subject: [PATCH 17/22] Replace removed base64.encodestring() --- intermine/results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intermine/results.py b/intermine/results.py index 7a49ffc2..3f8db79c 100644 --- a/intermine/results.py +++ b/intermine/results.py @@ -603,7 +603,7 @@ def __init__(self, credentials=None, token=None): self.token = token if credentials and len(credentials) == 2: encoded = '{0}:{1}'.format(*credentials).encode('utf8') - base64string = 'Basic {0}'.format(base64.encodestring(encoded)[:-1].decode('ascii')) + base64string = 'Basic {0}'.format(base64.encodebytes(encoded)[:-1].decode('ascii')) self.auth_header = base64string self.using_authentication = True elif self.token is not None: From 0daca8946a7b2183018953f9359193d6bd4bdd02 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 18 Nov 2025 13:32:41 +0000 Subject: [PATCH 18/22] Use Postgres 14 in workflow as 13 now EOL --- .github/workflows/im-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index d70f2702..4c3027f1 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -12,7 +12,7 @@ jobs: services: # Label used to access the service container postgres: - image: postgres:13 + image: postgres:14 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres From 84adc9d224d9a349581e962b9cd5fb64487eb04a Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 18 Nov 2025 13:53:18 +0000 Subject: [PATCH 19/22] Possible fix for GitHub workflow failure when fetching Java Error: Unexpected HTTP status code '500' when retrieving versions from 'https://static.azul.com/zulu/bin/'. error code: 500 --- .github/workflows/im-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/im-build.yml b/.github/workflows/im-build.yml index 4c3027f1..d4ffbc48 100644 --- a/.github/workflows/im-build.yml +++ b/.github/workflows/im-build.yml @@ -26,8 +26,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: + distribution: 'temurin' java-version: '11' - name: Set up python 3.10 uses: actions/setup-python@v5 From 377d7734a27ab5bc6e308e81b7bb060cefccec53 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 18 Nov 2025 13:55:35 +0000 Subject: [PATCH 20/22] Bump urllib3 to fix CVE-2025-50181 and CVE-2025-50182 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9815f6a4..fc1fb4ff 100755 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ pytz==2024.1 requests==2.32.3 simplejson==3.19.3 six==1.16.0 -urllib3==2.2.2 +urllib3==2.5.0 From 1f18e95abbea2e2f314e4078264b2ac928f039d9 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 18 Nov 2025 13:56:21 +0000 Subject: [PATCH 21/22] Bump requests to fix CVE-2024-47081 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fc1fb4ff..bda7ec4b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ pandas==2.2.2 pyparsing==3.1.4 python-dateutil==2.9.0 pytz==2024.1 -requests==2.32.3 +requests==2.32.4 simplejson==3.19.3 six==1.16.0 urllib3==2.5.0 From df146328dbb3175dc4598d13158e64129d1dc2d6 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Fri, 21 Nov 2025 18:17:53 +0000 Subject: [PATCH 22/22] Use dev branch in CI for now --- config/ci/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/ci/init.sh b/config/ci/init.sh index 3a23ac17..28835f93 100755 --- a/config/ci/init.sh +++ b/config/ci/init.sh @@ -18,7 +18,7 @@ cd "${WORKSPACE_DIR}" # Pull in the server code. if [ ! -d "${INTERMINE_DIR}" ]; then - git clone --single-branch --depth 1 -b build-workflows https://github.com/ucam-department-of-psychiatry/intermine.git "${INTERMINE_DIR}" + git clone --single-branch --depth 1 -b dev https://github.com/ucam-department-of-psychiatry/intermine.git "${INTERMINE_DIR}" fi export PSQL_USER=test