Docs: Restrict Payara http-listener-1 to localhost when proxied

.github/workflows/deploy_beta_testing.yml

@@ -50,7 +50,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Download war artifact
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           name: built-app
           path: ./

.github/workflows/maven_unit_test.yml

@@ -112,7 +112,7 @@ jobs:
                   cache: maven
 
             # Get the build output from the unit test job
-            - uses: actions/download-artifact@v4
+            - uses: actions/download-artifact@v5
               with:
                   name: java-artifacts
             - run: |
@@ -145,7 +145,7 @@ jobs:
                 cache: maven
 
           # Get the build output from the integration test job
-          - uses: actions/download-artifact@v4
+          - uses: actions/download-artifact@v5
             with:
                 name: java-reportdir
           - run: tar -xvf java-reportdir.tar

conf/keycloak/Dockerfile

@@ -14,10 +14,10 @@ RUN mvn clean package
 # ------------------------------------------
 # Stage 2: Build Keycloak Image
 # ------------------------------------------
-FROM quay.io/keycloak/keycloak:26.1.4
+FROM quay.io/keycloak/keycloak:26.3.2
 
 # Add the Oracle JDBC jars
-ARG ORACLE_JDBC_VERSION=23.7.0.25.01
+ARG ORACLE_JDBC_VERSION=23.8.0.25.04
 ADD --chown=keycloak:keycloak https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc11/${ORACLE_JDBC_VERSION}/ojdbc11-${ORACLE_JDBC_VERSION}.jar /opt/keycloak/providers/ojdbc11.jar
 ADD --chown=keycloak:keycloak https://repo1.maven.org/maven2/com/oracle/database/nls/orai18n/${ORACLE_JDBC_VERSION}/orai18n-${ORACLE_JDBC_VERSION}.jar /opt/keycloak/providers/orai18n.jar
 

conf/keycloak/builtin-users-spi/pom.xml

@@ -100,7 +100,7 @@
     </build>
 
     <properties>
-        <keycloak.version>26.1.4</keycloak.version>
+        <keycloak.version>26.3.2</keycloak.version>
         <java.version>17</java.version>
         <jakarta.persistence.version>3.2.0</jakarta.persistence.version>
         <mindrot.jbcrypt.version>0.4</mindrot.jbcrypt.version>

conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java

@@ -27,11 +27,16 @@ public DataverseUser getUserById(String id) {
 
         DataverseBuiltinUser builtinUser = em.find(DataverseBuiltinUser.class, persistenceId);
         if (builtinUser == null) {
-            logger.debugf("User not found for external ID: %s", persistenceId);
+            logger.debugf("Builtin user not found for external ID: %s", persistenceId);
             return null;
         }
 
-        DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(builtinUser.getUsername());
+        String username = builtinUser.getUsername();
+        DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(username);
+        if (authenticatedUser == null) {
+            logger.debugf("Authenticated user not found by username: %s", username);
+            return null;
+        }
 
         return new DataverseUser(authenticatedUser, builtinUser);
     }
@@ -43,11 +48,15 @@ public DataverseUser getUserByUsername(String username) {
                 .getResultList();
 
         if (users.isEmpty()) {
-            logger.debugf("User not found by username: %s", username);
+            logger.debugf("Builtin user not found by username: %s", username);
             return null;
         }
 
         DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(username);
+        if (authenticatedUser == null) {
+            logger.debugf("Authenticated user not found by username: %s", username);
+            return null;
+        }
 
         return new DataverseUser(authenticatedUser, users.get(0));
     }
@@ -59,7 +68,7 @@ public DataverseUser getUserByEmail(String email) {
                 .getResultList();
 
         if (authUsers.isEmpty()) {
-            logger.debugf("User not found by email: %s", email);
+            logger.debugf("Authenticated user not found by email: %s", email);
             return null;
         }
 
@@ -68,6 +77,11 @@ public DataverseUser getUserByEmail(String email) {
                 .setParameter("username", username)
                 .getResultList();
 
+        if (builtinUsers.isEmpty()) {
+            logger.debugf("Builtin user not found by username: %s", username);
+            return null;
+        }
+
         return new DataverseUser(authUsers.get(0), builtinUsers.get(0));
     }
 

conf/keycloak/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.9"
 services:
 
   keycloak:
-    image: 'quay.io/keycloak/keycloak:26.1.4'
+    image: 'quay.io/keycloak/keycloak:26.3.2'
     command:
       - "start-dev"
       - "--import-realm"

conf/keycloak/run-keycloak.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-DOCKER_IMAGE="quay.io/keycloak/keycloak:26.1.4"
+DOCKER_IMAGE="quay.io/keycloak/keycloak:26.3.2"
 KEYCLOAK_USER="kcadmin"
 KEYCLOAK_PASSWORD="kcpassword"
 KEYCLOAK_PORT=8090

conf/solr/schema.xml

@@ -242,6 +242,7 @@
 
     <field name="license" type="string" stored="true" indexed="true" multiValued="false"/>
     <field name="fileCount" type="plong" stored="true" indexed="true" multiValued="false"/>
+    <field name="datasetCount" type="plong" stored="true" indexed="true" multiValued="false"/>
 
     <!--
         METADATA SCHEMA FIELDS

conf/solr/update-fields.sh

@@ -68,10 +68,10 @@ while getopts ":hp" opt; do
 done
 
 # Check for ed and bc being present
-exists ed || error "Please ensure ed, bc, sed + awk are installed"
-exists bc || error "Please ensure ed, bc, sed + awk are installed"
-exists awk || error "Please ensure ed, bc, sed + awk are installed"
-exists sed || error "Please ensure ed, bc, sed + awk are installed"
+exists ed || error "Please ensure ed, bc, sed + awk are installed (ed is missing)"
+exists bc || error "Please ensure ed, bc, sed + awk are installed (bc is missing)"
+exists awk || error "Please ensure ed, bc, sed + awk are installed (awk is missing)"
+exists sed || error "Please ensure ed, bc, sed + awk are installed (sed is missing)"
 
 # remove all the parsed options
 shift $((OPTIND-1))

doc/release-notes/10190-dataset-count.md

@@ -0,0 +1,2 @@
+The search index now includes datasetCount for each collection, counting published, linked, and harvested datasets.
+Collections can be filtered using datasetCount (e.g., `datasetCount:[1000 TO *]`), and the value is returned in Dataverse search results via the Search API.

doc/release-notes/11243-editmetadata-api-extension.md

@@ -0,0 +1,7 @@
+### Edit Dataset Metadata API extension
+
+- This endpoint now allows removing fields (by sending empty values), as long as they are not required by the dataset.
+- New ``sourceLastUpdateTime`` optional query parameter, which prevents inconsistencies by managing updates that
+  may occur from other users while a dataset is being edited.
+
+NOTE: This release note was updated to conform to the refactoring of the validation as part of issue #11392

doc/release-notes/11392-edit-file-metadata-empty-values.md

@@ -0,0 +1,7 @@
+### Edit File Metadata empty values should clear data
+
+Previously the API POST /files/{id}/metadata would ignore fields with empty values. Now the API updates the fields with the empty values essentially clearing the data. Missing fields will still be ignored.
+
+An optional query parameter (sourceLastUpdateTime) was added to ensure the metadata update doesn't overwrite stale data.
+
+See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-metadata), #11392, and #11359.

doc/release-notes/11448-api-endpoint-for-analytics-html.md

@@ -0,0 +1,5 @@
+### Feature Request: API endpoint for analytics.html
+
+New API to get the analytics.html from settings for SPA (Also can be used to get homePage, header, footer, style, and logo)
+
+See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/installation/config.html#web-analytics-code), #11448.

doc/release-notes/11492-list-dataset-links.md

@@ -0,0 +1 @@
+The [API for listing the collections a dataset has been linked to](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#list-collections-that-are-linked-from-a-dataset) (`api/datasets/$linked-dataset-id/links`) is no longer restricted to superusers. For unpublished datasets, users need the "View Unpublished Dataset" permission to access the API. Unpublished collections in the list require the "View Unpublished Dataverse" permission; otherwise, they are hidden.

doc/release-notes/11534-link-permissions.md

@@ -0,0 +1,3 @@
+Linking or unlinking a dataset or dataverse now requires the new "Link Dataset/Dataverse" permission.
+Previously, this action was covered by the "Publish Dataset/Dataverse" permission.
+Linking and publishing permissions can now be granted separately, allowing for more fine-grained access control.

doc/release-notes/11558-show-collections.md

@@ -0,0 +1,12 @@
+The Search API now supports a `show_collections` parameter for dataset results.
+When the parameter is set, each result includes a `collections` array showing the dataset’s parent and linked collections. Each entry includes `id`, `name`, and `alias`, for example:
+
+```json
+"collections": [
+  {
+    "id": 11,
+    "name": "My cool collection",
+    "alias": "dvcb50a190"
+  }
+]
+```

doc/release-notes/11562-templates-api.md

@@ -0,0 +1,4 @@
+New endpoints have been implemented in the Dataverses API for the management of dataverse templates:
+
+- POST `/dataverses/{id}/templates`: Creates a template for a given Dataverse collection ``id``.
+- GET `/dataverses/{id}/templates`: Lists the templates for a given Dataverse collection ``id``.

doc/release-notes/11605-existing-shib-external-users-auth.md

@@ -0,0 +1 @@
+Implemented a new feature flag ``dataverse.feature.api-bearer-auth-use-shib-user-on-id-match``, which supports the use of the new Dataverse client in instances that have historically allowed login via Shibboleth. Specifically, with this flag enabled, when an OIDC bridge is configured to allow OIDC login with validation by the bridged Shibboleth providers, users with existing Shibboleth-based accounts in Dataverse can log in to those accounts, thereby maintaining access to their existing content and retaining their roles. (For security reasons, Dataverse's current support for direct login via Shibboleth cannot be used in browser-based clients.)

doc/release-notes/11614-include-isAdvancedSeachField-property.md

@@ -0,0 +1,3 @@
+The API endpoints `api/{dataverse-alias}/metadatablocks` and `/api/metadatablocks/{block_id}` have been extended to include the following field:
+
+- `isAdvancedSearchFieldType`: Whether the field can be used in advanced search or not.

doc/release-notes/11629-CSLFix.md

@@ -0,0 +1 @@
+The styled citations available through the "View Styled Citations" menu were including extra characters, e.g. 'doi:' in the URL form of the PIDs in the citation. This is now fixed.

doc/release-notes/11632-commons-lang3-update.md

@@ -0,0 +1,4 @@
+Due to changes in how the commons-lang3 library handles a non-ascii chararacter, two keys in the citation.properties and citation.tsv files have changed to include i instead of ɨ. Translations will need to address this.
+
+controlledvocabulary.language.magɨ_(madang_province) => controlledvocabulary.language.magi_(madang_province)
+controlledvocabulary.language.magɨyi =>  controlledvocabulary.language.magiyi

doc/release-notes/11633-list-dataverse-links-api-change.md

@@ -0,0 +1 @@
+The [API for listing the collections a dataverse has been linked to](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#list-dataverse-collection-links) (`api/dataverses/$dataverse-alias/links`) has been refactored to return a new Json format. This is a breaking API.

doc/release-notes/11634-get-dataset-file-available-categories-api.md

@@ -0,0 +1,4 @@
+### Get Dataset File Available Categories API
+
+- This  new endpoint allows the user to get all of the available file categories for a dataset, both built-in and custom.
+

doc/release-notes/11648-notifications-api-extension.md

@@ -0,0 +1,7 @@
+# getAllNotificationsForUser API extension
+
+- Extended endpoint getAllNotificationsForUser(``/notifications/all``), which now supports an optional query parameter ``inAppNotificationFormat`` which, if sent as ``true``, retrieves the fields needed to build the in-app notifications for the Notifications section of the Dataverse UI, omitting fields related to email notifications. See also #11648 and #11696.
+
+# Notifications triggered by API endpoints
+
+The addDataset and addDataverse API endpoints now trigger user notifications upon successful execution. See also #1342 and #11696.

doc/release-notes/11650-unread.md

@@ -0,0 +1,11 @@
+## API Updates 
+
+### Support read/unread status for notifications
+
+The API for managing notifications has been extended. 
+
+- displayAsRead boolean added to "get all"
+- new GET unreadCount API endpoint
+- new PUT markAsRead API endpoint
+
+See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#notifications), #11650, and #11664.

doc/release-notes/11685-CurationStatus_fix.md

@@ -0,0 +1,6 @@
+The updates to support keeping the history of curation status labels added in #11268
+will incorrectly show curation statuses added prior to v6.7 as the current one, regardless of 
+whether newer statuses exist. This PR corrects the problem.
+
+(As a work-around for 6.7 admins can add createtime dates (must be prior to when 6.7 was installed) to the curationstatus table
+ for entries that have null createtimes. The code fix in this version properly handles null dates as indicating older/pre v6.7 curationstatuses.)

doc/release-notes/11722-configbaker-bc-missing.md

@@ -0,0 +1 @@
+When following the container demo tutorial, it was not possible to update Solr fields after adding additional metadata blocks. This has been fixed. See #11722 and #11723

doc/release-notes/11724-extend-list-dataverse-collection-links.md

@@ -0,0 +1 @@
+The [API for listing the collections a dataverse has been linked to](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#list-dataverse-collection-links) (`api/dataverses/$dataverse-alias/links`) has been refactored to return a new Json format. This is a breaking API.

doc/release-notes/issue-10523.md

@@ -0,0 +1,6 @@
+
+## Developer Updates
+
+### Writing External Exporters
+
+The getDatasetFileDetails data structure now contains "directoryLabel" (file path). See #10523 and #11618.

doc/sphinx-guides/source/_static/api/dataverse-template.json

@@ -0,0 +1,27 @@
+{
+  "name": "Dataverse template",
+  "isDefault": true,
+  "fields": [
+    {
+      "typeName": "author",
+      "value": [
+        {
+          "authorName": {
+            "typeName": "authorName",
+            "value": "Belicheck, Bill"
+          },
+          "authorAffiliation": {
+            "typeName": "authorIdentifierScheme",
+            "value": "ORCID"
+          }
+        }
+      ]
+    }
+  ],
+  "instructions": [
+    {
+      "instructionField": "author",
+      "instructionText": "The author data"
+    }
+  ]
+}

doc/sphinx-guides/source/admin/dataverses-datasets.rst

@@ -118,7 +118,7 @@ Moves a dataset whose id is passed to a Dataverse collection whose alias is pass
 Link a Dataset
 ^^^^^^^^^^^^^^
 
-Creates a link between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). ::
+Creates a link between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). Accessible to users with Link Dataset permission on the Dataverse collection. ::
 
     curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/datasets/$linked-dataset-id/link/$linking-dataverse-alias
 
@@ -155,7 +155,7 @@ It returns a list in the following format (new format as of v6.4):
 Unlink a Dataset
 ^^^^^^^^^^^^^^^^
 
-Removes a link between a dataset and a Dataverse collection. Accessible to users with Publish Dataset permissions. ::
+Removes a link between a dataset and a Dataverse collection. Accessible to users with Link Dataset permission on the Dataverse collection. ::
 
     curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$linked-dataset-id/deleteLink/$linking-dataverse-alias
 

doc/sphinx-guides/source/api/changelog.rst

@@ -7,6 +7,13 @@ This API changelog is experimental and we would love feedback on its usefulness.
     :local:
     :depth: 1
 
+v6.8
+----
+
+- For POST /api/files/{id}/metadata passing an empty string ("description":"")  or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string.
+- For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale.
+- For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API.
+
 v6.7
 ----