Skip to content

Reduce Schema.org jsonld for datasets with many files #12118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
qqmyers wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

Reduce Schema.org jsonld for datasets with many files #12118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 60 additions & 20 deletions src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2129,30 +2129,70 @@ public String getJsonLd() {
}

List<FileMetadata> fileMetadatasSorted = getFileMetadatasSorted();

if (fileMetadatasSorted != null && !fileMetadatasSorted.isEmpty()) {
JsonArrayBuilder fileArray = Json.createArrayBuilder();
String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic();
for (FileMetadata fileMetadata : fileMetadatasSorted) {
JsonObjectBuilder fileObject = NullSafeJsonBuilder.jsonObjectBuilder();
String filePidUrlAsString = null;
GlobalId gid = fileMetadata.getDataFile().getGlobalId();
filePidUrlAsString = gid != null ? gid.asURL() : null;
fileObject.add("@type", "DataDownload");
fileObject.add("name", fileMetadata.getLabel());
fileObject.add("encodingFormat", fileMetadata.getDataFile().getContentType());
fileObject.add("contentSize", fileMetadata.getDataFile().getFilesize());
fileObject.add("description", MarkupChecker.stripAllTags(fileMetadata.getDescription()));
fileObject.add("@id", filePidUrlAsString);
fileObject.add("identifier", filePidUrlAsString);
boolean hideFilesBoolean = JvmSettings.HIDE_SCHEMA_DOT_ORG_DOWNLOAD_URLS.lookupOptional(Boolean.class).orElse(false);
if (!hideFilesBoolean) {
String nullDownloadType = null;
fileObject.add("contentUrl", dataverseSiteUrl + FileUtil.getFileDownloadUrlPath(nullDownloadType, fileMetadata.getDataFile().getId(), false, fileMetadata.getId()));
Integer maxFilesForDownloadEntries = JvmSettings.EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES.lookupOptional(Integer.class).orElse(Integer.MAX_VALUE);
if (fileMetadatasSorted.size() <= maxFilesForDownloadEntries) {

JsonArrayBuilder fileArray = Json.createArrayBuilder();
String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic();
for (FileMetadata fileMetadata : fileMetadatasSorted) {
JsonObjectBuilder fileObject = NullSafeJsonBuilder.jsonObjectBuilder();
String filePidUrlAsString = null;
GlobalId gid = fileMetadata.getDataFile().getGlobalId();
filePidUrlAsString = gid != null ? gid.asURL() : null;
fileObject.add("@type", "DataDownload");
fileObject.add("name", fileMetadata.getLabel());
fileObject.add("encodingFormat", fileMetadata.getDataFile().getContentType());
fileObject.add("contentSize", fileMetadata.getDataFile().getFilesize());
fileObject.add("description", MarkupChecker.stripAllTags(fileMetadata.getDescription()));
fileObject.add("@id", filePidUrlAsString);
fileObject.add("identifier", filePidUrlAsString);
boolean hideFilesBoolean = JvmSettings.HIDE_SCHEMA_DOT_ORG_DOWNLOAD_URLS.lookupOptional(Boolean.class).orElse(false);
if (!hideFilesBoolean) {
String nullDownloadType = null;
fileObject.add("contentUrl", dataverseSiteUrl + FileUtil.getFileDownloadUrlPath(nullDownloadType, fileMetadata.getDataFile().getId(), false, fileMetadata.getId()));
}
fileArray.add(fileObject);
}
job.add("distribution", fileArray);
} else {
// If we have too many files, we don't include individual distribution entries
// but we can still provide a search action for file downloads
StringBuilder valuePattern = new StringBuilder();
boolean first = true;

// Build the pattern of all file IDs joined with OR operator
for (FileMetadata fileMetadata : fileMetadatas) {
if (!first) {
valuePattern.append("|");
} else {
first = false;
}
valuePattern.append(fileMetadata.getDataFile().getId());
}
fileArray.add(fileObject);

// Create the potentialAction object
JsonObjectBuilder potentialAction = Json.createObjectBuilder()
.add("@type", "SearchAction")
.add("target", Json.createObjectBuilder()
.add("@type", "EntryPoint")
.add("contentType", Json.createArrayBuilder().add("*/*"))
.add("urlTemplate", SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/{fileId}")
.add("description", "Download each file from the dataset based on file id")
.add("httpMethod", Json.createArrayBuilder().add("GET")))
.add("query-input", Json.createArrayBuilder()
.add(Json.createObjectBuilder()
.add("@type", "PropertyValueSpecification")
.add("valueName", "fileId")
.add("description", "Id of the desired file")
.add("valueRequired", true)
.add("valuePattern", "(" + valuePattern.toString() + ")")));

job.add("potentialAction", potentialAction);
}
job.add("distribution", fileArray);
}

jsonLd = job.build().toString();

//Most fields above should be stripped/sanitized but, since this is output in the dataset page as header metadata, do a final sanitize step to make sure
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,12 @@ public enum JvmSettings {
SCOPE_COARNOTIFY(SCOPE_LINKEDDATANOTIFICATION, "coar-notify"),
SCOPE_COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT(SCOPE_COARNOTIFY, "relationship-announcement"),
COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERUSERS_ONLY(SCOPE_COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT, "notify-superusers-only"),

//EXPORTS
SCOPE_EXPORTS(PREFIX, "exports"),
SCOPE_EXPORTS_SCHEMA_DOT_ORG(SCOPE_EXPORTS, "schema-dot-org"),
EXPORTS_SCHEMA_DOT_ORG_MAX_FILES_FOR_DOWNLOAD_ENTRIES(SCOPE_EXPORTS_SCHEMA_DOT_ORG, "max-files-for-download-entries"),

;

private static final String SCOPE_SEPARATOR = ".";
Expand Down