From 423097c5399bcd887c8f5f8ca4a18561f5c0f9a5 Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Tue, 4 Aug 2015 10:34:07 +0100 Subject: [PATCH 01/13] Added SafeNet-specific version of ServeContent, and configuration to enable it (org.lockss.contentui.safenet) --- .../BaseEntitlementRegistryClient.java | 66 ++- .../CachingEntitlementRegistryClient.java | 8 +- .../safenet/EntitlementRegistryClient.java | 2 +- .../lockss/servlet/ContentServletManager.java | 18 +- .../lockss/servlet/SafeNetServeContent.java | 236 ++++++++++ src/org/lockss/servlet/ServeContent.java | 77 ++-- src/org/lockss/servlet/ServletUtil.java | 3 + .../TestCachingEntitlementRegistryClient.java | 16 +- .../TestEntitlementRegistryClient.java | 192 +------- .../servlet/TestSafeNetServeContent.java | 411 ++++++++++++++++++ 10 files changed, 776 insertions(+), 253 deletions(-) create mode 100644 src/org/lockss/servlet/SafeNetServeContent.java create mode 100644 test/src/org/lockss/servlet/TestSafeNetServeContent.java diff --git a/src/org/lockss/safenet/BaseEntitlementRegistryClient.java b/src/org/lockss/safenet/BaseEntitlementRegistryClient.java index d2815630ead..66e77256e75 100644 --- a/src/org/lockss/safenet/BaseEntitlementRegistryClient.java +++ b/src/org/lockss/safenet/BaseEntitlementRegistryClient.java @@ -52,29 +52,35 @@ public void setConfig(Configuration config, Configuration oldConfig, Configurati } public boolean isUserEntitled(String issn, String institution, String start, String end) throws IOException { + JsonNode entitlement = this.findMatchingEntitlement(issn, institution, start, end); + return entitlement != null; + } + + private JsonNode findMatchingEntitlement(String issn, String institution, String start, String end) throws IOException { Map parameters = new HashMap(); - parameters.put("api_key", apiKey); parameters.put("identifier_value", issn); parameters.put("institution", institution); parameters.put("start", start); parameters.put("end", end); + parameters.put("validate", "1"); JsonNode entitlements = callEntitlementRegistry("/entitlements", parameters); if (entitlements != null) { for(JsonNode entitlement : entitlements) { JsonNode entitlementInstitution = entitlement.get("institution"); - if (entitlementInstitution != null && entitlementInstitution.asText().equals(institution)) { + log.debug("Checking entitlement " + entitlement.toString()); + if (entitlementInstitution != null && entitlementInstitution.asText().endsWith(institution + "/")) { log.warning("TODO: Verify title and dates"); - return true; + return entitlement; } } // Valid request, but the entitlements don't match the information we passed, which should never happen - throw new IOException("No matching entitlements returned from entitlement registry"); + log.error("Entitlements returned from entitlement registry do not match passed parameters"); } //Valid request, no entitlements found - return false; + return null; } private Date extractDate(String value) throws IOException { @@ -96,46 +102,22 @@ private Date extractDate(JsonNode node, String key) throws IOException { return extractDate(value.asText()); } - public String getPublisher(String issn, String start, String end) throws IOException { - Map parameters = new HashMap(); - parameters.put("identifier", issn); - Date startDate = extractDate(start); - Date endDate = extractDate(end); - JsonNode titles = callEntitlementRegistry("/titles", parameters); - if (titles != null) { - List foundPublishers = new ArrayList(); - for(JsonNode title : titles) { - JsonNode publishers = title.get("publishers"); - for(JsonNode publisher : publishers) { - Date foundStartDate = extractDate(publisher, "start"); - Date foundEndDate = extractDate(publisher, "end"); - - if ( foundStartDate != null && ( startDate == null || foundStartDate.after(startDate) ) ) { - continue; - } - if ( foundEndDate != null && ( endDate == null || foundEndDate.before(endDate) ) ) { - continue; - } - foundPublishers.add(publisher.get("id").asText()); - } - } - if (foundPublishers.size() > 1) { - // Valid request, but there are multiple publishers for the date range, which should never happen - throw new IOException("Multiple matching publishers returned from entitlement registry"); - } - if (foundPublishers.size() == 1) { - return foundPublishers.get(0); - } + public String getPublisher(String issn, String institution, String start, String end) throws IOException { + JsonNode entitlement = this.findMatchingEntitlement(issn, institution, start, end); + if ( entitlement == null ) { + return null; } - // Valid request, no publisher found - return null; + + String url = entitlement.get("publisher").asText(); + String[] parts = url.split("/"); + return parts[parts.length - 1]; } public PublisherWorkflow getPublisherWorkflow(String publisherGuid) throws IOException { Map parameters = new HashMap(); JsonNode publisher = callEntitlementRegistry("/publishers/"+publisherGuid, parameters); if (publisher != null) { - JsonNode foundGuid = publisher.get("id"); + JsonNode foundGuid = publisher.get("guid"); if (foundGuid != null && foundGuid.asText().equals(publisherGuid)) { JsonNode foundWorkflow = publisher.get("workflow"); if(foundWorkflow != null) { @@ -147,6 +129,10 @@ public PublisherWorkflow getPublisherWorkflow(String publisherGuid) throws IOExc throw new IOException("No valid workflow returned from entitlement registry: " + foundWorkflow.asText().toUpperCase()); } } + else { + log.warning("No workflow set for publisher, defaulting to PRIMARY_SAFENET"); + return PublisherWorkflow.PRIMARY_SAFENET; + } } } // Valid request, but no valid workflow information was returned, which should never happen @@ -169,7 +155,7 @@ public String getInstitution(String scope) throws IOException { if (!scope.equals(institution.get("scope").asText())) { throw new IOException("No matching institutions returned from entitlement registry"); } - return institution.get("id").asText(); + return institution.get("guid").asText(); } throw new IOException("No matching institutions returned from entitlement registry"); } @@ -190,6 +176,8 @@ private JsonNode callEntitlementRegistry(String endpoint, List pa String url = builder.toString(); log.debug("Connecting to ER at " + url); connection = openConnection(url); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty("Authorization", "Token " + apiKey); connection.execute(); int responseCode = connection.getResponseCode(); if (responseCode == 200) { diff --git a/src/org/lockss/safenet/CachingEntitlementRegistryClient.java b/src/org/lockss/safenet/CachingEntitlementRegistryClient.java index d63330d6c66..6624cdadf2f 100644 --- a/src/org/lockss/safenet/CachingEntitlementRegistryClient.java +++ b/src/org/lockss/safenet/CachingEntitlementRegistryClient.java @@ -41,11 +41,11 @@ public boolean isUserEntitled(String issn, String institution, String start, Str return (Boolean) result; } - public String getPublisher(String issn, String start, String end) throws IOException { - Object result = this.cache.get("getPublisher", issn, start, end); + public String getPublisher(String issn, String institution, String start, String end) throws IOException { + Object result = this.cache.get("getPublisher", issn, institution, start, end); if(result == null) { - result = this.client.getPublisher(issn, start, end); - this.cache.put("getPublisher", issn, start, end, result); + result = this.client.getPublisher(issn, institution, start, end); + this.cache.put("getPublisher", issn, institution, start, end, result); } return (String) result; } diff --git a/src/org/lockss/safenet/EntitlementRegistryClient.java b/src/org/lockss/safenet/EntitlementRegistryClient.java index b32a029c4a7..4df6a75632e 100644 --- a/src/org/lockss/safenet/EntitlementRegistryClient.java +++ b/src/org/lockss/safenet/EntitlementRegistryClient.java @@ -7,6 +7,6 @@ public interface EntitlementRegistryClient extends LockssManager { boolean isUserEntitled(String issn, String institution, String start, String end) throws IOException; String getInstitution(String scope) throws IOException; - String getPublisher(String issn, String start, String end) throws IOException; + String getPublisher(String issn, String institution, String start, String end) throws IOException; PublisherWorkflow getPublisherWorkflow(String publisherGuid) throws IOException; } diff --git a/src/org/lockss/servlet/ContentServletManager.java b/src/org/lockss/servlet/ContentServletManager.java index c44d0e9629a..a810e0e3112 100644 --- a/src/org/lockss/servlet/ContentServletManager.java +++ b/src/org/lockss/servlet/ContentServletManager.java @@ -112,6 +112,10 @@ protected ManagerInfo getManagerInfo() { public static final String PARAM_CONTENT_ONLY = PREFIX + "contentOnly"; public static final boolean DEFAULT_CONTENT_ONLY = false; + /** Use the SafeNet Content server */ + public static final String PARAM_SAFENET = PREFIX + "safenet"; + public static final boolean DEFAULT_SAFENET = false; + static final String COMPRESSOR_PREFIX = PREFIX + "compressor."; public static final String PARAM_COMPRESSOR_ENABLED = @@ -139,6 +143,12 @@ protected ManagerInfo getManagerInfo() { "Serve Content", ServletDescr.NO_NAV_TABLE); + public static final ServletDescr SERVLET_SERVE_CONTENT_SAFENET = + new ServletDescr("ServeContent", + SafeNetServeContent.class, + "Serve Content", + ServletDescr.NO_NAV_TABLE); + public static final ServletDescr SERVLET_SERVE_CONTENT = new ServletDescr("ServeContent", ServeContent.class, @@ -192,6 +202,10 @@ static String mailtoUrl(String addr) { SERVLET_SERVE_CONTENT_NO_NAV, }; + static final ServletDescr servletDescrsSafeNet[] = { + SERVLET_SERVE_CONTENT_SAFENET, + }; + // All servlets must be listed here (even if not in nav table). // Order of descrs determines order in nav table. static final ServletDescr servletDescrs[] = { @@ -203,7 +217,9 @@ static String mailtoUrl(String addr) { }; public ServletDescr[] getServletDescrs() { - if (CurrentConfig.getBooleanParam(PARAM_CONTENT_ONLY, + if (LockssDaemon.getLockssDaemon().isSafenet()) { + return servletDescrsSafeNet; + } else if (CurrentConfig.getBooleanParam(PARAM_CONTENT_ONLY, DEFAULT_CONTENT_ONLY)) { return servletDescrsNoNav; } else { diff --git a/src/org/lockss/servlet/SafeNetServeContent.java b/src/org/lockss/servlet/SafeNetServeContent.java new file mode 100644 index 00000000000..0ed4fb02033 --- /dev/null +++ b/src/org/lockss/servlet/SafeNetServeContent.java @@ -0,0 +1,236 @@ +/* + * $Id$ + */ + +/* + +Copyright (c) 2000-2016 Board of Trustees of Leland Stanford Jr. University, +all rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of Stanford University shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization from Stanford University. + +*/ + +package org.lockss.servlet; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.List; +import java.util.regex.*; + +import javax.servlet.*; + +import org.apache.commons.collections.*; +import org.lockss.app.LockssDaemon; +import org.lockss.config.*; +import org.lockss.daemon.*; +import org.lockss.exporter.counter.*; +import org.lockss.plugin.*; +import org.lockss.plugin.PluginManager.CuContentReq; +import org.lockss.safenet.EntitlementRegistryClient; +import org.lockss.safenet.PublisherWorkflow; +import org.lockss.util.*; +import org.lockss.util.urlconn.*; +import org.mortbay.html.*; +import org.mortbay.http.*; + +@SuppressWarnings("serial") +public class SafeNetServeContent extends ServeContent { + + private static final Logger log = Logger.getLogger(SafeNetServeContent.class); + + private static final String INSTITUTION_HEADER = "X-SafeNet-Institution"; + + private PublisherWorkflow workflow; + private String institution; + private EntitlementRegistryClient entitlementRegistry; + + // don't hold onto objects after request finished + protected void resetLocals() { + workflow = null; + super.resetLocals(); + } + + public void init(ServletConfig config) throws ServletException { + super.init(config); + LockssDaemon daemon = getLockssDaemon(); + entitlementRegistry = daemon.getEntitlementRegistryClient(); + } + + /** Called by ServletUtil.setConfig() */ + static void setConfig(Configuration config, + Configuration oldConfig, + Configuration.Differences diffs) { + ServeContent.setConfig(config, oldConfig, diffs); + if (diffs.contains(PREFIX)) { + } + } + + protected boolean isNeverProxyForAu(ArchivalUnit au) { + return super.isNeverProxyForAu(au) || workflow == PublisherWorkflow.PRIMARY_SAFENET; + } + + /** + * Handle a request + * @throws IOException + */ + public void lockssHandleRequest() throws IOException { + updateInstitution(); + + super.lockssHandleRequest(); + } + + protected boolean setCachedUrlAndAu() throws IOException { + // Find a CU that the user is entitled to access, and with content + List cachedUrls = pluginMgr.findCachedUrls(url, CuContentReq.HasContent); + if(cachedUrls != null && !cachedUrls.isEmpty()) { + for(CachedUrl cachedUrl: cachedUrls) { + try { + if(isUserEntitled(cachedUrl.getArchivalUnit())) { + cu = cachedUrl; + au = cu.getArchivalUnit(); + if (log.isDebug3()) log.debug3("cu: " + cu + " au: " + au); + break; + } + } + catch (IOException e) { + // We can't communicate with the ER, so we have to assume that we can't give the user access to the content at the moment + log.error("Error communicating with entitlement registry: " + e); + handleEntitlementRegistryErrorUrlRequest(url); + return false; + } + catch (IllegalArgumentException e) { + // We don't have enough information about the AU to determine if the user is entitled, but there's nothing they can do about it + log.error("Error with AU configuration: " + e); + handleMissingUrlRequest(url, PubState.KnownDown); + return false; + } + } + if(cu == null) { + // We found at least one CachedUrl, which means the content is preserved, but the user wasn't entitled to any of them + handleUnauthorisedUrlRequest(url); + return false; + } + } + return true; + } + + + /** + * Handle request for content that belongs to one of our AUs, whether or not + * we have content for that URL. If this request contains a version param, + * serve it from cache with a Memento-Datetime header and no + * link-rewriting. For requests without a version param, rewrite links, + * and serve from publisher if publisher provides it and the daemon options + * allow it; otherwise, try to serve from cache. + * + * @throws IOException for IO errors + */ + protected void handleAuRequest() throws IOException { + try { + if (!isUserEntitled(au)) { + handleUnauthorisedUrlRequest(url); + return; + } + workflow = getPublisherWorkflow(au); + if (workflow == PublisherWorkflow.LIBRARY_NOTIFICATION) { + handleUnauthorisedUrlRequest(url); + return; + } + } + catch (IOException e) { + // We can't communicate with the ER, so we have to assume that we can't give the user access to the content at the moment + log.error("Error communicating with entitlement registry: " + e); + handleEntitlementRegistryErrorUrlRequest(url); + return; + } + catch (IllegalArgumentException e) { + // We don't have enough information about the AU to determine if the user is entitled, but there's nothing they can do about it + log.error("Error with AU configuration: " + e); + handleMissingUrlRequest(url, PubState.KnownDown); + return; + } + + super.handleAuRequest(); + } + + protected LockssUrlConnection doOpenConnection(String url, LockssUrlConnectionPool pool) throws IOException { + return super.openConnection(url, pool); + } + + protected LockssUrlConnection openConnection(String url, LockssUrlConnectionPool pool) throws IOException { + LockssUrlConnection conn = doOpenConnection(url, pool); + return conn; + } + + protected void handleEntitlementRegistryErrorUrlRequest(String missingUrl) + throws IOException { + handleUrlRequestError(missingUrl, PubState.KnownDown, "An error occurred trying to access the requested URL on this LOCKSS box. This may be temporary and you may wish to report this, and try again later. ", HttpResponse.__503_Service_Unavailable, "entitlement registry error"); + } + + protected void handleUnauthorisedUrlRequest(String missingUrl) + throws IOException { + handleUrlRequestError(missingUrl, PubState.KnownDown, "You are not authorised to access the requested URL on this LOCKSS box. ", HttpResponse.__403_Forbidden, "unauthorised"); + } + + + void updateInstitution() throws IOException { + //This is currently called in lockssHandleRequest, it needs to be called from wherever we do the SAML authentication + institutionScope = "ed.ac.uk"; + institution = entitlementRegistry.getInstitution(institutionScope); + } + + boolean isUserEntitled(ArchivalUnit au) throws IOException, IllegalArgumentException { + TdbAu tdbAu = au.getTdbAu(); + String issn = tdbAu.getIssn(); + if(StringUtil.isNullString(issn)) { + throw new IllegalArgumentException("ArchivalUnit has no ISSN"); + } + String start = tdbAu.getStartYear() + "0101"; + String end = tdbAu.getEndYear() + "1231"; + + return entitlementRegistry.isUserEntitled(issn, institution, start, end); + } + + PublisherWorkflow getPublisherWorkflow(ArchivalUnit au) throws IOException, IllegalArgumentException { + TdbAu tdbAu = au.getTdbAu(); + String issn = tdbAu.getIssn(); + if(StringUtil.isNullString(issn)) { + throw new IllegalArgumentException("ArchivalUnit has no ISSN"); + } + String start = tdbAu.getStartYear() + "0101"; + String end = tdbAu.getEndYear() + "1231"; + + String publisher = entitlementRegistry.getPublisher(issn, institution, start, end); + if(StringUtil.isNullString(publisher)) { + throw new IllegalArgumentException("No publisher found"); + } + + return entitlementRegistry.getPublisherWorkflow(publisher); + } + + void logAccess(String url, String msg) { + super.logAccess(url, "UA: \"" + req.getHeader("User-Agent") + "\" " + msg); + } +} + diff --git a/src/org/lockss/servlet/ServeContent.java b/src/org/lockss/servlet/ServeContent.java index 81efb9fd9c0..74e05adced3 100644 --- a/src/org/lockss/servlet/ServeContent.java +++ b/src/org/lockss/servlet/ServeContent.java @@ -303,28 +303,28 @@ public static enum RewriteStyle { private static boolean paramAccessAlertsEnabled = DEFAULT_ACCESS_ALERTS_ENABLED; private static boolean processForms = DEFAULT_PROCESS_FORMS; - private static String candidates404Msg = DEFAULT_404_CANDIDATES_MSG; + protected static String candidates404Msg = DEFAULT_404_CANDIDATES_MSG; private static int loginCheckerBufSize = BaseUrlFetcher.DEFAULT_LOGIN_CHECKER_MARK_LIMIT; - private ArchivalUnit au; + protected ArchivalUnit au; private ArchivalUnit explicitAu; - private String url; + protected String url; private String cuUrl; // CU's url (might differ from incoming url due to // normalizaton) private String baseUrl; // The base URL to use for resolving relative // links when rewriting. If redirected, this is // the URL from which the content was served. - private String versionStr; // non-null iff handling a (possibly-invalid) + protected String versionStr; // non-null iff handling a (possibly-invalid) // Memento request - private CachedUrl cu; - private boolean enabledPluginsOnly; + protected CachedUrl cu; + protected boolean enabledPluginsOnly; private String accessLogInfo; private AccessLogType requestType = AccessLogType.None; - private PluginManager pluginMgr; - private ProxyManager proxyMgr; + protected PluginManager pluginMgr; + protected ProxyManager proxyMgr; private OpenUrlResolver openUrlResolver; // don't hold onto objects after request finished @@ -751,14 +751,11 @@ protected void handleUrlRequest() throws IOException { } } else if (!isMementoRequest()) { - // Find a CU with content if possible. If none, find an AU where - // it would fit so can rewrite content from publisher if necessary. - cu = pluginMgr.findCachedUrl(url, CuContentReq.PreferContent); - if (cu != null) { - cuUrl = cu.getUrl(); - au = cu.getArchivalUnit(); - if (log.isDebug3()) log.debug3("cu: " + cu + " au: " + au); - } + boolean findCachedUrl = setCachedUrlAndAu(); + // Returns false if there is an error that has already been handled + if (!findCachedUrl) { + return; + } } else { /* * This is a Memento request, and the AU param was provided, but we @@ -790,6 +787,18 @@ protected void handleUrlRequest() throws IOException { } } + protected boolean setCachedUrlAndAu() throws IOException { + // Find a CU with content if possible. If none, find an AU where + // it would fit so can rewrite content from publisher if necessary. + cu = pluginMgr.findCachedUrl(url, CuContentReq.PreferContent); + if (cu != null) { + cuUrl = cu.getUrl(); + au = cu.getArchivalUnit(); + if (log.isDebug3()) log.debug3("cu: " + cu + " au: " + au); + } + return true; + } + /** * Given a CachedUrl and a string representation of a version number, returns * that version of the CachedUrl. Has no side effects within this instance. @@ -801,7 +810,7 @@ protected void handleUrlRequest() throws IOException { * @throws VersionNotFoundException if cachedUrl lacks the requested version * @throws RuntimeException */ - private static CachedUrl getHistoricalCu(CachedUrl cachedUrl, String verStr) + protected static CachedUrl getHistoricalCu(CachedUrl cachedUrl, String verStr) throws NumberFormatException, VersionNotFoundException, RuntimeException { CachedUrl result; int version = Integer.parseInt(verStr); @@ -818,7 +827,7 @@ private static CachedUrl getHistoricalCu(CachedUrl cachedUrl, String verStr) return result; } - private static class VersionNotFoundException extends Exception {} + protected static class VersionNotFoundException extends Exception {} /** * Redirect to the current URL. Uses response redirection @@ -1096,7 +1105,7 @@ protected void handleAuRequest() throws IOException { /** * @return true iff the user is requesting a particular version of the content */ - private boolean isMementoRequest() { + protected boolean isMementoRequest() { return !StringUtil.isNullString(versionStr); } @@ -1417,7 +1426,7 @@ protected LockssUrlConnection openLockssUrlConnection(LockssUrlConnectionPool } else { fwdUrl = url; } - LockssUrlConnection conn = UrlUtil.openConnection(fwdUrl, pool); + LockssUrlConnection conn = openConnection(fwdUrl, pool); // check connection header String connectionHdr = req.getHeader(HttpFields.__Connection); @@ -1559,6 +1568,10 @@ LinkRewriterFactory getLinkRewriterFactory(String mimeType) { return null; } + protected LockssUrlConnection openConnection(String url, LockssUrlConnectionPool pool) throws IOException { + return UrlUtil.openConnection(url, pool); + } + protected void handleRewriteInputStream(InputStream original, String mimeType, String charset, @@ -2151,12 +2164,18 @@ protected Collection getCandidateAus( protected void handleMissingUrlRequest(String missingUrl, PubState pstate) throws IOException { + handleUrlRequestError(missingUrl, pstate, "The requested URL is not preserved on this LOCKSS box. ", HttpResponse.__404_Not_Found, "not present"); + } + + protected void handleUrlRequestError(String missingUrl, PubState pstate, String errorMessage, int responseCode, String logMessage) + throws IOException { String missing = missingUrl + ((au != null) ? " in AU: " + au.getName() : ""); Block block = new Block(Block.Center); // display publisher page - block.add("

The requested URL is not preserved on this LOCKSS box. "); + block.add("

"); + block.add(errorMessage); block.add("Select link"); block.add(addFootnote( "Selecting publisher link takes you away from this LOCKSS box.")); @@ -2165,9 +2184,9 @@ protected void handleMissingUrlRequest(String missingUrl, PubState pstate) switch (getMissingFileAction(pstate)) { case Error_404: - resp.sendError(HttpServletResponse.SC_NOT_FOUND, + resp.sendError(responseCode, missing + " is not preserved on this LOCKSS box"); - logAccess("not present, 404"); + logAccess(logMessage + ", " + responseCode); break; case Redirect: case AlwaysRedirect: @@ -2183,24 +2202,24 @@ protected void handleMissingUrlRequest(String missingUrl, PubState pstate) } if (candidateAus != null && !candidateAus.isEmpty()) { displayIndexPage(candidateAus, - HttpResponse.__404_Not_Found, + responseCode, block, candidates404Msg); - logAccess("not present, 404 with index"); + logAccess(logMessage + ", " + responseCode + " with index"); } else { displayIndexPage(Collections.emptyList(), - HttpResponse.__404_Not_Found, + responseCode, block, null); - logAccess("not present, 404"); + logAccess(logMessage + ", " + responseCode); } break; case AuIndex: displayIndexPage(pluginMgr.getAllAus(), - HttpResponse.__404_Not_Found, + responseCode, block, null); - logAccess("not present, 404 with index"); + logAccess(logMessage + ", " + responseCode + " with index"); break; } } diff --git a/src/org/lockss/servlet/ServletUtil.java b/src/org/lockss/servlet/ServletUtil.java index ad1cb771f50..dad0492fbb0 100644 --- a/src/org/lockss/servlet/ServletUtil.java +++ b/src/org/lockss/servlet/ServletUtil.java @@ -373,6 +373,9 @@ protected String getLink() { public static void setConfig(Configuration config, Configuration oldConfig, Configuration.Differences diffs) { + if (diffs.contains(SafeNetServeContent.PREFIX)) { + SafeNetServeContent.setConfig(config, oldConfig, diffs); + } if (diffs.contains(ServeContent.PREFIX)) { ServeContent.setConfig(config, oldConfig, diffs); } diff --git a/test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java b/test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java index 06b7e25674d..34d74c23459 100644 --- a/test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java +++ b/test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java @@ -27,8 +27,8 @@ public void testCaching() throws Exception { Mockito.when(mock.isUserEntitled("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenReturn(true); Mockito.when(mock.isUserEntitled("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenReturn(false); Mockito.when(mock.isUserEntitled("0000-0002", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenThrow(new IOException("Error communicating with entitlement registry. Response was 500 null")); - Mockito.when(mock.getPublisher("0000-0000", "20120101", "20151231")).thenReturn("33333333-0000-0000-0000-000000000000"); - Mockito.when(mock.getPublisher("0000-0001", "20120101", "20151231")).thenReturn("33333333-0000-0000-0000-000000000001"); + Mockito.when(mock.getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(mock.getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenReturn("33333333-0000-0000-0000-000000000001"); Mockito.when(mock.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_SAFENET); Mockito.when(mock.getPublisherWorkflow("33333333-1111-1111-1111-111111111111")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); @@ -41,15 +41,15 @@ public void testCaching() throws Exception { catch(IOException e) { assertEquals("Error communicating with entitlement registry. Response was 500 null", e.getMessage()); } - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0000-0000", "20120101", "20151231")); - assertEquals("33333333-0000-0000-0000-000000000001", client.getPublisher("0000-0001", "20120101", "20151231")); + assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); + assertEquals("33333333-0000-0000-0000-000000000001", client.getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); assertEquals(PublisherWorkflow.PRIMARY_PUBLISHER, client.getPublisherWorkflow("33333333-1111-1111-1111-111111111111")); assertEquals(PublisherWorkflow.PRIMARY_PUBLISHER, client.getPublisherWorkflow("33333333-1111-1111-1111-111111111111")); assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); - assertEquals("33333333-0000-0000-0000-000000000001", client.getPublisher("0000-0001", "20120101", "20151231")); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0000-0000", "20120101", "20151231")); + assertEquals("33333333-0000-0000-0000-000000000001", client.getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); + assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); try { client.isUserEntitled("0000-0002", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); fail("Expected exception not thrown"); @@ -63,8 +63,8 @@ public void testCaching() throws Exception { Mockito.verify(mock, Mockito.times(2)).isUserEntitled("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); Mockito.verify(mock).isUserEntitled("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); Mockito.verify(mock, Mockito.times(2)).isUserEntitled("0000-0002", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); - Mockito.verify(mock).getPublisher("0000-0000", "20120101", "20151231"); - Mockito.verify(mock).getPublisher("0000-0001", "20120101", "20151231"); + Mockito.verify(mock).getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); + Mockito.verify(mock).getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); Mockito.verify(mock).getPublisherWorkflow("33333333-0000-0000-0000-000000000000"); Mockito.verify(mock).getPublisherWorkflow("33333333-1111-1111-1111-111111111111"); } diff --git a/test/src/org/lockss/safenet/TestEntitlementRegistryClient.java b/test/src/org/lockss/safenet/TestEntitlementRegistryClient.java index 52499d0b246..18381931bb0 100644 --- a/test/src/org/lockss/safenet/TestEntitlementRegistryClient.java +++ b/test/src/org/lockss/safenet/TestEntitlementRegistryClient.java @@ -25,9 +25,11 @@ import org.lockss.util.urlconn.LockssUrlConnection; public class TestEntitlementRegistryClient extends LockssTestCase { + private static final String ER_URI = "http://dev-safenet.edina.ac.uk"; private BaseEntitlementRegistryClient client; private Map validEntitlementParams; + private Map validResponseParams; private Map validPublisherParams; public void setUp() throws Exception { @@ -37,21 +39,25 @@ public void setUp() throws Exception { daemon.setEntitlementRegistryClient(client); daemon.setDaemonInited(true); Properties p = new Properties(); - p.setProperty(BaseEntitlementRegistryClient.PARAM_ER_URI, "http://dev-safenet.edina.ac.uk"); + p.setProperty(BaseEntitlementRegistryClient.PARAM_ER_URI, ER_URI); p.setProperty(BaseEntitlementRegistryClient.PARAM_ER_APIKEY, "00000000-0000-0000-0000-000000000000"); ConfigurationUtil.setCurrentConfigFromProps(p); client.initService(daemon); client.startService(); validEntitlementParams = new HashMap(); - validEntitlementParams.put("api_key", "00000000-0000-0000-0000-000000000000"); validEntitlementParams.put("identifier_value", "0123-456X"); validEntitlementParams.put("institution", "11111111-1111-1111-1111-111111111111"); validEntitlementParams.put("start", "20120101"); validEntitlementParams.put("end", "20151231"); + validEntitlementParams.put("validate", "1"); + + validResponseParams = new HashMap(validEntitlementParams); + validResponseParams.put("institution", ER_URI + "/institutions/11111111-1111-1111-1111-111111111111/"); + validResponseParams.put("publisher", ER_URI + "/publishers/33333333-0000-0000-0000-000000000000/"); validPublisherParams = new HashMap(); - validPublisherParams.put("id", "33333333-0000-0000-0000-000000000000"); + validPublisherParams.put("guid", "33333333-0000-0000-0000-000000000000"); validPublisherParams.put("name", "Wiley"); } @@ -73,13 +79,7 @@ public void testEntitlementRegistryInvalidResponse() throws Exception { String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "[]")).when(client).openConnection(url); - try { - client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); - fail("Expected exception not thrown"); - } - catch(IOException e) { - assertEquals("No matching entitlements returned from entitlement registry", e.getMessage()); - } + assertFalse(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); Mockito.verify(client).openConnection(url); } @@ -101,21 +101,13 @@ public void testEntitlementRegistryUnexpectedJson() throws Exception { String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "{\"surprise\": \"object\"}")).when(client).openConnection(url); - try { - client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231"); - fail("Expected exception not thrown"); - } - catch(IOException e) { - assertTrue(e.getMessage().startsWith("No matching entitlements returned from entitlement registry")); - } + assertFalse(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); Mockito.verify(client).openConnection(url); } public void testUserEntitled() throws Exception { - Map responseParams = new HashMap(validEntitlementParams); - responseParams.remove("api_key"); String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); + Mockito.doReturn(connection(url, 200, "[" + mapToJson(validResponseParams) + "]")).when(client).openConnection(url); assertTrue(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); Mockito.verify(client).openConnection(url); @@ -133,7 +125,7 @@ public void testGetInstitution() throws Exception { Map queryParams = new HashMap(); queryParams.put("scope", "ed.ac.uk"); Map institution = new HashMap(); - institution.put("id", "11111111-0000-0000-0000-000000000000"); + institution.put("guid", "11111111-0000-0000-0000-000000000000"); institution.put("name", "University of Edinburgh"); institution.put("scope", "ed.ac.uk"); @@ -187,154 +179,18 @@ public void testGetInstitutionMultipleResults() throws Exception { } public void testGetPublisher() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - Map publisher = new HashMap(); - publisher.put("id", "33333333-0000-0000-0000-000000000000"); - publisher.put("start", null); - publisher.put("end", null); - List> publishers = new ArrayList>(); - publishers.add(publisher); - Map responseParams = new HashMap(); - responseParams.put("publishers", publishers); - - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "20120101", "20151231")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", null, "20151231")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "20120101", null)); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", null, null)); - - Mockito.verify(client, Mockito.times(4)).openConnection(url); - } - - public void testGetPublisherDateLimited() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - Map publisher = new HashMap(); - publisher.put("id", "33333333-0000-0000-0000-000000000000"); - publisher.put("start", "20120101"); - publisher.put("end", "20151231"); - - List> publishers = new ArrayList>(); - publishers.add(publisher); - Map responseParams = new HashMap(); - responseParams.put("publishers", publishers); - - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "20120101", "20151231")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", "20111231", "20151231")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", "20120101", "20160101")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "20120102", "20151230")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", null, "20151231")); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", "20120101", null)); - - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", null, null)); - - Mockito.verify(client, Mockito.times(7)).openConnection(url); - } - - public void testGetPublisherNoResponse() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", "20120101", "20151231")); - } - - private List> getMultiplePublishers() { - Map publisher = new HashMap(); - publisher.put("id", "33333333-0000-0000-0000-000000000000"); - publisher.put("start", "20120101"); - publisher.put("end", "20151231"); - Map publisher2 = new HashMap(); - publisher2.put("id", "33333333-1111-1111-1111-111111111111"); - publisher2.put("start", "20160101"); - publisher2.put("end", null); - List> publishers = new ArrayList>(); - publishers.add(publisher); - publishers.add(publisher2); - return publishers; - } - - public void testGetPublisherMultipleResponses() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - Map responseParams = new HashMap(); - List> publishers = getMultiplePublishers(); - responseParams.put("publishers", publishers); - - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "20150101", "20151231")); - - Mockito.verify(client).openConnection(url); - } - - public void testGetPublisherMultipleResponses2() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - Map responseParams = new HashMap(); - List> publishers = getMultiplePublishers(); - responseParams.put("publishers", publishers); - - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals("33333333-1111-1111-1111-111111111111", client.getPublisher("0123-456X", "20160101", "20161231")); - - Mockito.verify(client).openConnection(url); - } - - public void testGetPublisherMultipleResponsesOutsideRange() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - Map responseParams = new HashMap(); - List> publishers = getMultiplePublishers(); - responseParams.put("publishers", publishers); - - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - assertEquals(null, client.getPublisher("0123-456X", "20150101", "20161231")); + String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + Mockito.doReturn(connection(url, 200, "[" + mapToJson(validResponseParams) + "]")).when(client).openConnection(url); + assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); Mockito.verify(client).openConnection(url); } - public void testGetPublisherMultipleResponsesMultipleMatching() throws Exception { - Map queryParams = new HashMap(); - queryParams.put("identifier", "0123-456X"); - Map responseParams = new HashMap(); - List> publishers = getMultiplePublishers(); - responseParams.put("publishers", publishers); - - publishers.get(1).put("start", null); - String url = url("/titles", BaseEntitlementRegistryClient.mapToPairs(queryParams)); - Mockito.doReturn(connection(url, 200, "[" + mapToJson(responseParams) + "]")).when(client).openConnection(url); - try { - client.getPublisher("0123-456X", "20150101", "20151231"); - fail("Expected exception not thrown"); - } - catch (IOException e) { - assertEquals("Multiple matching publishers returned from entitlement registry", e.getMessage()); - } + public void testGetPublisherNotEntitled() throws Exception { + String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + Mockito.doReturn(connection(url, 204, "")).when(client).openConnection(url); + assertEquals(null, client.getPublisher("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); Mockito.verify(client).openConnection(url); } @@ -353,13 +209,7 @@ public void testGetPublisherWorkflowMissingWorkflow() throws Exception { String url = url("/publishers/33333333-0000-0000-0000-000000000000"); Mockito.doReturn(connection(url, 200, mapToJson(responseParams))).when(client).openConnection(url); - try { - client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000"); - fail("Expected exception not thrown"); - } - catch(IOException e) { - assertTrue(e.getMessage().startsWith("No valid workflow returned from entitlement registry")); - } + assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); Mockito.verify(client).openConnection(url); } diff --git a/test/src/org/lockss/servlet/TestSafeNetServeContent.java b/test/src/org/lockss/servlet/TestSafeNetServeContent.java new file mode 100644 index 00000000000..af9cfe33acd --- /dev/null +++ b/test/src/org/lockss/servlet/TestSafeNetServeContent.java @@ -0,0 +1,411 @@ +/* + * $Id$ + */ + +/* + +Copyright (c) 2000-2014 Board of Trustees of Leland Stanford Jr. University, +all rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of Stanford University shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization from Stanford University. + +*/ + +package org.lockss.servlet; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.ServletException; + +import org.lockss.app.LockssDaemon; +import org.lockss.config.ConfigManager; +import org.lockss.config.Tdb; +import org.lockss.config.TdbAu; +import org.lockss.daemon.TitleConfig; +import org.lockss.plugin.ArchivalUnit; +import org.lockss.plugin.CachedUrl; +import org.lockss.plugin.Plugin; +import org.lockss.plugin.PluginManager; +import org.lockss.safenet.BaseEntitlementRegistryClient; +import org.lockss.safenet.EntitlementRegistryClient; +import org.lockss.safenet.PublisherWorkflow; +import org.lockss.test.ConfigurationUtil; +import org.lockss.test.MockArchivalUnit; +import org.lockss.test.MockCachedUrl; +import org.lockss.test.MockLockssDaemon; +import org.lockss.test.MockLockssUrlConnection; +import org.lockss.test.MockNodeManager; +import org.lockss.test.MockPlugin; +import org.lockss.test.StringInputStream; +import org.lockss.util.urlconn.LockssUrlConnection; +import org.lockss.util.urlconn.LockssUrlConnectionPool; + +import com.meterware.httpunit.WebRequest; +import com.meterware.httpunit.WebResponse; +import com.meterware.httpunit.WebTable; +import com.meterware.httpunit.GetMethodWebRequest; +import com.meterware.servletunit.InvocationContext; +import org.mockito.Mockito; +import org.apache.commons.collections.map.MultiKeyMap; + +public class TestSafeNetServeContent extends LockssServletTestCase { + + private MockPluginManager pluginMgr = null; + private EntitlementRegistryClient entitlementRegistryClient = null; + + public void setUp() throws Exception { + super.setUp(); + String tempDirPath = setUpDiskSpace(); + pluginMgr = new MockPluginManager(theDaemon); + theDaemon.setPluginManager(pluginMgr); + theDaemon.setIdentityManager(new org.lockss.protocol.MockIdentityManager()); + entitlementRegistryClient = Mockito.mock(EntitlementRegistryClient.class); + theDaemon.setEntitlementRegistryClient(entitlementRegistryClient); + theDaemon.getServletManager(); + theDaemon.setDaemonInited(true); + theDaemon.setAusStarted(true); + theDaemon.getRemoteApi().startService(); + + pluginMgr.initService(theDaemon); + pluginMgr.startService(); + + ConfigurationUtil.addFromArgs(ConfigManager.PARAM_PLATFORM_PROJECT, "safenet"); + ConfigurationUtil.addFromArgs(SafeNetServeContent.PARAM_MISSING_FILE_ACTION, "Redirect"); + //ConfigurationUtil.addFromArgs("org.lockss.log.default.level", "debug3"); + } + + private MockArchivalUnit makeAu() throws Exception { + return makeAu(null); + } + + private MockArchivalUnit makeAu(Properties override) throws Exception { + Plugin plugin = new MockPlugin(theDaemon); + Tdb tdb = new Tdb(); + + // Tdb with values for some metadata fields + Properties tdbProps = new Properties(); + tdbProps.setProperty("title", "Air and Space Volume 1"); + tdbProps.setProperty("journalTitle", "Air and Space"); + tdbProps.setProperty("attributes.isbn", "976-1-58562-317-7"); + tdbProps.setProperty("issn", "0740-2783"); + tdbProps.setProperty("eissn", "0740-2783"); + tdbProps.setProperty("attributes.year", "2014"); + tdbProps.setProperty("attributes.publisher", "Wiley"); + tdbProps.setProperty("attributes.provider", "Provider[10.0135/12345678]"); + + tdbProps.setProperty("param.1.key", "base_url"); + tdbProps.setProperty("param.1.value", "http://dev-safenet.edina.ac.uk/test_journal/"); + tdbProps.setProperty("param.2.key", "volume"); + tdbProps.setProperty("param.2.value", "vol1"); + tdbProps.setProperty("plugin", plugin.getClass().toString()); + + if(override != null) { + tdbProps.putAll(override); + } + + TdbAu tdbAu = tdb.addTdbAuFromProperties(tdbProps); + TitleConfig titleConfig = new TitleConfig(tdbAu, plugin); + MockArchivalUnit au = new MockArchivalUnit(plugin, "TestAU"); + au.setStartUrls(Arrays.asList("http://dev-safenet.edina.ac.uk/test_journal/")); + au.setTitleConfig(titleConfig); + return au; + } + + protected void initServletRunner() { + super.initServletRunner(); + sRunner.setServletContextAttribute(ServletManager.CONTEXT_ATTR_SERVLET_MGR, new ContentServletManager()); + sRunner.registerServlet("/SafeNetServeContent", MockSafeNetServeContent.class.getName() ); + sRunner.registerServlet("/test_journal/", RedirectServlet.class.getName()); + } + + public void testIndex() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu(), null); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("content type", "text/html", resp1.getContentType()); + WebTable auTable = resp1.getTableStartingWith("Archival Unit"); + assertNotNull(auTable); + assertEquals(2, auTable.getRowCount()); + assertEquals(3, auTable.getColumnCount()); + assertEquals("MockAU", auTable.getCellAsText(1, 0)); + assertEquals("/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F&auid=TestAU", auTable.getTableCell(1, 1).getLinks()[0].getURLString()); + } + + public void testMissingUrl() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu(), null); + sClient.setExceptionsThrownOnErrorStatus(false); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahRedirected content", resp1.getText()); + } + + public void testMissingUrlExplicitAU() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu(), null); + sClient.setExceptionsThrownOnErrorStatus(false); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F&auid=TestAU" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahRedirected content", resp1.getText()); + } + + public void testCachedUrlPrimaryPublisherResponse() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); + Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); + snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahFetched content", resp1.getText()); + Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); + Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + } + + public void testCachedUrlPrimaryPublisherError() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); + Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + LockssUrlConnection connection = mockConnection(403, "BlahContent refused"); + snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahCached content", resp1.getText()); + Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); + Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + } + + public void testCachedUrlPrimarySafenet() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); + Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_SAFENET); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahCached content", resp1.getText()); + } + + public void testCachedUrlLibraryNotification() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); + Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.LIBRARY_NOTIFICATION); + sClient.setExceptionsThrownOnErrorStatus(false); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertEquals(403, resp1.getResponseCode()); + assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + } + + public void testUnauthorisedUrl() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(false); + sClient.setExceptionsThrownOnErrorStatus(false); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + System.out.println(resp1.getText()); + assertEquals(403, resp1.getResponseCode()); + assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + } + + public void testEntitlementRegistryError() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenThrow(new IOException("Could not contact entitlement registry")); + sClient.setExceptionsThrownOnErrorStatus(false); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertEquals(503, resp1.getResponseCode()); + assertTrue(resp1.getText().contains("

An error occurred trying to access the requested URL on this LOCKSS box. This may be temporary and you may wish to report this, and try again later. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + } + + public void testInvalidArchivalUnit() throws Exception { + initServletRunner(); + Properties props = new Properties(); + props.setProperty("issn", ""); + props.setProperty("eissn", ""); + sClient.setExceptionsThrownOnErrorStatus(false); + pluginMgr.addAu(makeAu(props)); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + + WebResponse resp1 = sClient.getResponse(request); + assertEquals(404, resp1.getResponseCode()); + assertTrue(resp1.getText().contains("

The requested URL is not preserved on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + } + + public void testCachedUrlMultipleAUs() throws Exception { + initServletRunner(); + pluginMgr.addAu(makeAu()); + Properties props = new Properties(); + props.setProperty("issn", "0000-0000"); + props.setProperty("eissn", "0000-0000"); + pluginMgr.addAu(makeAu(props)); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(false); + Mockito.when(entitlementRegistryClient.isUserEntitled("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); + Mockito.when(entitlementRegistryClient.getPublisher("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-00000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); + WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); + snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahFetched content", resp1.getText()); + Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); + Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + } + + private static class MockPluginManager extends PluginManager { + private Map> aus; + private MockLockssDaemon theDaemon; + private MockNodeManager nodeManager; + + public MockPluginManager(MockLockssDaemon theDaemon) { + this.aus = new HashMap>(); + this.theDaemon = theDaemon; + this.nodeManager = new MockNodeManager(); + } + + public void addAu(ArchivalUnit au) { + aus.put(au, new ArrayList(au.getStartUrls())); + theDaemon.setNodeManager(nodeManager, au); + } + + public void addAu(ArchivalUnit au, List urls) { + aus.put(au, urls); + theDaemon.setNodeManager(nodeManager, au); + } + + @Override + public List getAllAus() { + return new ArrayList(aus.keySet()); + } + + @Override + public List findCachedUrls(String url, CuContentReq req) { + return this.findCachedUrls(url); + } + + @Override + public List findCachedUrls(String url) { + List cached = new ArrayList(); + for(ArchivalUnit au : aus.keySet()) { + List urls = aus.get(au); + if(urls != null && urls.contains(url)) { + MockCachedUrl cu = new MockCachedUrl(url, au); + cu.addProperty(CachedUrl.PROPERTY_CONTENT_TYPE, "text/html"); + cu.setContent("BlahCached content"); + cached.add(cu); + } + } + return cached; + } + } + + public static class RedirectServlet extends HttpServlet { + public void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + resp.getWriter().print("BlahRedirected content"); + } + } + + private LockssUrlConnection mockConnection(int responseCode, String response) throws IOException { + LockssUrlConnection connection = Mockito.mock(LockssUrlConnection.class); + Mockito.when(connection.getResponseCode()).thenReturn(responseCode); + Mockito.when(connection.getResponseInputStream()).thenReturn(new StringInputStream(response)); + return connection; + } + + public static class MockSafeNetServeContent extends SafeNetServeContent { + private SafeNetServeContent delegate = Mockito.mock(SafeNetServeContent.class); + + public void expectRequest(String url, LockssUrlConnection connection) throws IOException { + Mockito.when(delegate.doOpenConnection(Mockito.eq(url), Mockito.any(LockssUrlConnectionPool.class))).thenReturn(connection); + } + + protected LockssUrlConnection doOpenConnection(String url, LockssUrlConnectionPool pool) throws IOException { + return delegate.doOpenConnection(url, pool); + } + } +} From a4c02ad24c1d8a7116c6ac56cefb0b6a0c88f47e Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Thu, 8 Jun 2017 14:06:53 +0100 Subject: [PATCH 02/13] Get bibliographic info from content, not TDB --- .../lockss/servlet/SafeNetServeContent.java | 356 +++++++++++++++++- src/org/lockss/servlet/ServeContent.java | 3 + .../servlet/TestSafeNetServeContent.java | 49 +-- 3 files changed, 369 insertions(+), 39 deletions(-) diff --git a/src/org/lockss/servlet/SafeNetServeContent.java b/src/org/lockss/servlet/SafeNetServeContent.java index 0ed4fb02033..ba716da8f98 100644 --- a/src/org/lockss/servlet/SafeNetServeContent.java +++ b/src/org/lockss/servlet/SafeNetServeContent.java @@ -34,17 +34,24 @@ of this software and associated documentation files (the "Software"), to deal import java.io.*; import java.net.*; +import java.sql.*; import java.util.*; import java.util.List; import java.util.regex.*; import javax.servlet.*; +import static org.lockss.db.SqlConstants.*; + import org.apache.commons.collections.*; import org.lockss.app.LockssDaemon; import org.lockss.config.*; import org.lockss.daemon.*; +import org.lockss.daemon.OpenUrlResolver.OpenUrlInfo; +import org.lockss.db.*; +import org.lockss.exporter.biblio.*; import org.lockss.exporter.counter.*; +import org.lockss.extractor.*; import org.lockss.plugin.*; import org.lockss.plugin.PluginManager.CuContentReq; import org.lockss.safenet.EntitlementRegistryClient; @@ -63,11 +70,18 @@ public class SafeNetServeContent extends ServeContent { private PublisherWorkflow workflow; private String institution; + private String issn; + private String start; + private String end; private EntitlementRegistryClient entitlementRegistry; // don't hold onto objects after request finished protected void resetLocals() { workflow = null; + institution = null; + issn = null; + start = null; + end = null; super.resetLocals(); } @@ -109,7 +123,7 @@ protected boolean setCachedUrlAndAu() throws IOException { if(isUserEntitled(cachedUrl.getArchivalUnit())) { cu = cachedUrl; au = cu.getArchivalUnit(); - if (log.isDebug3()) log.debug3("cu: " + cu + " au: " + au); + if (log.isDebug3()) log.debug("cu: " + cu + " au: " + au); break; } } @@ -136,6 +150,11 @@ protected boolean setCachedUrlAndAu() throws IOException { } + protected void handleOpenUrlInfo(OpenUrlInfo info) throws IOException { + setBibInfoFromOpenUrl(info); + super.handleOpenUrlInfo(info); + } + /** * Handle request for content that belongs to one of our AUs, whether or not * we have content for that URL. If this request contains a version param, @@ -201,25 +220,21 @@ void updateInstitution() throws IOException { } boolean isUserEntitled(ArchivalUnit au) throws IOException, IllegalArgumentException { - TdbAu tdbAu = au.getTdbAu(); - String issn = tdbAu.getIssn(); - if(StringUtil.isNullString(issn)) { - throw new IllegalArgumentException("ArchivalUnit has no ISSN"); - } - String start = tdbAu.getStartYear() + "0101"; - String end = tdbAu.getEndYear() + "1231"; + setBibInfoFromMetadataDB(cu); + setBibInfoFromCu(cu, au); + setBibInfoFromTdb(au); + setBibInfoFromArticleFiles(au, cu); + validateBibInfo(); return entitlementRegistry.isUserEntitled(issn, institution, start, end); } PublisherWorkflow getPublisherWorkflow(ArchivalUnit au) throws IOException, IllegalArgumentException { - TdbAu tdbAu = au.getTdbAu(); - String issn = tdbAu.getIssn(); - if(StringUtil.isNullString(issn)) { - throw new IllegalArgumentException("ArchivalUnit has no ISSN"); - } - String start = tdbAu.getStartYear() + "0101"; - String end = tdbAu.getEndYear() + "1231"; + setBibInfoFromMetadataDB(cu); + setBibInfoFromCu(cu, au); + setBibInfoFromTdb(au); + setBibInfoFromArticleFiles(au, cu); + validateBibInfo(); String publisher = entitlementRegistry.getPublisher(issn, institution, start, end); if(StringUtil.isNullString(publisher)) { @@ -229,6 +244,317 @@ PublisherWorkflow getPublisherWorkflow(ArchivalUnit au) throws IOException, Ille return entitlementRegistry.getPublisherWorkflow(publisher); } + private void setBibInfoFromOpenUrl(OpenUrlInfo info) throws IllegalArgumentException { + log.debug2("Setting bib info from OpenURL"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(info == null) { + log.debug2("No OpenUrlInfo"); + return; + } + + BibliographicItem item = info.getBibliographicItem(); + setBibInfoFromBibliographicItem(item); + } + + private void setBibInfoFromBibliographicItem(BibliographicItem item) { + log.debug2("Setting bib info from BibliographicItem"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(item == null) { + log.debug2("No BibliographicItem"); + return; + } + + if(StringUtil.isNullString(issn)) { + issn = item.getIssn(); + log.debug("Setting issn to " + issn); + } + + if(StringUtil.isNullString(start)) { + // Despite being called StartYear, this is actually a full date + start = item.getStartYear(); + log.debug("Setting start to " + start); + } + + if(StringUtil.isNullString(end)) { + end = item.getEndYear(); + log.debug("Setting end to " + end); + } + } + + private void setBibInfoFromMetadata(ArticleMetadata md) { + log.debug2("Setting bib info from TDB"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(md == null) { + log.debug2("No ArticleMetadata"); + return; + } + + BibliographicItemImpl item = new BibliographicItemImpl(); + item.setPrintIssn(md.get("issn")); + item.setEissn(md.get("eissn")); + item.setIssnL(md.get("issnl")); + item.setYear(md.get("date")); + setBibInfoFromBibliographicItem(item); + } + + private void setBibInfoFromArticleFiles(ArchivalUnit au, final CachedUrl cu) throws IllegalArgumentException { + log.debug2("Setting bib info from TDB"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(au == null) { + log.debug2("No ArchivalUnit"); + return; + } + if(cu == null) { + log.debug2("No CachedUrl"); + return; + } + + Plugin plugin = au.getPlugin(); + if(plugin == null) { + log.debug2("No Plugin"); + return; + } + + MetadataTarget target = new MetadataTarget(MetadataTarget.PURPOSE_OPENURL); + Iterator afs = au.getArticleIterator(); + ArticleMetadataExtractor mdExtractor = plugin.getArticleMetadataExtractor(target, au); + + if(afs == null) { + log.debug2("No ArticleIterator"); + return; + } + if(mdExtractor == null) { + log.debug2("No ArticleMetadataExtractor"); + return; + } + + ArticleMetadataExtractor.Emitter emitter = new ArticleMetadataExtractor.Emitter() { + public void emitMetadata(ArticleFiles af2, ArticleMetadata md) { + String mdUrl = md.get("access.url"); + String cuUrl = cu.getUrl(); + log.debug3("Comparing " + mdUrl + " to " + cuUrl); + if(mdUrl.equals(cuUrl)) { + log.debug2("Found matching URL"); + setBibInfoFromMetadata(md); + } + } + }; + + while(afs.hasNext()) { + ArticleFiles af = afs.next(); + try { + mdExtractor.extract(target, af, emitter); + } + catch(IOException e) { + log.error("Error extracting article metadata", e); + } + catch(PluginException e) { + log.error("Error extracting article metadata", e); + } + + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + } + } + + private void setBibInfoFromTdb(ArchivalUnit au) throws IllegalArgumentException { + log.debug2("Setting bib info from TDB"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(au == null) { + log.debug2("No ArchivalUnit"); + return; + } + + TdbAu tdbAu = au.getTdbAu(); + if(tdbAu == null) { + log.debug2("No TdbAu"); + return; + } + + if(StringUtil.isNullString(issn)) { + issn = tdbAu.getIssn(); + log.debug("Setting issn to " + issn); + } + + if(StringUtil.isNullString(start)) { + start = tdbAu.getStartYear(); + if(!StringUtil.isNullString(start)) { + start += "0101"; + } + log.debug("Setting start to " + start); + } + + if(StringUtil.isNullString(end)) { + end = tdbAu.getEndYear(); + if(!StringUtil.isNullString(end)) { + end += "1231"; + } + log.debug("Setting end to " + end); + } + } + + private void setBibInfoFromCu(CachedUrl cu, ArchivalUnit au) { + log.debug2("Setting bib info from CU"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(au == null) { + log.debug2("No ArchivalUnit"); + return; + } + if(cu == null) { + log.debug2("No CachedUrl"); + return; + } + + Plugin plugin = au.getPlugin(); + if(plugin == null) { + log.debug2("No Plugin"); + return; + } + + + MetadataTarget target = new MetadataTarget(MetadataTarget.PURPOSE_OPENURL); + FileMetadataExtractor mdExtractor = plugin.getFileMetadataExtractor(target, cu.getContentType(), au); + if(mdExtractor == null) { + log.debug2("No FileMetadataExtractor"); + return; + } + FileMetadataExtractor.Emitter emitter = new FileMetadataExtractor.Emitter() { + public void emitMetadata(CachedUrl cu, ArticleMetadata md) { + log.debug2("ArticleMetadata found"); + setBibInfoFromMetadata(md); + } + }; + try { + mdExtractor.extract(target, cu, emitter); + } + catch(IOException e) { + log.error("Error extracting CU metadata", e); + } + catch(PluginException e) { + log.error("Error extracting CU metadata", e); + } + } + + private void setBibInfoFromMetadataDB(CachedUrl cu) { + log.debug2("Setting bib info from database"); + if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { + log.debug2("Bib info already set"); + return; + } + + if(cu == null) { + log.debug2("No CachedUrl"); + return; + } + + Connection conn = null; + OpenUrlInfo resolved = null; + try { + LockssDaemon daemon = getLockssDaemon(); + conn = daemon.getDbManager().getConnection(); + + StringBuilder select = new StringBuilder("select distinct "); + StringBuilder from = new StringBuilder(" from "); + StringBuilder where = new StringBuilder(" where "); + ArrayList args = new ArrayList(); + + // return all related values for debugging purposes + select.append("mi1." + DATE_COLUMN); + select.append(",i." + ISSN_COLUMN); + select.append(",i." + ISSN_TYPE_COLUMN); + + from.append(URL_TABLE + " u"); + from.append(", " + MD_ITEM_TABLE + " mi1"); // publication md_item + from.append("," + MD_ITEM_TABLE + " mi2"); // article md_item + from.append("," + ISSN_TABLE + " i"); + + where.append("u." + URL_COLUMN + "= ?"); + args.add(cu.getUrl()); + where.append(" and u." + FEATURE_COLUMN + "='Access'"); + + where.append(" and u." + MD_ITEM_SEQ_COLUMN + "="); + where.append("mi1." + MD_ITEM_SEQ_COLUMN); + + where.append(" and mi1." + PARENT_SEQ_COLUMN + "="); + where.append("mi2." + MD_ITEM_SEQ_COLUMN); + + where.append(" and i." + MD_ITEM_SEQ_COLUMN + "="); + where.append("mi2." + MD_ITEM_SEQ_COLUMN); + + String query = select.toString() + from.toString() + where.toString(); + + PreparedStatement stmt = daemon.getDbManager().prepareStatement(conn, query); + + for (int i = 0; i < args.size(); i++) { + log.debug3("query arg: " + args.get(i)); + stmt.setString(i + 1, args.get(i)); + } + + ResultSet resultSet = daemon.getDbManager().executeQuery(stmt); + + BibliographicItemImpl item = new BibliographicItemImpl(); + while ( resultSet.next() ) { + String year = resultSet.getString(1); + String issn = resultSet.getString(2); + String issnType = resultSet.getString(3); + item.setYear(year); + if(P_ISSN_TYPE.equals(issnType)){ + item.setPrintIssn(issn); + } + else if(E_ISSN_TYPE.equals(issnType)){ + item.setEissn(issn); + } + else if(L_ISSN_TYPE.equals(issnType)){ + item.setIssnL(issn); + } + } + setBibInfoFromBibliographicItem(item); + + } catch (DbException e) { + log.error("Error fetching metadata", e); + } catch (SQLException e) { + log.error("Error fetching metadata", e); + } + } + + private void validateBibInfo() { + if(StringUtil.isNullString(issn)) { + throw new IllegalArgumentException("ArchivalUnit has no ISSN"); + } + if(StringUtil.isNullString(start)) { + throw new IllegalArgumentException("ArchivalUnit has no start year"); + } + if(StringUtil.isNullString(end)) { + throw new IllegalArgumentException("ArchivalUnit has no end year"); + } + } + void logAccess(String url, String msg) { super.logAccess(url, "UA: \"" + req.getHeader("User-Agent") + "\" " + msg); } diff --git a/src/org/lockss/servlet/ServeContent.java b/src/org/lockss/servlet/ServeContent.java index 74e05adced3..60bc81f83f4 100644 --- a/src/org/lockss/servlet/ServeContent.java +++ b/src/org/lockss/servlet/ServeContent.java @@ -421,6 +421,9 @@ protected boolean isInCache() { if (res && explicitAu != null) { pluginMgr.promoteAuInSearchSets(explicitAu); } + if(log.isDebug2()) { + log.debug2("isInCache: " + res + ", cu: " + (cu != null) + ", hasContent: " + (cu != null && cu.hasContent()) + ", explicitAu: " + (explicitAu != null)); + } return res; } diff --git a/test/src/org/lockss/servlet/TestSafeNetServeContent.java b/test/src/org/lockss/servlet/TestSafeNetServeContent.java index af9cfe33acd..96fbcbfff8d 100644 --- a/test/src/org/lockss/servlet/TestSafeNetServeContent.java +++ b/test/src/org/lockss/servlet/TestSafeNetServeContent.java @@ -312,30 +312,31 @@ public void testInvalidArchivalUnit() throws Exception { assertTrue(resp1.getText().contains("

The requested URL is not preserved on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); } - public void testCachedUrlMultipleAUs() throws Exception { - initServletRunner(); - pluginMgr.addAu(makeAu()); - Properties props = new Properties(); - props.setProperty("issn", "0000-0000"); - props.setProperty("eissn", "0000-0000"); - pluginMgr.addAu(makeAu(props)); - Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); - Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(false); - Mockito.when(entitlementRegistryClient.isUserEntitled("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); - Mockito.when(entitlementRegistryClient.getPublisher("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); - Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-00000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); - InvocationContext ic = sClient.newInvocation(request); - MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); - LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); - snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); - - WebResponse resp1 = sClient.getResponse(request); - assertResponseOk(resp1); - assertEquals("BlahFetched content", resp1.getText()); - Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); - Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); - } + // Test no longer valid due to changes to how we extract bibliographic data + // public void testCachedUrlMultipleAUs() throws Exception { + // initServletRunner(); + // pluginMgr.addAu(makeAu()); + // Properties props = new Properties(); + // props.setProperty("issn", "0000-0000"); + // props.setProperty("eissn", "0000-0000"); + // pluginMgr.addAu(makeAu(props)); + // Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + // Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(false); + // Mockito.when(entitlementRegistryClient.isUserEntitled("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); + // Mockito.when(entitlementRegistryClient.getPublisher("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); + // Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-00000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); + // WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + // InvocationContext ic = sClient.newInvocation(request); + // MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + // LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); + // snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + + // WebResponse resp1 = sClient.getResponse(request); + // assertResponseOk(resp1); + // assertEquals("BlahFetched content", resp1.getText()); + // Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); + // Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + // } private static class MockPluginManager extends PluginManager { private Map> aus; From 97670e3a33d32c66899bed33333afaece32dbfec Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Mon, 15 May 2017 15:50:31 +0100 Subject: [PATCH 03/13] Add ability to mock institution scope from parameters --- .../lockss/servlet/SafeNetServeContent.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/org/lockss/servlet/SafeNetServeContent.java b/src/org/lockss/servlet/SafeNetServeContent.java index ba716da8f98..f4755549a6d 100644 --- a/src/org/lockss/servlet/SafeNetServeContent.java +++ b/src/org/lockss/servlet/SafeNetServeContent.java @@ -66,8 +66,15 @@ public class SafeNetServeContent extends ServeContent { private static final Logger log = Logger.getLogger(SafeNetServeContent.class); + // If true, scope can be 'mocked' from the URL parameters. This is for testing purposes, and should never be true in production + public static final String PARAM_EDIAUTH_MOCK_SCOPE = PREFIX + "mockScope"; + private static final String INSTITUTION_HEADER = "X-SafeNet-Institution"; + public static final String INSTITUTION_SCOPE_SESSION_KEY = "scope"; + + private static boolean mockScope = false; + private PublisherWorkflow workflow; private String institution; private String issn; @@ -97,6 +104,7 @@ static void setConfig(Configuration config, Configuration.Differences diffs) { ServeContent.setConfig(config, oldConfig, diffs); if (diffs.contains(PREFIX)) { + mockScope = config.getBoolean(PARAM_EDIAUTH_MOCK_SCOPE, false); } } @@ -109,6 +117,15 @@ protected boolean isNeverProxyForAu(ArchivalUnit au) { * @throws IOException */ public void lockssHandleRequest() throws IOException { + + if ( mockScope ) { + String userInstScope = req.getParameter(INSTITUTION_SCOPE_SESSION_KEY); + if ( ! "".equals(userInstScope) ) { + log.warning("Setting scope from parameters:"+userInstScope+" This should not be done in production"); + this.getSession().setAttribute(INSTITUTION_SCOPE_SESSION_KEY, userInstScope); + } + } + updateInstitution(); super.lockssHandleRequest(); @@ -199,6 +216,7 @@ protected LockssUrlConnection doOpenConnection(String url, LockssUrlConnectionPo protected LockssUrlConnection openConnection(String url, LockssUrlConnectionPool pool) throws IOException { LockssUrlConnection conn = doOpenConnection(url, pool); + conn.addRequestProperty(INSTITUTION_HEADER, (String) getSession().getAttribute(INSTITUTION_SCOPE_SESSION_KEY)); return conn; } From 08e0c631579db86c9de04acd5a1a33d18404cd1c Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Mon, 14 Aug 2017 08:58:40 +0100 Subject: [PATCH 04/13] Remove references to Safenet from code --- src/org/lockss/app/LockssDaemon.java | 34 +++++-- .../CachingEntitlementRegistryClient.java | 26 ++--- .../EntitlementRegistryClient.java | 2 +- .../KeepsafeEntitlementRegistryClient.java} | 14 +-- .../lockss/entitlement/PublisherWorkflow.java | 15 +++ src/org/lockss/safenet/PublisherWorkflow.java | 6 -- .../lockss/servlet/ContentServletManager.java | 17 ++-- ...java => EntitlementCheckServeContent.java} | 21 ++--- src/org/lockss/servlet/ServletUtil.java | 4 +- .../TestCachingEntitlementRegistryClient.java | 23 +++-- ...estKeepsafeEntitlementRegistryClient.java} | 45 +++++---- ... => TestEntitlementCheckServeContent.java} | 94 +++++++++---------- .../src/org/lockss/test/MockLockssDaemon.java | 13 ++- 13 files changed, 174 insertions(+), 140 deletions(-) rename src/org/lockss/{safenet => entitlement}/CachingEntitlementRegistryClient.java (78%) rename src/org/lockss/{safenet => entitlement}/EntitlementRegistryClient.java (93%) rename src/org/lockss/{safenet/BaseEntitlementRegistryClient.java => entitlement/KeepsafeEntitlementRegistryClient.java} (94%) create mode 100644 src/org/lockss/entitlement/PublisherWorkflow.java delete mode 100644 src/org/lockss/safenet/PublisherWorkflow.java rename src/org/lockss/servlet/{SafeNetServeContent.java => EntitlementCheckServeContent.java} (96%) rename test/src/org/lockss/{safenet => entitlement}/TestCachingEntitlementRegistryClient.java (83%) rename test/src/org/lockss/{safenet/TestEntitlementRegistryClient.java => entitlement/TestKeepsafeEntitlementRegistryClient.java} (83%) rename test/src/org/lockss/servlet/{TestSafeNetServeContent.java => TestEntitlementCheckServeContent.java} (79%) diff --git a/src/org/lockss/app/LockssDaemon.java b/src/org/lockss/app/LockssDaemon.java index ed3ae7162f3..59daf1daf97 100644 --- a/src/org/lockss/app/LockssDaemon.java +++ b/src/org/lockss/app/LockssDaemon.java @@ -58,7 +58,7 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.crawler.*; import org.lockss.remote.*; import org.lockss.clockss.*; -import org.lockss.safenet.*; +import org.lockss.entitlement.*; import org.apache.commons.collections.map.LinkedMap; /** @@ -147,7 +147,8 @@ public class LockssDaemon extends LockssApp { public static final String ICP_MANAGER = "IcpManager"; public static final String CRON = "Cron"; public static final String CLOCKSS_PARAMS = "ClockssParams"; - public static final String SAFENET_MANAGER = "SafenetManager"; + public static final String ENTITLEMENT_REGISTRY_CLIENT = "EntitlementRegistryClient"; + public static final String CACHED_ENTITLEMENT_REGISTRY_CLIENT = "CachedEntitlementRegistryClient"; public static final String TRUEZIP_MANAGER = "TrueZipManager"; public static final String DB_MANAGER = "DbManager"; public static final String COUNTER_REPORTS_MANAGER = "CounterReportsManager"; @@ -237,9 +238,13 @@ public class LockssDaemon extends LockssApp { public boolean shouldStart() { return isClockss(); }}, - new ManagerDesc(SAFENET_MANAGER, "org.lockss.safenet.CachingEntitlementRegistryClient") { + new ManagerDesc(ENTITLEMENT_REGISTRY_CLIENT, "org.lockss.entitlement.KeepsafeEntitlementRegistryClient") { public boolean shouldStart() { - return isSafenet(); + return isKeepsafe(); + }}, + new ManagerDesc(CACHED_ENTITLEMENT_REGISTRY_CLIENT, "org.lockss.entitlement.CachingEntitlementRegistryClient") { + public boolean shouldStart() { + return isKeepsafe(); }}, // watchdog last new ManagerDesc(WATCHDOG_SERVICE, DEFAULT_WATCHDOG_SERVICE) @@ -271,7 +276,7 @@ public boolean shouldStart() { private static LockssDaemon theDaemon; private boolean isClockss; - private boolean isSafenet; + private boolean isKeepsafe; protected String testingMode; protected LockssDaemon(List propUrls) { @@ -338,10 +343,10 @@ public boolean isDetectClockssSubscription() { } /** - * True if running as a Safenet daemon + * True if running as a Keepsafe daemon */ - public boolean isSafenet() { - return isSafenet; + public boolean isKeepsafe() { + return isKeepsafe; } /** Stop the daemon. Currently only used in testing. */ @@ -630,7 +635,16 @@ public ClockssParams getClockssParams() { * @throws IllegalArgumentException if the manager is not available. */ public EntitlementRegistryClient getEntitlementRegistryClient() { - return (EntitlementRegistryClient) getManager(SAFENET_MANAGER); + return (EntitlementRegistryClient) getManager(ENTITLEMENT_REGISTRY_CLIENT); + } + + /** + * return the EntitlementRegistryClient instance. + * @return EntitlementRegistryClient instance. + * @throws IllegalArgumentException if the manager is not available. + */ + public EntitlementRegistryClient getCachedEntitlementRegistryClient() { + return (EntitlementRegistryClient) getManager(CACHED_ENTITLEMENT_REGISTRY_CLIENT); } // LockssAuManager accessors @@ -977,7 +991,7 @@ protected void setConfig(Configuration config, Configuration prevConfig, testingMode = config.get(PARAM_TESTING_MODE); String proj = ConfigManager.getPlatformProject(); isClockss = "clockss".equalsIgnoreCase(proj); - isSafenet = "safenet".equalsIgnoreCase(proj); + isKeepsafe = "keepsafe".equalsIgnoreCase(proj); super.setConfig(config, prevConfig, changedKeys); } diff --git a/src/org/lockss/safenet/CachingEntitlementRegistryClient.java b/src/org/lockss/entitlement/CachingEntitlementRegistryClient.java similarity index 78% rename from src/org/lockss/safenet/CachingEntitlementRegistryClient.java rename to src/org/lockss/entitlement/CachingEntitlementRegistryClient.java index 6624cdadf2f..cc6f9a7f21c 100644 --- a/src/org/lockss/safenet/CachingEntitlementRegistryClient.java +++ b/src/org/lockss/entitlement/CachingEntitlementRegistryClient.java @@ -1,4 +1,4 @@ -package org.lockss.safenet; +package org.lockss.entitlement; import java.io.IOException; import java.util.Collections; @@ -7,7 +7,7 @@ import org.apache.commons.collections.map.LRUMap; import org.apache.commons.collections.map.MultiKeyMap; -import org.lockss.app.BaseLockssManager; +import org.lockss.app.BaseLockssDaemonManager; import org.lockss.app.ConfigurableManager; import org.lockss.config.Configuration; @@ -15,21 +15,23 @@ * A very basic cache which just stores the results of the last 100 calls to the Entitlement Registry. * There's a strong chance this will need to become something more complicated down the line. */ -public class CachingEntitlementRegistryClient extends BaseLockssManager implements EntitlementRegistryClient, ConfigurableManager { - private BaseEntitlementRegistryClient client; +public class CachingEntitlementRegistryClient extends BaseLockssDaemonManager implements EntitlementRegistryClient, ConfigurableManager { + public static final String PREFIX = Configuration.PREFIX + "entitlement.cache."; + public static final String PARAM_CACHE_SIZE = PREFIX + "size"; + static final int DEFAULT_CACHE_SIZE = 100; + private EntitlementRegistryClient client; private MultiKeyMap cache; - public CachingEntitlementRegistryClient() { - this(new BaseEntitlementRegistryClient(), 100); - } - - protected CachingEntitlementRegistryClient(BaseEntitlementRegistryClient client, int size) { - this.client = client; - this.cache = MultiKeyMap.decorate(new LRUMap(size)); + public void startService() { + super.startService(); + this.client = getDaemon().getEntitlementRegistryClient(); } public void setConfig(Configuration config, Configuration oldConfig, Configuration.Differences diffs) { - client.setConfig(config, oldConfig, diffs); + if (diffs.contains(PREFIX)) { + int size = config.getInt(PARAM_CACHE_SIZE, DEFAULT_CACHE_SIZE); + this.cache = MultiKeyMap.decorate(new LRUMap(size)); + } } public boolean isUserEntitled(String issn, String institution, String start, String end) throws IOException { diff --git a/src/org/lockss/safenet/EntitlementRegistryClient.java b/src/org/lockss/entitlement/EntitlementRegistryClient.java similarity index 93% rename from src/org/lockss/safenet/EntitlementRegistryClient.java rename to src/org/lockss/entitlement/EntitlementRegistryClient.java index 4df6a75632e..f5016c60102 100644 --- a/src/org/lockss/safenet/EntitlementRegistryClient.java +++ b/src/org/lockss/entitlement/EntitlementRegistryClient.java @@ -1,4 +1,4 @@ -package org.lockss.safenet; +package org.lockss.entitlement; import java.io.IOException; diff --git a/src/org/lockss/safenet/BaseEntitlementRegistryClient.java b/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java similarity index 94% rename from src/org/lockss/safenet/BaseEntitlementRegistryClient.java rename to src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java index 66e77256e75..afdf3bf0aa1 100644 --- a/src/org/lockss/safenet/BaseEntitlementRegistryClient.java +++ b/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java @@ -1,4 +1,4 @@ -package org.lockss.safenet; +package org.lockss.entitlement; import java.io.IOException; import java.net.URISyntaxException; @@ -25,11 +25,11 @@ import org.lockss.util.UrlUtil; import org.lockss.util.urlconn.LockssUrlConnection; -public class BaseEntitlementRegistryClient extends BaseLockssManager implements EntitlementRegistryClient, ConfigurableManager { +public class KeepsafeEntitlementRegistryClient extends BaseLockssManager implements EntitlementRegistryClient, ConfigurableManager { - private static final Logger log = Logger.getLogger(BaseEntitlementRegistryClient.class); + private static final Logger log = Logger.getLogger(KeepsafeEntitlementRegistryClient.class); - public static final String PREFIX = Configuration.PREFIX + "safenet."; + public static final String PREFIX = Configuration.PREFIX + "entitlement.keepsafe."; public static final String PARAM_ER_URI = PREFIX + "registryUri"; static final String DEFAULT_ER_URI = ""; public static final String PARAM_ER_APIKEY = PREFIX + "apiKey"; @@ -40,7 +40,7 @@ public class BaseEntitlementRegistryClient extends BaseLockssManager implements private String erUri; private String apiKey; - public BaseEntitlementRegistryClient() { + public KeepsafeEntitlementRegistryClient() { this.objectMapper = new ObjectMapper(); } @@ -130,8 +130,8 @@ public PublisherWorkflow getPublisherWorkflow(String publisherGuid) throws IOExc } } else { - log.warning("No workflow set for publisher, defaulting to PRIMARY_SAFENET"); - return PublisherWorkflow.PRIMARY_SAFENET; + log.warning("No workflow set for publisher, defaulting to PRIMARY_LOCKSS"); + return PublisherWorkflow.PRIMARY_LOCKSS; } } } diff --git a/src/org/lockss/entitlement/PublisherWorkflow.java b/src/org/lockss/entitlement/PublisherWorkflow.java new file mode 100644 index 00000000000..7b48236c55b --- /dev/null +++ b/src/org/lockss/entitlement/PublisherWorkflow.java @@ -0,0 +1,15 @@ +package org.lockss.entitlement; + +/** + * When running a network where content is not available to all users, agreements with the publishers may limit how that content should be used + * This enum encapsulates the different options, allowing the EntitlementRegistry to determine, based on the publisher and other information where any content should be served from + */ +public enum PublisherWorkflow { + /* Serve the content from LOCKSS first, only attempting to pass the request to the publisher if it cannot be found */ + PRIMARY_LOCKSS, + /* Pass the request to the publisher first, only attempting to use the archived copy if the publisher does not serve it */ + PRIMARY_PUBLISHER, + /* Pass the request to the publisher first, if the publisher does not serve it then display some error */ + LIBRARY_NOTIFICATION +}; + diff --git a/src/org/lockss/safenet/PublisherWorkflow.java b/src/org/lockss/safenet/PublisherWorkflow.java deleted file mode 100644 index 8fa6186019f..00000000000 --- a/src/org/lockss/safenet/PublisherWorkflow.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.lockss.safenet; - -public enum PublisherWorkflow { - PRIMARY_SAFENET, PRIMARY_PUBLISHER, LIBRARY_NOTIFICATION -}; - diff --git a/src/org/lockss/servlet/ContentServletManager.java b/src/org/lockss/servlet/ContentServletManager.java index a810e0e3112..41bb69478df 100644 --- a/src/org/lockss/servlet/ContentServletManager.java +++ b/src/org/lockss/servlet/ContentServletManager.java @@ -112,10 +112,6 @@ protected ManagerInfo getManagerInfo() { public static final String PARAM_CONTENT_ONLY = PREFIX + "contentOnly"; public static final boolean DEFAULT_CONTENT_ONLY = false; - /** Use the SafeNet Content server */ - public static final String PARAM_SAFENET = PREFIX + "safenet"; - public static final boolean DEFAULT_SAFENET = false; - static final String COMPRESSOR_PREFIX = PREFIX + "compressor."; public static final String PARAM_COMPRESSOR_ENABLED = @@ -143,10 +139,11 @@ protected ManagerInfo getManagerInfo() { "Serve Content", ServletDescr.NO_NAV_TABLE); - public static final ServletDescr SERVLET_SERVE_CONTENT_SAFENET = + public static final ServletDescr SERVLET_ENTITLEMENT_CHECK_SERVE_CONTENT = new ServletDescr("ServeContent", - SafeNetServeContent.class, + EntitlementCheckServeContent.class, "Serve Content", + "ServeContent", ServletDescr.NO_NAV_TABLE); public static final ServletDescr SERVLET_SERVE_CONTENT = @@ -202,8 +199,8 @@ static String mailtoUrl(String addr) { SERVLET_SERVE_CONTENT_NO_NAV, }; - static final ServletDescr servletDescrsSafeNet[] = { - SERVLET_SERVE_CONTENT_SAFENET, + static final ServletDescr servletDescrsKeepsafe[] = { + SERVLET_ENTITLEMENT_CHECK_SERVE_CONTENT, }; // All servlets must be listed here (even if not in nav table). @@ -217,8 +214,8 @@ static String mailtoUrl(String addr) { }; public ServletDescr[] getServletDescrs() { - if (LockssDaemon.getLockssDaemon().isSafenet()) { - return servletDescrsSafeNet; + if (LockssDaemon.getLockssDaemon().isKeepsafe()) { + return servletDescrsKeepsafe; } else if (CurrentConfig.getBooleanParam(PARAM_CONTENT_ONLY, DEFAULT_CONTENT_ONLY)) { return servletDescrsNoNav; diff --git a/src/org/lockss/servlet/SafeNetServeContent.java b/src/org/lockss/servlet/EntitlementCheckServeContent.java similarity index 96% rename from src/org/lockss/servlet/SafeNetServeContent.java rename to src/org/lockss/servlet/EntitlementCheckServeContent.java index f4755549a6d..b2b3d8454bd 100644 --- a/src/org/lockss/servlet/SafeNetServeContent.java +++ b/src/org/lockss/servlet/EntitlementCheckServeContent.java @@ -54,22 +54,22 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.extractor.*; import org.lockss.plugin.*; import org.lockss.plugin.PluginManager.CuContentReq; -import org.lockss.safenet.EntitlementRegistryClient; -import org.lockss.safenet.PublisherWorkflow; +import org.lockss.entitlement.EntitlementRegistryClient; +import org.lockss.entitlement.PublisherWorkflow; import org.lockss.util.*; import org.lockss.util.urlconn.*; import org.mortbay.html.*; import org.mortbay.http.*; @SuppressWarnings("serial") -public class SafeNetServeContent extends ServeContent { +public class EntitlementCheckServeContent extends ServeContent { - private static final Logger log = Logger.getLogger(SafeNetServeContent.class); + private static final Logger log = Logger.getLogger(EntitlementCheckServeContent.class); // If true, scope can be 'mocked' from the URL parameters. This is for testing purposes, and should never be true in production - public static final String PARAM_EDIAUTH_MOCK_SCOPE = PREFIX + "mockScope"; + public static final String PARAM_MOCK_SCOPE = PREFIX + "mockScope"; - private static final String INSTITUTION_HEADER = "X-SafeNet-Institution"; + private static final String INSTITUTION_HEADER = "X-Lockss-Institution"; public static final String INSTITUTION_SCOPE_SESSION_KEY = "scope"; @@ -95,7 +95,7 @@ protected void resetLocals() { public void init(ServletConfig config) throws ServletException { super.init(config); LockssDaemon daemon = getLockssDaemon(); - entitlementRegistry = daemon.getEntitlementRegistryClient(); + entitlementRegistry = daemon.getCachedEntitlementRegistryClient(); } /** Called by ServletUtil.setConfig() */ @@ -104,12 +104,12 @@ static void setConfig(Configuration config, Configuration.Differences diffs) { ServeContent.setConfig(config, oldConfig, diffs); if (diffs.contains(PREFIX)) { - mockScope = config.getBoolean(PARAM_EDIAUTH_MOCK_SCOPE, false); + mockScope = config.getBoolean(PARAM_MOCK_SCOPE, false); } } protected boolean isNeverProxyForAu(ArchivalUnit au) { - return super.isNeverProxyForAu(au) || workflow == PublisherWorkflow.PRIMARY_SAFENET; + return super.isNeverProxyForAu(au) || workflow == PublisherWorkflow.PRIMARY_LOCKSS; } /** @@ -232,8 +232,7 @@ protected void handleUnauthorisedUrlRequest(String missingUrl) void updateInstitution() throws IOException { - //This is currently called in lockssHandleRequest, it needs to be called from wherever we do the SAML authentication - institutionScope = "ed.ac.uk"; + String institutionScope = (String) this.getSession().getAttribute(INSTITUTION_SCOPE_SESSION_KEY); institution = entitlementRegistry.getInstitution(institutionScope); } diff --git a/src/org/lockss/servlet/ServletUtil.java b/src/org/lockss/servlet/ServletUtil.java index dad0492fbb0..05ef59f4c26 100644 --- a/src/org/lockss/servlet/ServletUtil.java +++ b/src/org/lockss/servlet/ServletUtil.java @@ -373,8 +373,8 @@ protected String getLink() { public static void setConfig(Configuration config, Configuration oldConfig, Configuration.Differences diffs) { - if (diffs.contains(SafeNetServeContent.PREFIX)) { - SafeNetServeContent.setConfig(config, oldConfig, diffs); + if (diffs.contains(EntitlementCheckServeContent.PREFIX)) { + EntitlementCheckServeContent.setConfig(config, oldConfig, diffs); } if (diffs.contains(ServeContent.PREFIX)) { ServeContent.setConfig(config, oldConfig, diffs); diff --git a/test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java b/test/src/org/lockss/entitlement/TestCachingEntitlementRegistryClient.java similarity index 83% rename from test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java rename to test/src/org/lockss/entitlement/TestCachingEntitlementRegistryClient.java index 34d74c23459..4d1b2a82700 100644 --- a/test/src/org/lockss/safenet/TestCachingEntitlementRegistryClient.java +++ b/test/src/org/lockss/entitlement/TestCachingEntitlementRegistryClient.java @@ -1,24 +1,29 @@ -package org.lockss.safenet; +package org.lockss.entitlement; import java.io.IOException; +import java.util.Properties; import org.mockito.Mockito; -import org.lockss.safenet.PublisherWorkflow; +import org.lockss.test.ConfigurationUtil; import org.lockss.test.LockssTestCase; import org.lockss.test.MockLockssDaemon; public class TestCachingEntitlementRegistryClient extends LockssTestCase { private EntitlementRegistryClient client; - private BaseEntitlementRegistryClient mock; + private EntitlementRegistryClient mock; public void setUp() throws Exception { super.setUp(); - mock = Mockito.mock(BaseEntitlementRegistryClient.class); - client = new CachingEntitlementRegistryClient(mock, 5); + mock = Mockito.mock(EntitlementRegistryClient.class); + client = new CachingEntitlementRegistryClient(); MockLockssDaemon daemon = getMockLockssDaemon(); - daemon.setEntitlementRegistryClient(client); + daemon.setEntitlementRegistryClient(mock); + daemon.setCachedEntitlementRegistryClient(client); daemon.setDaemonInited(true); + Properties p = new Properties(); + p.setProperty(CachingEntitlementRegistryClient.PARAM_CACHE_SIZE, "5"); + ConfigurationUtil.setCurrentConfigFromProps(p); client.initService(daemon); client.startService(); } @@ -29,7 +34,7 @@ public void testCaching() throws Exception { Mockito.when(mock.isUserEntitled("0000-0002", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenThrow(new IOException("Error communicating with entitlement registry. Response was 500 null")); Mockito.when(mock.getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenReturn("33333333-0000-0000-0000-000000000000"); Mockito.when(mock.getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")).thenReturn("33333333-0000-0000-0000-000000000001"); - Mockito.when(mock.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_SAFENET); + Mockito.when(mock.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_LOCKSS); Mockito.when(mock.getPublisherWorkflow("33333333-1111-1111-1111-111111111111")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); assertTrue(client.isUserEntitled("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -43,11 +48,11 @@ public void testCaching() throws Exception { } assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); assertEquals("33333333-0000-0000-0000-000000000001", client.getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); - assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); + assertEquals(PublisherWorkflow.PRIMARY_LOCKSS, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); assertEquals(PublisherWorkflow.PRIMARY_PUBLISHER, client.getPublisherWorkflow("33333333-1111-1111-1111-111111111111")); assertEquals(PublisherWorkflow.PRIMARY_PUBLISHER, client.getPublisherWorkflow("33333333-1111-1111-1111-111111111111")); - assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); + assertEquals(PublisherWorkflow.PRIMARY_LOCKSS, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); assertEquals("33333333-0000-0000-0000-000000000001", client.getPublisher("0000-0001", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0000-0000", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); try { diff --git a/test/src/org/lockss/safenet/TestEntitlementRegistryClient.java b/test/src/org/lockss/entitlement/TestKeepsafeEntitlementRegistryClient.java similarity index 83% rename from test/src/org/lockss/safenet/TestEntitlementRegistryClient.java rename to test/src/org/lockss/entitlement/TestKeepsafeEntitlementRegistryClient.java index 18381931bb0..0d807a1a512 100644 --- a/test/src/org/lockss/safenet/TestEntitlementRegistryClient.java +++ b/test/src/org/lockss/entitlement/TestKeepsafeEntitlementRegistryClient.java @@ -1,4 +1,4 @@ -package org.lockss.safenet; +package org.lockss.entitlement; import java.io.IOException; import java.net.URISyntaxException; @@ -16,7 +16,6 @@ import org.apache.http.message.BasicNameValuePair; import org.mockito.Mockito; -import org.lockss.safenet.PublisherWorkflow; import org.lockss.test.ConfigurationUtil; import org.lockss.test.LockssTestCase; import org.lockss.test.MockLockssDaemon; @@ -24,9 +23,9 @@ import org.lockss.test.StringInputStream; import org.lockss.util.urlconn.LockssUrlConnection; -public class TestEntitlementRegistryClient extends LockssTestCase { - private static final String ER_URI = "http://dev-safenet.edina.ac.uk"; - private BaseEntitlementRegistryClient client; +public class TestKeepsafeEntitlementRegistryClient extends LockssTestCase { + private static final String ER_URI = "http://keepsafe.org"; + private KeepsafeEntitlementRegistryClient client; private Map validEntitlementParams; private Map validResponseParams; @@ -34,13 +33,13 @@ public class TestEntitlementRegistryClient extends LockssTestCase { public void setUp() throws Exception { super.setUp(); - client = Mockito.spy(new BaseEntitlementRegistryClient()); + client = Mockito.spy(new KeepsafeEntitlementRegistryClient()); MockLockssDaemon daemon = getMockLockssDaemon(); daemon.setEntitlementRegistryClient(client); daemon.setDaemonInited(true); Properties p = new Properties(); - p.setProperty(BaseEntitlementRegistryClient.PARAM_ER_URI, ER_URI); - p.setProperty(BaseEntitlementRegistryClient.PARAM_ER_APIKEY, "00000000-0000-0000-0000-000000000000"); + p.setProperty(KeepsafeEntitlementRegistryClient.PARAM_ER_URI, ER_URI); + p.setProperty(KeepsafeEntitlementRegistryClient.PARAM_ER_APIKEY, "00000000-0000-0000-0000-000000000000"); ConfigurationUtil.setCurrentConfigFromProps(p); client.initService(daemon); client.startService(); @@ -62,7 +61,7 @@ public void setUp() throws Exception { } public void testEntitlementRegistryError() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 500, "Internal server error")).when(client).openConnection(url); try { @@ -76,7 +75,7 @@ public void testEntitlementRegistryError() throws Exception { } public void testEntitlementRegistryInvalidResponse() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "[]")).when(client).openConnection(url); assertFalse(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -84,7 +83,7 @@ public void testEntitlementRegistryInvalidResponse() throws Exception { } public void testEntitlementRegistryInvalidJson() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "[{\"this\": isn't, JSON}]")).when(client).openConnection(url); try { @@ -98,7 +97,7 @@ public void testEntitlementRegistryInvalidJson() throws Exception { } public void testEntitlementRegistryUnexpectedJson() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "{\"surprise\": \"object\"}")).when(client).openConnection(url); assertFalse(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -106,7 +105,7 @@ public void testEntitlementRegistryUnexpectedJson() throws Exception { } public void testUserEntitled() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "[" + mapToJson(validResponseParams) + "]")).when(client).openConnection(url); assertTrue(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -114,7 +113,7 @@ public void testUserEntitled() throws Exception { } public void testUserNotEntitled() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 204, "")).when(client).openConnection(url); assertFalse(client.isUserEntitled("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -129,7 +128,7 @@ public void testGetInstitution() throws Exception { institution.put("name", "University of Edinburgh"); institution.put("scope", "ed.ac.uk"); - String url = url("/institutions", BaseEntitlementRegistryClient.mapToPairs(queryParams)); + String url = url("/institutions", KeepsafeEntitlementRegistryClient.mapToPairs(queryParams)); Mockito.doReturn(connection(url, 200, "[" + mapToJson(institution) + "]")).when(client).openConnection(url); assertEquals("11111111-0000-0000-0000-000000000000", client.getInstitution("ed.ac.uk")); @@ -140,7 +139,7 @@ public void testGetInstitutionNoResponse() throws Exception { Map queryParams = new HashMap(); queryParams.put("scope", "ed.ac.uk"); - String url = url("/institutions", BaseEntitlementRegistryClient.mapToPairs(queryParams)); + String url = url("/institutions", KeepsafeEntitlementRegistryClient.mapToPairs(queryParams)); Mockito.doReturn(connection(url, 200, "[]")).when(client).openConnection(url); try { client.getInstitution("ed.ac.uk"); @@ -165,7 +164,7 @@ public void testGetInstitutionMultipleResults() throws Exception { institution2.put("name", "University of Edinburgh 2"); institution2.put("scope", "ed.ac.uk"); - String url = url("/institutions", BaseEntitlementRegistryClient.mapToPairs(queryParams)); + String url = url("/institutions", KeepsafeEntitlementRegistryClient.mapToPairs(queryParams)); Mockito.doReturn(connection(url, 200, "[" + mapToJson(institution1) + "," + mapToJson(institution2) + "]")).when(client).openConnection(url); try { client.getInstitution("ed.ac.uk"); @@ -179,7 +178,7 @@ public void testGetInstitutionMultipleResults() throws Exception { } public void testGetPublisher() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 200, "[" + mapToJson(validResponseParams) + "]")).when(client).openConnection(url); assertEquals("33333333-0000-0000-0000-000000000000", client.getPublisher("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -187,7 +186,7 @@ public void testGetPublisher() throws Exception { } public void testGetPublisherNotEntitled() throws Exception { - String url = url("/entitlements", BaseEntitlementRegistryClient.mapToPairs(validEntitlementParams)); + String url = url("/entitlements", KeepsafeEntitlementRegistryClient.mapToPairs(validEntitlementParams)); Mockito.doReturn(connection(url, 204, "")).when(client).openConnection(url); assertEquals(null, client.getPublisher("0123-456X", "11111111-1111-1111-1111-111111111111", "20120101", "20151231")); @@ -196,11 +195,11 @@ public void testGetPublisherNotEntitled() throws Exception { public void testGetPublisherWorkflow() throws Exception { Map responseParams = new HashMap(validPublisherParams); - responseParams.put("workflow", "primary_safenet"); + responseParams.put("workflow", "primary_lockss"); String url = url("/publishers/33333333-0000-0000-0000-000000000000"); Mockito.doReturn(connection(url, 200, mapToJson(responseParams))).when(client).openConnection(url); - assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); + assertEquals(PublisherWorkflow.PRIMARY_LOCKSS, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); Mockito.verify(client).openConnection(url); } @@ -209,7 +208,7 @@ public void testGetPublisherWorkflowMissingWorkflow() throws Exception { String url = url("/publishers/33333333-0000-0000-0000-000000000000"); Mockito.doReturn(connection(url, 200, mapToJson(responseParams))).when(client).openConnection(url); - assertEquals(PublisherWorkflow.PRIMARY_SAFENET, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); + assertEquals(PublisherWorkflow.PRIMARY_LOCKSS, client.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")); Mockito.verify(client).openConnection(url); } @@ -236,7 +235,7 @@ private String url(String endpoint) { private String url(String endpoint, List params) { URIBuilder builder = new URIBuilder(); builder.setScheme("http"); - builder.setHost("dev-safenet.edina.ac.uk"); + builder.setHost("keepsafe.org"); builder.setPath(endpoint); if(params != null) { builder.setParameters(params); diff --git a/test/src/org/lockss/servlet/TestSafeNetServeContent.java b/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java similarity index 79% rename from test/src/org/lockss/servlet/TestSafeNetServeContent.java rename to test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java index 96fbcbfff8d..b3b3d589339 100644 --- a/test/src/org/lockss/servlet/TestSafeNetServeContent.java +++ b/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java @@ -56,9 +56,8 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.plugin.CachedUrl; import org.lockss.plugin.Plugin; import org.lockss.plugin.PluginManager; -import org.lockss.safenet.BaseEntitlementRegistryClient; -import org.lockss.safenet.EntitlementRegistryClient; -import org.lockss.safenet.PublisherWorkflow; +import org.lockss.entitlement.EntitlementRegistryClient; +import org.lockss.entitlement.PublisherWorkflow; import org.lockss.test.ConfigurationUtil; import org.lockss.test.MockArchivalUnit; import org.lockss.test.MockCachedUrl; @@ -78,7 +77,7 @@ of this software and associated documentation files (the "Software"), to deal import org.mockito.Mockito; import org.apache.commons.collections.map.MultiKeyMap; -public class TestSafeNetServeContent extends LockssServletTestCase { +public class TestEntitlementCheckServeContent extends LockssServletTestCase { private MockPluginManager pluginMgr = null; private EntitlementRegistryClient entitlementRegistryClient = null; @@ -90,7 +89,7 @@ public void setUp() throws Exception { theDaemon.setPluginManager(pluginMgr); theDaemon.setIdentityManager(new org.lockss.protocol.MockIdentityManager()); entitlementRegistryClient = Mockito.mock(EntitlementRegistryClient.class); - theDaemon.setEntitlementRegistryClient(entitlementRegistryClient); + theDaemon.setCachedEntitlementRegistryClient(entitlementRegistryClient); theDaemon.getServletManager(); theDaemon.setDaemonInited(true); theDaemon.setAusStarted(true); @@ -99,8 +98,9 @@ public void setUp() throws Exception { pluginMgr.initService(theDaemon); pluginMgr.startService(); - ConfigurationUtil.addFromArgs(ConfigManager.PARAM_PLATFORM_PROJECT, "safenet"); - ConfigurationUtil.addFromArgs(SafeNetServeContent.PARAM_MISSING_FILE_ACTION, "Redirect"); + ConfigurationUtil.addFromArgs(ConfigManager.PARAM_PLATFORM_PROJECT, "keepsafe"); + ConfigurationUtil.addFromArgs(EntitlementCheckServeContent.PARAM_MISSING_FILE_ACTION, "Redirect"); + ConfigurationUtil.addFromArgs(EntitlementCheckServeContent.PARAM_MOCK_SCOPE, "true"); //ConfigurationUtil.addFromArgs("org.lockss.log.default.level", "debug3"); } @@ -124,7 +124,7 @@ private MockArchivalUnit makeAu(Properties override) throws Exception { tdbProps.setProperty("attributes.provider", "Provider[10.0135/12345678]"); tdbProps.setProperty("param.1.key", "base_url"); - tdbProps.setProperty("param.1.value", "http://dev-safenet.edina.ac.uk/test_journal/"); + tdbProps.setProperty("param.1.value", "http://publisher.org/test_journal/"); tdbProps.setProperty("param.2.key", "volume"); tdbProps.setProperty("param.2.value", "vol1"); tdbProps.setProperty("plugin", plugin.getClass().toString()); @@ -136,7 +136,7 @@ private MockArchivalUnit makeAu(Properties override) throws Exception { TdbAu tdbAu = tdb.addTdbAuFromProperties(tdbProps); TitleConfig titleConfig = new TitleConfig(tdbAu, plugin); MockArchivalUnit au = new MockArchivalUnit(plugin, "TestAU"); - au.setStartUrls(Arrays.asList("http://dev-safenet.edina.ac.uk/test_journal/")); + au.setStartUrls(Arrays.asList("http://publisher.org/test_journal/")); au.setTitleConfig(titleConfig); return au; } @@ -144,16 +144,16 @@ private MockArchivalUnit makeAu(Properties override) throws Exception { protected void initServletRunner() { super.initServletRunner(); sRunner.setServletContextAttribute(ServletManager.CONTEXT_ATTR_SERVLET_MGR, new ContentServletManager()); - sRunner.registerServlet("/SafeNetServeContent", MockSafeNetServeContent.class.getName() ); + sRunner.registerServlet("/EntitlementCheckServeContent", MockEntitlementCheckServeContent.class.getName() ); sRunner.registerServlet("/test_journal/", RedirectServlet.class.getName()); } public void testIndex() throws Exception { initServletRunner(); pluginMgr.addAu(makeAu(), null); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertResponseOk(resp1); @@ -163,16 +163,16 @@ public void testIndex() throws Exception { assertEquals(2, auTable.getRowCount()); assertEquals(3, auTable.getColumnCount()); assertEquals("MockAU", auTable.getCellAsText(1, 0)); - assertEquals("/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F&auid=TestAU", auTable.getTableCell(1, 1).getLinks()[0].getURLString()); + assertEquals("/ServeContent?url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F&auid=TestAU", auTable.getTableCell(1, 1).getLinks()[0].getURLString()); } public void testMissingUrl() throws Exception { initServletRunner(); pluginMgr.addAu(makeAu(), null); sClient.setExceptionsThrownOnErrorStatus(false); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertResponseOk(resp1); @@ -183,9 +183,9 @@ public void testMissingUrlExplicitAU() throws Exception { initServletRunner(); pluginMgr.addAu(makeAu(), null); sClient.setExceptionsThrownOnErrorStatus(false); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F&auid=TestAU" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F&auid=TestAU" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertResponseOk(resp1); @@ -199,17 +199,17 @@ public void testCachedUrlPrimaryPublisherResponse() throws Exception { Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + MockEntitlementCheckServeContent snsc = (MockEntitlementCheckServeContent) ic.getServlet(); LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); - snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + snsc.expectRequest("http://publisher.org/test_journal/", connection); WebResponse resp1 = sClient.getResponse(request); assertResponseOk(resp1); assertEquals("BlahFetched content", resp1.getText()); Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); - Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + Mockito.verify(connection).addRequestProperty("X-Lockss-Institution", "ed.ac.uk"); } public void testCachedUrlPrimaryPublisherError() throws Exception { @@ -219,29 +219,29 @@ public void testCachedUrlPrimaryPublisherError() throws Exception { Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + MockEntitlementCheckServeContent snsc = (MockEntitlementCheckServeContent) ic.getServlet(); LockssUrlConnection connection = mockConnection(403, "BlahContent refused"); - snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + snsc.expectRequest("http://publisher.org/test_journal/", connection); WebResponse resp1 = sClient.getResponse(request); assertResponseOk(resp1); assertEquals("BlahCached content", resp1.getText()); Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); - Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + Mockito.verify(connection).addRequestProperty("X-Lockss-Institution", "ed.ac.uk"); } - public void testCachedUrlPrimarySafenet() throws Exception { + public void testCachedUrlPrimaryLockss() throws Exception { initServletRunner(); pluginMgr.addAu(makeAu()); Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); - Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_SAFENET); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_LOCKSS); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertResponseOk(resp1); @@ -256,13 +256,13 @@ public void testCachedUrlLibraryNotification() throws Exception { Mockito.when(entitlementRegistryClient.getPublisher("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.LIBRARY_NOTIFICATION); sClient.setExceptionsThrownOnErrorStatus(false); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertEquals(403, resp1.getResponseCode()); - assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://publisher.org/test_journal/")); } public void testUnauthorisedUrl() throws Exception { @@ -271,14 +271,14 @@ public void testUnauthorisedUrl() throws Exception { Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(false); sClient.setExceptionsThrownOnErrorStatus(false); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); System.out.println(resp1.getText()); assertEquals(403, resp1.getResponseCode()); - assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://publisher.org/test_journal/")); } public void testEntitlementRegistryError() throws Exception { @@ -287,13 +287,13 @@ public void testEntitlementRegistryError() throws Exception { Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); Mockito.when(entitlementRegistryClient.isUserEntitled("0740-2783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenThrow(new IOException("Could not contact entitlement registry")); sClient.setExceptionsThrownOnErrorStatus(false); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertEquals(503, resp1.getResponseCode()); - assertTrue(resp1.getText().contains("

An error occurred trying to access the requested URL on this LOCKSS box. This may be temporary and you may wish to report this, and try again later. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + assertTrue(resp1.getText().contains("

An error occurred trying to access the requested URL on this LOCKSS box. This may be temporary and you may wish to report this, and try again later. Select link1 to view it at the publisher:

http://publisher.org/test_journal/")); } public void testInvalidArchivalUnit() throws Exception { @@ -303,13 +303,13 @@ public void testInvalidArchivalUnit() throws Exception { props.setProperty("eissn", ""); sClient.setExceptionsThrownOnErrorStatus(false); pluginMgr.addAu(makeAu(props)); - WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); InvocationContext ic = sClient.newInvocation(request); - SafeNetServeContent snsc = (SafeNetServeContent) ic.getServlet(); + EntitlementCheckServeContent snsc = (EntitlementCheckServeContent) ic.getServlet(); WebResponse resp1 = sClient.getResponse(request); assertEquals(404, resp1.getResponseCode()); - assertTrue(resp1.getText().contains("

The requested URL is not preserved on this LOCKSS box. Select link1 to view it at the publisher:

http://dev-safenet.edina.ac.uk/test_journal/")); + assertTrue(resp1.getText().contains("

The requested URL is not preserved on this LOCKSS box. Select link1 to view it at the publisher:

http://publisher.org/test_journal/")); } // Test no longer valid due to changes to how we extract bibliographic data @@ -325,17 +325,17 @@ public void testInvalidArchivalUnit() throws Exception { // Mockito.when(entitlementRegistryClient.isUserEntitled("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn(true); // Mockito.when(entitlementRegistryClient.getPublisher("0000-0000", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "20140101", "20141231")).thenReturn("33333333-0000-0000-0000-000000000000"); // Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-00000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); - // WebRequest request = new GetMethodWebRequest("http://null/SafeNetServeContent?url=http%3A%2F%2Fdev-safenet.edina.ac.uk%2Ftest_journal%2F" ); + // WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); // InvocationContext ic = sClient.newInvocation(request); - // MockSafeNetServeContent snsc = (MockSafeNetServeContent) ic.getServlet(); + // MockEntitlementCheckServeContent snsc = (MockEntitlementCheckServeContent) ic.getServlet(); // LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); - // snsc.expectRequest("http://dev-safenet.edina.ac.uk/test_journal/", connection); + // snsc.expectRequest("http://publisher.org/test_journal/", connection); // WebResponse resp1 = sClient.getResponse(request); // assertResponseOk(resp1); // assertEquals("BlahFetched content", resp1.getText()); // Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); - // Mockito.verify(connection).addRequestProperty("X-SafeNet-Institution", "ed.ac.uk"); + // Mockito.verify(connection).addRequestProperty("X-Lockss-Institution", "ed.ac.uk"); // } private static class MockPluginManager extends PluginManager { @@ -398,8 +398,8 @@ private LockssUrlConnection mockConnection(int responseCode, String response) th return connection; } - public static class MockSafeNetServeContent extends SafeNetServeContent { - private SafeNetServeContent delegate = Mockito.mock(SafeNetServeContent.class); + public static class MockEntitlementCheckServeContent extends EntitlementCheckServeContent { + private EntitlementCheckServeContent delegate = Mockito.mock(EntitlementCheckServeContent.class); public void expectRequest(String url, LockssUrlConnection connection) throws IOException { Mockito.when(delegate.doOpenConnection(Mockito.eq(url), Mockito.any(LockssUrlConnectionPool.class))).thenReturn(connection); diff --git a/test/src/org/lockss/test/MockLockssDaemon.java b/test/src/org/lockss/test/MockLockssDaemon.java index 9eee37ed2c1..5888463c80e 100644 --- a/test/src/org/lockss/test/MockLockssDaemon.java +++ b/test/src/org/lockss/test/MockLockssDaemon.java @@ -63,7 +63,7 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.subscription.SubscriptionManager; import org.lockss.util.*; import org.lockss.clockss.*; -import org.lockss.safenet.*; +import org.lockss.entitlement.*; public class MockLockssDaemon extends LockssDaemon { private static Logger log = Logger.getLogger("MockLockssDaemon"); @@ -863,7 +863,16 @@ public void setCron(Cron cron) { */ public void setEntitlementRegistryClient(EntitlementRegistryClient entitlementRegistryClient) { this.entitlementRegistryClient = entitlementRegistryClient; - managerMap.put(LockssDaemon.SAFENET_MANAGER, entitlementRegistryClient); + managerMap.put(LockssDaemon.ENTITLEMENT_REGISTRY_CLIENT, entitlementRegistryClient); + } + + /** + * Set the EntitlementRegistryClient + * @param pluginMan the new manager + */ + public void setCachedEntitlementRegistryClient(EntitlementRegistryClient entitlementRegistryClient) { + this.entitlementRegistryClient = entitlementRegistryClient; + managerMap.put(LockssDaemon.CACHED_ENTITLEMENT_REGISTRY_CLIENT, entitlementRegistryClient); } // AU managers From 814980796a4bd2630adf8daf39f8b0ab9167981c Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Thu, 17 Aug 2017 11:24:45 +0100 Subject: [PATCH 05/13] Remove unnecessary/invalid methods for getting bibliographic data --- .../servlet/EntitlementCheckServeContent.java | 150 +----------------- 1 file changed, 6 insertions(+), 144 deletions(-) diff --git a/src/org/lockss/servlet/EntitlementCheckServeContent.java b/src/org/lockss/servlet/EntitlementCheckServeContent.java index b2b3d8454bd..614cedb9487 100644 --- a/src/org/lockss/servlet/EntitlementCheckServeContent.java +++ b/src/org/lockss/servlet/EntitlementCheckServeContent.java @@ -167,11 +167,6 @@ protected boolean setCachedUrlAndAu() throws IOException { } - protected void handleOpenUrlInfo(OpenUrlInfo info) throws IOException { - setBibInfoFromOpenUrl(info); - super.handleOpenUrlInfo(info); - } - /** * Handle request for content that belongs to one of our AUs, whether or not * we have content for that URL. If this request contains a version param, @@ -237,21 +232,13 @@ void updateInstitution() throws IOException { } boolean isUserEntitled(ArchivalUnit au) throws IOException, IllegalArgumentException { - setBibInfoFromMetadataDB(cu); - setBibInfoFromCu(cu, au); - setBibInfoFromTdb(au); - setBibInfoFromArticleFiles(au, cu); - validateBibInfo(); + validateBibInfo(cu, au); return entitlementRegistry.isUserEntitled(issn, institution, start, end); } PublisherWorkflow getPublisherWorkflow(ArchivalUnit au) throws IOException, IllegalArgumentException { - setBibInfoFromMetadataDB(cu); - setBibInfoFromCu(cu, au); - setBibInfoFromTdb(au); - setBibInfoFromArticleFiles(au, cu); - validateBibInfo(); + validateBibInfo(cu, au); String publisher = entitlementRegistry.getPublisher(issn, institution, start, end); if(StringUtil.isNullString(publisher)) { @@ -261,22 +248,6 @@ PublisherWorkflow getPublisherWorkflow(ArchivalUnit au) throws IOException, Ille return entitlementRegistry.getPublisherWorkflow(publisher); } - private void setBibInfoFromOpenUrl(OpenUrlInfo info) throws IllegalArgumentException { - log.debug2("Setting bib info from OpenURL"); - if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { - log.debug2("Bib info already set"); - return; - } - - if(info == null) { - log.debug2("No OpenUrlInfo"); - return; - } - - BibliographicItem item = info.getBibliographicItem(); - setBibInfoFromBibliographicItem(item); - } - private void setBibInfoFromBibliographicItem(BibliographicItem item) { log.debug2("Setting bib info from BibliographicItem"); if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { @@ -326,72 +297,6 @@ private void setBibInfoFromMetadata(ArticleMetadata md) { setBibInfoFromBibliographicItem(item); } - private void setBibInfoFromArticleFiles(ArchivalUnit au, final CachedUrl cu) throws IllegalArgumentException { - log.debug2("Setting bib info from TDB"); - if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { - log.debug2("Bib info already set"); - return; - } - - if(au == null) { - log.debug2("No ArchivalUnit"); - return; - } - if(cu == null) { - log.debug2("No CachedUrl"); - return; - } - - Plugin plugin = au.getPlugin(); - if(plugin == null) { - log.debug2("No Plugin"); - return; - } - - MetadataTarget target = new MetadataTarget(MetadataTarget.PURPOSE_OPENURL); - Iterator afs = au.getArticleIterator(); - ArticleMetadataExtractor mdExtractor = plugin.getArticleMetadataExtractor(target, au); - - if(afs == null) { - log.debug2("No ArticleIterator"); - return; - } - if(mdExtractor == null) { - log.debug2("No ArticleMetadataExtractor"); - return; - } - - ArticleMetadataExtractor.Emitter emitter = new ArticleMetadataExtractor.Emitter() { - public void emitMetadata(ArticleFiles af2, ArticleMetadata md) { - String mdUrl = md.get("access.url"); - String cuUrl = cu.getUrl(); - log.debug3("Comparing " + mdUrl + " to " + cuUrl); - if(mdUrl.equals(cuUrl)) { - log.debug2("Found matching URL"); - setBibInfoFromMetadata(md); - } - } - }; - - while(afs.hasNext()) { - ArticleFiles af = afs.next(); - try { - mdExtractor.extract(target, af, emitter); - } - catch(IOException e) { - log.error("Error extracting article metadata", e); - } - catch(PluginException e) { - log.error("Error extracting article metadata", e); - } - - if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { - log.debug2("Bib info already set"); - return; - } - } - } - private void setBibInfoFromTdb(ArchivalUnit au) throws IllegalArgumentException { log.debug2("Setting bib info from TDB"); if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { @@ -432,52 +337,6 @@ private void setBibInfoFromTdb(ArchivalUnit au) throws IllegalArgumentException } } - private void setBibInfoFromCu(CachedUrl cu, ArchivalUnit au) { - log.debug2("Setting bib info from CU"); - if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { - log.debug2("Bib info already set"); - return; - } - - if(au == null) { - log.debug2("No ArchivalUnit"); - return; - } - if(cu == null) { - log.debug2("No CachedUrl"); - return; - } - - Plugin plugin = au.getPlugin(); - if(plugin == null) { - log.debug2("No Plugin"); - return; - } - - - MetadataTarget target = new MetadataTarget(MetadataTarget.PURPOSE_OPENURL); - FileMetadataExtractor mdExtractor = plugin.getFileMetadataExtractor(target, cu.getContentType(), au); - if(mdExtractor == null) { - log.debug2("No FileMetadataExtractor"); - return; - } - FileMetadataExtractor.Emitter emitter = new FileMetadataExtractor.Emitter() { - public void emitMetadata(CachedUrl cu, ArticleMetadata md) { - log.debug2("ArticleMetadata found"); - setBibInfoFromMetadata(md); - } - }; - try { - mdExtractor.extract(target, cu, emitter); - } - catch(IOException e) { - log.error("Error extracting CU metadata", e); - } - catch(PluginException e) { - log.error("Error extracting CU metadata", e); - } - } - private void setBibInfoFromMetadataDB(CachedUrl cu) { log.debug2("Setting bib info from database"); if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { @@ -560,7 +419,10 @@ else if(L_ISSN_TYPE.equals(issnType)){ } } - private void validateBibInfo() { + private void validateBibInfo(CachedUrl cu, ArchivalUnit au) { + setBibInfoFromMetadataDB(cu); + setBibInfoFromTdb(au); + if(StringUtil.isNullString(issn)) { throw new IllegalArgumentException("ArchivalUnit has no ISSN"); } From 283df3cda25ad88dfa9b1be44a11955e1ab5aeb4 Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Thu, 17 Aug 2017 14:34:52 +0100 Subject: [PATCH 06/13] Improved thread-safety of EntitlementRegistryClients --- .../CachingEntitlementRegistryClient.java | 12 ++++++------ .../KeepsafeEntitlementRegistryClient.java | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/org/lockss/entitlement/CachingEntitlementRegistryClient.java b/src/org/lockss/entitlement/CachingEntitlementRegistryClient.java index cc6f9a7f21c..529b19750ed 100644 --- a/src/org/lockss/entitlement/CachingEntitlementRegistryClient.java +++ b/src/org/lockss/entitlement/CachingEntitlementRegistryClient.java @@ -13,7 +13,8 @@ /* * A very basic cache which just stores the results of the last 100 calls to the Entitlement Registry. - * There's a strong chance this will need to become something more complicated down the line. + * There's a strong chance this will need to become something more complicated down the line if performance isn't acceptable. + * In this case though, it would probably be best to replace it with something like Guava's caching, rather than building something custom. */ public class CachingEntitlementRegistryClient extends BaseLockssDaemonManager implements EntitlementRegistryClient, ConfigurableManager { public static final String PREFIX = Configuration.PREFIX + "entitlement.cache."; @@ -34,7 +35,7 @@ public void setConfig(Configuration config, Configuration oldConfig, Configurati } } - public boolean isUserEntitled(String issn, String institution, String start, String end) throws IOException { + public synchronized boolean isUserEntitled(String issn, String institution, String start, String end) throws IOException { Object result = this.cache.get("isUserEntitled", issn, institution, start, end); if(result == null) { result = this.client.isUserEntitled(issn, institution, start, end); @@ -43,7 +44,7 @@ public boolean isUserEntitled(String issn, String institution, String start, Str return (Boolean) result; } - public String getPublisher(String issn, String institution, String start, String end) throws IOException { + public synchronized String getPublisher(String issn, String institution, String start, String end) throws IOException { Object result = this.cache.get("getPublisher", issn, institution, start, end); if(result == null) { result = this.client.getPublisher(issn, institution, start, end); @@ -52,7 +53,7 @@ public String getPublisher(String issn, String institution, String start, String return (String) result; } - public PublisherWorkflow getPublisherWorkflow(String publisherName) throws IOException { + public synchronized PublisherWorkflow getPublisherWorkflow(String publisherName) throws IOException { Object result = this.cache.get("getPublisherWorkflow", publisherName); if(result == null) { result = this.client.getPublisherWorkflow(publisherName); @@ -61,7 +62,7 @@ public PublisherWorkflow getPublisherWorkflow(String publisherName) throws IOExc return (PublisherWorkflow) result; } - public String getInstitution(String scope) throws IOException { + public synchronized String getInstitution(String scope) throws IOException { Object result = this.cache.get("getInstitution", scope); if(result == null) { result = this.client.getInstitution(scope); @@ -69,6 +70,5 @@ public String getInstitution(String scope) throws IOException { } return (String) result; } - } diff --git a/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java b/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java index afdf3bf0aa1..be18b0be2fa 100644 --- a/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java +++ b/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java @@ -4,7 +4,6 @@ import java.net.URISyntaxException; import java.text.DateFormat; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -13,6 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.JsonNode; +import org.apache.commons.lang3.time.FastDateFormat; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URIBuilder; import org.apache.http.message.BasicNameValuePair; @@ -34,7 +34,7 @@ public class KeepsafeEntitlementRegistryClient extends BaseLockssManager impleme static final String DEFAULT_ER_URI = ""; public static final String PARAM_ER_APIKEY = PREFIX + "apiKey"; static final String DEFAULT_ER_APIKEY = ""; - private static final DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); + private static final FastDateFormat dateFormat = FastDateFormat.getInstance("yyyyMMdd"); private ObjectMapper objectMapper; private String erUri; From f159cdbdb38e1609a03ceeaf66f854a5748f1dcf Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Thu, 17 Aug 2017 15:08:19 +0100 Subject: [PATCH 07/13] Use connection pool when talking to entilement registry --- .../KeepsafeEntitlementRegistryClient.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java b/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java index be18b0be2fa..0a05daf4797 100644 --- a/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java +++ b/src/org/lockss/entitlement/KeepsafeEntitlementRegistryClient.java @@ -20,10 +20,12 @@ import org.lockss.app.BaseLockssManager; import org.lockss.app.ConfigurableManager; import org.lockss.config.Configuration; +import org.lockss.util.Constants; import org.lockss.util.IOUtil; import org.lockss.util.Logger; import org.lockss.util.UrlUtil; import org.lockss.util.urlconn.LockssUrlConnection; +import org.lockss.util.urlconn.LockssUrlConnectionPool; public class KeepsafeEntitlementRegistryClient extends BaseLockssManager implements EntitlementRegistryClient, ConfigurableManager { @@ -34,20 +36,35 @@ public class KeepsafeEntitlementRegistryClient extends BaseLockssManager impleme static final String DEFAULT_ER_URI = ""; public static final String PARAM_ER_APIKEY = PREFIX + "apiKey"; static final String DEFAULT_ER_APIKEY = ""; + public static final String PARAM_CONNECT_TIMEOUT = PREFIX + "timeout.connect"; + static final long DEFAULT_CONNECT_TIMEOUT = 30 * Constants.SECOND; + public static final String PARAM_DATA_TIMEOUT = PREFIX + "timeout.data"; + static final long DEFAULT_DATA_TIMEOUT = 120 * Constants.SECOND; + public static final String PARAM_CLOSE_IDLE_CONNECTION_IDLE_TIME = PREFIX + "closeIdleConnections.idleTime"; + static final long DEFAULT_CLOSE_IDLE_CONNECTION_IDLE_TIME = 10 * Constants.MINUTE; + private static final FastDateFormat dateFormat = FastDateFormat.getInstance("yyyyMMdd"); + private LockssUrlConnectionPool connectionPool; + private long paramCloseIdleConnectionsIdleTime = DEFAULT_CLOSE_IDLE_CONNECTION_IDLE_TIME; private ObjectMapper objectMapper; private String erUri; private String apiKey; public KeepsafeEntitlementRegistryClient() { this.objectMapper = new ObjectMapper(); + this.connectionPool = new LockssUrlConnectionPool(); } public void setConfig(Configuration config, Configuration oldConfig, Configuration.Differences diffs) { if (diffs.contains(PREFIX)) { erUri = config.get(PARAM_ER_URI, DEFAULT_ER_URI); apiKey = config.get(PARAM_ER_APIKEY, DEFAULT_ER_APIKEY); + long connectTimeout = config.getTimeInterval(PARAM_CONNECT_TIMEOUT, DEFAULT_CONNECT_TIMEOUT); + long dataTimeout = config.getTimeInterval(PARAM_DATA_TIMEOUT, DEFAULT_DATA_TIMEOUT); + paramCloseIdleConnectionsIdleTime = config.getTimeInterval(PARAM_CLOSE_IDLE_CONNECTION_IDLE_TIME, DEFAULT_CLOSE_IDLE_CONNECTION_IDLE_TIME); + connectionPool.setConnectTimeout(connectTimeout); + connectionPool.setDataTimeout(dataTimeout); } } @@ -198,12 +215,13 @@ else if (responseCode == 204) { if(connection != null) { IOUtil.safeRelease(connection); } + connectionPool.closeIdleConnections(paramCloseIdleConnectionsIdleTime); } } // protected so that it can be overriden with mock connections in tests protected LockssUrlConnection openConnection(String url) throws IOException { - return UrlUtil.openConnection(url); + return UrlUtil.openConnection(url, connectionPool); } protected static List mapToPairs(Map params) { From 44ecd15f815cfff57e42d58cdc7140e9d4b7515b Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Mon, 21 Aug 2017 16:15:23 +0100 Subject: [PATCH 08/13] Add comments about openConnection methods --- .../lockss/servlet/EntitlementCheckServeContent.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/org/lockss/servlet/EntitlementCheckServeContent.java b/src/org/lockss/servlet/EntitlementCheckServeContent.java index 614cedb9487..5975d5c8807 100644 --- a/src/org/lockss/servlet/EntitlementCheckServeContent.java +++ b/src/org/lockss/servlet/EntitlementCheckServeContent.java @@ -205,16 +205,18 @@ protected void handleAuRequest() throws IOException { super.handleAuRequest(); } - protected LockssUrlConnection doOpenConnection(String url, LockssUrlConnectionPool pool) throws IOException { - return super.openConnection(url, pool); - } - + @Override protected LockssUrlConnection openConnection(String url, LockssUrlConnectionPool pool) throws IOException { LockssUrlConnection conn = doOpenConnection(url, pool); conn.addRequestProperty(INSTITUTION_HEADER, (String) getSession().getAttribute(INSTITUTION_SCOPE_SESSION_KEY)); return conn; } + // Extracted out so that the connection can be mocked in test classes + protected LockssUrlConnection doOpenConnection(String url, LockssUrlConnectionPool pool) throws IOException { + return super.openConnection(url, pool); + } + protected void handleEntitlementRegistryErrorUrlRequest(String missingUrl) throws IOException { handleUrlRequestError(missingUrl, PubState.KnownDown, "An error occurred trying to access the requested URL on this LOCKSS box. This may be temporary and you may wish to report this, and try again later. ", HttpResponse.__503_Service_Unavailable, "entitlement registry error"); From 46c58d6bcb38bb75bd8a2714baf458ae6ca3386e Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Tue, 22 Aug 2017 09:55:08 +0100 Subject: [PATCH 09/13] Keepsafe config now enabled with org.lockss.entitlement.keepsafe.enabled, not org.lockss.platform.project --- src/org/lockss/app/LockssDaemon.java | 5 ++++- .../org/lockss/servlet/TestEntitlementCheckServeContent.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/org/lockss/app/LockssDaemon.java b/src/org/lockss/app/LockssDaemon.java index 59daf1daf97..0914c6fad26 100644 --- a/src/org/lockss/app/LockssDaemon.java +++ b/src/org/lockss/app/LockssDaemon.java @@ -103,6 +103,9 @@ public class LockssDaemon extends LockssApp { static final long DEFAULT_DAEMON_DEADLINE_REASONABLE_FUTURE = 20 * Constants.WEEK; + public static final String PARAM_KEEPSAFE_ENABLED = Configuration.PREFIX + "entitlement.keepsafe.enabled"; + static final boolean DEFAULT_KEEPSAFE_ENABLED = false; + /** List of local IP addresses to which to bind listen sockets for * servers (admin ui, content, proxy). If not set, servers listen on all * interfaces. Does not affect the port on which various servers listen. @@ -991,7 +994,7 @@ protected void setConfig(Configuration config, Configuration prevConfig, testingMode = config.get(PARAM_TESTING_MODE); String proj = ConfigManager.getPlatformProject(); isClockss = "clockss".equalsIgnoreCase(proj); - isKeepsafe = "keepsafe".equalsIgnoreCase(proj); + isKeepsafe = config.getBoolean(PARAM_KEEPSAFE_ENABLED, DEFAULT_KEEPSAFE_ENABLED); super.setConfig(config, prevConfig, changedKeys); } diff --git a/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java b/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java index b3b3d589339..4db9b675903 100644 --- a/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java +++ b/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java @@ -98,7 +98,7 @@ public void setUp() throws Exception { pluginMgr.initService(theDaemon); pluginMgr.startService(); - ConfigurationUtil.addFromArgs(ConfigManager.PARAM_PLATFORM_PROJECT, "keepsafe"); + ConfigurationUtil.addFromArgs(LockssDaemon.PARAM_KEEPSAFE_ENABLED, "true"); ConfigurationUtil.addFromArgs(EntitlementCheckServeContent.PARAM_MISSING_FILE_ACTION, "Redirect"); ConfigurationUtil.addFromArgs(EntitlementCheckServeContent.PARAM_MOCK_SCOPE, "true"); //ConfigurationUtil.addFromArgs("org.lockss.log.default.level", "debug3"); From eb570c403cce04902cca62a0335b26f0d377a5d5 Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Fri, 15 Sep 2017 09:11:40 +0100 Subject: [PATCH 10/13] Add test where bibliographic information is retrieved from DB --- .../TestEntitlementCheckServeContent.java | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java b/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java index 4db9b675903..a60fa1c85ee 100644 --- a/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java +++ b/test/src/org/lockss/servlet/TestEntitlementCheckServeContent.java @@ -32,8 +32,12 @@ of this software and associated documentation files (the "Software"), to deal package org.lockss.servlet; +import static org.lockss.db.SqlConstants.*; + import java.io.IOException; import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; import java.util.Arrays; import java.util.ArrayList; import java.util.HashMap; @@ -52,12 +56,15 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.config.Tdb; import org.lockss.config.TdbAu; import org.lockss.daemon.TitleConfig; +import org.lockss.db.DbManager; +import org.lockss.entitlement.EntitlementRegistryClient; +import org.lockss.entitlement.PublisherWorkflow; +import org.lockss.extractor.MetadataField; +import org.lockss.metadata.MetadataManager; import org.lockss.plugin.ArchivalUnit; import org.lockss.plugin.CachedUrl; import org.lockss.plugin.Plugin; import org.lockss.plugin.PluginManager; -import org.lockss.entitlement.EntitlementRegistryClient; -import org.lockss.entitlement.PublisherWorkflow; import org.lockss.test.ConfigurationUtil; import org.lockss.test.MockArchivalUnit; import org.lockss.test.MockCachedUrl; @@ -66,6 +73,7 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.test.MockNodeManager; import org.lockss.test.MockPlugin; import org.lockss.test.StringInputStream; +import org.lockss.util.Logger; import org.lockss.util.urlconn.LockssUrlConnection; import org.lockss.util.urlconn.LockssUrlConnectionPool; @@ -81,6 +89,8 @@ public class TestEntitlementCheckServeContent extends LockssServletTestCase { private MockPluginManager pluginMgr = null; private EntitlementRegistryClient entitlementRegistryClient = null; + private DbManager dbManager; + private MetadataManager metadataManager; public void setUp() throws Exception { super.setUp(); @@ -95,9 +105,14 @@ public void setUp() throws Exception { theDaemon.setAusStarted(true); theDaemon.getRemoteApi().startService(); + dbManager = getTestDbManager(tempDirPath); + pluginMgr.initService(theDaemon); pluginMgr.startService(); + metadataManager = theDaemon.getMetadataManager(); + metadataManager.startService(); + ConfigurationUtil.addFromArgs(LockssDaemon.PARAM_KEEPSAFE_ENABLED, "true"); ConfigurationUtil.addFromArgs(EntitlementCheckServeContent.PARAM_MISSING_FILE_ACTION, "Redirect"); ConfigurationUtil.addFromArgs(EntitlementCheckServeContent.PARAM_MOCK_SCOPE, "true"); @@ -265,6 +280,44 @@ public void testCachedUrlLibraryNotification() throws Exception { assertTrue(resp1.getText().contains("

You are not authorised to access the requested URL on this LOCKSS box. Select link1 to view it at the publisher:

http://publisher.org/test_journal/")); } + public void testCachedUrlIndexed() throws Exception { + initServletRunner(); + + Connection conn = dbManager.getConnection(); + Long publisherSeq = dbManager.addPublisher(conn, "Wiley"); + String proprietaryId = null; + Long mdItemTypeSeq = metadataManager.findMetadataItemType(conn, MetadataField.ARTICLE_TYPE_JOURNALARTICLE); + Long auMdSeq = null; + String date = "2014"; + String coverage = null; + long fetchTime = 0; + Long parentSeq = metadataManager.findOrCreateJournal(conn, publisherSeq, "07402783", "07402783", "Air and Space", proprietaryId); + Long mdItemSeq = metadataManager.addMdItem(conn, parentSeq, mdItemTypeSeq, auMdSeq, date, coverage, fetchTime); + metadataManager.addMdItemUrl(conn, mdItemSeq, MetadataManager.ACCESS_URL_FEATURE, "http://publisher.org/test_journal/"); + dbManager.commitOrRollback(conn, Logger.getLogger(DbManager.class)); + + Properties props = new Properties(); + props.setProperty("issn", ""); + props.setProperty("eissn", ""); + props.setProperty("attributes.year", ""); + pluginMgr.addAu(makeAu(props)); + Mockito.when(entitlementRegistryClient.getInstitution("ed.ac.uk")).thenReturn("03bd5fc6-97f0-11e4-b270-8932ea886a12"); + Mockito.when(entitlementRegistryClient.isUserEntitled("07402783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "2014", "2014")).thenReturn(true); + Mockito.when(entitlementRegistryClient.getPublisher("07402783", "03bd5fc6-97f0-11e4-b270-8932ea886a12", "2014", "2014")).thenReturn("33333333-0000-0000-0000-000000000000"); + Mockito.when(entitlementRegistryClient.getPublisherWorkflow("33333333-0000-0000-0000-000000000000")).thenReturn(PublisherWorkflow.PRIMARY_PUBLISHER); + WebRequest request = new GetMethodWebRequest("http://null/EntitlementCheckServeContent?scope=ed.ac.uk&url=http%3A%2F%2Fpublisher.org%2Ftest_journal%2F" ); + InvocationContext ic = sClient.newInvocation(request); + MockEntitlementCheckServeContent snsc = (MockEntitlementCheckServeContent) ic.getServlet(); + LockssUrlConnection connection = mockConnection(200, "BlahFetched content"); + snsc.expectRequest("http://publisher.org/test_journal/", connection); + + WebResponse resp1 = sClient.getResponse(request); + assertResponseOk(resp1); + assertEquals("BlahFetched content", resp1.getText()); + Mockito.verify(connection).addRequestProperty("X-Forwarded-For", "127.0.0.1"); + Mockito.verify(connection).addRequestProperty("X-Lockss-Institution", "ed.ac.uk"); + } + public void testUnauthorisedUrl() throws Exception { initServletRunner(); pluginMgr.addAu(makeAu()); From 52c95145ef87f8fffed9e2871bc839e13cd546c0 Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Fri, 15 Sep 2017 09:33:43 +0100 Subject: [PATCH 11/13] Avoid shadowing instance variables with parameters --- .../servlet/EntitlementCheckServeContent.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/org/lockss/servlet/EntitlementCheckServeContent.java b/src/org/lockss/servlet/EntitlementCheckServeContent.java index 5975d5c8807..9539d4e1718 100644 --- a/src/org/lockss/servlet/EntitlementCheckServeContent.java +++ b/src/org/lockss/servlet/EntitlementCheckServeContent.java @@ -108,8 +108,8 @@ static void setConfig(Configuration config, } } - protected boolean isNeverProxyForAu(ArchivalUnit au) { - return super.isNeverProxyForAu(au) || workflow == PublisherWorkflow.PRIMARY_LOCKSS; + protected boolean isNeverProxyForAu(ArchivalUnit archivalUnit) { + return super.isNeverProxyForAu(archivalUnit) || workflow == PublisherWorkflow.PRIMARY_LOCKSS; } /** @@ -233,14 +233,14 @@ void updateInstitution() throws IOException { institution = entitlementRegistry.getInstitution(institutionScope); } - boolean isUserEntitled(ArchivalUnit au) throws IOException, IllegalArgumentException { - validateBibInfo(cu, au); + boolean isUserEntitled(ArchivalUnit archivalUnit) throws IOException, IllegalArgumentException { + validateBibInfo(cu, archivalUnit); return entitlementRegistry.isUserEntitled(issn, institution, start, end); } - PublisherWorkflow getPublisherWorkflow(ArchivalUnit au) throws IOException, IllegalArgumentException { - validateBibInfo(cu, au); + PublisherWorkflow getPublisherWorkflow(ArchivalUnit archivalUnit) throws IOException, IllegalArgumentException { + validateBibInfo(cu, archivalUnit); String publisher = entitlementRegistry.getPublisher(issn, institution, start, end); if(StringUtil.isNullString(publisher)) { @@ -299,19 +299,19 @@ private void setBibInfoFromMetadata(ArticleMetadata md) { setBibInfoFromBibliographicItem(item); } - private void setBibInfoFromTdb(ArchivalUnit au) throws IllegalArgumentException { + private void setBibInfoFromTdb(ArchivalUnit archivalUnit) throws IllegalArgumentException { log.debug2("Setting bib info from TDB"); if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { log.debug2("Bib info already set"); return; } - if(au == null) { + if(archivalUnit == null) { log.debug2("No ArchivalUnit"); return; } - TdbAu tdbAu = au.getTdbAu(); + TdbAu tdbAu = archivalUnit.getTdbAu(); if(tdbAu == null) { log.debug2("No TdbAu"); return; @@ -339,14 +339,14 @@ private void setBibInfoFromTdb(ArchivalUnit au) throws IllegalArgumentException } } - private void setBibInfoFromMetadataDB(CachedUrl cu) { + private void setBibInfoFromMetadataDB(CachedUrl cachedUrl) { log.debug2("Setting bib info from database"); if(!StringUtil.isNullString(issn) && !StringUtil.isNullString(start) && !StringUtil.isNullString(end)) { log.debug2("Bib info already set"); return; } - if(cu == null) { + if(cachedUrl == null) { log.debug2("No CachedUrl"); return; } @@ -373,7 +373,7 @@ private void setBibInfoFromMetadataDB(CachedUrl cu) { from.append("," + ISSN_TABLE + " i"); where.append("u." + URL_COLUMN + "= ?"); - args.add(cu.getUrl()); + args.add(cachedUrl.getUrl()); where.append(" and u." + FEATURE_COLUMN + "='Access'"); where.append(" and u." + MD_ITEM_SEQ_COLUMN + "="); @@ -421,9 +421,9 @@ else if(L_ISSN_TYPE.equals(issnType)){ } } - private void validateBibInfo(CachedUrl cu, ArchivalUnit au) { - setBibInfoFromMetadataDB(cu); - setBibInfoFromTdb(au); + private void validateBibInfo(CachedUrl cachedUrl, ArchivalUnit archivalUnit) { + setBibInfoFromMetadataDB(cachedUrl); + setBibInfoFromTdb(archivalUnit); if(StringUtil.isNullString(issn)) { throw new IllegalArgumentException("ArchivalUnit has no ISSN"); From 9708b92671bc9467d9698c0397df1d22c318898c Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Fri, 15 Sep 2017 10:30:34 +0100 Subject: [PATCH 12/13] Use CachedUrl under examination when checking entitlement --- .../servlet/EntitlementCheckServeContent.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/org/lockss/servlet/EntitlementCheckServeContent.java b/src/org/lockss/servlet/EntitlementCheckServeContent.java index 9539d4e1718..7fe9605e8eb 100644 --- a/src/org/lockss/servlet/EntitlementCheckServeContent.java +++ b/src/org/lockss/servlet/EntitlementCheckServeContent.java @@ -137,7 +137,7 @@ protected boolean setCachedUrlAndAu() throws IOException { if(cachedUrls != null && !cachedUrls.isEmpty()) { for(CachedUrl cachedUrl: cachedUrls) { try { - if(isUserEntitled(cachedUrl.getArchivalUnit())) { + if(isUserEntitled(cachedUrl, cachedUrl.getArchivalUnit())) { cu = cachedUrl; au = cu.getArchivalUnit(); if (log.isDebug3()) log.debug("cu: " + cu + " au: " + au); @@ -179,11 +179,11 @@ protected boolean setCachedUrlAndAu() throws IOException { */ protected void handleAuRequest() throws IOException { try { - if (!isUserEntitled(au)) { + if (!isUserEntitled(cu, au)) { handleUnauthorisedUrlRequest(url); return; } - workflow = getPublisherWorkflow(au); + workflow = getPublisherWorkflow(cu, au); if (workflow == PublisherWorkflow.LIBRARY_NOTIFICATION) { handleUnauthorisedUrlRequest(url); return; @@ -233,14 +233,14 @@ void updateInstitution() throws IOException { institution = entitlementRegistry.getInstitution(institutionScope); } - boolean isUserEntitled(ArchivalUnit archivalUnit) throws IOException, IllegalArgumentException { - validateBibInfo(cu, archivalUnit); + boolean isUserEntitled(CachedUrl cachedUrl, ArchivalUnit archivalUnit) throws IOException, IllegalArgumentException { + validateBibInfo(cachedUrl, archivalUnit); return entitlementRegistry.isUserEntitled(issn, institution, start, end); } - PublisherWorkflow getPublisherWorkflow(ArchivalUnit archivalUnit) throws IOException, IllegalArgumentException { - validateBibInfo(cu, archivalUnit); + PublisherWorkflow getPublisherWorkflow(CachedUrl cachedUrl, ArchivalUnit archivalUnit) throws IOException, IllegalArgumentException { + validateBibInfo(cachedUrl, archivalUnit); String publisher = entitlementRegistry.getPublisher(issn, institution, start, end); if(StringUtil.isNullString(publisher)) { From 69225523615c0957d59c1ff563b341261c36ac85 Mon Sep 17 00:00:00 2001 From: Steven Davies Date: Thu, 14 Sep 2017 14:48:56 +0100 Subject: [PATCH 13/13] Extracted metadata query into MetadataManager --- src/org/lockss/metadata/MetadataManager.java | 36 +++++++++ .../lockss/metadata/MetadataManagerSql.java | 81 +++++++++++++++++++ .../servlet/EntitlementCheckServeContent.java | 69 ++-------------- 3 files changed, 123 insertions(+), 63 deletions(-) diff --git a/src/org/lockss/metadata/MetadataManager.java b/src/org/lockss/metadata/MetadataManager.java index f0ffd1c5612..6a54c325c19 100644 --- a/src/org/lockss/metadata/MetadataManager.java +++ b/src/org/lockss/metadata/MetadataManager.java @@ -50,12 +50,14 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.db.DbException; import org.lockss.db.DbManager; import org.lockss.db.PkNamePair; +import org.lockss.exporter.biblio.BibliographicItem; import org.lockss.extractor.ArticleMetadataExtractor; import org.lockss.extractor.BaseArticleMetadataExtractor; import org.lockss.extractor.MetadataField; import org.lockss.extractor.MetadataTarget; import org.lockss.plugin.ArchivalUnit; import org.lockss.plugin.AuUtil; +import org.lockss.plugin.CachedUrl; import org.lockss.plugin.Plugin; import org.lockss.plugin.Plugin.Feature; import org.lockss.plugin.PluginManager; @@ -4533,4 +4535,38 @@ public Collection> getDbArchivalUnitsDeletedFromDaemon() public boolean deleteDbAu(Long auSeq, String auKey) throws DbException { return getMetadataManagerSql().removeAu(auSeq, auKey); } + + /** + * Finds the bibliographic information held about a CachedUrl. + * + * @param cu + * A CachedUrl to lookup. + * @return a BibliographicItem with appropriate infomation populated if the CachedUrl was found in the database, + * null otherwise. + * @throws DbException + * if any problem occurred accessing the database. + */ + public BibliographicItem getCUBibliographicInfo(CachedUrl cu) throws DbException { + // Get a connection to the database. + Connection conn = dbManager.getConnection(); + + return getCUBibliographicInfo(conn, cu); + } + + /** + * Finds the bibliographic information held about a CachedUrl. + * + * @param conn + * A Connection with the database connection to be used. + * @param cu + * A CachedUrl to lookup. + * @return a BibliographicItem with appropriate infomation populated if the CachedUrl was found in the database, + * null otherwise. + * @throws DbException + * if any problem occurred accessing the database. + */ + public BibliographicItem getCUBibliographicInfo(Connection conn, CachedUrl cu) throws DbException { + return mdManagerSql.getCUBibliographicInfo(conn, cu); + } + } diff --git a/src/org/lockss/metadata/MetadataManagerSql.java b/src/org/lockss/metadata/MetadataManagerSql.java index 57a519ad0c4..801d693de65 100644 --- a/src/org/lockss/metadata/MetadataManagerSql.java +++ b/src/org/lockss/metadata/MetadataManagerSql.java @@ -46,9 +46,12 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.db.DbException; import org.lockss.db.DbManager; import org.lockss.db.PkNamePair; +import org.lockss.exporter.biblio.BibliographicItem; +import org.lockss.exporter.biblio.BibliographicItemImpl; import org.lockss.metadata.MetadataManager.PrioritizedAuId; import org.lockss.plugin.ArchivalUnit; import org.lockss.plugin.AuUtil; +import org.lockss.plugin.CachedUrl; import org.lockss.plugin.PluginManager; import org.lockss.util.KeyPair; import org.lockss.util.Logger; @@ -1575,6 +1578,21 @@ public class MetadataManagerSql { + AU_SEQ_COLUMN + " = ?" + " and " + AU_KEY_COLUMN + " = ?"; + // Query to fetch the date and ISSN of a CachedUrl + private static final String GET_CU_BIB_INFO_QUERY = "select distinct " + + "mi." + DATE_COLUMN + + ", i." + ISSN_COLUMN + + ", i." + ISSN_TYPE_COLUMN + + " from " + + URL_TABLE + " u" + + ", " + MD_ITEM_TABLE + " mi" + + ", " + ISSN_TABLE + " i" + + " where " + + "u." + URL_COLUMN + "= ?" + + " and u." + FEATURE_COLUMN + "='" + MetadataManager.ACCESS_URL_FEATURE + "'" + + " and u." + MD_ITEM_SEQ_COLUMN + "=" + "mi." + MD_ITEM_SEQ_COLUMN + + " and i." + MD_ITEM_SEQ_COLUMN + "=" + "mi." + PARENT_SEQ_COLUMN; + private DbManager dbManager; private MetadataManager metadataManager; @@ -7215,4 +7233,67 @@ private boolean removeAu(Connection conn, Long auSeq, String auKey) log.debug2(DEBUG_HEADER + "result = " + (deletedCount > 0)); return deletedCount > 0; } + + /** + * Finds the bibliographic information held about a CachedUrl. + * + * @param conn + * A Connection with the database connection to be used. + * @param cu + * A CachedUrl to lookup. + * @return a BibliographicItem with appropriate infomation populated if the CachedUrl was found in the database, + * null otherwise. + * @throws DbException + * if any problem occurred accessing the database. + */ + public BibliographicItem getCUBibliographicInfo(Connection conn, CachedUrl cu) throws DbException { + final String DEBUG_HEADER = "getCUBibliographicInfo(): "; + if (log.isDebug2()) log.debug2(DEBUG_HEADER + "cu = " + cu); + String query = GET_CU_BIB_INFO_QUERY; + PreparedStatement stmt = null; + ResultSet resultSet = null; + try { + // return all related values for debugging purposes + + stmt = dbManager.prepareStatement(conn, query); + + ArrayList args = new ArrayList(); + args.add(cu.getUrl()); + for (int i = 0; i < args.size(); i++) { + if (log.isDebug3()) log.debug3(DEBUG_HEADER + "arg " + i + " = " + args.get(i)); + stmt.setString(i + 1, args.get(i)); + } + + resultSet = dbManager.executeQuery(stmt); + + boolean foundResults = false; + BibliographicItemImpl item = new BibliographicItemImpl(); + while ( resultSet.next() ) { + foundResults = true; + String year = resultSet.getString(1); + String issn = resultSet.getString(2); + String issnType = resultSet.getString(3); + item.setYear(year); + if(P_ISSN_TYPE.equals(issnType)){ + item.setPrintIssn(issn); + } + else if(E_ISSN_TYPE.equals(issnType)){ + item.setEissn(issn); + } + else if(L_ISSN_TYPE.equals(issnType)){ + item.setIssnL(issn); + } + } + if (log.isDebug2()) log.debug2(DEBUG_HEADER + "result = " + item); + return foundResults ? item : null; + } catch (SQLException sqle) { + String message = "Cannot get the BibliogaphicItem"; + log.error(message, sqle); + log.error("SQL = '" + query + "'."); + throw new DbException(message, sqle); + } finally { + DbManager.safeCloseResultSet(resultSet); + DbManager.safeCloseStatement(stmt); + } + } } diff --git a/src/org/lockss/servlet/EntitlementCheckServeContent.java b/src/org/lockss/servlet/EntitlementCheckServeContent.java index 7fe9605e8eb..3c3e19c29f7 100644 --- a/src/org/lockss/servlet/EntitlementCheckServeContent.java +++ b/src/org/lockss/servlet/EntitlementCheckServeContent.java @@ -52,6 +52,7 @@ of this software and associated documentation files (the "Software"), to deal import org.lockss.exporter.biblio.*; import org.lockss.exporter.counter.*; import org.lockss.extractor.*; +import org.lockss.metadata.MetadataManager; import org.lockss.plugin.*; import org.lockss.plugin.PluginManager.CuContentReq; import org.lockss.entitlement.EntitlementRegistryClient; @@ -81,6 +82,7 @@ public class EntitlementCheckServeContent extends ServeContent { private String start; private String end; private EntitlementRegistryClient entitlementRegistry; + private MetadataManager metadataManager; // don't hold onto objects after request finished protected void resetLocals() { @@ -96,6 +98,7 @@ public void init(ServletConfig config) throws ServletException { super.init(config); LockssDaemon daemon = getLockssDaemon(); entitlementRegistry = daemon.getCachedEntitlementRegistryClient(); + metadataManager = daemon.getMetadataManager(); } /** Called by ServletUtil.setConfig() */ @@ -351,73 +354,13 @@ private void setBibInfoFromMetadataDB(CachedUrl cachedUrl) { return; } - Connection conn = null; - OpenUrlInfo resolved = null; try { - LockssDaemon daemon = getLockssDaemon(); - conn = daemon.getDbManager().getConnection(); - - StringBuilder select = new StringBuilder("select distinct "); - StringBuilder from = new StringBuilder(" from "); - StringBuilder where = new StringBuilder(" where "); - ArrayList args = new ArrayList(); - - // return all related values for debugging purposes - select.append("mi1." + DATE_COLUMN); - select.append(",i." + ISSN_COLUMN); - select.append(",i." + ISSN_TYPE_COLUMN); - - from.append(URL_TABLE + " u"); - from.append(", " + MD_ITEM_TABLE + " mi1"); // publication md_item - from.append("," + MD_ITEM_TABLE + " mi2"); // article md_item - from.append("," + ISSN_TABLE + " i"); - - where.append("u." + URL_COLUMN + "= ?"); - args.add(cachedUrl.getUrl()); - where.append(" and u." + FEATURE_COLUMN + "='Access'"); - - where.append(" and u." + MD_ITEM_SEQ_COLUMN + "="); - where.append("mi1." + MD_ITEM_SEQ_COLUMN); - - where.append(" and mi1." + PARENT_SEQ_COLUMN + "="); - where.append("mi2." + MD_ITEM_SEQ_COLUMN); - - where.append(" and i." + MD_ITEM_SEQ_COLUMN + "="); - where.append("mi2." + MD_ITEM_SEQ_COLUMN); - - String query = select.toString() + from.toString() + where.toString(); - - PreparedStatement stmt = daemon.getDbManager().prepareStatement(conn, query); - - for (int i = 0; i < args.size(); i++) { - log.debug3("query arg: " + args.get(i)); - stmt.setString(i + 1, args.get(i)); + BibliographicItem item = metadataManager.getCUBibliographicInfo(cachedUrl); + if ( item != null ) { + setBibInfoFromBibliographicItem(item); } - - ResultSet resultSet = daemon.getDbManager().executeQuery(stmt); - - BibliographicItemImpl item = new BibliographicItemImpl(); - while ( resultSet.next() ) { - String year = resultSet.getString(1); - String issn = resultSet.getString(2); - String issnType = resultSet.getString(3); - item.setYear(year); - if(P_ISSN_TYPE.equals(issnType)){ - item.setPrintIssn(issn); - } - else if(E_ISSN_TYPE.equals(issnType)){ - item.setEissn(issn); - } - else if(L_ISSN_TYPE.equals(issnType)){ - item.setIssnL(issn); - } - } - setBibInfoFromBibliographicItem(item); - } catch (DbException e) { log.error("Error fetching metadata", e); - } catch (SQLException e) { - log.error("Error fetching metadata", e); } }