From 52f19e442b00d3a223e14f499bd4d092fe7b2645 Mon Sep 17 00:00:00 2001 From: Arnaud Charleroy Date: Mon, 24 Feb 2020 16:37:58 +0100 Subject: [PATCH 1/3] optimization test --- .../opensilex/service/dao/DatasetDAO.java | 2 +- .../service/dao/ScientificObjectRdf4jDAO.java | 27 +++++++++---- .../ScientificObjectResourceService.java | 21 +++++++--- .../brapi/StudiesResourceService.java | 4 +- .../brapi/VariableResourceService.java | 2 +- .../src/main/profiles/dev/config.properties | 39 ++++++++++++------- 6 files changed, 65 insertions(+), 30 deletions(-) diff --git a/phis2-ws/src/main/java/opensilex/service/dao/DatasetDAO.java b/phis2-ws/src/main/java/opensilex/service/dao/DatasetDAO.java index 522365c78..66dd2a40b 100644 --- a/phis2-ws/src/main/java/opensilex/service/dao/DatasetDAO.java +++ b/phis2-ws/src/main/java/opensilex/service/dao/DatasetDAO.java @@ -205,7 +205,7 @@ protected BasicDBObject prepareSearchProvenance() { private void updateScientificObjectsWithExperimentsScientificObjects() { ScientificObjectRdf4jDAO agronomicalObjectDao = new ScientificObjectRdf4jDAO(); - ArrayList scientificObjectsSearched = agronomicalObjectDao.find(null, null, null, null, experiment, null); + ArrayList scientificObjectsSearched = agronomicalObjectDao.find(null, null, null, null, experiment, null, true); scientificObjectsSearched.forEach((scientificObject) -> { this.scientificObjects.add(scientificObject.getUri()); diff --git a/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java b/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java index 9215bdf7c..512e41c77 100644 --- a/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java +++ b/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java @@ -606,8 +606,11 @@ public ArrayList findScientificObjectProperties(String uri) { * @param alias * @return scientific objects list, result of the user query, empty if no result */ - public ArrayList find(Integer page, Integer pageSize, String uri, String rdfType, String experiment, String alias) { + public ArrayList find(Integer page, Integer pageSize, String uri, String rdfType, String experiment, String alias, Boolean withProperties) { try { + long startFind = System.nanoTime(); + + SPARQLQueryBuilder sparqlQuery = prepareSearchQuery(false, page, pageSize, uri, rdfType, experiment, alias); //SILEX:test //For pool connection issues @@ -618,7 +621,9 @@ public ArrayList find(Integer page, Integer pageSize, String u TupleQuery tupleQuery = getConnection().prepareTupleQuery(QueryLanguage.SPARQL, sparqlQuery.toString()); Map foundedScientificObjects = new HashMap<>(); - + long query = System.nanoTime(); + + System.out.println("Execution time in milliseconds (query) : " + ( query - startFind) / 1000000); try (TupleQueryResult result = tupleQuery.evaluate()) { while (result.hasNext()) { BindingSet bindingSet = result.next(); @@ -652,14 +657,17 @@ public ArrayList find(Integer page, Integer pageSize, String u scientificObject.setRdfType(bindingSet.getValue(RDF_TYPE).stringValue()); } } - - //Get scientific object properties - scientificObject.setProperties(findScientificObjectProperties(actualUri)); + if(withProperties){ + //Get scientific object properties + scientificObject.setProperties(findScientificObjectProperties(actualUri)); + } foundedScientificObjects.put(actualUri, scientificObject); } } - + long loopObject = System.nanoTime(); + + System.out.println("Execution time in milliseconds (loopObject) : " + ( loopObject - query) / 1000000); ArrayList scientificObjectsUris = new ArrayList<>(); ArrayList scientificObjects = new ArrayList<>(); foundedScientificObjects.entrySet().forEach((entry) -> { @@ -667,6 +675,8 @@ public ArrayList find(Integer page, Integer pageSize, String u scientificObjectsUris.add(entry.getKey()); }); + long geom = System.nanoTime(); + //Get geometries in relational database ScientificObjectSQLDAO scientificObjectDao = new ScientificObjectSQLDAO(); HashMap geometries = scientificObjectDao.getGeometries(scientificObjectsUris); @@ -674,7 +684,8 @@ public ArrayList find(Integer page, Integer pageSize, String u scientificObjects.forEach((scientificObject) -> { scientificObject.setGeometry(geometries.get(scientificObject.getUri())); }); - + System.out.println("Execution time in milliseconds (geom) : " + ( geom- loopObject) / 1000000); + return scientificObjects; } catch (SQLException ex) { java.util.logging.Logger.getLogger(ScientificObjectRdf4jDAO.class.getName()).log(Level.SEVERE, null, ex); @@ -996,7 +1007,7 @@ public void delete(List objects) throws DAOPersistenceExceptio * null if this scientific object does not exist. */ public ScientificObject getScientificObjectInContext(String uri, String context) { - ArrayList scientificObjects = find(null, null, uri, null, context, null); + ArrayList scientificObjects = find(null, null, uri, null, context, null,true); if (!scientificObjects.isEmpty()) { return scientificObjects.get(0); } else { diff --git a/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java b/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java index 90c0766f9..4f312c2d0 100644 --- a/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java +++ b/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java @@ -255,11 +255,13 @@ public Response getScientificObjectsBySearch( @ApiParam(value = "Search by URI", example = DocumentationAnnotation.EXAMPLE_SCIENTIFIC_OBJECT_URI) @QueryParam("uri") String uri, @ApiParam(value = "Search by experiment URI", example = DocumentationAnnotation.EXAMPLE_EXPERIMENT_URI) @QueryParam("experiment") @URL String experimentURI, @ApiParam(value = "Search by alias", example = DocumentationAnnotation.EXAMPLE_EXPERIMENT_ALIAS) @QueryParam("alias") String alias, - @ApiParam(value = "Search by rdfType", example = DocumentationAnnotation.EXAMPLE_SCIENTIFIC_OBJECT_TYPE) @QueryParam("rdfType") @URL String rdfType + @ApiParam(value = "Search by rdfType", example = DocumentationAnnotation.EXAMPLE_SCIENTIFIC_OBJECT_TYPE) @QueryParam("rdfType") @URL String rdfType, + @ApiParam(value = "Retreive detailled properties", example = "true") @DefaultValue("true") @QueryParam("withProperties") Boolean withProperties + ) { ArrayList scientificObjectsToReturn = new ArrayList<>(); ArrayList scientificObjects = new ArrayList<>(); - + long startTime = System.nanoTime(); ArrayList statusList = new ArrayList<>(); ResultForm getResponse; @@ -270,13 +272,19 @@ public Response getScientificObjectsBySearch( //1. Get count Integer totalCount = scientificObjectDaoSesame.count(uri, rdfType, experimentURI, alias); + long count = System.nanoTime(); + + System.out.println("Execution time in milliseconds (count) : " + ( count - startTime) / 1000000); + // If scientific objects found if(totalCount > 0){ //2. Get list of scientific objects - scientificObjects = scientificObjectDaoSesame.find(page, pageSize, uri, rdfType, experimentURI, alias); + scientificObjects = scientificObjectDaoSesame.find(page, pageSize, uri, rdfType, experimentURI, alias, withProperties); } - + + long get = System.nanoTime(); + System.out.println("Execution time in milliseconds (get) : " + ( get - count) / 1000000); if (scientificObjects == null) { //Request failure getResponse = new ResultForm<>(0, 0, scientificObjectsToReturn, true); return noResultFound(getResponse, statusList); @@ -284,11 +292,14 @@ public Response getScientificObjectsBySearch( getResponse = new ResultForm<>(0, 0, scientificObjectsToReturn, true); return noResultFound(getResponse, statusList); } else { + + //Convert all scientific objects to DTO scientificObjects.forEach((scientificObject) -> { scientificObjectsToReturn.add(new ScientificObjectDTO(scientificObject)); }); - + long scientific = System.nanoTime(); + System.out.println("Execution time in milliseconds (scientific) : " + ( scientific - get) / 1000000); getResponse = new ResultForm<>(scientificObjectDaoSesame.getPageSize(), scientificObjectDaoSesame.getPage(), scientificObjectsToReturn, true, totalCount); if (getResponse.getResult().dataSize() == 0) { return noResultFound(getResponse, statusList); diff --git a/phis2-ws/src/main/java/opensilex/service/resource/brapi/StudiesResourceService.java b/phis2-ws/src/main/java/opensilex/service/resource/brapi/StudiesResourceService.java index 696209a18..a6846ff42 100644 --- a/phis2-ws/src/main/java/opensilex/service/resource/brapi/StudiesResourceService.java +++ b/phis2-ws/src/main/java/opensilex/service/resource/brapi/StudiesResourceService.java @@ -664,7 +664,7 @@ public Response getObservationUnits ( rdfType = Oeso.NAMESPACE + observationLevel; } - ArrayList scientificObjects = scientificObjectsDAO.find(null, null, null, rdfType, studyDbId, null); + ArrayList scientificObjects = scientificObjectsDAO.find(null, null, null, rdfType, studyDbId, null, true); ExperimentSQLDAO experimentDAO = new ExperimentSQLDAO(); experimentDAO.uri = studyDbId; @@ -735,7 +735,7 @@ private ArrayList getObservationsList(StudySQLDAO studyDAO, ArrayList observations = new ArrayList(); ScientificObjectRdf4jDAO objectDAO = new ScientificObjectRdf4jDAO(); - ArrayList objectsList = objectDAO.find(null, null, null, null, studyDAO.studyDbIds.get(0), null); + ArrayList objectsList = objectDAO.find(null, null, null, null, studyDAO.studyDbIds.get(0), null, true); ArrayList variablesList = new ArrayList(); if (variableURIs.isEmpty()) { diff --git a/phis2-ws/src/main/java/opensilex/service/resource/brapi/VariableResourceService.java b/phis2-ws/src/main/java/opensilex/service/resource/brapi/VariableResourceService.java index a3bb033d6..29af19763 100644 --- a/phis2-ws/src/main/java/opensilex/service/resource/brapi/VariableResourceService.java +++ b/phis2-ws/src/main/java/opensilex/service/resource/brapi/VariableResourceService.java @@ -390,7 +390,7 @@ private ArrayList getObservationsList(StudySQLDAO studyDAO, ArrayList observations = new ArrayList(); ScientificObjectRdf4jDAO objectDAO = new ScientificObjectRdf4jDAO(); - ArrayList objectsList = objectDAO.find(null, null, null, null, studyDAO.studyDbIds.get(0), null); + ArrayList objectsList = objectDAO.find(null, null, null, null, studyDAO.studyDbIds.get(0), null, true); ArrayList variablesList = new ArrayList(); if (variableURIs.isEmpty()) { diff --git a/phis2-ws/src/main/profiles/dev/config.properties b/phis2-ws/src/main/profiles/dev/config.properties index 3a2ae8ca5..7db575937 100644 --- a/phis2-ws/src/main/profiles/dev/config.properties +++ b/phis2-ws/src/main/profiles/dev/config.properties @@ -10,26 +10,28 @@ mongo.db=dbname mongo.user= mongo.password= mongo.authdb= +mongo.logDataQuery=true + # PostgreSQL configuration pg.host=localhost -pg.port=5432 -pg.db=database -pg.user=username -pg.password=password +pg.port=5433 +pg.db=sunagri +pg.user=postgres +pg.password=postgres # RDF4J Configuration rdf.base.uri=http://www.opensilex.org/ -rdf.host=localhost +rdf.host=138.102.159.36 rdf.port=8080 rdf.path=rdf4j-server -rdf.infra=platform -rdf.infra.code=DMO -rdf.repo=repository +rdf.infra=sunagri +rdf.infra.code=SUA +rdf.repo=sunagri rdf.vocabulary.context=http://www.opensilex.org/vocabulary/oeso # Webservice configuration -ws.log.dir=/home/tomcat/opensilex/logs +ws.log.dir=/tmp/opensilex/logs ws.host=localhost ws.port=8080 @@ -42,12 +44,23 @@ ws.doc.port=8080 ws.doc.name=opensilex ws.updir.host=localhost -ws.updir.user=username -ws.updir.password=password -ws.updir.doc=/home/opensilex/documents/instance +ws.updir.user=charlero +ws.updir.password=pic3.14 +ws.updir.doc=/tmp/ ws.images.dir=/var/www/html/images ws.images.url=http://localhost/images ws.layers.dir=/var/www/html/layers -ws.layers.url=http://localhost/layers \ No newline at end of file +ws.layers.url=http://localhost/layers + + +opencpu.host=http://localhost:8005 + +shinyproxy.run=true +shinyproxy.host=127.0.0.1 +shinyproxy.port=8004 +# must be a writable directory +shinyproxy.jarFilePath=/tmp/opensilex/shinyproxy/shinyproxy-2.3.0.jar +shinyproxy.webFilePath=https://www.shinyproxy.io/downloads/shinyproxy-2.3.0.jar + From 31e45c08764cdf62af2c967371c9f4cc2d0ed0ca Mon Sep 17 00:00:00 2001 From: Arnaud Charleroy Date: Wed, 26 Feb 2020 17:23:42 +0100 Subject: [PATCH 2/3] remove profiling variables --- .../service/dao/ScientificObjectRdf4jDAO.java | 42 ++++++++----------- .../service/dao/ScientificObjectSQLDAO.java | 5 +-- .../ScientificObjectResourceService.java | 11 ++--- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java b/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java index 512e41c77..5bfe17602 100644 --- a/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java +++ b/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectRdf4jDAO.java @@ -608,8 +608,6 @@ public ArrayList findScientificObjectProperties(String uri) { */ public ArrayList find(Integer page, Integer pageSize, String uri, String rdfType, String experiment, String alias, Boolean withProperties) { try { - long startFind = System.nanoTime(); - SPARQLQueryBuilder sparqlQuery = prepareSearchQuery(false, page, pageSize, uri, rdfType, experiment, alias); //SILEX:test @@ -621,9 +619,7 @@ public ArrayList find(Integer page, Integer pageSize, String u TupleQuery tupleQuery = getConnection().prepareTupleQuery(QueryLanguage.SPARQL, sparqlQuery.toString()); Map foundedScientificObjects = new HashMap<>(); - long query = System.nanoTime(); - System.out.println("Execution time in milliseconds (query) : " + ( query - startFind) / 1000000); try (TupleQueryResult result = tupleQuery.evaluate()) { while (result.hasNext()) { BindingSet bindingSet = result.next(); @@ -665,9 +661,7 @@ public ArrayList find(Integer page, Integer pageSize, String u foundedScientificObjects.put(actualUri, scientificObject); } } - long loopObject = System.nanoTime(); - System.out.println("Execution time in milliseconds (loopObject) : " + ( loopObject - query) / 1000000); ArrayList scientificObjectsUris = new ArrayList<>(); ArrayList scientificObjects = new ArrayList<>(); foundedScientificObjects.entrySet().forEach((entry) -> { @@ -675,17 +669,15 @@ public ArrayList find(Integer page, Integer pageSize, String u scientificObjectsUris.add(entry.getKey()); }); - long geom = System.nanoTime(); - - //Get geometries in relational database + //Get geometries in relational database ScientificObjectSQLDAO scientificObjectDao = new ScientificObjectSQLDAO(); HashMap geometries = scientificObjectDao.getGeometries(scientificObjectsUris); - + scientificObjects.forEach((scientificObject) -> { scientificObject.setGeometry(geometries.get(scientificObject.getUri())); }); - System.out.println("Execution time in milliseconds (geom) : " + ( geom- loopObject) / 1000000); - + + return scientificObjects; } catch (SQLException ex) { java.util.logging.Logger.getLogger(ScientificObjectRdf4jDAO.class.getName()).log(Level.SEVERE, null, ex); @@ -732,7 +724,20 @@ protected SPARQLQueryBuilder prepareSearchQuery(boolean count, Integer page, Int sparqlQuery.appendAndFilter("REGEX ( str(?" + URI + "),\".*" + uri + ".*\",\"i\")"); } - //Label filter + + //Rdf type filter + if (rdfType != null) { + sparqlQuery.appendTriplet("?" + URI, Rdf.RELATION_TYPE.toString(), rdfType, null); + } else { + sparqlQuery.appendTriplet("?" + URI, Rdf.RELATION_TYPE.toString(), "?" + RDF_TYPE, null); + sparqlQuery.appendTriplet( + "?" + RDF_TYPE, + "<" + Rdfs.RELATION_SUBCLASS_OF.toString() + ">*", + Oeso.CONCEPT_SCIENTIFIC_OBJECT.toString(), null); + sparqlQuery.appendSelect(" ?" + RDF_TYPE); + } + + //Label filter sparqlQuery.appendSelect("?" + ALIAS); if (alias == null && !count) { sparqlQuery.beginBodyOptional(); @@ -751,17 +756,6 @@ protected SPARQLQueryBuilder prepareSearchQuery(boolean count, Integer page, Int sparqlQuery.appendOptional("?" + URI + " <" + Oeso.RELATION_PARTICIPATES_IN.toString() + "> " + "?" + EXPERIMENT + " . "); } - //Rdf type filter - if (rdfType != null) { - sparqlQuery.appendTriplet("?" + URI, Rdf.RELATION_TYPE.toString(), rdfType, null); - } else { - sparqlQuery.appendSelect(" ?" + RDF_TYPE); - sparqlQuery.appendTriplet("?" + URI, Rdf.RELATION_TYPE.toString(), "?" + RDF_TYPE, null); - sparqlQuery.appendTriplet( - "?" + RDF_TYPE, - "<" + Rdfs.RELATION_SUBCLASS_OF.toString() + ">*", - Oeso.CONCEPT_SCIENTIFIC_OBJECT.toString(), null); - } if (page != null && pageSize != null) { sparqlQuery.appendLimit(pageSize); diff --git a/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectSQLDAO.java b/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectSQLDAO.java index 73969f2e4..55aabda0c 100644 --- a/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectSQLDAO.java +++ b/phis2-ws/src/main/java/opensilex/service/dao/ScientificObjectSQLDAO.java @@ -253,10 +253,7 @@ public HashMap getGeometries(ArrayList scientificObjects query.appendSelect("ST_AsGeoJSON(ST_Transform(" + GEOMETRY + ", 4326)), ao." + URI); query.appendFrom(table, tableAlias); - for (String scientificObjectURI : scientificObjectsURIs) { - query.appendORWhereConditionIfNeeded(URI, scientificObjectURI, "=", null, tableAlias); - } - + query.appendINConditions(URI, scientificObjectsURIs, tableAlias); LOGGER.debug(getTraceabilityLogs() + " quert : " + query.toString()); ResultSet queryResult = statement.executeQuery(query.toString()); diff --git a/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java b/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java index 4f312c2d0..d1269ad95 100644 --- a/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java +++ b/phis2-ws/src/main/java/opensilex/service/resource/ScientificObjectResourceService.java @@ -261,7 +261,7 @@ public Response getScientificObjectsBySearch( ) { ArrayList scientificObjectsToReturn = new ArrayList<>(); ArrayList scientificObjects = new ArrayList<>(); - long startTime = System.nanoTime(); + ArrayList statusList = new ArrayList<>(); ResultForm getResponse; @@ -272,9 +272,6 @@ public Response getScientificObjectsBySearch( //1. Get count Integer totalCount = scientificObjectDaoSesame.count(uri, rdfType, experimentURI, alias); - long count = System.nanoTime(); - - System.out.println("Execution time in milliseconds (count) : " + ( count - startTime) / 1000000); // If scientific objects found @@ -283,8 +280,7 @@ public Response getScientificObjectsBySearch( scientificObjects = scientificObjectDaoSesame.find(page, pageSize, uri, rdfType, experimentURI, alias, withProperties); } - long get = System.nanoTime(); - System.out.println("Execution time in milliseconds (get) : " + ( get - count) / 1000000); + if (scientificObjects == null) { //Request failure getResponse = new ResultForm<>(0, 0, scientificObjectsToReturn, true); return noResultFound(getResponse, statusList); @@ -298,8 +294,7 @@ public Response getScientificObjectsBySearch( scientificObjects.forEach((scientificObject) -> { scientificObjectsToReturn.add(new ScientificObjectDTO(scientificObject)); }); - long scientific = System.nanoTime(); - System.out.println("Execution time in milliseconds (scientific) : " + ( scientific - get) / 1000000); + getResponse = new ResultForm<>(scientificObjectDaoSesame.getPageSize(), scientificObjectDaoSesame.getPage(), scientificObjectsToReturn, true, totalCount); if (getResponse.getResult().dataSize() == 0) { return noResultFound(getResponse, statusList); From 502fcd3ed99c915d40559bc006e36180d6108d81 Mon Sep 17 00:00:00 2001 From: Arnaud Charleroy Date: Wed, 26 Feb 2020 18:07:33 +0100 Subject: [PATCH 3/3] Update config file --- .../src/main/profiles/dev/config.properties | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/phis2-ws/src/main/profiles/dev/config.properties b/phis2-ws/src/main/profiles/dev/config.properties index 7db575937..3a2ae8ca5 100644 --- a/phis2-ws/src/main/profiles/dev/config.properties +++ b/phis2-ws/src/main/profiles/dev/config.properties @@ -10,28 +10,26 @@ mongo.db=dbname mongo.user= mongo.password= mongo.authdb= -mongo.logDataQuery=true - # PostgreSQL configuration pg.host=localhost -pg.port=5433 -pg.db=sunagri -pg.user=postgres -pg.password=postgres +pg.port=5432 +pg.db=database +pg.user=username +pg.password=password # RDF4J Configuration rdf.base.uri=http://www.opensilex.org/ -rdf.host=138.102.159.36 +rdf.host=localhost rdf.port=8080 rdf.path=rdf4j-server -rdf.infra=sunagri -rdf.infra.code=SUA -rdf.repo=sunagri +rdf.infra=platform +rdf.infra.code=DMO +rdf.repo=repository rdf.vocabulary.context=http://www.opensilex.org/vocabulary/oeso # Webservice configuration -ws.log.dir=/tmp/opensilex/logs +ws.log.dir=/home/tomcat/opensilex/logs ws.host=localhost ws.port=8080 @@ -44,23 +42,12 @@ ws.doc.port=8080 ws.doc.name=opensilex ws.updir.host=localhost -ws.updir.user=charlero -ws.updir.password=pic3.14 -ws.updir.doc=/tmp/ +ws.updir.user=username +ws.updir.password=password +ws.updir.doc=/home/opensilex/documents/instance ws.images.dir=/var/www/html/images ws.images.url=http://localhost/images ws.layers.dir=/var/www/html/layers -ws.layers.url=http://localhost/layers - - -opencpu.host=http://localhost:8005 - -shinyproxy.run=true -shinyproxy.host=127.0.0.1 -shinyproxy.port=8004 -# must be a writable directory -shinyproxy.jarFilePath=/tmp/opensilex/shinyproxy/shinyproxy-2.3.0.jar -shinyproxy.webFilePath=https://www.shinyproxy.io/downloads/shinyproxy-2.3.0.jar - +ws.layers.url=http://localhost/layers \ No newline at end of file