From 0dd2997e835013d81a2b67d601521114d3598c64 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 13:59:09 -0400 Subject: [PATCH 01/48] add init debian dir --- root/debian/changelog | 6 ++ root/debian/channelfinder.init | 41 ++++++++ root/debian/channelfinder.install | 2 + root/debian/channelfinder.postinst | 21 ++++ root/debian/channelfinder.postrm | 16 +++ root/debian/compat | 1 + root/debian/control | 12 +++ root/debian/copyright | 19 ++++ root/debian/rules | 6 ++ .../etc/channelfinder/glassfish_config_create | 12 +++ .../etc/channelfinder/glassfish_config_remove | 6 ++ root/etc/channelfinder/mapping_definitions.sh | 98 +++++++++++++++++++ root/etc/default/channelfinder | 15 +++ 13 files changed, 255 insertions(+) create mode 100644 root/debian/changelog create mode 100644 root/debian/channelfinder.init create mode 100644 root/debian/channelfinder.install create mode 100644 root/debian/channelfinder.postinst create mode 100644 root/debian/channelfinder.postrm create mode 100644 root/debian/compat create mode 100644 root/debian/control create mode 100644 root/debian/copyright create mode 100644 root/debian/rules create mode 100644 root/etc/channelfinder/glassfish_config_create create mode 100644 root/etc/channelfinder/glassfish_config_remove create mode 100644 root/etc/channelfinder/mapping_definitions.sh create mode 100644 root/etc/default/channelfinder diff --git a/root/debian/changelog b/root/debian/changelog new file mode 100644 index 0000000..92ed308 --- /dev/null +++ b/root/debian/changelog @@ -0,0 +1,6 @@ +channelfinder (1.0.0-0) unstable; urgency=low + + * initial + + -- Eric Berryman Mon, 09 May 2016 09:34:33 -0400 + diff --git a/root/debian/channelfinder.init b/root/debian/channelfinder.init new file mode 100644 index 0000000..3b46b90 --- /dev/null +++ b/root/debian/channelfinder.init @@ -0,0 +1,41 @@ +#!/bin/sh +# /etc/rc.d/init.d/channelfinder: Controls channelfinder +# +# Version: channelfinder 1.0 +# +# chkconfig: 345 99 01 +# description: EPICS Directory Service +# + +### BEGIN INIT INFO +# Provides: channelfinder +# Required-Start: $remote_fs $local_fs $network $syslog $time +# Required-Stop: $remote_fs $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Channelfinder +# Description: EPICS Directory Service +### END INIT INFO + +PROG=/usr/share/channelfinder/ChannelFinder.war + + +case $1 in +start) + asadmin deploy ${PROG} + ;; +stop) + asadmin undeploy ${PROG} + ;; +status) + asadmin list-applications --type web + ;; +restart) + asadmin undeploy ${PROG} + asadmin deploy ${PROG} + ;; +*) + echo $0 "start | stop | status" +esac + + diff --git a/root/debian/channelfinder.install b/root/debian/channelfinder.install new file mode 100644 index 0000000..197b8cc --- /dev/null +++ b/root/debian/channelfinder.install @@ -0,0 +1,2 @@ +usr +etc diff --git a/root/debian/channelfinder.postinst b/root/debian/channelfinder.postinst new file mode 100644 index 0000000..e557dce --- /dev/null +++ b/root/debian/channelfinder.postinst @@ -0,0 +1,21 @@ +#!/bin/sh -e + + +case "$1" in + configure) + source /etc/channelfinder/glassfish_config_create + ;; + abort-upgrade) + : + ;; + abort-remove) + : + ;; + abort-deconfigure) + : + ;; + *) echo "$0: didn't understand" + exit 0 + ;; +esac +exit 0 diff --git a/root/debian/channelfinder.postrm b/root/debian/channelfinder.postrm new file mode 100644 index 0000000..27288df --- /dev/null +++ b/root/debian/channelfinder.postrm @@ -0,0 +1,16 @@ +#!/bin/sh + + +case "$1" in + remove) + ;; + failed-upgrade) + ;; + deconfigure) + source /etc/channelfinder/glassfish_config_remove + ;; + *) + ;; +esac + +exit 0 diff --git a/root/debian/compat b/root/debian/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/root/debian/compat @@ -0,0 +1 @@ +7 diff --git a/root/debian/control b/root/debian/control new file mode 100644 index 0000000..5adcdcd --- /dev/null +++ b/root/debian/control @@ -0,0 +1,12 @@ +Source: channelfinder +Section: java +Priority: optional +Maintainer: Eric Berryman +Build-Depends: debhelper (>= 7), +Standards-Version: 3.8.0 +Homepage: https://channelfinder.github.io/ + +Package: channelfinder +Depends: default-jre-headless +Architecture: amd64 +Description: Directory service for EPICS diff --git a/root/debian/copyright b/root/debian/copyright new file mode 100644 index 0000000..e7e5376 --- /dev/null +++ b/root/debian/copyright @@ -0,0 +1,19 @@ +This package was debianized by Eric Berryman on +Mon, 09 May 2016 09:37:22 -0400. + + +Upstream Authors: + + Ralph Lange, Kunal Shroff + +Copyright: + + +License: + + +On Debian systems, the complete text of the GNU General +Public License can be found in `/usr/share/common-licenses/GPL'. + +The Debian packaging is (C) 2011, Eric Berryman and +is licensed under the MIT, see above. diff --git a/root/debian/rules b/root/debian/rules new file mode 100644 index 0000000..cb336fd --- /dev/null +++ b/root/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f + + +%: + dh $@ + diff --git a/root/etc/channelfinder/glassfish_config_create b/root/etc/channelfinder/glassfish_config_create new file mode 100644 index 0000000..7fe5d87 --- /dev/null +++ b/root/etc/channelfinder/glassfish_config_create @@ -0,0 +1,12 @@ +#!/bin/bash + +exec sh -ac '. /etc/default/channelfinder + +asadmin create-auth-realm --classname ${CLASSNAME} --property "jaas-context=${JAAS_CONTEXT}:directory=${DIRECTORY}:base-dn=${BASE_DN}:group-base-dn=${GROUP_BASE_DN}:group-search-filter=${GROUP_SEARCH_FILTER}:java.naming.referral=${JAVA_NAMING_REFERRAL}:search-filter=${SEARCH_FILTER}:search-bind-password=${SEARCH_BIND_PASSWORD}:search-bind-dn=${SEARCH_BIND_DN}" channelfinder + +asadmin create-custom-resource --restype javax.naming.directory.Directory --factoryclass com.sun.jndi.ldap.LdapCtxFactory --property "URL=${DIRECTORY}/${BASE_DN}:javax.naming.security.principal=${SEACRH_BIND_DN}:javax.naming.security.credentials=${SEARCH_BIND_PASSWORD}:java.naming.referral=${JAVA_NAMING_REFERRAL}" channelfinder/ldapManagerConnection + +asadmin create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property "Value=${USER_MANAGER}" channelfinder/userManager + +/etc/channelfinder/mapping_definitions.sh +' diff --git a/root/etc/channelfinder/glassfish_config_remove b/root/etc/channelfinder/glassfish_config_remove new file mode 100644 index 0000000..b45fb05 --- /dev/null +++ b/root/etc/channelfinder/glassfish_config_remove @@ -0,0 +1,6 @@ +#!/bin/bash + + +asadmin delete-auth-realm channelfinder +asadmin delete-custom-resource channelfinder/ldapManagerConnection +asadmin delete-custom-resource channelfinder/userManager diff --git a/root/etc/channelfinder/mapping_definitions.sh b/root/etc/channelfinder/mapping_definitions.sh new file mode 100644 index 0000000..829b46b --- /dev/null +++ b/root/etc/channelfinder/mapping_definitions.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +### +# #%L +# ChannelFinder Directory Service +# %% +# Copyright (C) 2010 - 2016 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH +# %% +# Copyright (C) 2010 - 2012 Brookhaven National Laboratory +# All rights reserved. Use is subject to license terms. +# #L% +### + +# The mapping definition for the Indexes associated with the channelfinder v2 + + +#Create the Index +curl -XPUT '${ELASTIC_HOST}/tags' +#Set the mapping +curl -XPUT '${ELASTIC_HOST}/tags/_mapping/tag' -d' +{ + "tag" : { + "properties" : { + "name" : { + "type" : "string" + }, + "owner" : { + "type" : "string" + } + } + } +}' + +curl -XPUT '${ELASTIC_HOST}/properties' +curl -XPUT '${ELASTIC_HOST}/properties/_mapping/property' -d' +{ + "property" : { + "properties" : { + "name" : { + "type" : "string" + }, + "owner" : { + "type" : "string" + } + } + } +}' + +curl -XPUT '${ELASTIC_HOST}/channelfinder' +curl -XPUT '${ELASTIC_HOST}/channelfinder/_mapping/channel' -d' +{ + "channel" : { + "properties" : { + "name" : { + "type" : "string", + "analyzer" : "whitespace" + }, + "owner" : { + "type" : "string", + "analyzer" : "whitespace" + }, + "script" : { + "type" : "string" + }, + "properties" : { + "type" : "nested", + "include_in_parent" : true, + "properties" : { + "name" : { + "type" : "string", + "analyzer" : "whitespace" + }, + "owner" : { + "type" : "string" + }, + "value" : { + "type" : "string", + "analyzer" : "whitespace" + } + } + }, + "tags" : { + "type" : "nested", + "include_in_parent" : true, + "properties" : { + "name" : { + "type" : "string", + "analyzer" : "whitespace" + }, + "owner" : { + "type" : "string", + "analyzer" : "whitespace" + } + } + } + } + } +}' diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder new file mode 100644 index 0000000..78b6993 --- /dev/null +++ b/root/etc/default/channelfinder @@ -0,0 +1,15 @@ +#Channelfinder defaults + +ELASTIC_HOST="http://localhost:9200" + +CLASSNAME="com.sun.enterprise.security.auth.realm.ldap.LDAPRealm" +JAAS_CONTEXT="ldapRealm" +DIRECTORY="ldap://localhost:389" +BASE_DN="dc=cf-test,dc=local" +GROUP_BASE_DN="dc=cf-test,dc=local" +GROUP_SEARCH_FILTER="(&(objectClass\=group)(member\=%d))" +JAVA_NAMING_REFERRAL="follow" +SEARCH_FILTER="(&(objectClass\=user)(sAMAccountName\=%s))" +SEARCH_BIND_PASSWORD="1234" +SEARCH_BIND_DN="cn=channelfinder,dc=cf-test,dc=local" +USER_MANAGER="gov.bnl.channelfinder.LDAPUserManager" From 109f6e0bceec0c894574c6b8f9a15393b77ee4a2 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 14:20:13 -0400 Subject: [PATCH 02/48] change deb install file --- root/debian/channelfinder.install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/debian/channelfinder.install b/root/debian/channelfinder.install index 197b8cc..cd37ffb 100644 --- a/root/debian/channelfinder.install +++ b/root/debian/channelfinder.install @@ -1,2 +1,2 @@ -usr +ChannelFinder.war usr/share/channelfinder etc From 005531be73e03cd941051d2e2bcc2e194b39d4c9 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 15:08:46 -0400 Subject: [PATCH 03/48] dpkg: add glassfish home to defaults --- root/etc/channelfinder/glassfish_config_create | 6 +++--- root/etc/channelfinder/glassfish_config_remove | 9 +++++---- root/etc/default/channelfinder | 3 +++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/root/etc/channelfinder/glassfish_config_create b/root/etc/channelfinder/glassfish_config_create index 7fe5d87..107ae29 100644 --- a/root/etc/channelfinder/glassfish_config_create +++ b/root/etc/channelfinder/glassfish_config_create @@ -2,11 +2,11 @@ exec sh -ac '. /etc/default/channelfinder -asadmin create-auth-realm --classname ${CLASSNAME} --property "jaas-context=${JAAS_CONTEXT}:directory=${DIRECTORY}:base-dn=${BASE_DN}:group-base-dn=${GROUP_BASE_DN}:group-search-filter=${GROUP_SEARCH_FILTER}:java.naming.referral=${JAVA_NAMING_REFERRAL}:search-filter=${SEARCH_FILTER}:search-bind-password=${SEARCH_BIND_PASSWORD}:search-bind-dn=${SEARCH_BIND_DN}" channelfinder +${ASADMIN} create-auth-realm --classname ${CLASSNAME} --property "jaas-context=${JAAS_CONTEXT}:directory=${DIRECTORY}:base-dn=${BASE_DN}:group-base-dn=${GROUP_BASE_DN}:group-search-filter=${GROUP_SEARCH_FILTER}:java.naming.referral=${JAVA_NAMING_REFERRAL}:search-filter=${SEARCH_FILTER}:search-bind-password=${SEARCH_BIND_PASSWORD}:search-bind-dn=${SEARCH_BIND_DN}" channelfinder -asadmin create-custom-resource --restype javax.naming.directory.Directory --factoryclass com.sun.jndi.ldap.LdapCtxFactory --property "URL=${DIRECTORY}/${BASE_DN}:javax.naming.security.principal=${SEACRH_BIND_DN}:javax.naming.security.credentials=${SEARCH_BIND_PASSWORD}:java.naming.referral=${JAVA_NAMING_REFERRAL}" channelfinder/ldapManagerConnection +${ASADMIN} create-custom-resource --restype javax.naming.directory.Directory --factoryclass com.sun.jndi.ldap.LdapCtxFactory --property "URL=${DIRECTORY}/${BASE_DN}:javax.naming.security.principal=${SEACRH_BIND_DN}:javax.naming.security.credentials=${SEARCH_BIND_PASSWORD}:java.naming.referral=${JAVA_NAMING_REFERRAL}" channelfinder/ldapManagerConnection -asadmin create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property "Value=${USER_MANAGER}" channelfinder/userManager +${ASADMIN} create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property "Value=${USER_MANAGER}" channelfinder/userManager /etc/channelfinder/mapping_definitions.sh ' diff --git a/root/etc/channelfinder/glassfish_config_remove b/root/etc/channelfinder/glassfish_config_remove index b45fb05..976f406 100644 --- a/root/etc/channelfinder/glassfish_config_remove +++ b/root/etc/channelfinder/glassfish_config_remove @@ -1,6 +1,7 @@ #!/bin/bash - -asadmin delete-auth-realm channelfinder -asadmin delete-custom-resource channelfinder/ldapManagerConnection -asadmin delete-custom-resource channelfinder/userManager +exec sh -ac '. /etc/default/channelfinder +${ASADMIN} delete-auth-realm channelfinder +${ASADMIN} delete-custom-resource channelfinder/ldapManagerConnection +${ASADMIN} delete-custom-resource channelfinder/userManager +' diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder index 78b6993..4f8246e 100644 --- a/root/etc/default/channelfinder +++ b/root/etc/default/channelfinder @@ -1,5 +1,8 @@ #Channelfinder defaults +GF_HOME=/home/glassfish/glassfish4/glassfish +ASADMIN=$GF_HOME/bin/asadmin + ELASTIC_HOST="http://localhost:9200" CLASSNAME="com.sun.enterprise.security.auth.realm.ldap.LDAPRealm" From ef427287e27e3523024d7eafa21175832cf14f1b Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 15:20:01 -0400 Subject: [PATCH 04/48] dpkg: add glassfish home to init --- root/debian/channelfinder.init | 13 +++++++------ root/debian/channelfinder.postinst | 2 +- root/debian/channelfinder.postrm | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/root/debian/channelfinder.init b/root/debian/channelfinder.init index 3b46b90..cc88093 100644 --- a/root/debian/channelfinder.init +++ b/root/debian/channelfinder.init @@ -18,21 +18,22 @@ ### END INIT INFO PROG=/usr/share/channelfinder/ChannelFinder.war - +GF_HOME=/home/glassfish/glassfish4/glassfish +ASADMIN=$GF_HOME/bin/asadmin case $1 in start) - asadmin deploy ${PROG} + ${ASADMIN} deploy ${PROG} ;; stop) - asadmin undeploy ${PROG} + ${ASADMIN} undeploy ${PROG} ;; status) - asadmin list-applications --type web + ${ASADMIN} list-applications --type web ;; restart) - asadmin undeploy ${PROG} - asadmin deploy ${PROG} + ${ASADMIN} undeploy ${PROG} + ${ASADMIN} deploy ${PROG} ;; *) echo $0 "start | stop | status" diff --git a/root/debian/channelfinder.postinst b/root/debian/channelfinder.postinst index e557dce..b7f4454 100644 --- a/root/debian/channelfinder.postinst +++ b/root/debian/channelfinder.postinst @@ -3,7 +3,7 @@ case "$1" in configure) - source /etc/channelfinder/glassfish_config_create + . /etc/channelfinder/glassfish_config_create ;; abort-upgrade) : diff --git a/root/debian/channelfinder.postrm b/root/debian/channelfinder.postrm index 27288df..2dc0f7d 100644 --- a/root/debian/channelfinder.postrm +++ b/root/debian/channelfinder.postrm @@ -7,7 +7,7 @@ case "$1" in failed-upgrade) ;; deconfigure) - source /etc/channelfinder/glassfish_config_remove + . /etc/channelfinder/glassfish_config_remove ;; *) ;; From 56fdecbc8c8e81e834746073fa1ba7db050b853b Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 15:25:23 -0400 Subject: [PATCH 05/48] dpkg: escape : and = in defaults --- root/etc/default/channelfinder | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder index 4f8246e..3ccbdc3 100644 --- a/root/etc/default/channelfinder +++ b/root/etc/default/channelfinder @@ -3,16 +3,16 @@ GF_HOME=/home/glassfish/glassfish4/glassfish ASADMIN=$GF_HOME/bin/asadmin -ELASTIC_HOST="http://localhost:9200" +ELASTIC_HOST="http\://localhost\:9200" CLASSNAME="com.sun.enterprise.security.auth.realm.ldap.LDAPRealm" JAAS_CONTEXT="ldapRealm" -DIRECTORY="ldap://localhost:389" -BASE_DN="dc=cf-test,dc=local" -GROUP_BASE_DN="dc=cf-test,dc=local" +DIRECTORY="ldap\://localhost\:389" +BASE_DN="dc\=cf-test,dc\=local" +GROUP_BASE_DN="dc\=cf-test,dc\=local" GROUP_SEARCH_FILTER="(&(objectClass\=group)(member\=%d))" JAVA_NAMING_REFERRAL="follow" SEARCH_FILTER="(&(objectClass\=user)(sAMAccountName\=%s))" SEARCH_BIND_PASSWORD="1234" -SEARCH_BIND_DN="cn=channelfinder,dc=cf-test,dc=local" +SEARCH_BIND_DN="cn\=channelfinder,dc\=cf-test,dc\=local" USER_MANAGER="gov.bnl.channelfinder.LDAPUserManager" From 8b0bb07e175d149b95c3c9724b9ab944272bfb64 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 15:35:24 -0400 Subject: [PATCH 06/48] dpkg: add asadmin export --- root/etc/channelfinder/glassfish_config_create | 4 +++- root/etc/channelfinder/glassfish_config_remove | 3 +++ root/etc/default/channelfinder | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/root/etc/channelfinder/glassfish_config_create b/root/etc/channelfinder/glassfish_config_create index 107ae29..c3278ea 100644 --- a/root/etc/channelfinder/glassfish_config_create +++ b/root/etc/channelfinder/glassfish_config_create @@ -1,10 +1,12 @@ #!/bin/bash exec sh -ac '. /etc/default/channelfinder +${ASADMIN} export AS_ADMIN_USER=${ADMIN} +${ASADMIN} export AS_ADMIN_PASSWORD=${PASS} ${ASADMIN} create-auth-realm --classname ${CLASSNAME} --property "jaas-context=${JAAS_CONTEXT}:directory=${DIRECTORY}:base-dn=${BASE_DN}:group-base-dn=${GROUP_BASE_DN}:group-search-filter=${GROUP_SEARCH_FILTER}:java.naming.referral=${JAVA_NAMING_REFERRAL}:search-filter=${SEARCH_FILTER}:search-bind-password=${SEARCH_BIND_PASSWORD}:search-bind-dn=${SEARCH_BIND_DN}" channelfinder -${ASADMIN} create-custom-resource --restype javax.naming.directory.Directory --factoryclass com.sun.jndi.ldap.LdapCtxFactory --property "URL=${DIRECTORY}/${BASE_DN}:javax.naming.security.principal=${SEACRH_BIND_DN}:javax.naming.security.credentials=${SEARCH_BIND_PASSWORD}:java.naming.referral=${JAVA_NAMING_REFERRAL}" channelfinder/ldapManagerConnection +${ASADMIN} create-custom-resource --restype javax.naming.directory.Directory --factoryclass com.sun.jndi.ldap.LdapCtxFactory --property "URL=${DIRECTORY}/${BASE_DN}:javax.naming.security.principal=${SEARCH_BIND_DN}:javax.naming.security.credentials=${SEARCH_BIND_PASSWORD}:java.naming.referral=${JAVA_NAMING_REFERRAL}" channelfinder/ldapManagerConnection ${ASADMIN} create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property "Value=${USER_MANAGER}" channelfinder/userManager diff --git a/root/etc/channelfinder/glassfish_config_remove b/root/etc/channelfinder/glassfish_config_remove index 976f406..257951d 100644 --- a/root/etc/channelfinder/glassfish_config_remove +++ b/root/etc/channelfinder/glassfish_config_remove @@ -1,6 +1,9 @@ #!/bin/bash exec sh -ac '. /etc/default/channelfinder +${ASADMIN} export AS_ADMIN_USER=${ADMIN} +${ASADMIN} export AS_ADMIN_PASSWORD=${PASS} + ${ASADMIN} delete-auth-realm channelfinder ${ASADMIN} delete-custom-resource channelfinder/ldapManagerConnection ${ASADMIN} delete-custom-resource channelfinder/userManager diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder index 3ccbdc3..8b9935f 100644 --- a/root/etc/default/channelfinder +++ b/root/etc/default/channelfinder @@ -2,6 +2,8 @@ GF_HOME=/home/glassfish/glassfish4/glassfish ASADMIN=$GF_HOME/bin/asadmin +ADMIN=admin +PASS=adminadmin ELASTIC_HOST="http\://localhost\:9200" From ae22575ceaca9afd619003f3b9ba52f571b87712 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 9 May 2016 15:40:30 -0400 Subject: [PATCH 07/48] dpkg: remove asadmin export --- root/etc/channelfinder/glassfish_config_create | 4 +--- root/etc/channelfinder/glassfish_config_remove | 2 -- root/etc/default/channelfinder | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/root/etc/channelfinder/glassfish_config_create b/root/etc/channelfinder/glassfish_config_create index c3278ea..f9e4427 100644 --- a/root/etc/channelfinder/glassfish_config_create +++ b/root/etc/channelfinder/glassfish_config_create @@ -1,8 +1,6 @@ #!/bin/bash exec sh -ac '. /etc/default/channelfinder -${ASADMIN} export AS_ADMIN_USER=${ADMIN} -${ASADMIN} export AS_ADMIN_PASSWORD=${PASS} ${ASADMIN} create-auth-realm --classname ${CLASSNAME} --property "jaas-context=${JAAS_CONTEXT}:directory=${DIRECTORY}:base-dn=${BASE_DN}:group-base-dn=${GROUP_BASE_DN}:group-search-filter=${GROUP_SEARCH_FILTER}:java.naming.referral=${JAVA_NAMING_REFERRAL}:search-filter=${SEARCH_FILTER}:search-bind-password=${SEARCH_BIND_PASSWORD}:search-bind-dn=${SEARCH_BIND_DN}" channelfinder @@ -10,5 +8,5 @@ ${ASADMIN} create-custom-resource --restype javax.naming.directory.Directory --f ${ASADMIN} create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property "Value=${USER_MANAGER}" channelfinder/userManager -/etc/channelfinder/mapping_definitions.sh +. /etc/channelfinder/mapping_definitions.sh ' diff --git a/root/etc/channelfinder/glassfish_config_remove b/root/etc/channelfinder/glassfish_config_remove index 257951d..9fffe6b 100644 --- a/root/etc/channelfinder/glassfish_config_remove +++ b/root/etc/channelfinder/glassfish_config_remove @@ -1,8 +1,6 @@ #!/bin/bash exec sh -ac '. /etc/default/channelfinder -${ASADMIN} export AS_ADMIN_USER=${ADMIN} -${ASADMIN} export AS_ADMIN_PASSWORD=${PASS} ${ASADMIN} delete-auth-realm channelfinder ${ASADMIN} delete-custom-resource channelfinder/ldapManagerConnection diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder index 8b9935f..3ccbdc3 100644 --- a/root/etc/default/channelfinder +++ b/root/etc/default/channelfinder @@ -2,8 +2,6 @@ GF_HOME=/home/glassfish/glassfish4/glassfish ASADMIN=$GF_HOME/bin/asadmin -ADMIN=admin -PASS=adminadmin ELASTIC_HOST="http\://localhost\:9200" From 5b9ef20a6e6146296ac47f4f0caf11d000ccb5d1 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 10 May 2016 09:38:35 -0400 Subject: [PATCH 08/48] dpkg: moved configure to init --- root/debian/channelfinder.init | 12 +++++++++++- root/debian/channelfinder.postinst | 21 --------------------- root/debian/channelfinder.postrm | 16 ---------------- 3 files changed, 11 insertions(+), 38 deletions(-) delete mode 100644 root/debian/channelfinder.postinst delete mode 100644 root/debian/channelfinder.postrm diff --git a/root/debian/channelfinder.init b/root/debian/channelfinder.init index cc88093..96a68ee 100644 --- a/root/debian/channelfinder.init +++ b/root/debian/channelfinder.init @@ -35,8 +35,18 @@ restart) ${ASADMIN} undeploy ${PROG} ${ASADMIN} deploy ${PROG} ;; +configure) + . /etc/channelfinder/glassfish_config_create + ;; +deconfigure) + . /etc/channelfinder/glassfish_config_remove + ;; +reconfigure) + . /etc/channelfinder/glassfish_config_remove + . /etc/channelfinder/glassfish_config_create + ;; *) - echo $0 "start | stop | status" + echo $0 "start | stop | status | configure | deconfigure | reconfigure" esac diff --git a/root/debian/channelfinder.postinst b/root/debian/channelfinder.postinst deleted file mode 100644 index b7f4454..0000000 --- a/root/debian/channelfinder.postinst +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -e - - -case "$1" in - configure) - . /etc/channelfinder/glassfish_config_create - ;; - abort-upgrade) - : - ;; - abort-remove) - : - ;; - abort-deconfigure) - : - ;; - *) echo "$0: didn't understand" - exit 0 - ;; -esac -exit 0 diff --git a/root/debian/channelfinder.postrm b/root/debian/channelfinder.postrm deleted file mode 100644 index 2dc0f7d..0000000 --- a/root/debian/channelfinder.postrm +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - - -case "$1" in - remove) - ;; - failed-upgrade) - ;; - deconfigure) - . /etc/channelfinder/glassfish_config_remove - ;; - *) - ;; -esac - -exit 0 From b52ede7da2d1df12f376ee0ff7fcbe005e701b0b Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 10 May 2016 10:28:26 -0400 Subject: [PATCH 09/48] dpkg: fix undeploy --- root/debian/channelfinder.init | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/root/debian/channelfinder.init b/root/debian/channelfinder.init index 96a68ee..8037350 100644 --- a/root/debian/channelfinder.init +++ b/root/debian/channelfinder.init @@ -17,13 +17,14 @@ # Description: EPICS Directory Service ### END INIT INFO -PROG=/usr/share/channelfinder/ChannelFinder.war +PROG=/usr/share/channelfinder/ChannelFinder +PROG_WAR=/usr/share/channelfinder/ChannelFinder.war GF_HOME=/home/glassfish/glassfish4/glassfish ASADMIN=$GF_HOME/bin/asadmin case $1 in start) - ${ASADMIN} deploy ${PROG} + ${ASADMIN} deploy ${PROG_WAR} ;; stop) ${ASADMIN} undeploy ${PROG} @@ -33,7 +34,7 @@ status) ;; restart) ${ASADMIN} undeploy ${PROG} - ${ASADMIN} deploy ${PROG} + ${ASADMIN} deploy ${PROG_WAR} ;; configure) . /etc/channelfinder/glassfish_config_create From 4a09fadf292d6bad3d38abed30194806b4851e0a Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 10 May 2016 10:53:33 -0400 Subject: [PATCH 10/48] release --- root/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index 92ed308..d0c739f 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,10 @@ +channelfinder (1.0.0-0.1) unstable; urgency=medium + + * Non-maintainer upload. + * release + + -- Eric Berryman Tue, 10 May 2016 10:53:33 -0400 + channelfinder (1.0.0-0) unstable; urgency=low * initial From d07b828ba8b3b75922dc1b8c97f5fa28bd5340c9 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 10 May 2016 11:06:39 -0400 Subject: [PATCH 11/48] release --- root/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index d0c739f..24b1a3d 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,10 @@ +channelfinder (1.0.0-0.2) unstable; urgency=medium + + * Non-maintainer upload. + * release + + -- Eric Berryman Tue, 10 May 2016 11:06:39 -0400 + channelfinder (1.0.0-0.1) unstable; urgency=medium * Non-maintainer upload. From 6d7fe0c4bd0487c11f60e9a2e28f58d77f0e40e3 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 10 May 2016 11:09:27 -0400 Subject: [PATCH 12/48] release --- root/debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index 24b1a3d..f478b24 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,9 @@ +channelfinder (1.0.0-1) unstable; urgency=medium + + * release + + -- Eric Berryman Tue, 10 May 2016 11:09:27 -0400 + channelfinder (1.0.0-0.2) unstable; urgency=medium * Non-maintainer upload. From 91df93cb140dec2949a8abc4e65494e24aa8a0f0 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 10 May 2016 11:12:23 -0400 Subject: [PATCH 13/48] release --- root/debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index f478b24..53a6679 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,9 @@ +channelfinder (1.0.0-2) unstable; urgency=medium + + * release + + -- Eric Berryman Tue, 10 May 2016 11:12:23 -0400 + channelfinder (1.0.0-1) unstable; urgency=medium * release From 9f2220b22c480ef0ad9cdc1f2e45386d4292ee15 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 10 May 2016 14:16:22 -0400 Subject: [PATCH 14/48] dpkg: change single to double quotes --- root/etc/channelfinder/mapping_definitions.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/root/etc/channelfinder/mapping_definitions.sh b/root/etc/channelfinder/mapping_definitions.sh index 829b46b..3e25b7c 100644 --- a/root/etc/channelfinder/mapping_definitions.sh +++ b/root/etc/channelfinder/mapping_definitions.sh @@ -15,9 +15,9 @@ #Create the Index -curl -XPUT '${ELASTIC_HOST}/tags' +curl -XPUT "${ELASTIC_HOST}/tags" #Set the mapping -curl -XPUT '${ELASTIC_HOST}/tags/_mapping/tag' -d' +curl -XPUT "${ELASTIC_HOST}/tags/_mapping/tag" -d' { "tag" : { "properties" : { @@ -31,8 +31,8 @@ curl -XPUT '${ELASTIC_HOST}/tags/_mapping/tag' -d' } }' -curl -XPUT '${ELASTIC_HOST}/properties' -curl -XPUT '${ELASTIC_HOST}/properties/_mapping/property' -d' +curl -XPUT "${ELASTIC_HOST}/properties" +curl -XPUT "${ELASTIC_HOST}/properties/_mapping/property" -d' { "property" : { "properties" : { @@ -46,8 +46,8 @@ curl -XPUT '${ELASTIC_HOST}/properties/_mapping/property' -d' } }' -curl -XPUT '${ELASTIC_HOST}/channelfinder' -curl -XPUT '${ELASTIC_HOST}/channelfinder/_mapping/channel' -d' +curl -XPUT "${ELASTIC_HOST}/channelfinder" +curl -XPUT "${ELASTIC_HOST}/channelfinder/_mapping/channel" -d' { "channel" : { "properties" : { From 0b0f1a12bcba5c5a42a21bcd1e72d7c7dcb29503 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 10 May 2016 14:19:25 -0400 Subject: [PATCH 15/48] dpkg init script fixes --- root/debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index 53a6679..ba0d06e 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,9 @@ +channelfinder (1.0.0-3) unstable; urgency=medium + + * dpkg init script fixes + + -- Eric Berryman Tue, 10 May 2016 14:19:25 -0400 + channelfinder (1.0.0-2) unstable; urgency=medium * release From f8629eb16e391e04410de3d5cd28c56cc97be36c Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Wed, 11 May 2016 10:30:39 -0400 Subject: [PATCH 16/48] dpkg: add Elastic configuration file to war on start --- root/debian/channelfinder.init | 1 + .../WEB-INF/classes/elasticsearch.yml | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml diff --git a/root/debian/channelfinder.init b/root/debian/channelfinder.init index 8037350..ddb4652 100644 --- a/root/debian/channelfinder.init +++ b/root/debian/channelfinder.init @@ -24,6 +24,7 @@ ASADMIN=$GF_HOME/bin/asadmin case $1 in start) + jar -uf ${PROG_WAR} -C /etc/channelfinder WEB-INF/classes/elasticsearch.yml ${ASADMIN} deploy ${PROG_WAR} ;; stop) diff --git a/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml b/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml new file mode 100644 index 0000000..cb3d72e --- /dev/null +++ b/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml @@ -0,0 +1,40 @@ +############################## Network And HTTP ############################### + +# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens +# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). + +# Set the bind address specifically (IPv4 or IPv6): +# +#network.bind_host: 192.168.0.1 + +# Set the address other nodes will use to communicate with this node. If not +# set, it is automatically derived. It must point to an actual IP address. +# +#network.publish_host: 192.168.0.1 + +# Set both 'bind_host' and 'publish_host': +# +network.host: 127.0.0.1 + +# Set a custom port for the node to node communication (9300 by default): +# +transport.tcp.port: 9300 + +# Enable compression for all communication between nodes (disabled by default): +# +#transport.tcp.compress: true + +# Set a custom port to listen for HTTP traffic: +# +#http.port: 9200 + +# Set a custom allowed content length: +# +#http.max_content_length: 100mb + +# Disable HTTP completely: +# +#http.enabled: false + From e872e3daa73eb2d77d69d439daa27f68fb7da120 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Wed, 11 May 2016 12:38:06 -0400 Subject: [PATCH 17/48] release --- root/debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index ba0d06e..3007c53 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,9 @@ +channelfinder (1.0.0-4) unstable; urgency=medium + + * release + + -- Eric Berryman Wed, 11 May 2016 12:38:06 -0400 + channelfinder (1.0.0-3) unstable; urgency=medium * dpkg init script fixes From 860a1aaaf22f909ea4dbb24027e6a0c1c93cf0a0 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 6 Sep 2016 16:50:15 -0400 Subject: [PATCH 18/48] added JACCUserManager --- .../bnl/channelfinder/JACCUserManager.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java b/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java new file mode 100644 index 0000000..66117ae --- /dev/null +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2010 Brookhaven National Laboratory + * Copyright (c) 2010-2011 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH + * All rights reserved. Use is subject to license terms and conditions. + */ +package gov.bnl.channelfinder; + +import java.security.Principal; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.security.auth.Subject; +import javax.security.jacc.PolicyContext; +import javax.security.jacc.PolicyContextException; + + +/** + * Owner (group) membership management: JACC connection and binding. + * + * @author beryman + */ +public class JACCUserManager extends UserManager { + + private static final String SUBJECT_HANDLER_KEY = "javax.security.auth.Subject.container"; + + @Override + protected Set getGroups(Principal user) { + + try { + Subject s = (Subject) PolicyContext.getContext(SUBJECT_HANDLER_KEY); + Principal principals[] = (s == null ? new Principal[0] : s.getPrincipals().toArray(new Principal[0])); + + HashSet roleSet = new HashSet(); + for(Principal principal :principals){ + roleSet.add(principal.getName()); + } + return roleSet; + +// Do we want to look at only Web Role Permissions? Checking againist all principals for now. +// CodeSource cs = new CodeSource(null, (java.security.cert.Certificate[]) null); +// ProtectionDomain pd = new ProtectionDomain(cs, null, null, principals); +// +// Policy policy = Policy.getPolicy(); +// PermissionCollection pc = policy.getPermissions(pd); +// pc.implies(new WebRoleRefPermission(null, null)); +// +// HashSet roleSet = null; +// +// Enumeration e = pc.elements(); +// while (e.hasMoreElements()) { +// Object p = e.nextElement(); +// if (p instanceof WebRoleRefPermission) { +// String roleRef = ((WebRoleRefPermission) p).getActions(); +// if (roleSet == null) { +// roleSet = new HashSet(); +// } +// roleSet.add(((WebRoleRefPermission) p).getActions()); +// } +// } +// if (roleSet != null) { +// roleSet.toArray(new String[0]); +// } +// return roleSet; +// + } catch (PolicyContextException ex) { + Logger.getLogger(JACCUserManager.class.getName()).log(Level.SEVERE, null, ex); + } + return null; + } +} From 2ead96c200db80156a0c1c55e81c932e451136c4 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 6 Sep 2016 16:51:26 -0400 Subject: [PATCH 19/48] added JACCUserManager --- root/debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index 3007c53..b821d2d 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,9 @@ +channelfinder (1.0.0-5) unstable; urgency=medium + + * added JACCUserManager + + -- Eric Berryman Tue, 06 Sep 2016 16:51:26 -0400 + channelfinder (1.0.0-4) unstable; urgency=medium * release From 1a36a8cdce567dedc038e079b8cfba30b2549711 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Wed, 17 May 2017 15:29:12 -0400 Subject: [PATCH 20/48] move to elastic 5.4 java api --- channelfinder/pom.xml | 73 +++++++++++++++---- .../bnl/channelfinder/ChannelsResource.java | 23 +++--- .../channelfinder/ElasticSearchClient.java | 14 ++-- .../bnl/channelfinder/JACCUserManager.java | 12 +++ .../bnl/channelfinder/PropertiesResource.java | 67 ++++++++++------- .../bnl/channelfinder/ResponseCorsFilter.java | 11 +++ .../gov/bnl/channelfinder/TagsResource.java | 73 +++++++++++-------- .../serverresources/config/logging.properties | 10 +++ 8 files changed, 193 insertions(+), 90 deletions(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 3fa2bbe..45d0a4a 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -67,7 +67,7 @@ - epics.releases @@ -141,19 +141,24 @@ org.elasticsearch elasticsearch - 1.5.2 + 5.4.0 + + + org.elasticsearch.client + transport + 5.4.0 org.glassfish.jersey.core jersey-common 2.10.4 - provided + provided org.glassfish.jersey.media jersey-media-multipart 2.10.4 - provided + provided com.fasterxml.jackson.jaxrs @@ -164,13 +169,13 @@ com.fasterxml.jackson.core jackson-core 2.5.1 - provided + provided com.fasterxml.jackson.core jackson-databind 2.5.1 - provided + provided com.google.guava @@ -235,16 +240,52 @@ - - maven-deploy-plugin - 2.8.2 - - - org.apache.maven.wagon - wagon-ssh - 2.8 - - + + org.codehaus.mojo + exec-maven-plugin + 1.5.0 + + + fish.payara.extras + payara-micro + 4.1.1.162 + + + + + payara-uber-jar + package + + java + + + fish.payara.micro.PayaraMicro + + --deploy + ${basedir}/target/${project.build.finalName}.war + --outputUberJar + ${basedir}/target/${project.build.finalName}.jar + + false + true + + fish.payara.extras + payara-micro + + + + + + + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.wagon + wagon-ssh + 2.8 + + maven-assembly-plugin diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index b7a8e7c..e3c46c6 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -66,8 +65,11 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; /** * Top level Jersey HTTP methods for the .../channels URL * @@ -126,7 +128,7 @@ public Response query() { for (String pattern : value.split("\\|")) { tagQuery.add(wildcardQuery("tags.name", pattern.trim())); } - qb.must(nestedQuery("tags", tagQuery)); + qb.must(nestedQuery("tags", tagQuery, ScoreMode.Avg)); } break; case "~size": @@ -143,7 +145,8 @@ public Response query() { propertyQuery.add(nestedQuery("properties", boolQuery() .must(matchQuery("properties.name", parameter.getKey().trim())) - .must(wildcardQuery("properties.value", pattern.trim())))); + .must(wildcardQuery("properties.value", pattern.trim())), + ScoreMode.Avg)); } } qb.must(propertyQuery); @@ -219,7 +222,7 @@ public Response create(List data) throws IOException { } String prepare = "|Prepare: " + (System.currentTimeMillis()-start) + "|"; start = System.currentTimeMillis(); - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); String execute = "|Execute: " + (System.currentTimeMillis()-start)+"|"; start = System.currentTimeMillis(); @@ -317,7 +320,7 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { IndexRequest indexRequest = new IndexRequest("channelfinder", "channel", chan) .source(mapper.writeValueAsBytes(data)); UpdateRequest updateRequest = new UpdateRequest("channelfinder", "channel", chan) - .doc(mapper.writeValueAsBytes(data)).upsert(indexRequest).refresh(true); + .doc(mapper.writeValueAsBytes(data)).upsert(indexRequest).setRefreshPolicy(RefreshPolicy.IMMEDIATE); UpdateResponse result = client.update(updateRequest).actionGet(); Response r = Response.noContent().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|PUT|OK|" + r.getStatus() + "|data=" + XmlChannel.toLog(data)); @@ -376,7 +379,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { }).collect(Collectors.toList())); channel.setTags(data.getTags()); UpdateRequest updateRequest = new UpdateRequest("channelfinder", "channel", chan) - .doc(mapper.writeValueAsBytes(channel)).refresh(true); + .doc(mapper.writeValueAsBytes(channel)).setRefreshPolicy(RefreshPolicy.IMMEDIATE); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|POST|prepare : "+ (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); UpdateResponse result = client.update(updateRequest).actionGet(); @@ -421,7 +424,7 @@ private Response renameChannel(UserManager um, Client client, String chan, XmlCh IndexRequest indexRequest = new IndexRequest("channelfinder", "channel", originalChannel.getName()) .source(mapper.writeValueAsBytes(originalChannel)); bulkRequest.add(indexRequest); - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -461,8 +464,8 @@ public Response remove(@PathParam("chName") String chan) { UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { - DeleteResponse response = client.prepareDelete("channelfinder", "channel", chan).setRefresh(true).execute().get(); - if(response.isFound()){ + DeleteResponse deleteResponse = client.prepareDelete("channelfinder", "channel", chan).setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute().get(); + if(deleteResponse.getResult() == DocWriteResponse.Result.DELETED){ Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); return r; diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java index 5348832..7669567 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java @@ -19,9 +19,11 @@ import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; +import java.net.InetSocketAddress; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; @@ -38,7 +40,7 @@ public class ElasticSearchClient implements ServletContextListener { private static TransportClient searchClient; private static TransportClient indexClient; - public static TransportClient getSearchClient() { + public static TransportClient getSearchClient() { return searchClient; } @@ -56,7 +58,7 @@ public static TransportClient getNewClient() { String host = settings.get("network.host"); int port = Integer.valueOf(settings.get("transport.tcp.port")); try { - return new TransportClient().addTransportAddress(new InetSocketTransportAddress(host, port)); + return new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); } catch (ElasticsearchException e) { log.severe(e.getDetailedMessage()); return null; @@ -70,13 +72,13 @@ public static Settings getSettings(){ @Override public void contextInitialized(ServletContextEvent sce) { log.info("Initializing a new Transport clients."); - searchClient = new TransportClient(); - indexClient = new TransportClient(); + searchClient = new PreBuiltTransportClient(settings); + indexClient = new PreBuiltTransportClient(settings); settings = searchClient.settings(); String host = settings.get("network.host"); int port = Integer.valueOf(settings.get("transport.tcp.port")); - searchClient.addTransportAddress(new InetSocketTransportAddress(host, port)); - indexClient.addTransportAddress(new InetSocketTransportAddress(host, port)); + searchClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); + indexClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); } @Override diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java b/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java index 66117ae..1083b83 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java @@ -5,6 +5,18 @@ */ package gov.bnl.channelfinder; +/* + * #%L + * ChannelFinder Directory Service + * %% + * Copyright (C) 2010 - 2017 Brookhaven National Laboratory / National Synchrotron Light Source II + * %% + * Copyright (C) 2010 - 2012 Brookhaven National Laboratory + * All rights reserved. Use is subject to license terms. + * #L% + */ + + import java.security.Principal; import java.util.HashSet; import java.util.Set; diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 4e42691..6f4efb0 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -41,7 +41,6 @@ import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; -import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -63,6 +62,9 @@ import gov.bnl.channelfinder.ChannelsResource.OnlyXmlProperty; import gov.bnl.channelfinder.TagsResource.MyMixInForXmlChannels; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; /** * Top level Jersey HTTP methods for the .../properties URL @@ -112,8 +114,7 @@ public Response list() { final SearchResponse response = client.prepareSearch("properties") .setTypes("property") .setQuery(new MatchAllQueryBuilder()) - .setSize(size) - .execute().actionGet(); + .setSize(size).execute().actionGet(); StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, WebApplicationException { @@ -164,7 +165,7 @@ public Response create(List data) throws IOException { .source(mapper.writeValueAsBytes(property))) ); } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, @@ -214,7 +215,7 @@ public Response update(List data) throws IOException { param.put("value", property.getValue()); for (XmlChannel channel : property.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProperty = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == property.name) { removeProperty = xmlProp} }; " @@ -224,7 +225,7 @@ public Response update(List data) throws IOException { } } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -276,8 +277,8 @@ public Response read(@PathParam("propName") String prop) { //TODO iterator or scrolling needed if (parameters.containsKey("withChannels")) { final SearchResponse channelResult = client.prepareSearch("channelfinder") - .setQuery(matchQuery("properties.name", prop.trim())).setSize(10000).execute() - .actionGet(); + .setQuery(matchQuery("properties.name", prop.trim())) + .setSize(10000).execute().actionGet(); List channels = new ArrayList(); if (channelResult != null) { for (SearchHit hit : channelResult.getHits()) { @@ -331,15 +332,18 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .startObject().field("name", data.getName()).field("owner", data.getOwner()).endObject()); UpdateRequest updateRequest = new UpdateRequest("properties", "property", prop).doc(jsonBuilder() .startObject().field("name", data.getName()).field("owner", data.getOwner()).endObject()) - .upsert(indexRequest).refresh(true); + .upsert(indexRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("properties.name", prop)).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("properties.name", prop)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).refresh(true) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " @@ -360,12 +364,12 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { } param.put("value", ChannelUtil.getProperty(channel, data.getName()).getValue()); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("ctx._source.properties.add(prop)") .addScriptParam("prop", param)); } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -444,11 +448,13 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { param.put("name", data.getName()); param.put("owner", propOwner); SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("properties.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("properties.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("origProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == prop.name) { origProp = xmlProp} }; " @@ -471,7 +477,7 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { } param.put("value", value); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProp = new Object();" + "for (property in ctx._source.properties) " + "{ if (property.name == prop.name) { removeProp = property} }; " @@ -480,7 +486,7 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { .addScriptParam("prop", param)); } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -506,8 +512,10 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { private Response renameProperty(UserManager um, Client client, XmlProperty original, XmlProperty data) { try { SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("properties.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("properties.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); List channelNames = new ArrayList(); for (SearchHit hit : queryResponse.getHits()) { channelNames.add(hit.getId()); @@ -527,7 +535,7 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi param.put("name", data.getName()); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("origProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == originalProp.name) { origProp = xmlProp} }; " @@ -538,7 +546,7 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi .addScriptParam("newProp", param)); } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -578,11 +586,14 @@ public Response remove(@PathParam("propName") String prop) { BulkRequestBuilder bulkRequest = client.prepareBulk(); bulkRequest.add(new DeleteRequest("properties", "property", prop)); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("properties.name", prop)).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("properties.name", prop)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).refresh(true) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " @@ -591,14 +602,14 @@ public Response remove(@PathParam("propName") String prop) { } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); throw new Exception(); } else { DeleteResponse deleteResponse = bulkResponse.getItems()[0].getResponse(); - if (deleteResponse.isFound()) { + if (deleteResponse.getResult() == DocWriteResponse.Result.DELETED) { Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); return r; @@ -652,7 +663,7 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName param.put("owner", result.getOwner()); UpdateResponse updateResponse = client.update(new UpdateRequest("channelfinder", "channel", chan) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProps = new java.util.ArrayList(); " + "for (property in ctx._source.properties) " + "{ if (property.name == prop.name) { removeProps.add(property)} }; " @@ -696,7 +707,7 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN .script(" removeProps = new java.util.ArrayList();" + "for (property in ctx._source.properties)" + "{ if (property.name == prop) { removeProps.add(property)} };" + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}") - .addScriptParam("prop", prop).refresh(true)).actionGet(); + .addScriptParam("prop", prop).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); Response r = Response.ok().build(); return r; } else { diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java index 8bbe73f..830fc83 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java @@ -1,5 +1,16 @@ package gov.bnl.channelfinder; +/* + * #%L + * ChannelFinder Directory Service + * %% + * Copyright (C) 2010 - 2016 Brookhaven National Laboratory / National Synchrotron Light Source II + * %% + * Copyright (C) 2010 - 2012 Brookhaven National Laboratory + * All rights reserved. Use is subject to license terms. + * #L% + */ + import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerResponseContext; import javax.ws.rs.container.ContainerResponseFilter; diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index 78821c0..814bc70 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -53,7 +53,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -69,6 +68,9 @@ import com.google.common.collect.Collections2; import gov.bnl.channelfinder.ChannelsResource.OnlyXmlTag; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; /** * Top level Jersey HTTP methods for the .../tags URL @@ -118,8 +120,7 @@ public Response list() { final SearchResponse response = client.prepareSearch("tags") .setTypes("tag") .setQuery(new MatchAllQueryBuilder()) - .setSize(size) - .execute().actionGet(); + .setSize(size).execute().actionGet(); StreamingOutput stream = new StreamingOutput(){ @Override public void write(OutputStream os) throws IOException, WebApplicationException { @@ -232,9 +233,11 @@ public Response create(@PathParam("tagName") String tag, XmlTag data) { .upsert(indexRequest); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("tags.name", tag)).addField("name").setSize(10000).execute() - .actionGet(); - + .setQuery(QueryBuilders.matchQuery("tags.name", tag)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); + Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { existingChannels.add(hit.field("name").getValue().toString()); @@ -260,18 +263,18 @@ public String apply(XmlChannel channel) { param.put("name", data.getName()); param.put("owner", data.getOwner()); for (String ch : remove) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).refresh(true) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " + "ctx._source.tags.remove(removeTag);") .addScriptParam("tag", param)); } for (String ch : add) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).refresh(true) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("ctx._source.tags.add(tag)").addScriptParam("tag", param)); } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -335,7 +338,10 @@ public Response createTags(List data) { .field("name", xmlTag.getName()).field("owner", xmlTag.getOwner()).endObject()).upsert(indexRequest); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("tags.name", xmlTag.getName())).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("tags.name", xmlTag.getName())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { @@ -363,7 +369,7 @@ public String apply(XmlChannel channel) { param.put("name", xmlTag.getName()); param.put("owner", xmlTag.getOwner()); for (String ch : remove) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).refresh(true) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -377,7 +383,7 @@ public String apply(XmlChannel channel) { } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -445,11 +451,13 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { param.put("owner", tagOwner); if(!original.getOwner().equals(data.getOwner())){ SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("tags.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("tags.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -462,7 +470,7 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { if (data.getChannels() != null) { for (XmlChannel channel : data.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -471,7 +479,7 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { .addScriptParam("tag", param)); } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -509,8 +517,10 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { private Response renameTag(UserManager um, Client client, XmlTag original, XmlTag data) { try { SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("tags.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("tags.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); List channelNames = new ArrayList(); for (SearchHit hit : queryResponse.getHits()) { channelNames.add(hit.getId()); @@ -533,7 +543,7 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa param.put("owner", tagOwner); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == originalTag.name) { removeTag = xmltag} }; " @@ -543,7 +553,7 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa .addScriptParam("tag", param)); } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -594,7 +604,7 @@ public Response updateTags(List data) throws IOException { param.put("owner", tag.getOwner()); for (XmlChannel channel : tag.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -604,7 +614,7 @@ public Response updateTags(List data) throws IOException { } } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -645,11 +655,14 @@ public Response remove(@PathParam("tagName") String tag) { BulkRequestBuilder bulkRequest = client.prepareBulk(); bulkRequest.add(new DeleteRequest("tags", "tag", tag)); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("tags.name", tag)).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("tags.name", tag)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).refresh(true) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag) { removeTag = xmltag} }; " @@ -657,7 +670,7 @@ public Response remove(@PathParam("tagName") String tag) { .addScriptParam("tag", tag)); } } - bulkRequest.setRefresh(true); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); @@ -665,7 +678,7 @@ public Response remove(@PathParam("tagName") String tag) { bulkResponse.buildFailureMessage()); } else { DeleteResponse deleteResponse = bulkResponse.getItems()[0].getResponse(); - if (deleteResponse.isFound()) { + if (deleteResponse.getResult() == DocWriteResponse.Result.DELETED) { Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); return r; @@ -718,8 +731,8 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") param.put("name", result.getName()); param.put("owner", result.getOwner()); UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan).refresh(true) - .refresh(true) + .update(new UpdateRequest("channelfinder", "channel", chan).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)};" @@ -778,7 +791,7 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( try { if (client.prepareGet("tags", "tag", tag).get().isExists()) { UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan).refresh(true) + .update(new UpdateRequest("channelfinder", "channel", chan).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script(" removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)}") diff --git a/channelfinder/src/test/serverresources/config/logging.properties b/channelfinder/src/test/serverresources/config/logging.properties index 53ca421..82ad9ce 100644 --- a/channelfinder/src/test/serverresources/config/logging.properties +++ b/channelfinder/src/test/serverresources/config/logging.properties @@ -1,3 +1,13 @@ +### +# #%L +# ChannelFinder Directory Service +# %% +# Copyright (C) 2010 - 2016 Brookhaven National Laboratory / National Synchrotron Light Source II +# %% +# Copyright (C) 2010 - 2012 Brookhaven National Laboratory +# All rights reserved. Use is subject to license terms. +# #L% +### # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # From e712b01d1676aee9f095df3cbae764bc83d5c967 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 19 May 2017 11:37:09 -0400 Subject: [PATCH 21/48] fix null pointer --- .../main/java/gov/bnl/channelfinder/ElasticSearchClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java index 7669567..e6f8fb6 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java @@ -72,8 +72,8 @@ public static Settings getSettings(){ @Override public void contextInitialized(ServletContextEvent sce) { log.info("Initializing a new Transport clients."); - searchClient = new PreBuiltTransportClient(settings); - indexClient = new PreBuiltTransportClient(settings); + searchClient = new PreBuiltTransportClient(Settings.EMPTY); + indexClient = new PreBuiltTransportClient(Settings.EMPTY); settings = searchClient.settings(); String host = settings.get("network.host"); int port = Integer.valueOf(settings.get("transport.tcp.port")); From 6359d6f4b6268c003d3ef3fc9b2bd8f6a52e1233 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 19 May 2017 14:34:33 -0400 Subject: [PATCH 22/48] changed settings to api, added new depends --- channelfinder/pom.xml | 17 ++++++++++---- .../bnl/channelfinder/ChannelsResource.java | 1 - .../channelfinder/ElasticSearchClient.java | 23 ++++++++++++------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 45d0a4a..917f0b9 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -163,18 +163,17 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - 2.5.1 + 2.8.8 com.fasterxml.jackson.core jackson-core - 2.5.1 - provided + 2.8.8 com.fasterxml.jackson.core jackson-databind - 2.5.1 + 2.8.8 provided @@ -188,6 +187,16 @@ 1.4.17 maven-plugin + + org.apache.logging.log4j + log4j-api + 2.7 + + + org.apache.logging.log4j + log4j-core + 2.7 + diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index df421c0..811091d 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -70,7 +70,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; /** * Top level Jersey HTTP methods for the .../channels URL diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java index e6f8fb6..69a7194 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java @@ -15,6 +15,7 @@ */ +import java.io.IOException; import java.util.logging.Logger; import javax.servlet.ServletContextEvent; @@ -71,14 +72,20 @@ public static Settings getSettings(){ @Override public void contextInitialized(ServletContextEvent sce) { - log.info("Initializing a new Transport clients."); - searchClient = new PreBuiltTransportClient(Settings.EMPTY); - indexClient = new PreBuiltTransportClient(Settings.EMPTY); - settings = searchClient.settings(); - String host = settings.get("network.host"); - int port = Integer.valueOf(settings.get("transport.tcp.port")); - searchClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); - indexClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); + try { + log.info("Initializing a new Transport clients."); + String yaml = "elasticsearch.yml"; + settings = Settings.builder().loadFromStream(yaml,getClass().getClassLoader().getResourceAsStream(yaml)).build(); + String host = settings.get("network.host"); + int port = Integer.valueOf(settings.get("transport.tcp.port")); + + searchClient = new PreBuiltTransportClient(settings); + indexClient = new PreBuiltTransportClient(settings); + searchClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); + indexClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); + } catch (IOException e) { + log.severe(e.getMessage()); + } } @Override From da22751d162863ae5cd6af749e237f10d86b712d Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 19 May 2017 14:43:55 -0400 Subject: [PATCH 23/48] update travis and readme to elastic 5.4 --- .travis.yml | 6 +++--- README.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52bb803..21a59b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jdk: - oraclejdk8 before_install: - - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.3.deb && sudo dpkg -i --force-confnew elasticsearch-1.7.3.deb + - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-5.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.4.0.deb install: - sudo apt-get update @@ -13,8 +13,8 @@ install: before_script: - sleep 10 - - 'echo "script.inline: on" | sudo tee -a /etc/elasticsearch/elasticsearch.yml' - - 'echo "script.inline: on" | sudo tee -a /etc/elasticsearch/indexed.yml' + - 'echo "script.inline: true" | sudo tee -a /etc/elasticsearch/elasticsearch.yml' + - 'echo "script.inline: true" | sudo tee -a /etc/elasticsearch/indexed.yml' - sudo /etc/init.d/elasticsearch start - sleep 25 - curl -XGET 'http://localhost:9200/_cluster/health?pretty' diff --git a/README.md b/README.md index 5192952..46472a8 100644 --- a/README.md +++ b/README.md @@ -69,12 +69,12 @@ in the web container. ``` ################################# Scripting ############################### - script.inline: on - script.indexed: on + script.inline: true + script.stored: true ``` - * For more information of how Index and mappings can be setup using any rest client as described here [create elastic index](https://www.elastic.co/guide/en/elasticsearch/reference/1.4/_create_an_index.html) + * For more information of how Index and mappings can be setup using any rest client as described here [create elastic index](https://www.elastic.co/guide/en/elasticsearch/reference/5.4/_create_an_index.html) 4. Authentication/Authorization From c094161afa08234b5a0bd1982848c4f71c8202c5 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 19 May 2017 14:50:06 -0400 Subject: [PATCH 24/48] travis: url change --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 21a59b9..cd52d10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jdk: - oraclejdk8 before_install: - - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-5.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.4.0.deb + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.4.0.deb install: - sudo apt-get update From 9948a3dd2102b161ce70a2541f0c6953ba9bfd81 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 22 May 2017 08:47:20 -0400 Subject: [PATCH 25/48] updated mapping_definitions to use text/keyword, then sort from keyword --- .../bnl/channelfinder/ChannelsResource.java | 2 +- .../src/main/resources/mapping_definitions.sh | 49 ++++++++++++++----- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index 811091d..4c08ef8 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -169,7 +169,7 @@ public Response query() { start = System.currentTimeMillis(); SearchRequestBuilder builder = client.prepareSearch("channelfinder").setQuery(qb).setSize(size); if(from >= 0){ - builder.addSort(SortBuilders.fieldSort("name")); + builder.addSort(SortBuilders.fieldSort("name.keyword")); builder.setFrom(from); } final SearchResponse qbResult = builder.execute().actionGet(); diff --git a/channelfinder/src/main/resources/mapping_definitions.sh b/channelfinder/src/main/resources/mapping_definitions.sh index fee1465..2a68496 100644 --- a/channelfinder/src/main/resources/mapping_definitions.sh +++ b/channelfinder/src/main/resources/mapping_definitions.sh @@ -22,10 +22,15 @@ curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' "tag" : { "properties" : { "name" : { - "type" : "string" + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "owner" : { - "type" : "string" + "type" : "text" } } } @@ -37,10 +42,15 @@ curl -XPUT 'http://localhost:9200/properties/_mapping/property' -d' "property" : { "properties" : { "name" : { - "type" : "string" + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "owner" : { - "type" : "string" + "type" : "text" } } } @@ -52,29 +62,39 @@ curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' "channel" : { "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" }, "script" : { - "type" : "string" + "type" : "text" }, "properties" : { "type" : "nested", "include_in_parent" : true, "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string" + "type" : "text" }, "value" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" } } @@ -84,11 +104,16 @@ curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' "include_in_parent" : true, "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" } } From 06c67551a9f27468fc26f5b1bda71d15985a4646 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 22 May 2017 10:58:44 -0400 Subject: [PATCH 26/48] added base.dir to relative path in pom.xml --- channelfinder/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 917f0b9..8a7221b 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -399,7 +399,7 @@ nosetests - ../test + ${project.basedir}/../test ${skipNoseTests} -v From c14165248c5bc375530a5b73d2bc187fe707bc7d Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 22 May 2017 11:28:35 -0400 Subject: [PATCH 27/48] travis: use addon to install python-nose --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd52d10..46d3903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,14 @@ sudo: required jdk: - oraclejdk8 +addons: + apt: + packages: + - python-nose + before_install: - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.4.0.deb -install: - - sudo apt-get update - - sudo apt-get install python-nose - before_script: - sleep 10 - 'echo "script.inline: true" | sudo tee -a /etc/elasticsearch/elasticsearch.yml' From d50047a7ab685fc6b5b7704d5f7fafe98904728e Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 22 May 2017 11:41:22 -0400 Subject: [PATCH 28/48] travis: add debug mvn --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 46d3903..6ff4b43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_script: script: - channelfinder/src/main/resources/mapping_definitions.sh - - mvn clean verify -fchannelfinder/pom.xml + - mvn -X clean verify -fchannelfinder/pom.xml after_script: - sudo /etc/init.d/elasticsearch stop From 0941173bcdb1945c4bd2b9d8cb898b0e04fe08de Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 22 May 2017 12:07:52 -0400 Subject: [PATCH 29/48] removing nosetests arg causing issue on travis --- channelfinder/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 8a7221b..54a2e32 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -405,7 +405,7 @@ -v -s --with-xunit - --xunit-file=${project.build.directory}/failsafe-reports/nosetests.xml + From 17dfb6b4dd03f7b69a686f5cf036285ec7b036f0 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 22 May 2017 12:13:46 -0400 Subject: [PATCH 30/48] travis: remove debug --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6ff4b43..46d3903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_script: script: - channelfinder/src/main/resources/mapping_definitions.sh - - mvn -X clean verify -fchannelfinder/pom.xml + - mvn clean verify -fchannelfinder/pom.xml after_script: - sudo /etc/init.d/elasticsearch stop From 624f343ffdede9bf62568d547c9c086483f4a8dd Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Wed, 24 May 2017 12:32:53 -0400 Subject: [PATCH 31/48] refresh only on bulk requests, change property put to use XContentBuilder, instead of script --- channelfinder/pom.xml | 79 ++++++++++--------- .../bnl/channelfinder/ChannelsResource.java | 6 +- .../bnl/channelfinder/PropertiesResource.java | 40 +++++----- .../gov/bnl/channelfinder/TagsResource.java | 17 ++-- .../test/serverresources/config/domain.xml | 44 ++++++----- 5 files changed, 94 insertions(+), 92 deletions(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 54a2e32..c6e6cf8 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -250,37 +250,27 @@ - org.codehaus.mojo - exec-maven-plugin - 1.5.0 - - - fish.payara.extras - payara-micro - 4.1.1.162 - - + maven-dependency-plugin + 2.6 - payara-uber-jar + copy-payara-micro package - java + copy - fish.payara.micro.PayaraMicro - - --deploy - ${basedir}/target/${project.build.finalName}.war - --outputUberJar - ${basedir}/target/${project.build.finalName}.jar - - false - true - - fish.payara.extras - payara-micro - + target + true + true + + + fish.payara.extras + payara-micro + 4.1.2.172 + jar + + @@ -395,19 +385,36 @@ exec + + nosetests + ${project.basedir}/../test + ${skipNoseTests} + + -v + -s + --with-xunit + + + + + payara-uber-jar + package + + exec + + + java + + -jar + ${basedir}/target/payara-micro.jar + --deploy + ${basedir}/target/${project.build.finalName}.war + --outputUberJar + ${basedir}/target/${project.build.finalName}.jar + + - - nosetests - ${project.basedir}/../test - ${skipNoseTests} - - -v - -s - --with-xunit - - - ChannelFinder diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index 4c08ef8..9c62188 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -335,7 +335,7 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { IndexRequest indexRequest = new IndexRequest("channelfinder", "channel", chan) .source(mapper.writeValueAsBytes(data)); UpdateRequest updateRequest = new UpdateRequest("channelfinder", "channel", chan) - .doc(mapper.writeValueAsBytes(data)).upsert(indexRequest).setRefreshPolicy(RefreshPolicy.IMMEDIATE); + .doc(mapper.writeValueAsBytes(data)).upsert(indexRequest); UpdateResponse result = client.update(updateRequest).actionGet(); Response r = Response.noContent().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|PUT|OK|" + r.getStatus() + "|data=" + XmlChannel.toLog(data)); @@ -394,7 +394,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { }).collect(Collectors.toList())); channel.setTags(data.getTags()); UpdateRequest updateRequest = new UpdateRequest("channelfinder", "channel", chan) - .doc(mapper.writeValueAsBytes(channel)).setRefreshPolicy(RefreshPolicy.IMMEDIATE); + .doc(mapper.writeValueAsBytes(channel)); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|POST|prepare : "+ (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); UpdateResponse result = client.update(updateRequest).actionGet(); @@ -479,7 +479,7 @@ public Response remove(@PathParam("chName") String chan) { UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { - DeleteResponse deleteResponse = client.prepareDelete("channelfinder", "channel", chan).setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute().get(); + DeleteResponse deleteResponse = client.prepareDelete("channelfinder", "channel", chan).execute().get(); if(deleteResponse.getResult() == DocWriteResponse.Result.DELETED){ Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 6f4efb0..ca844b6 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -65,6 +65,8 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; /** * Top level Jersey HTTP methods for the .../properties URL @@ -215,7 +217,6 @@ public Response update(List data) throws IOException { param.put("value", property.getValue()); for (XmlChannel channel : property.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProperty = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == property.name) { removeProperty = xmlProp} }; " @@ -332,7 +333,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .startObject().field("name", data.getName()).field("owner", data.getOwner()).endObject()); UpdateRequest updateRequest = new UpdateRequest("properties", "property", prop).doc(jsonBuilder() .startObject().field("name", data.getName()).field("owner", data.getOwner()).endObject()) - .upsert(indexRequest).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + .upsert(indexRequest); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") @@ -343,7 +344,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) .script("removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " @@ -364,7 +365,6 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { } param.put("value", ChannelUtil.getProperty(channel, data.getName()).getValue()); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("ctx._source.properties.add(prop)") .addScriptParam("prop", param)); } @@ -454,7 +454,6 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { .setSize(10000).execute().actionGet(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("origProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == prop.name) { origProp = xmlProp} }; " @@ -477,7 +476,6 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { } param.put("value", value); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeProp = new Object();" + "for (property in ctx._source.properties) " + "{ if (property.name == prop.name) { removeProp = property} }; " @@ -535,7 +533,6 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi param.put("name", data.getName()); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("origProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == originalProp.name) { origProp = xmlProp} }; " @@ -593,7 +590,7 @@ public Response remove(@PathParam("propName") String prop) { if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) .script("removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " @@ -656,21 +653,20 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName mapper.addMixIn(XmlChannel.class, MyMixInForXmlChannels.class); result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); if (result != null) { - HashMap param = new HashMap(); - param.put("name", data.getName()); - param.put("value", data.getValue()); + XContentBuilder xb = XContentFactory.jsonBuilder().startObject(); + xb.startArray("properties"); + xb.startObject(); + xb.field("name", data.getName()); + xb.field("value", data.getValue()); // ignores the provided user and matches the one present in the properties index - param.put("owner", result.getOwner()); + xb.field("owner", result.getOwner()); + xb.endObject(); + xb.endArray(); + xb.endObject(); + + client.update(new UpdateRequest("channelfinder", "channel", chan) + .doc(xb)).get(); - UpdateResponse updateResponse = client.update(new UpdateRequest("channelfinder", "channel", chan) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .script("removeProps = new java.util.ArrayList(); " - + "for (property in ctx._source.properties) " - + "{ if (property.name == prop.name) { removeProps.add(property)} }; " - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}; " - + "ctx._source.properties.add(prop)") - .addScriptParam("prop", param)) - .actionGet(); Response r = Response.ok().build(); return r; }else{ @@ -707,7 +703,7 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN .script(" removeProps = new java.util.ArrayList();" + "for (property in ctx._source.properties)" + "{ if (property.name == prop) { removeProps.add(property)} };" + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}") - .addScriptParam("prop", prop).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet(); + .addScriptParam("prop", prop)).actionGet(); Response r = Response.ok().build(); return r; } else { diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index 814bc70..72e0331 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -263,14 +263,14 @@ public String apply(XmlChannel channel) { param.put("name", data.getName()); param.put("owner", data.getOwner()); for (String ch : remove) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " + "ctx._source.tags.remove(removeTag);") .addScriptParam("tag", param)); } for (String ch : add) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script("ctx._source.tags.add(tag)").addScriptParam("tag", param)); } @@ -369,7 +369,7 @@ public String apply(XmlChannel channel) { param.put("name", xmlTag.getName()); param.put("owner", xmlTag.getOwner()); for (String ch : remove) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -457,7 +457,6 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { .setSize(10000).execute().actionGet(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -470,7 +469,6 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { if (data.getChannels() != null) { for (XmlChannel channel : data.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -543,7 +541,6 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa param.put("owner", tagOwner); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == originalTag.name) { removeTag = xmltag} }; " @@ -604,7 +601,6 @@ public Response updateTags(List data) throws IOException { param.put("owner", tag.getOwner()); for (XmlChannel channel : tag.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " @@ -662,7 +658,7 @@ public Response remove(@PathParam("tagName") String tag) { if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + "{ if (xmltag.name == tag) { removeTag = xmltag} }; " @@ -731,8 +727,7 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") param.put("name", result.getName()); param.put("owner", result.getOwner()); UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .update(new UpdateRequest("channelfinder", "channel", chan) .script("removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)};" @@ -791,7 +786,7 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( try { if (client.prepareGet("tags", "tag", tag).get().isExists()) { UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .update(new UpdateRequest("channelfinder", "channel", chan) .script(" removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)}") diff --git a/channelfinder/src/test/serverresources/config/domain.xml b/channelfinder/src/test/serverresources/config/domain.xml index 7d314ef..34f9a4d 100644 --- a/channelfinder/src/test/serverresources/config/domain.xml +++ b/channelfinder/src/test/serverresources/config/domain.xml @@ -2,23 +2,23 @@ #%L ChannelFinder Directory Service %% - Copyright (C) 2010 - 2012 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH + Copyright (C) 2010 - 2017 Brookhaven National Laboratory / National Synchrotron Light Source II %% Copyright (C) 2010 - 2012 Brookhaven National Laboratory All rights reserved. Use is subject to license terms. #L% --> - + - + - + @@ -32,7 +32,7 @@ - + @@ -54,7 +54,7 @@ - + @@ -62,7 +62,7 @@ - + @@ -105,9 +105,6 @@ - - - @@ -120,8 +117,18 @@ - + + + + + + + + + + + @@ -250,7 +257,7 @@ - + @@ -298,9 +305,6 @@ - - - @@ -391,7 +395,6 @@ - @@ -439,7 +442,7 @@ - + @@ -466,9 +469,10 @@ - - - + + + + From d19eb113156acda0f728cd137f52a3f3a47e618e Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Wed, 24 May 2017 16:36:31 -0400 Subject: [PATCH 32/48] add exceptions to cover non existing channel for property resource --- .../main/java/gov/bnl/channelfinder/PropertiesResource.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index ca844b6..48764fc 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -62,11 +62,13 @@ import gov.bnl.channelfinder.ChannelsResource.OnlyXmlProperty; import gov.bnl.channelfinder.TagsResource.MyMixInForXmlChannels; +import java.util.concurrent.ExecutionException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.transport.RemoteTransportException; /** * Top level Jersey HTTP methods for the .../properties URL @@ -673,9 +675,9 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName return handleException(um.getUserName(), "PUT", Status.BAD_REQUEST, "Property " +prop+ " does not exist "); } - } catch (DocumentMissingException e) { + } catch (DocumentMissingException | RemoteTransportException | ExecutionException e) { return handleException(um.getUserName(), "PUT", Response.Status.BAD_REQUEST, - "Channels specified in property update do not exist" + e.getDetailedMessage()); + "Channels specified in property update do not exist" + e.getMessage()); } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { From e796c105534038523e667b6d8a7026e8858e7e5d Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Thu, 25 May 2017 08:14:35 -0400 Subject: [PATCH 33/48] remove script from TagsResource --- .../gov/bnl/channelfinder/TagsResource.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index 72e0331..cbeddfe 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -68,9 +68,13 @@ import com.google.common.collect.Collections2; import gov.bnl.channelfinder.ChannelsResource.OnlyXmlTag; +import java.util.concurrent.ExecutionException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.transport.RemoteTransportException; /** * Top level Jersey HTTP methods for the .../tags URL @@ -722,18 +726,20 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") if (result != null) { // if(um.userHasAdminRole() || um.userIsInGroup(result.getOwner())){ - if (validateTag(result, data)) { - HashMap param = new HashMap(); - param.put("name", result.getName()); - param.put("owner", result.getOwner()); - UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan) - .script("removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " - + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)};" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)) - .actionGet(); + if (validateTag(result, data)) { + XContentBuilder xb = XContentFactory.jsonBuilder().startObject(); + xb.startArray("tags"); + xb.startObject(); + xb.field("name", data.getName()); + // ignores the provided user and matches the one present in the properties index + xb.field("owner", result.getOwner()); + xb.endObject(); + xb.endArray(); + xb.endObject(); + + client.update(new UpdateRequest("channelfinder", "channel", chan) + .doc(xb)).get(); + Response r = Response.ok().build(); return r; } else { @@ -749,8 +755,8 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") }else{ return Response.status(Status.BAD_REQUEST).entity(tag + " Does not exist").build(); } - } catch (DocumentMissingException e) { - return Response.status(Status.BAD_REQUEST).entity("Channels specified in tag update do not exist"+e.getDetailedMessage()).build(); + } catch (DocumentMissingException | RemoteTransportException | ExecutionException e) { + return Response.status(Status.BAD_REQUEST).entity("Channels specified in tag update do not exist"+e.getMessage()).build(); } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { From d2935619d6c0354a8ed0c8b0393cc8590c1c2fb7 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 26 May 2017 12:06:52 -0400 Subject: [PATCH 34/48] updating to new script api --- .../bnl/channelfinder/PropertiesResource.java | 101 ++++++++++------- .../gov/bnl/channelfinder/TagsResource.java | 105 +++++++++++------- 2 files changed, 121 insertions(+), 85 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 48764fc..00bb587 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -62,12 +62,15 @@ import gov.bnl.channelfinder.ChannelsResource.OnlyXmlProperty; import gov.bnl.channelfinder.TagsResource.MyMixInForXmlChannels; +import java.util.Collections; import java.util.concurrent.ExecutionException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.transport.RemoteTransportException; /** @@ -213,18 +216,21 @@ public Response update(List data) throws IOException { .source(mapper.writeValueAsBytes(property))) ); if (property.getChannels() != null) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", property.getName()); param.put("owner", property.getOwner()); param.put("value", property.getValue()); + HashMap params = new HashMap<>(); + params.put("property", param); for (XmlChannel channel : property.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .script("removeProperty = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeProperty = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == property.name) { removeProperty = xmlProp} }; " + + "{ if (xmlProp.name == params.property.name) { removeProperty = xmlProp} }; " + "ctx._source.tags.remove(removeProperty);" - + "ctx._source.tags.add(property)") - .addScriptParam("property", param)); + + "ctx._source.tags.add(params.property)", + params))); } } } @@ -282,7 +288,7 @@ public Response read(@PathParam("propName") String prop) { final SearchResponse channelResult = client.prepareSearch("channelfinder") .setQuery(matchQuery("properties.name", prop.trim())) .setSize(10000).execute().actionGet(); - List channels = new ArrayList(); + List channels = new ArrayList<>(); if (channelResult != null) { for (SearchHit hit : channelResult.getHits()) { channels.add(mapper.readValue(hit.source(), XmlChannel.class)); @@ -345,19 +351,18 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.field("name").getValue().toString(); + String channelName = hit.getField("name").getValue().toString(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) - .script("removeProp = new Object();" + .script(new Script("removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " - + "ctx._source.properties.remove(removeProp);") - .addScriptParam("prop", prop)); + + "{ if (xmlProp.name == "+prop+") { removeProp = xmlProp} }; " + + "ctx._source.properties.remove(removeProp);"))); } } if (data.getChannels() != null) { for (XmlChannel channel : data.getChannels()) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", data.getOwner()); String value = ChannelUtil.getProperty(channel, data.getName()).getValue(); @@ -366,9 +371,12 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { "Invalid property value (missing or null or empty string) for '"+data.getName()+"'"); } param.put("value", ChannelUtil.getProperty(channel, data.getName()).getValue()); + HashMap params = new HashMap<>(); + params.put("prop", param); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .script("ctx._source.properties.add(prop)") - .addScriptParam("prop", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.add(params.prop)", + params))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -446,9 +454,11 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { .endObject()); if(!original.getOwner().equals(data.getOwner())){ - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", propOwner); + HashMap params = new HashMap<>(); + params.put("prop", param); SearchResponse queryResponse = client.prepareSearch("channelfinder") .setQuery(wildcardQuery("properties.name", original.getName().trim())) .setSearchType(SearchType.QUERY_THEN_FETCH) @@ -456,13 +466,14 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { .setSize(10000).execute().actionGet(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .script("origProp = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "origProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == prop.name) { origProp = xmlProp} }; " + + "{ if (xmlProp.name == params.prop.name) { origProp = xmlProp} }; " + "ctx._source.properties.remove(origProp);" - + "origProp.owner = prop.owner;" - + "ctx._source.properties.add(origProp)") - .addScriptParam("prop", param)); + + "origProp.owner = params.prop.owner;" + + "ctx._source.properties.add(origProp)", + params))); } } bulkRequest.add(updateRequest); @@ -477,13 +488,16 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { "Invalid property value (missing or null or empty string) for '"+data.getName()+"'"); } param.put("value", value); + HashMap params = new HashMap<>(); + params.put("prop", param); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .script("removeProp = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeProp = new Object();" + "for (property in ctx._source.properties) " - + "{ if (property.name == prop.name) { removeProp = property} }; " + + "{ if (property.name == params.prop.name) { removeProp = property} }; " + "ctx._source.properties.remove(removeProp);" - + "ctx._source.properties.add(prop)") - .addScriptParam("prop", param)); + + "ctx._source.properties.add(params.prop)", + params))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -529,20 +543,23 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi .field("name", data.getName()).field("owner", data.getOwner()).endObject()).upsert(indexRequest); bulkRequest.add(updateRequest); if (!channelNames.isEmpty()) { - HashMap originalParam = new HashMap(); + HashMap originalParam = new HashMap<>(); originalParam.put("name", original.getName()); - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); + HashMap params = new HashMap<>(); + params.put("originalProp", originalParam); + params.put("newProp", param); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .script("origProp = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "origProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == originalProp.name) { origProp = xmlProp} }; " + + "{ if (xmlProp.name == params.originalProp.name) { origProp = xmlProp} }; " + "ctx._source.properties.remove(origProp);" - + "origProp.name = newProp.name;" - + "ctx._source.properties.add(origProp)") - .addScriptParam("originalProp", originalParam) - .addScriptParam("newProp", param)); + + "origProp.name = params.newProp.name;" + + "ctx._source.properties.add(origProp)", + params))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -591,13 +608,12 @@ public Response remove(@PathParam("propName") String prop) { .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.field("name").getValue().toString(); + String channelName = hit.getField("name").getValue().toString(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) - .script("removeProp = new Object();" + .script(new Script("removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " - + "ctx._source.properties.remove(removeProp);") - .addScriptParam("prop", prop)); + + "{ if (xmlProp.name == "+prop+") { removeProp = xmlProp} }; " + + "ctx._source.properties.remove(removeProp);"))); } } @@ -701,11 +717,12 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { if(client.prepareGet("properties", "property", prop).get().isExists()){ - UpdateResponse updateResponse = client.update(new UpdateRequest("channelfinder", "channel", chan) - .script(" removeProps = new java.util.ArrayList();" + "for (property in ctx._source.properties)" - + "{ if (property.name == prop) { removeProps.add(property)} };" - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}") - .addScriptParam("prop", prop)).actionGet(); + client.update(new UpdateRequest("channelfinder", "channel", chan) + .script(new Script("removeProps = new java.util.ArrayList();" + + "for (property in ctx._source.properties)" + + "{ if (property.name == "+prop+") { removeProps.add(property)} };" + + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}"))) + .actionGet(); Response r = Response.ok().build(); return r; } else { diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index cbeddfe..3090926 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -74,6 +74,8 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.transport.RemoteTransportException; /** @@ -257,25 +259,30 @@ public String apply(XmlChannel channel) { })); } - Set remove = new HashSet(existingChannels); + Set remove = new HashSet<>(existingChannels); remove.removeAll(newChannels); - Set add = new HashSet(newChannels); + Set add = new HashSet<>(newChannels); add.removeAll(existingChannels); - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", data.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); for (String ch : remove) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) - .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);") - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " + + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " + + "ctx._source.tags.remove(removeTag);", + params))); } for (String ch : add) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) - .script("ctx._source.tags.add(tag)").addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.add(params.tag)", + params))); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -372,18 +379,22 @@ public String apply(XmlChannel channel) { HashMap param = new HashMap(); param.put("name", xmlTag.getName()); param.put("owner", xmlTag.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); for (String ch : remove) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) - .script("removeTag = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);") - .addScriptParam("tag", param)); + + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " + + "ctx._source.tags.remove(removeTag);", + params))); } for (String ch : add) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) - .script("ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.add(params.tag)", + params))); } } @@ -453,6 +464,8 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { HashMap param = new HashMap(); param.put("name", data.getName()); param.put("owner", tagOwner); + HashMap params = new HashMap<>(); + params.put("tag", param); if(!original.getOwner().equals(data.getOwner())){ SearchResponse queryResponse = client.prepareSearch("channelfinder") .setQuery(wildcardQuery("tags.name", original.getName().trim())) @@ -461,24 +474,26 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { .setSize(10000).execute().actionGet(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .script("removeTag = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " + + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + + "ctx._source.tags.add(params.tag)", + params))); } } bulkRequest.add(updateRequest); if (data.getChannels() != null) { for (XmlChannel channel : data.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .script("removeTag = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " + + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + + "ctx._source.tags.add(params.tag)", + params))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -538,20 +553,23 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa .field("name", data.getName()).field("owner", data.getOwner()).endObject()).upsert(indexRequest); bulkRequest.add(updateRequest); if (!channelNames.isEmpty()) { - HashMap originalParam = new HashMap(); + HashMap originalParam = new HashMap<>(); originalParam.put("name", original.getName()); - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", tagOwner); + HashMap params = new HashMap<>(); + params.put("originalTag", originalParam); + params.put("tag", param); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .script("removeTag = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == originalTag.name) { removeTag = xmltag} }; " + + "{ if (xmltag.name == params.originalTag.name) { removeTag = xmltag} }; " + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("originalTag", originalParam) - .addScriptParam("tag", param)); + + "ctx._source.tags.add(params.tag)", + params))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -600,17 +618,20 @@ public Response updateTags(List data) throws IOException { .field("owner", tag.getOwner()) .endObject())); if (tag.getChannels() != null) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", tag.getName()); param.put("owner", tag.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); for (XmlChannel channel : tag.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .script("removeTag = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " + + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + + "ctx._source.tags.add(params.tag)", + params))); } } } @@ -661,13 +682,12 @@ public Response remove(@PathParam("tagName") String tag) { .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.field("name").getValue().toString(); + String channelName = hit.getField("name").getValue().toString(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) - .script("removeTag = new Object();" + .script(new Script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);") - .addScriptParam("tag", tag)); + + "{ if (xmltag.name == "+tag+") { removeTag = xmltag} }; " + + "ctx._source.tags.remove(removeTag);"))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -793,10 +813,9 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( if (client.prepareGet("tags", "tag", tag).get().isExists()) { UpdateResponse updateResponse = client .update(new UpdateRequest("channelfinder", "channel", chan) - .script(" removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " - + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)}") - .addScriptParam("tagName", tag)).actionGet(); + .script(new Script(" removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " + + "{ if (tag.name == "+tag+") { removeTags.add(tag)} }; " + + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)}"))).actionGet(); Response r = Response.ok().build(); return r; } else { From 282641c4158da28c322f69bfd2f5d9eeb1afaae6 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 30 May 2017 16:48:20 -0400 Subject: [PATCH 35/48] updating scripts --- .../bnl/channelfinder/PropertiesResource.java | 76 ++++++++------- .../gov/bnl/channelfinder/TagsResource.java | 92 ++++++++----------- 2 files changed, 80 insertions(+), 88 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 00bb587..768cdb1 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -71,6 +71,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHitField; import org.elasticsearch.transport.RemoteTransportException; /** @@ -225,11 +226,11 @@ public Response update(List data) throws IOException { for (XmlChannel channel : property.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeProperty = new Object();" + "Object removeProperty = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.property.name) { removeProperty = xmlProp} }; " + + "{ if (xmlProp.name == params.property.name) { removeProperty = xmlProp;} } " + "ctx._source.tags.remove(removeProperty);" - + "ctx._source.tags.add(params.property)", + + "ctx._source.tags.add(params.property);", params))); } } @@ -353,10 +354,12 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.getField("name").getValue().toString(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) - .script(new Script("removeProp = new Object();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "Object removeProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == "+prop+") { removeProp = xmlProp} }; " - + "ctx._source.properties.remove(removeProp);"))); + + "{ if (xmlProp.name == params.prop) { removeProp = xmlProp;} } " + + "ctx._source.properties.remove(removeProp);", + Collections.singletonMap("prop", prop)))); } } @@ -467,12 +470,12 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "origProp = new Object();" + "Object origProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.prop.name) { origProp = xmlProp} }; " + + "{ if (xmlProp.name == params.prop.name) { origProp = xmlProp;} } " + "ctx._source.properties.remove(origProp);" + "origProp.owner = params.prop.owner;" - + "ctx._source.properties.add(origProp)", + + "ctx._source.properties.add(origProp);", params))); } } @@ -492,11 +495,11 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { params.put("prop", param); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeProp = new Object();" + "Object removeProp = new Object();" + "for (property in ctx._source.properties) " - + "{ if (property.name == params.prop.name) { removeProp = property} }; " + + "{ if (property.name == params.prop.name) { removeProp = property;} } " + "ctx._source.properties.remove(removeProp);" - + "ctx._source.properties.add(params.prop)", + + "ctx._source.properties.add(params.prop);", params))); } } @@ -553,12 +556,12 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "origProp = new Object();" + "Object origProp = new Object();" + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.originalProp.name) { origProp = xmlProp} }; " + + "{ if (xmlProp.name == params.originalProp.name) { origProp = xmlProp;} } " + "ctx._source.properties.remove(origProp);" + "origProp.name = params.newProp.name;" - + "ctx._source.properties.add(origProp)", + + "ctx._source.properties.add(origProp);", params))); } } @@ -608,12 +611,11 @@ public Response remove(@PathParam("propName") String prop) { .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.getField("name").getValue().toString(); + String channelName = hit.getId(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) - .script(new Script("removeProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == "+prop+") { removeProp = xmlProp} }; " - + "ctx._source.properties.remove(removeProp);"))); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop); ", + Collections.singletonMap("prop", prop)))); } } @@ -671,27 +673,29 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName mapper.addMixIn(XmlChannel.class, MyMixInForXmlChannels.class); result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); if (result != null) { - XContentBuilder xb = XContentFactory.jsonBuilder().startObject(); - xb.startArray("properties"); - xb.startObject(); - xb.field("name", data.getName()); - xb.field("value", data.getValue()); + HashMap param = new HashMap<>(); + param.put("name", data.getName()); + param.put("value", data.getValue()); // ignores the provided user and matches the one present in the properties index - xb.field("owner", result.getOwner()); - xb.endObject(); - xb.endArray(); - xb.endObject(); + param.put("owner", result.getOwner()); + HashMap params = new HashMap<>(); + params.put("prop", param); client.update(new UpdateRequest("channelfinder", "channel", chan) - .doc(xb)).get(); - + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ArrayList removeProps = new ArrayList(); " + + "for (property in ctx._source.properties) " + + "{ if (property.name == params.prop.name) { removeProps.add(property);} } " + + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp);} " + + "ctx._source.properties.add(params.prop)",params))) + .actionGet(); Response r = Response.ok().build(); return r; }else{ return handleException(um.getUserName(), "PUT", Status.BAD_REQUEST, "Property " +prop+ " does not exist "); } - } catch (DocumentMissingException | RemoteTransportException | ExecutionException e) { + } catch (DocumentMissingException | RemoteTransportException e) { return handleException(um.getUserName(), "PUT", Response.Status.BAD_REQUEST, "Channels specified in property update do not exist" + e.getMessage()); } catch (Exception e) { @@ -718,10 +722,12 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN try { if(client.prepareGet("properties", "property", prop).get().isExists()){ client.update(new UpdateRequest("channelfinder", "channel", chan) - .script(new Script("removeProps = new java.util.ArrayList();" + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ArrayList removeProps = new ArrayList();" + "for (property in ctx._source.properties)" - + "{ if (property.name == "+prop+") { removeProps.add(property)} };" - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}"))) + + "{ if (property.name == params.prop) { removeProps.add(property);} }" + + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp);}", + Collections.singletonMap("prop", prop)))) .actionGet(); Response r = Response.ok().build(); return r; diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index 3090926..bfd9c86 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -68,6 +68,7 @@ import com.google.common.collect.Collections2; import gov.bnl.channelfinder.ChannelsResource.OnlyXmlTag; +import java.util.Collections; import java.util.concurrent.ExecutionException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; @@ -246,7 +247,7 @@ public Response create(@PathParam("tagName") String tag, XmlTag data) { Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { - existingChannels.add(hit.field("name").getValue().toString()); + existingChannels.add(hit.getId()); } Set newChannels = new HashSet(); @@ -273,15 +274,13 @@ public String apply(XmlChannel channel) { for (String ch : remove) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);", + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);", params))); } for (String ch : add) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ctx._source.tags.add(params.tag)", + "ctx._source.tags.add(params.tag);", params))); } @@ -356,10 +355,10 @@ public Response createTags(List data) { Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { - existingChannels.add(hit.field("name").getValue().toString()); + existingChannels.add(hit.getId()); } - Set newChannels = new HashSet(); + Set newChannels = new HashSet<>(); if (xmlTag.getChannels() != null) { newChannels.addAll( Collections2.transform(xmlTag.getChannels(), new Function() { @@ -384,16 +383,13 @@ public String apply(XmlChannel channel) { for (String ch : remove) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);", + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);", params))); } for (String ch : add) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ctx._source.tags.add(params.tag)", + "ctx._source.tags.add(params.tag);", params))); } } @@ -475,11 +471,8 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(params.tag)", + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);", params))); } } @@ -488,11 +481,8 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { for (XmlChannel channel : data.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(params.tag)", + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);", params))); } } @@ -564,11 +554,8 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == params.originalTag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(params.tag)", + "ctx._source.tags.removeIf(item -> item.name == params.originalTag.name);" + + "ctx._source.tags.add(params.tag);", params))); } } @@ -626,11 +613,8 @@ public Response updateTags(List data) throws IOException { for (XmlChannel channel : tag.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == params.tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(params.tag)", + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);", params))); } } @@ -682,12 +666,11 @@ public Response remove(@PathParam("tagName") String tag) { .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.getField("name").getValue().toString(); + String channelName = hit.getId(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) - .script(new Script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == "+tag+") { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);"))); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag);", + Collections.singletonMap("tag", tag)))); } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -747,19 +730,19 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") if (result != null) { // if(um.userHasAdminRole() || um.userIsInGroup(result.getOwner())){ if (validateTag(result, data)) { - XContentBuilder xb = XContentFactory.jsonBuilder().startObject(); - xb.startArray("tags"); - xb.startObject(); - xb.field("name", data.getName()); - // ignores the provided user and matches the one present in the properties index - xb.field("owner", result.getOwner()); - xb.endObject(); - xb.endArray(); - xb.endObject(); - + HashMap param = new HashMap<>(); + param.put("name", result.getName()); + param.put("owner", result.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); client.update(new UpdateRequest("channelfinder", "channel", chan) - .doc(xb)).get(); - + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ArrayList removeTags = new ArrayList();" + + "for (tag in ctx._source.tags) " + + "{ if (tag.name == params.tag.name) { removeTags.add(tag);} } " + + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag);}" + + "ctx._source.tags.add(params.tag);",params))) + .actionGet(); Response r = Response.ok().build(); return r; } else { @@ -775,7 +758,7 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") }else{ return Response.status(Status.BAD_REQUEST).entity(tag + " Does not exist").build(); } - } catch (DocumentMissingException | RemoteTransportException | ExecutionException e) { + } catch (DocumentMissingException | RemoteTransportException e) { return Response.status(Status.BAD_REQUEST).entity("Channels specified in tag update do not exist"+e.getMessage()).build(); } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); @@ -813,9 +796,12 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( if (client.prepareGet("tags", "tag", tag).get().isExists()) { UpdateResponse updateResponse = client .update(new UpdateRequest("channelfinder", "channel", chan) - .script(new Script(" removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " - + "{ if (tag.name == "+tag+") { removeTags.add(tag)} }; " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)}"))).actionGet(); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ArrayList removeTags = new ArrayList();" + + "for (tag in ctx._source.tags) " + + "{ if (tag.name == params.tag) { removeTags.add(tag);} } " + + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag);}", + Collections.singletonMap("tag", tag)))).actionGet(); Response r = Response.ok().build(); return r; } else { From 1f8a55f6076a66eac46ee62f0a82398695980368 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 2 Jun 2017 08:55:45 -0400 Subject: [PATCH 36/48] updated scripts to use lambdas, and fixed the document missing tests to match the new output --- .../bnl/channelfinder/ChannelsResource.java | 1 - .../bnl/channelfinder/PropertiesResource.java | 54 +++++-------------- .../gov/bnl/channelfinder/TagsResource.java | 13 +---- test/testcf.py | 8 +-- test/testcftags.py | 2 +- 5 files changed, 20 insertions(+), 58 deletions(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index 9c62188..bd7c16d 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -189,7 +189,6 @@ public void write(OutputStream os) throws IOException, WebApplicationException { if(qbResult != null){ for (SearchHit hit : qbResult.getHits()) { jg.writeObject(mapper.readValue(hit.source(), XmlChannel.class)); - jg.flush(); } } jg.writeEndArray(); diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 768cdb1..81e869d 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -49,7 +49,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -63,15 +62,11 @@ import gov.bnl.channelfinder.ChannelsResource.OnlyXmlProperty; import gov.bnl.channelfinder.TagsResource.MyMixInForXmlChannels; import java.util.Collections; -import java.util.concurrent.ExecutionException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.SearchHitField; import org.elasticsearch.transport.RemoteTransportException; /** @@ -222,15 +217,12 @@ public Response update(List data) throws IOException { param.put("owner", property.getOwner()); param.put("value", property.getValue()); HashMap params = new HashMap<>(); - params.put("property", param); + params.put("prop", param); for (XmlChannel channel : property.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "Object removeProperty = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.property.name) { removeProperty = xmlProp;} } " - + "ctx._source.tags.remove(removeProperty);" - + "ctx._source.tags.add(params.property);", + "ctx._source.properties.removeIf(item -> item.name == params.prop.name);" + + "ctx._source.tags.add(params.prop);", params))); } } @@ -352,13 +344,10 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .setSize(10000).execute().actionGet(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.getField("name").getValue().toString(); + String channelName = hit.getId(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "Object removeProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.prop) { removeProp = xmlProp;} } " - + "ctx._source.properties.remove(removeProp);", + "ctx._source.properties.removeIf(item -> item.name == params.prop);", Collections.singletonMap("prop", prop)))); } } @@ -470,12 +459,8 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "Object origProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.prop.name) { origProp = xmlProp;} } " - + "ctx._source.properties.remove(origProp);" - + "origProp.owner = params.prop.owner;" - + "ctx._source.properties.add(origProp);", + "ctx._source.properties.stream().filter(item -> item.name == params.prop.name)" + + ".forEach(item -> item.owner = params.prop.owner);", params))); } } @@ -495,10 +480,7 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { params.put("prop", param); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "Object removeProp = new Object();" - + "for (property in ctx._source.properties) " - + "{ if (property.name == params.prop.name) { removeProp = property;} } " - + "ctx._source.properties.remove(removeProp);" + "ctx._source.properties.removeIf(item -> item.name == params.prop.name);" + "ctx._source.properties.add(params.prop);", params))); } @@ -556,12 +538,8 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "Object origProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == params.originalProp.name) { origProp = xmlProp;} } " - + "ctx._source.properties.remove(origProp);" - + "origProp.name = params.newProp.name;" - + "ctx._source.properties.add(origProp);", + "ctx._source.properties.stream().filter(item -> item.name == params.originalProp.name)" + + ".forEach(item -> item.name = params.newProp.name);", params))); } } @@ -614,7 +592,7 @@ public Response remove(@PathParam("propName") String prop) { String channelName = hit.getId(); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ctx._source.properties.removeIf(item -> item.name == params.prop); ", + "ctx._source.properties.removeIf(item -> item.name == params.prop);", Collections.singletonMap("prop", prop)))); } } @@ -683,10 +661,7 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName client.update(new UpdateRequest("channelfinder", "channel", chan) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ArrayList removeProps = new ArrayList(); " - + "for (property in ctx._source.properties) " - + "{ if (property.name == params.prop.name) { removeProps.add(property);} } " - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp);} " + "ctx._source.properties.removeIf(item -> item.name == params.prop.name);" + "ctx._source.properties.add(params.prop)",params))) .actionGet(); Response r = Response.ok().build(); @@ -723,10 +698,7 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN if(client.prepareGet("properties", "property", prop).get().isExists()){ client.update(new UpdateRequest("channelfinder", "channel", chan) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ArrayList removeProps = new ArrayList();" - + "for (property in ctx._source.properties)" - + "{ if (property.name == params.prop) { removeProps.add(property);} }" - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp);}", + "ctx._source.properties.removeIf(item -> item.name == params.prop);", Collections.singletonMap("prop", prop)))) .actionGet(); Response r = Response.ok().build(); diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index bfd9c86..cc7136e 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -69,12 +69,9 @@ import gov.bnl.channelfinder.ChannelsResource.OnlyXmlTag; import java.util.Collections; -import java.util.concurrent.ExecutionException; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.transport.RemoteTransportException; @@ -737,10 +734,7 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") params.put("tag", param); client.update(new UpdateRequest("channelfinder", "channel", chan) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ArrayList removeTags = new ArrayList();" - + "for (tag in ctx._source.tags) " - + "{ if (tag.name == params.tag.name) { removeTags.add(tag);} } " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag);}" + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + "ctx._source.tags.add(params.tag);",params))) .actionGet(); Response r = Response.ok().build(); @@ -797,10 +791,7 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( UpdateResponse updateResponse = client .update(new UpdateRequest("channelfinder", "channel", chan) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, - "ArrayList removeTags = new ArrayList();" - + "for (tag in ctx._source.tags) " - + "{ if (tag.name == params.tag) { removeTags.add(tag);} } " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag);}", + "ctx._source.tags.removeIf(item -> item.name == params.tag);", Collections.singletonMap("tag", tag)))).actionGet(); Response r = Response.ok().build(); return r; diff --git a/test/testcf.py b/test/testcf.py index f8ae10c..d8d85ef 100644 --- a/test/testcf.py +++ b/test/testcf.py @@ -883,7 +883,7 @@ def test_AuthorizedAsAdminNewChannel(self): # Adding tag to non-existing channel def test_AuthorizedAsTagNonexChannel(self): - doPutAndFailMessageJSON(self, conn_tag, self.T1, T1_C3, 404, "DocumentMissingException[[channelfinder][-1] [channel][C3]") + doPutAndFailMessageJSON(self, conn_tag, self.T1, T1_C3, 404, "DocumentMissingException[[channel][C3]") # Payload and URL names do not match def test_AuthorizedAsTagLcPayload(self): @@ -1004,7 +1004,7 @@ def test_AuthorizedAsTagNonexTag(self): # Non-existing channel def test_AuthorizedAsTagNonexChannel(self): - doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channelfinder][-1] [channel][C4]") + doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channel][C4]") def tearDown(self): response = conn_admin.request_delete(self.T1, headers=copy(jsonheader)) @@ -1312,7 +1312,7 @@ def test_AuthorizedAsAdminNewChannel(self): # Adding property to non existing channel def test_AuthorizedAsPropNonexChannel(self): - doPutAndFailMessageJSON(self, conn_prop, self.P1, P1_C3, 404, "DocumentMissingException[[channelfinder][-1] [channel][C3]: document missing") + doPutAndFailMessageJSON(self, conn_prop, self.P1, P1_C3, 404, "DocumentMissingException[[channel][C3]: document missing") # Payload and URL names do not match def test_AuthorizedAsPropLcPayload(self): @@ -1460,7 +1460,7 @@ def test_AuthorizedAsPropNonexTag(self): # Non-existing channel def test_AuthorizedAsPropNonexChannel(self): - doPostAndFailMessageJSON(self, conn_prop, self.P2, P2_C4, 404, "DocumentMissingException[[channelfinder][-1] [channel][C4]: document missing") + doPostAndFailMessageJSON(self, conn_prop, self.P2, P2_C4, 404, "DocumentMissingException[[channel][C4]: document missing") def tearDown(self): response = conn_admin.request_delete(self.P1, headers=copy(jsonheader)) diff --git a/test/testcftags.py b/test/testcftags.py index 0b353f0..952a8a6 100644 --- a/test/testcftags.py +++ b/test/testcftags.py @@ -403,7 +403,7 @@ def test_AuthorizedAsTagNonexTag(self): # Non-existing channel def test_AuthorizedAsTagNonexChannel(self): - doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channelfinder][-1] [channel][C4]: document missing]") + doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channel][C4]: document missing]") def tearDown(self): response = conn_admin.request_delete(self.T1, headers=copy(jsonheader)) From 67e911edfe82dc45e544ae315a986f2d1065a1a0 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 2 Jun 2017 09:05:23 -0400 Subject: [PATCH 37/48] asadmin commands for integration tests --- root/etc/channelfinder/asadmin_commands | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 root/etc/channelfinder/asadmin_commands diff --git a/root/etc/channelfinder/asadmin_commands b/root/etc/channelfinder/asadmin_commands new file mode 100644 index 0000000..6d732d8 --- /dev/null +++ b/root/etc/channelfinder/asadmin_commands @@ -0,0 +1,6 @@ +create-auth-realm --classname com.sun.enterprise.security.auth.realm.file.FileRealm --property file=${com.sun.aas.instanceRoot}/keyfile:jaas-context=fileRealm channelfinder + +create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property Value=gov.bnl.channelfinder.JACCUserManager channelfinder/userManager + +set-log-levels gov.bnl.channelfinder.ChannelsResource=FINEST +set-log-levels gov.bnl.channelfinder.ChannelsResource.audit=FINEST From 2aec369150e16cc0f45d93ab9955b2d83a37cb35 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 2 Jun 2017 11:14:07 -0400 Subject: [PATCH 38/48] running tests from payara micro --- channelfinder/pom.xml | 101 ++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 58 deletions(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index c6e6cf8..945b164 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -302,18 +302,6 @@ single - - cargo-config-filtering - process-test-resources - - single - - - - src/main/assembly/cargo.xml - - - @@ -329,52 +317,6 @@ - - org.codehaus.cargo - cargo-maven2-plugin - 1.4.17 - - ${skipCargoAndRun} - - glassfish4x - - http://download.java.net/glassfish/4.1.1/release/glassfish-4.1.1.zip - - - - - - - ${project.build.directory}/ChannelFinder-cargo-inject/config - cargo-domain/config - - - - - - - start-cargo - pre-integration-test - - start - - - - stop-cargo - post-integration-test - - stop - - - - org.codehaus.mojo exec-maven-plugin @@ -416,6 +358,49 @@ + + com.bazaarvoice.maven.plugins + process-exec-maven-plugin + 0.8 + + + + start-jar + pre-integration-test + + start + + + ${basedir} + + 120 + + java + -jar + ${basedir}/target/${project.build.finalName}.jar + --rootdir + ${basedir}/src/test/serverresources/config + --sslPort + 8181 + --nocluster + true + --port + 8080 + --prebootcommandfile + ${basedir}/../root/etc/channelfinder/asadmin_commands + + + + + + stop-jar-process + post-integration-test + + stop-all + + + + ChannelFinder From 3a643ab156503b613127e07aae526ba9f2409bee Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Fri, 16 Jun 2017 12:58:03 -0400 Subject: [PATCH 39/48] shorter wait for payara startup --- channelfinder/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 945b164..a20cc46 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -373,7 +373,7 @@ ${basedir} - 120 + 60 java -jar From 771ecd1f0412b504cb2ea992f76a9fae6ffe612e Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Mon, 19 Jun 2017 11:19:02 -0400 Subject: [PATCH 40/48] moved debian package to use payara and systemd --- channelfinder/LICENSE.txt | 688 +++++++++--------- root/debian/channelfinder.init | 54 -- root/debian/channelfinder.install | 2 +- root/debian/channelfinder.service | 15 + root/debian/compat | 2 +- root/debian/control | 4 +- root/debian/rules | 2 +- .../classes/elasticsearch.yml | 0 8 files changed, 364 insertions(+), 403 deletions(-) delete mode 100644 root/debian/channelfinder.init create mode 100644 root/debian/channelfinder.service mode change 100644 => 100755 root/debian/rules rename root/etc/channelfinder/{WEB-INF => MICRO-INF}/classes/elasticsearch.yml (100%) diff --git a/channelfinder/LICENSE.txt b/channelfinder/LICENSE.txt index 532eda6..8084d19 100644 --- a/channelfinder/LICENSE.txt +++ b/channelfinder/LICENSE.txt @@ -1,344 +1,344 @@ -GNU General Public License, version 2, -with the Classpath Exception - -The GNU General Public License (GPL) - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny -you these rights or to ask you to surrender the rights. These restrictions -translate to certain responsibilities for you if you distribute copies of the -software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered by -this License; they are outside its scope. The act of running the Program is -not restricted, and the output from the Program is covered only if its contents -constitute a work based on the Program (independent of having been made by -running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus -forming a work based on the Program, and copy and distribute such modifications -or work under the terms of Section 1 above, provided that you also meet all of -these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or - in part contains or is derived from the Program or any part thereof, to be - licensed as a whole at no charge to all third parties under the terms of - this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a copy - of this License. (Exception: if the Program itself is interactive but does - not normally print such an announcement, your work based on the Program is - not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, and -its terms, do not apply to those sections when you distribute them as separate -works. But when you distribute the same sections as part of a whole which is a -work based on the Program, the distribution of the whole must be on the terms -of this License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with the -Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and installation -of the executable. However, as a special exception, the source code -distributed need not include anything that is normally distributed (in either -source or binary form) with the major components (compiler, kernel, and so on) -of the operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the source -code from the same place counts as distribution of the source code, even though -third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the Program -or its derivative works. These actions are prohibited by law if you do not -accept this License. Therefore, by modifying or distributing the Program (or -any work based on the Program), you indicate your acceptance of this License to -do so, and all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), conditions -are imposed on you (whether by court order, agreement or otherwise) that -contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar in -spirit to the present version, but may differ in detail to address new problems -or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software Foundation. -If the Program does not specify a version number of this License, you may -choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software Foundation, -write to the Free Software Foundation; we sometimes make exceptions for this. -Our decision will be guided by the two goals of preserving the free status of -all derivatives of our free software and of promoting the sharing and reuse of -software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA -BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER -OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision comes - with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free - software, and you are welcome to redistribute it under certain conditions; - type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, -if any, to sign a "copyright disclaimer" for the program, if necessary. Here -is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Linking this library statically or dynamically with other modules is making -a combined work based on this library. Thus, the terms and conditions of -the GNU General Public License cover the whole combination. - -As a special exception, the copyright holders of this library give you -permission to link this library with independent modules to produce an -executable, regardless of the license terms of these independent modules, -and to copy and distribute the resulting executable under terms of your -choice, provided that you also meet, for each linked independent module, -the terms and conditions of the license of that module. An independent -module is a module which is not derived from or based on this library. If -you modify this library, you may extend this exception to your version of -the library, but you are not obligated to do so. If you do not wish to do -so, delete this exception statement from your version. +GNU General Public License, version 2, +with the Classpath Exception + +The GNU General Public License (GPL) + +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to +most of the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for this service if you wish), +that you receive source code or can get it if you want it, that you can change +the software or use pieces of it in new free programs; and that you know you +can do these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for +a fee, you must give the recipients all the rights that you have. You must +make sure that they, too, receive or can get the source code. And you must +show them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) +offer you this license which gives you legal permission to copy, distribute +and/or modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced +by others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We +wish to avoid the danger that redistributors of a free program will +individually obtain patent licenses, in effect making the program proprietary. +To prevent this, we have made it clear that any patent must be licensed for +everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included +without limitation in the term "modification".) Each licensee is addressed as +"you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is +not restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such modifications +or work under the terms of Section 1 above, provided that you also meet all of +these conditions: + + a) You must cause the modified files to carry prominent notices stating + that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole or + in part contains or is derived from the Program or any part thereof, to be + licensed as a whole at no charge to all third parties under the terms of + this License. + + c) If the modified program normally reads commands interactively when run, + you must cause it, when started running for such interactive use in the + most ordinary way, to print or display an announcement including an + appropriate copyright notice and a notice that there is no warranty (or + else, saying that you provide a warranty) and that users may redistribute + the program under these conditions, and telling the user how to view a copy + of this License. (Exception: if the Program itself is interactive but does + not normally print such an announcement, your work based on the Program is + not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +works. But when you distribute the same sections as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source + code, which must be distributed under the terms of Sections 1 and 2 above + on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to + give any third party, for a charge no more than your cost of physically + performing source distribution, a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed only + for noncommercial distribution and only if you received the program in + object code or executable form with such an offer, in accord with + Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all +the source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code +distributed need not include anything that is normally distributed (in either +source or binary form) with the major components (compiler, kernel, and so on) +of the operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In +such case, this License incorporates the limitation as if written in the body +of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any later +version", you have the option of following the terms and conditions either of +that version or of any later version published by the Free Software Foundation. +If the Program does not specify a version number of this License, you may +choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of +all derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible +use to the public, the best way to achieve this is to make it free software +which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively convey the exclusion +of warranty; and each file should have at least the "copyright" line and a +pointer to where the full notice is found. + + One line to give the program's name and a brief idea of what it does. + + Copyright (C) + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., 59 + Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this when it +starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author Gnomovision comes + with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free + software, and you are welcome to redistribute it under certain conditions; + type 'show c' for details. + +The hypothetical commands 'show w' and 'show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may be +called something other than 'show w' and 'show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your school, +if any, to sign a "copyright disclaimer" for the program, if necessary. Here +is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + 'Gnomovision' (which makes passes at compilers) written by James Hacker. + + signature of Ty Coon, 1 April 1989 + + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General Public +License instead of this License. + + +"CLASSPATH" EXCEPTION TO THE GPL + +Linking this library statically or dynamically with other modules is making +a combined work based on this library. Thus, the terms and conditions of +the GNU General Public License cover the whole combination. + +As a special exception, the copyright holders of this library give you +permission to link this library with independent modules to produce an +executable, regardless of the license terms of these independent modules, +and to copy and distribute the resulting executable under terms of your +choice, provided that you also meet, for each linked independent module, +the terms and conditions of the license of that module. An independent +module is a module which is not derived from or based on this library. If +you modify this library, you may extend this exception to your version of +the library, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. diff --git a/root/debian/channelfinder.init b/root/debian/channelfinder.init deleted file mode 100644 index ddb4652..0000000 --- a/root/debian/channelfinder.init +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -# /etc/rc.d/init.d/channelfinder: Controls channelfinder -# -# Version: channelfinder 1.0 -# -# chkconfig: 345 99 01 -# description: EPICS Directory Service -# - -### BEGIN INIT INFO -# Provides: channelfinder -# Required-Start: $remote_fs $local_fs $network $syslog $time -# Required-Stop: $remote_fs $local_fs $network $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Channelfinder -# Description: EPICS Directory Service -### END INIT INFO - -PROG=/usr/share/channelfinder/ChannelFinder -PROG_WAR=/usr/share/channelfinder/ChannelFinder.war -GF_HOME=/home/glassfish/glassfish4/glassfish -ASADMIN=$GF_HOME/bin/asadmin - -case $1 in -start) - jar -uf ${PROG_WAR} -C /etc/channelfinder WEB-INF/classes/elasticsearch.yml - ${ASADMIN} deploy ${PROG_WAR} - ;; -stop) - ${ASADMIN} undeploy ${PROG} - ;; -status) - ${ASADMIN} list-applications --type web - ;; -restart) - ${ASADMIN} undeploy ${PROG} - ${ASADMIN} deploy ${PROG_WAR} - ;; -configure) - . /etc/channelfinder/glassfish_config_create - ;; -deconfigure) - . /etc/channelfinder/glassfish_config_remove - ;; -reconfigure) - . /etc/channelfinder/glassfish_config_remove - . /etc/channelfinder/glassfish_config_create - ;; -*) - echo $0 "start | stop | status | configure | deconfigure | reconfigure" -esac - - diff --git a/root/debian/channelfinder.install b/root/debian/channelfinder.install index cd37ffb..9c20b61 100644 --- a/root/debian/channelfinder.install +++ b/root/debian/channelfinder.install @@ -1,2 +1,2 @@ -ChannelFinder.war usr/share/channelfinder +ChannelFinder.jar usr/share/channelfinder etc diff --git a/root/debian/channelfinder.service b/root/debian/channelfinder.service new file mode 100644 index 0000000..d5de939 --- /dev/null +++ b/root/debian/channelfinder.service @@ -0,0 +1,15 @@ +[Unit] +Description=ChannelFinder Service +After=network.target remote-fs.target + +[Service] +user=channelfinder +ExecStartPre=/usr/bin/jar -uf /usr/share/channelfinder/ChannelFinder.jar -C /etc/channelfinder MICRO-INF/classes/elasticsearch.yml +ExecStartPre=mkdir -p /tmp/channelfinder +ExecStart=/usr/bin/java -Xmx2G -jar /usr/share/channelfinder/ChannelFinder.jar --sslport 8181 --nocluster true --port 8080 --prebootcommandfile /etc/channelfinder/asadmin_commands --logtofile /tmp/channelfinder/log.txt +ExecStop=/bin/kill -SIGINT $MAINPID +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/root/debian/compat b/root/debian/compat index 7f8f011..ec63514 100644 --- a/root/debian/compat +++ b/root/debian/compat @@ -1 +1 @@ -7 +9 diff --git a/root/debian/control b/root/debian/control index 5adcdcd..216e152 100644 --- a/root/debian/control +++ b/root/debian/control @@ -2,11 +2,11 @@ Source: channelfinder Section: java Priority: optional Maintainer: Eric Berryman -Build-Depends: debhelper (>= 7), +Build-Depends: debhelper (>= 9), dh-systemd (>=1.5), Standards-Version: 3.8.0 Homepage: https://channelfinder.github.io/ Package: channelfinder -Depends: default-jre-headless +Depends: oracle-java8-jdk Architecture: amd64 Description: Directory service for EPICS diff --git a/root/debian/rules b/root/debian/rules old mode 100644 new mode 100755 index cb336fd..953471c --- a/root/debian/rules +++ b/root/debian/rules @@ -2,5 +2,5 @@ %: - dh $@ + dh $@ --with systemd diff --git a/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml b/root/etc/channelfinder/MICRO-INF/classes/elasticsearch.yml similarity index 100% rename from root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml rename to root/etc/channelfinder/MICRO-INF/classes/elasticsearch.yml From 09d264664c9fca5122d1843a7dfb242b2a38b34a Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Mon, 19 Jun 2017 11:27:23 -0400 Subject: [PATCH 41/48] update to elastic 5.4, payara, and systemd --- root/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index b821d2d..e3515ad 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,10 @@ +channelfinder (1.0.0-5.1) unstable; urgency=medium + + * Non-maintainer upload. + * update to elastic 5.4, payara, and systemd + + -- Eric Berryman Mon, 19 Jun 2017 11:27:23 -0400 + channelfinder (1.0.0-5) unstable; urgency=medium * added JACCUserManager From 5f48a246225e6c54529b1b641be11458ad38bdba Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 20 Jun 2017 11:16:33 -0400 Subject: [PATCH 42/48] update debian packaging --- root/debian/channelfinder.service | 9 ++++++--- root/debian/control | 2 +- root/debian/dirs | 1 + .../{MICRO-INF => WEB-INF}/classes/elasticsearch.yml | 0 root/etc/default/channelfinder | 3 +++ 5 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 root/debian/dirs rename root/etc/channelfinder/{MICRO-INF => WEB-INF}/classes/elasticsearch.yml (100%) diff --git a/root/debian/channelfinder.service b/root/debian/channelfinder.service index d5de939..1a0be23 100644 --- a/root/debian/channelfinder.service +++ b/root/debian/channelfinder.service @@ -4,9 +4,12 @@ After=network.target remote-fs.target [Service] user=channelfinder -ExecStartPre=/usr/bin/jar -uf /usr/share/channelfinder/ChannelFinder.jar -C /etc/channelfinder MICRO-INF/classes/elasticsearch.yml -ExecStartPre=mkdir -p /tmp/channelfinder -ExecStart=/usr/bin/java -Xmx2G -jar /usr/share/channelfinder/ChannelFinder.jar --sslport 8181 --nocluster true --port 8080 --prebootcommandfile /etc/channelfinder/asadmin_commands --logtofile /tmp/channelfinder/log.txt +EnvironmentFile=/etc/default/channelfinder +ExecStartPre=/bin/mkdir -p /tmp/channelfinder +ExecStartPre=/usr/bin/unzip -o /usr/share/channelfinder/ChannelFinder.jar MICRO-INF/deploy/ChannelFinder.war -d /tmp/channelfinder +ExecStartPre=/usr/bin/jar -uf /tmp/channelfinder/MICRO-INF/deploy/ChannelFinder.war -C /etc/channelfinder WEB-INF/classes/elasticsearch.yml +ExecStartPre=/usr/bin/jar -uf /usr/share/channelfinder/ChannelFinder.jar -C /tmp/channelfinder MICRO-INF/deploy/ChannelFinder.war +ExecStart=/usr/bin/java -Xmx2G -jar /usr/share/channelfinder/ChannelFinder.jar --sslport ${SERVICESSLPORT} --nocluster true --port ${SERVICEPORT} --prebootcommandfile /etc/channelfinder/asadmin_commands --logtofile /var/log/channelfinder/log.txt ExecStop=/bin/kill -SIGINT $MAINPID Restart=on-failure RestartSec=10 diff --git a/root/debian/control b/root/debian/control index 216e152..19557d2 100644 --- a/root/debian/control +++ b/root/debian/control @@ -7,6 +7,6 @@ Standards-Version: 3.8.0 Homepage: https://channelfinder.github.io/ Package: channelfinder -Depends: oracle-java8-jdk +Depends: oracle-java8-jdk, unzip Architecture: amd64 Description: Directory service for EPICS diff --git a/root/debian/dirs b/root/debian/dirs new file mode 100644 index 0000000..4936bd6 --- /dev/null +++ b/root/debian/dirs @@ -0,0 +1 @@ +/var/log/channelfinder diff --git a/root/etc/channelfinder/MICRO-INF/classes/elasticsearch.yml b/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml similarity index 100% rename from root/etc/channelfinder/MICRO-INF/classes/elasticsearch.yml rename to root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder index 3ccbdc3..0fc42b9 100644 --- a/root/etc/default/channelfinder +++ b/root/etc/default/channelfinder @@ -1,5 +1,8 @@ #Channelfinder defaults +SERVICEPORT=8080 +SERVICESSLPORT=8181 + GF_HOME=/home/glassfish/glassfish4/glassfish ASADMIN=$GF_HOME/bin/asadmin From 977e514f1cf8b12af2bf94d65ba4ffdbc80ddedc Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 20 Jun 2017 11:23:43 -0400 Subject: [PATCH 43/48] fix version --- root/debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/debian/changelog b/root/debian/changelog index e3515ad..a463647 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,4 +1,4 @@ -channelfinder (1.0.0-5.1) unstable; urgency=medium +channelfinder (1.0.1) unstable; urgency=medium * Non-maintainer upload. * update to elastic 5.4, payara, and systemd From b1add60cd9dff04be9b160ae4ba619ddedf4c7fc Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 27 Jun 2017 13:00:14 -0400 Subject: [PATCH 44/48] update mapping file for deb package --- root/etc/channelfinder/mapping_definitions.sh | 61 +++++++++++++------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/root/etc/channelfinder/mapping_definitions.sh b/root/etc/channelfinder/mapping_definitions.sh index 3e25b7c..2a68496 100644 --- a/root/etc/channelfinder/mapping_definitions.sh +++ b/root/etc/channelfinder/mapping_definitions.sh @@ -15,66 +15,86 @@ #Create the Index -curl -XPUT "${ELASTIC_HOST}/tags" +curl -XPUT 'http://localhost:9200/tags' #Set the mapping -curl -XPUT "${ELASTIC_HOST}/tags/_mapping/tag" -d' +curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' { "tag" : { "properties" : { "name" : { - "type" : "string" + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "owner" : { - "type" : "string" + "type" : "text" } } } }' -curl -XPUT "${ELASTIC_HOST}/properties" -curl -XPUT "${ELASTIC_HOST}/properties/_mapping/property" -d' +curl -XPUT 'http://localhost:9200/properties' +curl -XPUT 'http://localhost:9200/properties/_mapping/property' -d' { "property" : { "properties" : { "name" : { - "type" : "string" + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "owner" : { - "type" : "string" + "type" : "text" } } } }' -curl -XPUT "${ELASTIC_HOST}/channelfinder" -curl -XPUT "${ELASTIC_HOST}/channelfinder/_mapping/channel" -d' +curl -XPUT 'http://localhost:9200/channelfinder' +curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' { "channel" : { "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" }, "script" : { - "type" : "string" + "type" : "text" }, "properties" : { "type" : "nested", "include_in_parent" : true, "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string" + "type" : "text" }, "value" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" } } @@ -84,11 +104,16 @@ curl -XPUT "${ELASTIC_HOST}/channelfinder/_mapping/channel" -d' "include_in_parent" : true, "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" } } From 61fb4397273581575fd24e25379078b52c51eb09 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 27 Jun 2017 16:12:04 -0400 Subject: [PATCH 45/48] add initialized settings, not empty settings --- .../main/java/gov/bnl/channelfinder/ElasticSearchClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java index 69a7194..8ce88b4 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java @@ -59,7 +59,7 @@ public static TransportClient getNewClient() { String host = settings.get("network.host"); int port = Integer.valueOf(settings.get("transport.tcp.port")); try { - return new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); + return new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); } catch (ElasticsearchException e) { log.severe(e.getDetailedMessage()); return null; From c559f3d2992e8165e1f4edfc7c9c006da0860815 Mon Sep 17 00:00:00 2001 From: Jenkins User Date: Tue, 27 Jun 2017 16:15:20 -0400 Subject: [PATCH 46/48] update settings --- root/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/root/debian/changelog b/root/debian/changelog index a463647..5fb5c58 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,3 +1,10 @@ +channelfinder (1.0.1+nmu1) unstable; urgency=medium + + * Non-maintainer upload. + * update settings + + -- Eric Berryman Tue, 27 Jun 2017 16:15:20 -0400 + channelfinder (1.0.1) unstable; urgency=medium * Non-maintainer upload. From 9f1e7be00297ac5011a75f0a1612e8bc2d218d47 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Wed, 28 Jun 2017 08:22:28 -0400 Subject: [PATCH 47/48] release debian 1.0.2 --- root/debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/debian/changelog b/root/debian/changelog index 5fb5c58..33a779c 100644 --- a/root/debian/changelog +++ b/root/debian/changelog @@ -1,4 +1,4 @@ -channelfinder (1.0.1+nmu1) unstable; urgency=medium +channelfinder (1.0.2) unstable; urgency=medium * Non-maintainer upload. * update settings From fdc70ec0eac56ca0717da445b5e9c0d831fe7b49 Mon Sep 17 00:00:00 2001 From: Eric Berryman Date: Tue, 27 Mar 2018 11:43:36 -0400 Subject: [PATCH 48/48] ES multi field added to help searching null property values --- channelfinder/.classpath | 1 - channelfinder/pom.xml | 4 +- .../bnl/channelfinder/ChannelsResource.java | 40 +++++---- .../channelfinder/ElasticSearchClient.java | 29 +------ .../bnl/channelfinder/PropertiesResource.java | 81 +++++++++--------- .../gov/bnl/channelfinder/TagsResource.java | 82 ++++++++++--------- .../src/main/resources/mapping_definitions.sh | 9 ++ test/testcf.py | 2 + 8 files changed, 121 insertions(+), 127 deletions(-) diff --git a/channelfinder/.classpath b/channelfinder/.classpath index fb139fd..669b749 100644 --- a/channelfinder/.classpath +++ b/channelfinder/.classpath @@ -20,7 +20,6 @@ - diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index a20cc46..79bf3a3 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -267,7 +267,7 @@ fish.payara.extras payara-micro - 4.1.2.172 + 5.181 jar @@ -372,10 +372,12 @@ ${basedir} + ${basedir}/target/channelfinder.log 60 java + -Xmx4g -jar ${basedir}/target/${project.build.finalName}.jar --rootdir diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index bd7c16d..410a50c 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -10,7 +10,6 @@ * #L% */ -import static gov.bnl.channelfinder.ElasticSearchClient.getNewClient; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.disMaxQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -158,6 +157,11 @@ public Response query() { .must(matchQuery("properties.name", parameter.getKey().trim())) .must(wildcardQuery("properties.value", pattern.trim())), ScoreMode.Avg)); + propertyQuery.add(nestedQuery("properties", + boolQuery() + .must(matchQuery("properties.name", parameter.getKey().trim())) + .must(wildcardQuery("properties.value.keyword", pattern.trim())), + ScoreMode.Avg)); } } qb.must(propertyQuery); @@ -172,7 +176,7 @@ public Response query() { builder.addSort(SortBuilders.fieldSort("name.keyword")); builder.setFrom(from); } - final SearchResponse qbResult = builder.execute().actionGet(); + final SearchResponse qbResult = builder.get(); performance.append("|query:("+qbResult.getHits().getTotalHits()+")" + (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); final ObjectMapper mapper = new ObjectMapper(); @@ -220,7 +224,7 @@ public void write(OutputStream os) throws IOException, WebApplicationException { @PUT @Consumes({"application/json"}) public Response create(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); ObjectMapper mapper = new ObjectMapper(); @@ -237,7 +241,7 @@ public Response create(List data) throws IOException { String prepare = "|Prepare: " + (System.currentTimeMillis()-start) + "|"; start = System.currentTimeMillis(); bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); String execute = "|Execute: " + (System.currentTimeMillis()-start)+"|"; start = System.currentTimeMillis(); if (bulkResponse.hasFailures()) { @@ -253,7 +257,6 @@ public Response create(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -271,7 +274,7 @@ public Response read(@PathParam("chName") String chan) { Client client = ElasticSearchClient.getSearchClient(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; try { - final GetResponse response = client.prepareGet("channelfinder", "channel", chan).execute().actionGet(); + final GetResponse response = client.prepareGet("channelfinder", "channel", chan).get(); Response r; if (response.isExists()) { final ObjectMapper mapper = new ObjectMapper(); @@ -325,7 +328,7 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { + chan + "' and payload channel name '" + data.getName() + "' do not match"); } long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); ObjectMapper mapper = new ObjectMapper(); try { start = System.currentTimeMillis(); @@ -344,7 +347,6 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -363,7 +365,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { long start = System.currentTimeMillis(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); if(data.getName()==null || data.getName().isEmpty()){ return handleException(um.getUserName(), "PUT", Response.Status.BAD_REQUEST, "Specified channel name '" + chan + "' and payload channel name '" + data.getName() + "' do not match"); @@ -377,7 +379,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { data = validateChannel(data, client); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|POST|validation : "+ (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); - GetResponse response = client.prepareGet("channelfinder", "channel", chan).execute().actionGet(); + GetResponse response = client.prepareGet("channelfinder", "channel", chan).get(); if(response.isExists()){ XmlChannel channel= mapper.readValue(response.getSourceAsBytes(), XmlChannel.class); channel.setName(data.getName()); @@ -409,13 +411,12 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { } catch (Exception e) { return handleException(um.getUserName(), "POST", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } private Response renameChannel(UserManager um, Client client, String chan, XmlChannel data) { - GetResponse response = client.prepareGet("channelfinder", "channel", chan).execute().actionGet(); + GetResponse response = client.prepareGet("channelfinder", "channel", chan).get(); if(!response.isExists()){ handleException(um.getUserName(), "POST", Response.Status.NOT_FOUND, "Specified channel '"+chan+"' does not exist"); } @@ -439,7 +440,7 @@ private Response renameChannel(UserManager um, Client client, String chan, XmlCh .source(mapper.writeValueAsBytes(originalChannel)); bulkRequest.add(indexRequest); bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -474,7 +475,7 @@ private boolean validateChannelName(String chan, XmlChannel data) { @Path("{chName: "+chNameRegex+"}") public Response remove(@PathParam("chName") String chan) { audit.info("deleting ch:" + chan); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -489,7 +490,6 @@ public Response remove(@PathParam("chName") String chan) { } catch (Exception e) { return handleException(um.getUserName(), "DELETE", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -522,13 +522,12 @@ private List validateChannels(List channels, Client clie mapper.addMixIn(XmlTag.class, OnlyXmlTag.class); SearchResponse response = client.prepareSearch("properties").setTypes("property") - .setQuery(new MatchAllQueryBuilder()).setSize(1000).execute().actionGet(); + .setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlProperty prop = mapper.readValue(hit.getSourceAsString(), XmlProperty.class); properties.put(prop.getName(), prop); } - response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).execute() - .actionGet(); + response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlTag tag = mapper.readValue(hit.getSourceAsString(), XmlTag.class); tags.put(tag.getName(), tag); @@ -592,13 +591,12 @@ private XmlChannel validateChannel(XmlChannel channel, Client client) throws Jso mapper.addMixIn(XmlTag.class, OnlyXmlTag.class); SearchResponse response = client.prepareSearch("properties").setTypes("property") - .setQuery(new MatchAllQueryBuilder()).setSize(1000).execute().actionGet(); + .setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlProperty prop = mapper.readValue(hit.getSourceAsString(), XmlProperty.class); properties.put(prop.getName(), prop); } - response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).execute() - .actionGet(); + response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlTag tag = mapper.readValue(hit.getSourceAsString(), XmlTag.class); tags.put(tag.getName(), tag); diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java index 8ce88b4..6667d01 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java @@ -20,6 +20,8 @@ import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; + +import java.net.InetAddress; import java.net.InetSocketAddress; import org.elasticsearch.ElasticsearchException; @@ -39,33 +41,11 @@ public class ElasticSearchClient implements ServletContextListener { private static Settings settings; private static TransportClient searchClient; - private static TransportClient indexClient; public static TransportClient getSearchClient() { return searchClient; } - public static TransportClient getIndexClient() { - return indexClient; - } - - /** - * Returns a new {@link TransportClient} using the default settings - * **IMPORTANT** it is the responsibility of the caller to close this client - * @return es transport client - */ - @SuppressWarnings("resource") - public static TransportClient getNewClient() { - String host = settings.get("network.host"); - int port = Integer.valueOf(settings.get("transport.tcp.port")); - try { - return new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); - } catch (ElasticsearchException e) { - log.severe(e.getDetailedMessage()); - return null; - } - } - public static Settings getSettings(){ return settings; } @@ -80,9 +60,7 @@ public void contextInitialized(ServletContextEvent sce) { int port = Integer.valueOf(settings.get("transport.tcp.port")); searchClient = new PreBuiltTransportClient(settings); - indexClient = new PreBuiltTransportClient(settings); - searchClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); - indexClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(host,port))); + searchClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host),port)); } catch (IOException e) { log.severe(e.getMessage()); } @@ -92,7 +70,6 @@ public void contextInitialized(ServletContextEvent sce) { public void contextDestroyed(ServletContextEvent sce) { log.info("Closeing the default Transport clients."); searchClient.close(); - indexClient.close(); } } diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 81e869d..3176317 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -10,7 +10,6 @@ * #L% */ -import static gov.bnl.channelfinder.ElasticSearchClient.getNewClient; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; @@ -47,13 +46,14 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.index.engine.DocumentMissingException; -import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonGenerator; @@ -98,7 +98,10 @@ public PropertiesResource() { @GET @Produces({ MediaType.APPLICATION_JSON }) public Response list() { - Client client = getNewClient(); + StringBuffer performance = new StringBuffer(); + long start = System.currentTimeMillis(); + long totalStart = System.currentTimeMillis(); + Client client = ElasticSearchClient.getSearchClient(); final String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; final ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlProperty.class, OnlyXmlProperty.class); @@ -114,10 +117,16 @@ public Response list() { } } - final SearchResponse response = client.prepareSearch("properties") + performance.append("|prepare:" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); + SearchRequestBuilder builder = client.prepareSearch("properties") .setTypes("property") - .setQuery(new MatchAllQueryBuilder()) - .setSize(size).execute().actionGet(); + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(size); + builder.addSort(SortBuilders.fieldSort("name.keyword")); + final SearchResponse response = builder.get(); + performance.append("|query:("+response.getHits().getTotalHits()+")" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, WebApplicationException { @@ -133,13 +142,15 @@ public void write(OutputStream os) throws IOException, WebApplicationException { jg.close(); } }; + performance.append("|parse:" + (System.currentTimeMillis() - start)); Response r = Response.ok(stream).build(); - audit.info(user + "|" + uriInfo.getPath() + "|GET|OK|" + r.getStatus() + "|returns " + response.getHits().getTotalHits()+ " properties"); + audit.fine(user + "|" + uriInfo.getPath() + "|GET|OK|" + performance.toString() + "|total:" + + (System.currentTimeMillis() - totalStart) + "|" + r.getStatus() + + "|returns " + response.getHits().getTotalHits()+ " properties"); return r; } catch (Exception e) { return handleException(user, "GET", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -154,7 +165,7 @@ public void write(OutputStream os) throws IOException, WebApplicationException { @PUT @Consumes("application/json") public Response create(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); ObjectMapper mapper = new ObjectMapper(); @@ -169,7 +180,7 @@ public Response create(List data) throws IOException { ); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, bulkResponse.buildFailureMessage()); @@ -181,7 +192,6 @@ public Response create(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -198,7 +208,7 @@ public Response create(List data) throws IOException { @POST @Consumes("application/json") public Response update(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); ObjectMapper mapper = new ObjectMapper(); @@ -228,7 +238,7 @@ public Response update(List data) throws IOException { } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -246,7 +256,6 @@ public Response update(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), "POST", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } /** @@ -264,11 +273,11 @@ public Response update(List data) throws IOException { @Produces("application/json") public Response read(@PathParam("propName") String prop) { MultivaluedMap parameters = uriInfo.getQueryParameters(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; XmlProperty result = null; try { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); if (response.isExists()) { ObjectMapper mapper = new ObjectMapper(); result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); @@ -280,7 +289,7 @@ public Response read(@PathParam("propName") String prop) { if (parameters.containsKey("withChannels")) { final SearchResponse channelResult = client.prepareSearch("channelfinder") .setQuery(matchQuery("properties.name", prop.trim())) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); List channels = new ArrayList<>(); if (channelResult != null) { for (SearchHit hit : channelResult.getHits()) { @@ -299,7 +308,6 @@ public Response read(@PathParam("propName") String prop) { } catch (Exception e) { return handleException(user, "GET", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -325,7 +333,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .entity("Specified property name '"+prop+"' and payload property name '"+data.getName()+"' do not match").build(); } long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -341,7 +349,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .setQuery(QueryBuilders.matchQuery("properties.name", prop)) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.getId(); @@ -372,7 +380,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -383,7 +391,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { bulkResponse.buildFailureMessage()); } } else { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); ObjectMapper mapper = new ObjectMapper(); XmlProperty result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); Response r; @@ -398,7 +406,6 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -417,14 +424,14 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { @Path("{propName : " + propertyNameRegex + "}") @Consumes("application/json") public Response update(@PathParam("propName") String prop, XmlProperty data) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); if(data.getName() == null || data.getName().isEmpty()){ handleException(um.getUserName(), "POST", Status.BAD_REQUEST, "payload data has invalid/incorrect property name " + data.getName()); } try { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); if(!response.isExists()){ return handleException(um.getUserName(), "POST", Response.Status.NOT_FOUND, "A property named '"+prop+"' does not exist"); } @@ -455,7 +462,7 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { .setQuery(wildcardQuery("properties.name", original.getName().trim())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, @@ -486,7 +493,7 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -504,7 +511,6 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { } catch (Exception e) { return handleException(um.getUserName(), "POST", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -514,7 +520,7 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi .setQuery(wildcardQuery("properties.name", original.getName().trim())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); List channelNames = new ArrayList(); for (SearchHit hit : queryResponse.getHits()) { channelNames.add(hit.getId()); @@ -544,7 +550,7 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -576,7 +582,7 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi @DELETE @Path("{propName : " + propertyNameRegex + "}") public Response remove(@PathParam("propName") String prop) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -586,7 +592,7 @@ public Response remove(@PathParam("propName") String prop) { .setQuery(QueryBuilders.matchQuery("properties.name", prop)) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.getId(); @@ -598,7 +604,7 @@ public Response remove(@PathParam("propName") String prop) { } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); throw new Exception(); @@ -616,7 +622,6 @@ public Response remove(@PathParam("propName") String prop) { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -633,7 +638,7 @@ public Response remove(@PathParam("propName") String prop) { @Path("{propName}/{chName}") @Consumes({ "application/xml", "application/json" }) public Response addSingle(@PathParam("propName") String prop, @PathParam("chName") String chan, XmlProperty data) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); if (data.getValue() == null || data.getValue().isEmpty()) { return Response.status(Response.Status.BAD_REQUEST) @@ -646,7 +651,7 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); XmlProperty result = null; try { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlChannel.class, MyMixInForXmlChannels.class); result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); @@ -676,7 +681,6 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -691,7 +695,7 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName @DELETE @Path("{propName}/{chName}") public Response removeSingle(@PathParam("propName") String prop, @PathParam("chName") String chan) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -711,7 +715,6 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN } catch (Exception e) { return handleException(um.getUserName(), "DELETE", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index cc7136e..c98ff35 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -13,7 +13,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; -import static gov.bnl.channelfinder.ElasticSearchClient.getNewClient; import java.io.IOException; import java.io.OutputStream; @@ -49,6 +48,7 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -57,6 +57,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.core.JsonEncoding; @@ -105,7 +106,10 @@ public TagsResource() { @GET @Produces({MediaType.APPLICATION_JSON}) public Response list() { - Client client = getNewClient(); + StringBuffer performance = new StringBuffer(); + long start = System.currentTimeMillis(); + long totalStart = System.currentTimeMillis(); + Client client = ElasticSearchClient.getSearchClient(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; final ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlTag.class, OnlyXmlTag.class); @@ -121,10 +125,16 @@ public Response list() { } } - final SearchResponse response = client.prepareSearch("tags") + performance.append("|prepare:" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); + SearchRequestBuilder builder = client.prepareSearch("tags") .setTypes("tag") - .setQuery(new MatchAllQueryBuilder()) - .setSize(size).execute().actionGet(); + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(size); + builder.addSort(SortBuilders.fieldSort("name.keyword")); + final SearchResponse response = builder.get(); + performance.append("|query:("+response.getHits().getTotalHits()+")" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); StreamingOutput stream = new StreamingOutput(){ @Override public void write(OutputStream os) throws IOException, WebApplicationException { @@ -140,13 +150,15 @@ public void write(OutputStream os) throws IOException, WebApplicationException { jg.close(); } }; + performance.append("|parse:" + (System.currentTimeMillis() - start)); Response r = Response.ok(stream).build(); - log.fine(user + "|" + uriInfo.getPath() + "|GET|OK|" + r.getStatus() + response.getTook() + "|returns " + response.getHits().getTotalHits() + " tags"); + audit.fine(user + "|" + uriInfo.getPath() + "|GET|OK|" + performance.toString() + "|total:" + + (System.currentTimeMillis() - totalStart) + "|" + r.getStatus() + + "|returns " + response.getHits().getTotalHits()+ " tags"); return r; } catch (Exception e) { return handleException(user,Response.Status.INTERNAL_SERVER_ERROR , e); } finally { - client.close(); } } @@ -165,12 +177,12 @@ public void write(OutputStream os) throws IOException, WebApplicationException { public Response read(@PathParam("tagName") String tag) { MultivaluedMap parameters = uriInfo.getQueryParameters(); long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; XmlTag result = null; try { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); if (response.isExists()) { ObjectMapper mapper = new ObjectMapper(); result = mapper.readValue(response.getSourceAsBytes(), XmlTag.class); @@ -181,7 +193,7 @@ public Response read(@PathParam("tagName") String tag) { if (parameters.containsKey("withChannels")) { // TODO iterator or scrolling needed final SearchResponse channelResult = client.prepareSearch("channelfinder") - .setQuery(matchQuery("tags.name", tag.trim())).setSize(10000).execute().actionGet(); + .setQuery(matchQuery("tags.name", tag.trim())).setSize(10000).get(); List channels = new ArrayList(); if (channelResult != null) { for (SearchHit hit : channelResult.getHits()) { @@ -204,7 +216,6 @@ public Response read(@PathParam("tagName") String tag) { } catch (IOException e) { return handleException(user, Response.Status.INTERNAL_SERVER_ERROR , e); } finally { - client.close(); } } @@ -223,7 +234,7 @@ public Response read(@PathParam("tagName") String tag) { @Consumes({MediaType.APPLICATION_JSON}) public Response create(@PathParam("tagName") String tag, XmlTag data) { long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); @@ -240,7 +251,7 @@ public Response create(@PathParam("tagName") String tag, XmlTag data) { .setQuery(QueryBuilders.matchQuery("tags.name", tag)) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { @@ -282,7 +293,7 @@ public String apply(XmlChannel channel) { } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -293,7 +304,7 @@ public String apply(XmlChannel channel) { bulkResponse.buildFailureMessage()); } } else { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); ObjectMapper mapper = new ObjectMapper(); XmlTag result = mapper.readValue(response.getSourceAsBytes(), XmlTag.class); Response r; @@ -314,7 +325,6 @@ public String apply(XmlChannel channel) { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -331,7 +341,7 @@ public String apply(XmlChannel channel) { @Consumes({MediaType.APPLICATION_JSON}) public Response createTags(List data) { long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); @@ -348,7 +358,7 @@ public Response createTags(List data) { .setQuery(QueryBuilders.matchQuery("tags.name", xmlTag.getName())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { @@ -392,7 +402,7 @@ public String apply(XmlChannel channel) { } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -408,7 +418,6 @@ public String apply(XmlChannel channel) { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -430,12 +439,12 @@ public String apply(XmlChannel channel) { @Consumes({"application/json"}) public Response update(@PathParam("tagName") String tag, XmlTag data) { long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); if(!response.isExists()){ return handleException(um.getUserName(), Response.Status.NOT_FOUND, "A tag named '"+tag+"' does not exist"); } @@ -464,7 +473,7 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { .setQuery(wildcardQuery("tags.name", original.getName().trim())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, @@ -484,7 +493,7 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -503,7 +512,6 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { } catch (Exception e) { return handleException(um.getUserName() , Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -524,7 +532,7 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa .setQuery(wildcardQuery("tags.name", original.getName().trim())) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); List channelNames = new ArrayList(); for (SearchHit hit : queryResponse.getHits()) { channelNames.add(hit.getId()); @@ -557,7 +565,7 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -590,7 +598,7 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa @POST @Consumes({MediaType.APPLICATION_JSON}) public Response updateTags(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -617,7 +625,7 @@ public Response updateTags(List data) throws IOException { } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -636,7 +644,6 @@ public Response updateTags(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR , e); } finally { - client.close(); } } @@ -650,7 +657,7 @@ public Response updateTags(List data) throws IOException { @DELETE @Path("{tagName: "+tagNameRegex+"}") public Response remove(@PathParam("tagName") String tag) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -660,7 +667,7 @@ public Response remove(@PathParam("tagName") String tag) { .setQuery(QueryBuilders.matchQuery("tags.name", tag)) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFetchSource(new String[]{"name"}, null) - .setSize(10000).execute().actionGet(); + .setSize(10000).get(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { String channelName = hit.getId(); @@ -671,7 +678,7 @@ public Response remove(@PathParam("tagName") String tag) { } } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, @@ -690,7 +697,6 @@ public Response remove(@PathParam("tagName") String tag) { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -714,12 +720,12 @@ public Response remove(@PathParam("tagName") String tag) { @Path("{tagName}/{chName}") @Consumes({"application/xml", "application/json"}) public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") String chan, XmlTag data) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); XmlTag result = null; try { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlChannel.class, MyMixInForXmlChannels.class); result = mapper.readValue(response.getSourceAsBytes(), XmlTag.class); @@ -757,7 +763,6 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -783,7 +788,7 @@ private boolean validateTag(XmlTag existing, XmlTag request) { @DELETE @Path("{tagName}/{chName}") public Response removeSingle(@PathParam("tagName") final String tag, @PathParam("chName") String chan) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -803,7 +808,6 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } diff --git a/channelfinder/src/main/resources/mapping_definitions.sh b/channelfinder/src/main/resources/mapping_definitions.sh index 2a68496..2f2a3c1 100644 --- a/channelfinder/src/main/resources/mapping_definitions.sh +++ b/channelfinder/src/main/resources/mapping_definitions.sh @@ -15,6 +15,7 @@ #Create the Index + curl -XPUT 'http://localhost:9200/tags' #Set the mapping curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' @@ -36,6 +37,7 @@ curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' } }' + curl -XPUT 'http://localhost:9200/properties' curl -XPUT 'http://localhost:9200/properties/_mapping/property' -d' { @@ -56,6 +58,7 @@ curl -XPUT 'http://localhost:9200/properties/_mapping/property' -d' } }' + curl -XPUT 'http://localhost:9200/channelfinder' curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' { @@ -95,6 +98,12 @@ curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' }, "value" : { "type" : "text", + "fields": { + "keyword": { + "type": "keyword", + "null_value": "null" + } + }, "analyzer" : "whitespace" } } diff --git a/test/testcf.py b/test/testcf.py index d8d85ef..4b77cd5 100644 --- a/test/testcf.py +++ b/test/testcf.py @@ -7,6 +7,7 @@ import sys, os import json from _testConf import _testConf +import time base_url = os.getenv("BASEURL") if base_url is None: @@ -213,6 +214,7 @@ def ordered(obj): return obj def doGetJSON(self, conn, g_url, g_result, g_resp): + time.sleep(1) response = conn_none.request_get(g_url, headers=copy(jsonheader)) self.assertEqual(`g_resp`, response[u'headers']['status'], 'unexpected return code for get operation - expected ' + `g_resp` + ', received ' + response[u'headers']['status'] + ', message body:\n' + response[u'body'])