diff --git a/.travis.yml b/.travis.yml index 52bb803..46d3903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,17 +4,18 @@ sudo: required jdk: - oraclejdk8 +addons: + apt: + packages: + - python-nose + before_install: - - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.3.deb && sudo dpkg -i --force-confnew elasticsearch-1.7.3.deb + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.4.0.deb -install: - - sudo apt-get update - - sudo apt-get install python-nose - before_script: - sleep 10 - - 'echo "script.inline: on" | sudo tee -a /etc/elasticsearch/elasticsearch.yml' - - 'echo "script.inline: on" | sudo tee -a /etc/elasticsearch/indexed.yml' + - 'echo "script.inline: true" | sudo tee -a /etc/elasticsearch/elasticsearch.yml' + - 'echo "script.inline: true" | sudo tee -a /etc/elasticsearch/indexed.yml' - sudo /etc/init.d/elasticsearch start - sleep 25 - curl -XGET 'http://localhost:9200/_cluster/health?pretty' diff --git a/README.md b/README.md index 5192952..46472a8 100644 --- a/README.md +++ b/README.md @@ -69,12 +69,12 @@ in the web container. ``` ################################# Scripting ############################### - script.inline: on - script.indexed: on + script.inline: true + script.stored: true ``` - * For more information of how Index and mappings can be setup using any rest client as described here [create elastic index](https://www.elastic.co/guide/en/elasticsearch/reference/1.4/_create_an_index.html) + * For more information of how Index and mappings can be setup using any rest client as described here [create elastic index](https://www.elastic.co/guide/en/elasticsearch/reference/5.4/_create_an_index.html) 4. Authentication/Authorization diff --git a/channelfinder/.classpath b/channelfinder/.classpath index fb139fd..669b749 100644 --- a/channelfinder/.classpath +++ b/channelfinder/.classpath @@ -20,7 +20,6 @@ - diff --git a/channelfinder/LICENSE.txt b/channelfinder/LICENSE.txt index 532eda6..8084d19 100644 --- a/channelfinder/LICENSE.txt +++ b/channelfinder/LICENSE.txt @@ -1,344 +1,344 @@ -GNU General Public License, version 2, -with the Classpath Exception - -The GNU General Public License (GPL) - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to deny -you these rights or to ask you to surrender the rights. These restrictions -translate to certain responsibilities for you if you distribute copies of the -software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered by -this License; they are outside its scope. The act of running the Program is -not restricted, and the output from the Program is covered only if its contents -constitute a work based on the Program (independent of having been made by -running the Program). Whether that is true depends on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, thus -forming a work based on the Program, and copy and distribute such modifications -or work under the terms of Section 1 above, provided that you also meet all of -these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole or - in part contains or is derived from the Program or any part thereof, to be - licensed as a whole at no charge to all third parties under the terms of - this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a copy - of this License. (Exception: if the Program itself is interactive but does - not normally print such an announcement, your work based on the Program is - not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, and -its terms, do not apply to those sections when you distribute them as separate -works. But when you distribute the same sections as part of a whole which is a -work based on the Program, the distribution of the whole must be on the terms -of this License, whose permissions for other licensees extend to the entire -whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with the -Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and installation -of the executable. However, as a special exception, the source code -distributed need not include anything that is normally distributed (in either -source or binary form) with the major components (compiler, kernel, and so on) -of the operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the source -code from the same place counts as distribution of the source code, even though -third parties are not compelled to copy the source along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the Program -or its derivative works. These actions are prohibited by law if you do not -accept this License. Therefore, by modifying or distributing the Program (or -any work based on the Program), you indicate your acceptance of this License to -do so, and all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), conditions -are imposed on you (whether by court order, agreement or otherwise) that -contradict the conditions of this License, they do not excuse you from the -conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar in -spirit to the present version, but may differ in detail to address new problems -or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software Foundation. -If the Program does not specify a version number of this License, you may -choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software Foundation, -write to the Free Software Foundation; we sometimes make exceptions for this. -Our decision will be guided by the two goals of preserving the free status of -all derivatives of our free software and of promoting the sharing and reuse of -software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA -BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER -OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision comes - with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free - software, and you are welcome to redistribute it under certain conditions; - type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your school, -if any, to sign a "copyright disclaimer" for the program, if necessary. Here -is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Linking this library statically or dynamically with other modules is making -a combined work based on this library. Thus, the terms and conditions of -the GNU General Public License cover the whole combination. - -As a special exception, the copyright holders of this library give you -permission to link this library with independent modules to produce an -executable, regardless of the license terms of these independent modules, -and to copy and distribute the resulting executable under terms of your -choice, provided that you also meet, for each linked independent module, -the terms and conditions of the license of that module. An independent -module is a module which is not derived from or based on this library. If -you modify this library, you may extend this exception to your version of -the library, but you are not obligated to do so. If you do not wish to do -so, delete this exception statement from your version. +GNU General Public License, version 2, +with the Classpath Exception + +The GNU General Public License (GPL) + +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to +most of the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for this service if you wish), +that you receive source code or can get it if you want it, that you can change +the software or use pieces of it in new free programs; and that you know you +can do these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for +a fee, you must give the recipients all the rights that you have. You must +make sure that they, too, receive or can get the source code. And you must +show them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) +offer you this license which gives you legal permission to copy, distribute +and/or modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced +by others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We +wish to avoid the danger that redistributors of a free program will +individually obtain patent licenses, in effect making the program proprietary. +To prevent this, we have made it clear that any patent must be licensed for +everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included +without limitation in the term "modification".) Each licensee is addressed as +"you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is +not restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such modifications +or work under the terms of Section 1 above, provided that you also meet all of +these conditions: + + a) You must cause the modified files to carry prominent notices stating + that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole or + in part contains or is derived from the Program or any part thereof, to be + licensed as a whole at no charge to all third parties under the terms of + this License. + + c) If the modified program normally reads commands interactively when run, + you must cause it, when started running for such interactive use in the + most ordinary way, to print or display an announcement including an + appropriate copyright notice and a notice that there is no warranty (or + else, saying that you provide a warranty) and that users may redistribute + the program under these conditions, and telling the user how to view a copy + of this License. (Exception: if the Program itself is interactive but does + not normally print such an announcement, your work based on the Program is + not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +works. But when you distribute the same sections as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source + code, which must be distributed under the terms of Sections 1 and 2 above + on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to + give any third party, for a charge no more than your cost of physically + performing source distribution, a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed only + for noncommercial distribution and only if you received the program in + object code or executable form with such an offer, in accord with + Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all +the source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code +distributed need not include anything that is normally distributed (in either +source or binary form) with the major components (compiler, kernel, and so on) +of the operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In +such case, this License incorporates the limitation as if written in the body +of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any later +version", you have the option of following the terms and conditions either of +that version or of any later version published by the Free Software Foundation. +If the Program does not specify a version number of this License, you may +choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of +all derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible +use to the public, the best way to achieve this is to make it free software +which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively convey the exclusion +of warranty; and each file should have at least the "copyright" line and a +pointer to where the full notice is found. + + One line to give the program's name and a brief idea of what it does. + + Copyright (C) + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., 59 + Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this when it +starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author Gnomovision comes + with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free + software, and you are welcome to redistribute it under certain conditions; + type 'show c' for details. + +The hypothetical commands 'show w' and 'show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may be +called something other than 'show w' and 'show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your school, +if any, to sign a "copyright disclaimer" for the program, if necessary. Here +is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + 'Gnomovision' (which makes passes at compilers) written by James Hacker. + + signature of Ty Coon, 1 April 1989 + + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General Public +License instead of this License. + + +"CLASSPATH" EXCEPTION TO THE GPL + +Linking this library statically or dynamically with other modules is making +a combined work based on this library. Thus, the terms and conditions of +the GNU General Public License cover the whole combination. + +As a special exception, the copyright holders of this library give you +permission to link this library with independent modules to produce an +executable, regardless of the license terms of these independent modules, +and to copy and distribute the resulting executable under terms of your +choice, provided that you also meet, for each linked independent module, +the terms and conditions of the license of that module. An independent +module is a module which is not derived from or based on this library. If +you modify this library, you may extend this exception to your version of +the library, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. diff --git a/channelfinder/pom.xml b/channelfinder/pom.xml index 3fa2bbe..79bf3a3 100644 --- a/channelfinder/pom.xml +++ b/channelfinder/pom.xml @@ -67,7 +67,7 @@ - epics.releases @@ -141,36 +141,40 @@ org.elasticsearch elasticsearch - 1.5.2 + 5.4.0 + + + org.elasticsearch.client + transport + 5.4.0 org.glassfish.jersey.core jersey-common 2.10.4 - provided + provided org.glassfish.jersey.media jersey-media-multipart 2.10.4 - provided + provided com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - 2.5.1 + 2.8.8 com.fasterxml.jackson.core jackson-core - 2.5.1 - provided + 2.8.8 com.fasterxml.jackson.core jackson-databind - 2.5.1 - provided + 2.8.8 + provided com.google.guava @@ -183,6 +187,16 @@ 1.4.17 maven-plugin + + org.apache.logging.log4j + log4j-api + 2.7 + + + org.apache.logging.log4j + log4j-core + 2.7 + @@ -235,16 +249,42 @@ - - maven-deploy-plugin - 2.8.2 - - - org.apache.maven.wagon - wagon-ssh - 2.8 - - + + maven-dependency-plugin + 2.6 + + + copy-payara-micro + package + + copy + + + target + true + true + + + fish.payara.extras + payara-micro + 5.181 + jar + + + + + + + + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.wagon + wagon-ssh + 2.8 + + maven-assembly-plugin @@ -262,18 +302,6 @@ single - - cargo-config-filtering - process-test-resources - - single - - - - src/main/assembly/cargo.xml - - - @@ -290,74 +318,90 @@ - org.codehaus.cargo - cargo-maven2-plugin - 1.4.17 - - ${skipCargoAndRun} - - glassfish4x - - http://download.java.net/glassfish/4.1.1/release/glassfish-4.1.1.zip - - - - - - - ${project.build.directory}/ChannelFinder-cargo-inject/config - cargo-domain/config - - - - + org.codehaus.mojo + exec-maven-plugin + 1.3.2 - start-cargo - pre-integration-test + integration-test - start + exec + + nosetests + ${project.basedir}/../test + ${skipNoseTests} + + -v + -s + --with-xunit + + - stop-cargo - post-integration-test + payara-uber-jar + package - stop + exec + + java + + -jar + ${basedir}/target/payara-micro.jar + --deploy + ${basedir}/target/${project.build.finalName}.war + --outputUberJar + ${basedir}/target/${project.build.finalName}.jar + + - org.codehaus.mojo - exec-maven-plugin - 1.3.2 + com.bazaarvoice.maven.plugins + process-exec-maven-plugin + 0.8 + - integration-test + start-jar + pre-integration-test - exec + start + + + ${basedir} + ${basedir}/target/channelfinder.log + + 60 + + java + -Xmx4g + -jar + ${basedir}/target/${project.build.finalName}.jar + --rootdir + ${basedir}/src/test/serverresources/config + --sslPort + 8181 + --nocluster + true + --port + 8080 + --prebootcommandfile + ${basedir}/../root/etc/channelfinder/asadmin_commands + + + + + + stop-jar-process + post-integration-test + + stop-all - - nosetests - ../test - ${skipNoseTests} - - -v - -s - --with-xunit - --xunit-file=${project.build.directory}/failsafe-reports/nosetests.xml - - ChannelFinder diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java index 8bec8a2..410a50c 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ChannelsResource.java @@ -10,7 +10,6 @@ * #L% */ -import static gov.bnl.channelfinder.ElasticSearchClient.getNewClient; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.disMaxQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -68,6 +67,9 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; /** * Top level Jersey HTTP methods for the .../channels URL * @@ -127,7 +129,7 @@ public Response query() { for (String pattern : value.split("\\|")) { tagQuery.add(wildcardQuery("tags.name", pattern.trim())); } - qb.must(nestedQuery("tags", tagQuery)); + qb.must(nestedQuery("tags", tagQuery, ScoreMode.Avg)); } break; case "~size": @@ -153,7 +155,13 @@ public Response query() { propertyQuery.add(nestedQuery("properties", boolQuery() .must(matchQuery("properties.name", parameter.getKey().trim())) - .must(wildcardQuery("properties.value", pattern.trim())))); + .must(wildcardQuery("properties.value", pattern.trim())), + ScoreMode.Avg)); + propertyQuery.add(nestedQuery("properties", + boolQuery() + .must(matchQuery("properties.name", parameter.getKey().trim())) + .must(wildcardQuery("properties.value.keyword", pattern.trim())), + ScoreMode.Avg)); } } qb.must(propertyQuery); @@ -165,10 +173,10 @@ public Response query() { start = System.currentTimeMillis(); SearchRequestBuilder builder = client.prepareSearch("channelfinder").setQuery(qb).setSize(size); if(from >= 0){ - builder.addSort(SortBuilders.fieldSort("name")); + builder.addSort(SortBuilders.fieldSort("name.keyword")); builder.setFrom(from); } - final SearchResponse qbResult = builder.execute().actionGet(); + final SearchResponse qbResult = builder.get(); performance.append("|query:("+qbResult.getHits().getTotalHits()+")" + (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); final ObjectMapper mapper = new ObjectMapper(); @@ -185,7 +193,6 @@ public void write(OutputStream os) throws IOException, WebApplicationException { if(qbResult != null){ for (SearchHit hit : qbResult.getHits()) { jg.writeObject(mapper.readValue(hit.source(), XmlChannel.class)); - jg.flush(); } } jg.writeEndArray(); @@ -217,7 +224,7 @@ public void write(OutputStream os) throws IOException, WebApplicationException { @PUT @Consumes({"application/json"}) public Response create(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); ObjectMapper mapper = new ObjectMapper(); @@ -233,8 +240,8 @@ public Response create(List data) throws IOException { } String prepare = "|Prepare: " + (System.currentTimeMillis()-start) + "|"; start = System.currentTimeMillis(); - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); String execute = "|Execute: " + (System.currentTimeMillis()-start)+"|"; start = System.currentTimeMillis(); if (bulkResponse.hasFailures()) { @@ -250,7 +257,6 @@ public Response create(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -268,7 +274,7 @@ public Response read(@PathParam("chName") String chan) { Client client = ElasticSearchClient.getSearchClient(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; try { - final GetResponse response = client.prepareGet("channelfinder", "channel", chan).execute().actionGet(); + final GetResponse response = client.prepareGet("channelfinder", "channel", chan).get(); Response r; if (response.isExists()) { final ObjectMapper mapper = new ObjectMapper(); @@ -322,7 +328,7 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { + chan + "' and payload channel name '" + data.getName() + "' do not match"); } long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); ObjectMapper mapper = new ObjectMapper(); try { start = System.currentTimeMillis(); @@ -331,7 +337,7 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { IndexRequest indexRequest = new IndexRequest("channelfinder", "channel", chan) .source(mapper.writeValueAsBytes(data)); UpdateRequest updateRequest = new UpdateRequest("channelfinder", "channel", chan) - .doc(mapper.writeValueAsBytes(data)).upsert(indexRequest).refresh(true); + .doc(mapper.writeValueAsBytes(data)).upsert(indexRequest); UpdateResponse result = client.update(updateRequest).actionGet(); Response r = Response.noContent().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|PUT|OK|" + r.getStatus() + "|data=" + XmlChannel.toLog(data)); @@ -341,7 +347,6 @@ public Response create(@PathParam("chName") String chan, XmlChannel data) { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -360,7 +365,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { long start = System.currentTimeMillis(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); if(data.getName()==null || data.getName().isEmpty()){ return handleException(um.getUserName(), "PUT", Response.Status.BAD_REQUEST, "Specified channel name '" + chan + "' and payload channel name '" + data.getName() + "' do not match"); @@ -374,7 +379,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { data = validateChannel(data, client); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|POST|validation : "+ (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); - GetResponse response = client.prepareGet("channelfinder", "channel", chan).execute().actionGet(); + GetResponse response = client.prepareGet("channelfinder", "channel", chan).get(); if(response.isExists()){ XmlChannel channel= mapper.readValue(response.getSourceAsBytes(), XmlChannel.class); channel.setName(data.getName()); @@ -390,7 +395,7 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { }).collect(Collectors.toList())); channel.setTags(data.getTags()); UpdateRequest updateRequest = new UpdateRequest("channelfinder", "channel", chan) - .doc(mapper.writeValueAsBytes(channel)).refresh(true); + .doc(mapper.writeValueAsBytes(channel)); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|POST|prepare : "+ (System.currentTimeMillis() - start)); start = System.currentTimeMillis(); UpdateResponse result = client.update(updateRequest).actionGet(); @@ -406,13 +411,12 @@ public Response update(@PathParam("chName") String chan, XmlChannel data) { } catch (Exception e) { return handleException(um.getUserName(), "POST", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } private Response renameChannel(UserManager um, Client client, String chan, XmlChannel data) { - GetResponse response = client.prepareGet("channelfinder", "channel", chan).execute().actionGet(); + GetResponse response = client.prepareGet("channelfinder", "channel", chan).get(); if(!response.isExists()){ handleException(um.getUserName(), "POST", Response.Status.NOT_FOUND, "Specified channel '"+chan+"' does not exist"); } @@ -435,8 +439,8 @@ private Response renameChannel(UserManager um, Client client, String chan, XmlCh IndexRequest indexRequest = new IndexRequest("channelfinder", "channel", originalChannel.getName()) .source(mapper.writeValueAsBytes(originalChannel)); bulkRequest.add(indexRequest); - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -471,12 +475,12 @@ private boolean validateChannelName(String chan, XmlChannel data) { @Path("{chName: "+chNameRegex+"}") public Response remove(@PathParam("chName") String chan) { audit.info("deleting ch:" + chan); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { - DeleteResponse response = client.prepareDelete("channelfinder", "channel", chan).setRefresh(true).execute().get(); - if(response.isFound()){ + DeleteResponse deleteResponse = client.prepareDelete("channelfinder", "channel", chan).execute().get(); + if(deleteResponse.getResult() == DocWriteResponse.Result.DELETED){ Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); return r; @@ -486,7 +490,6 @@ public Response remove(@PathParam("chName") String chan) { } catch (Exception e) { return handleException(um.getUserName(), "DELETE", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -519,13 +522,12 @@ private List validateChannels(List channels, Client clie mapper.addMixIn(XmlTag.class, OnlyXmlTag.class); SearchResponse response = client.prepareSearch("properties").setTypes("property") - .setQuery(new MatchAllQueryBuilder()).setSize(1000).execute().actionGet(); + .setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlProperty prop = mapper.readValue(hit.getSourceAsString(), XmlProperty.class); properties.put(prop.getName(), prop); } - response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).execute() - .actionGet(); + response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlTag tag = mapper.readValue(hit.getSourceAsString(), XmlTag.class); tags.put(tag.getName(), tag); @@ -589,13 +591,12 @@ private XmlChannel validateChannel(XmlChannel channel, Client client) throws Jso mapper.addMixIn(XmlTag.class, OnlyXmlTag.class); SearchResponse response = client.prepareSearch("properties").setTypes("property") - .setQuery(new MatchAllQueryBuilder()).setSize(1000).execute().actionGet(); + .setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlProperty prop = mapper.readValue(hit.getSourceAsString(), XmlProperty.class); properties.put(prop.getName(), prop); } - response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).execute() - .actionGet(); + response = client.prepareSearch("tags").setTypes("tag").setQuery(new MatchAllQueryBuilder()).setSize(1000).get(); for (SearchHit hit : response.getHits()) { XmlTag tag = mapper.readValue(hit.getSourceAsString(), XmlTag.class); tags.put(tag.getName(), tag); diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java index 5348832..6667d01 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ElasticSearchClient.java @@ -15,13 +15,18 @@ */ +import java.io.IOException; import java.util.logging.Logger; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; +import java.net.InetAddress; +import java.net.InetSocketAddress; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; @@ -36,54 +41,35 @@ public class ElasticSearchClient implements ServletContextListener { private static Settings settings; private static TransportClient searchClient; - private static TransportClient indexClient; - public static TransportClient getSearchClient() { + public static TransportClient getSearchClient() { return searchClient; } - public static TransportClient getIndexClient() { - return indexClient; - } - - /** - * Returns a new {@link TransportClient} using the default settings - * **IMPORTANT** it is the responsibility of the caller to close this client - * @return es transport client - */ - @SuppressWarnings("resource") - public static TransportClient getNewClient() { - String host = settings.get("network.host"); - int port = Integer.valueOf(settings.get("transport.tcp.port")); - try { - return new TransportClient().addTransportAddress(new InetSocketTransportAddress(host, port)); - } catch (ElasticsearchException e) { - log.severe(e.getDetailedMessage()); - return null; - } - } - public static Settings getSettings(){ return settings; } @Override public void contextInitialized(ServletContextEvent sce) { - log.info("Initializing a new Transport clients."); - searchClient = new TransportClient(); - indexClient = new TransportClient(); - settings = searchClient.settings(); - String host = settings.get("network.host"); - int port = Integer.valueOf(settings.get("transport.tcp.port")); - searchClient.addTransportAddress(new InetSocketTransportAddress(host, port)); - indexClient.addTransportAddress(new InetSocketTransportAddress(host, port)); + try { + log.info("Initializing a new Transport clients."); + String yaml = "elasticsearch.yml"; + settings = Settings.builder().loadFromStream(yaml,getClass().getClassLoader().getResourceAsStream(yaml)).build(); + String host = settings.get("network.host"); + int port = Integer.valueOf(settings.get("transport.tcp.port")); + + searchClient = new PreBuiltTransportClient(settings); + searchClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host),port)); + } catch (IOException e) { + log.severe(e.getMessage()); + } } @Override public void contextDestroyed(ServletContextEvent sce) { log.info("Closeing the default Transport clients."); searchClient.close(); - indexClient.close(); } } diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java b/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java new file mode 100644 index 0000000..1083b83 --- /dev/null +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/JACCUserManager.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2010 Brookhaven National Laboratory + * Copyright (c) 2010-2011 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH + * All rights reserved. Use is subject to license terms and conditions. + */ +package gov.bnl.channelfinder; + +/* + * #%L + * ChannelFinder Directory Service + * %% + * Copyright (C) 2010 - 2017 Brookhaven National Laboratory / National Synchrotron Light Source II + * %% + * Copyright (C) 2010 - 2012 Brookhaven National Laboratory + * All rights reserved. Use is subject to license terms. + * #L% + */ + + +import java.security.Principal; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.security.auth.Subject; +import javax.security.jacc.PolicyContext; +import javax.security.jacc.PolicyContextException; + + +/** + * Owner (group) membership management: JACC connection and binding. + * + * @author beryman + */ +public class JACCUserManager extends UserManager { + + private static final String SUBJECT_HANDLER_KEY = "javax.security.auth.Subject.container"; + + @Override + protected Set getGroups(Principal user) { + + try { + Subject s = (Subject) PolicyContext.getContext(SUBJECT_HANDLER_KEY); + Principal principals[] = (s == null ? new Principal[0] : s.getPrincipals().toArray(new Principal[0])); + + HashSet roleSet = new HashSet(); + for(Principal principal :principals){ + roleSet.add(principal.getName()); + } + return roleSet; + +// Do we want to look at only Web Role Permissions? Checking againist all principals for now. +// CodeSource cs = new CodeSource(null, (java.security.cert.Certificate[]) null); +// ProtectionDomain pd = new ProtectionDomain(cs, null, null, principals); +// +// Policy policy = Policy.getPolicy(); +// PermissionCollection pc = policy.getPermissions(pd); +// pc.implies(new WebRoleRefPermission(null, null)); +// +// HashSet roleSet = null; +// +// Enumeration e = pc.elements(); +// while (e.hasMoreElements()) { +// Object p = e.nextElement(); +// if (p instanceof WebRoleRefPermission) { +// String roleRef = ((WebRoleRefPermission) p).getActions(); +// if (roleSet == null) { +// roleSet = new HashSet(); +// } +// roleSet.add(((WebRoleRefPermission) p).getActions()); +// } +// } +// if (roleSet != null) { +// roleSet.toArray(new String[0]); +// } +// return roleSet; +// + } catch (PolicyContextException ex) { + Logger.getLogger(JACCUserManager.class.getName()).log(Level.SEVERE, null, ex); + } + return null; + } +} diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java index 4e42691..3176317 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/PropertiesResource.java @@ -10,7 +10,6 @@ * #L% */ -import static gov.bnl.channelfinder.ElasticSearchClient.getNewClient; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; @@ -41,21 +40,20 @@ import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; -import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.index.engine.DocumentMissingException; -import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonGenerator; @@ -63,6 +61,13 @@ import gov.bnl.channelfinder.ChannelsResource.OnlyXmlProperty; import gov.bnl.channelfinder.TagsResource.MyMixInForXmlChannels; +import java.util.Collections; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.transport.RemoteTransportException; /** * Top level Jersey HTTP methods for the .../properties URL @@ -93,7 +98,10 @@ public PropertiesResource() { @GET @Produces({ MediaType.APPLICATION_JSON }) public Response list() { - Client client = getNewClient(); + StringBuffer performance = new StringBuffer(); + long start = System.currentTimeMillis(); + long totalStart = System.currentTimeMillis(); + Client client = ElasticSearchClient.getSearchClient(); final String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; final ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlProperty.class, OnlyXmlProperty.class); @@ -109,11 +117,16 @@ public Response list() { } } - final SearchResponse response = client.prepareSearch("properties") + performance.append("|prepare:" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); + SearchRequestBuilder builder = client.prepareSearch("properties") .setTypes("property") - .setQuery(new MatchAllQueryBuilder()) - .setSize(size) - .execute().actionGet(); + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(size); + builder.addSort(SortBuilders.fieldSort("name.keyword")); + final SearchResponse response = builder.get(); + performance.append("|query:("+response.getHits().getTotalHits()+")" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, WebApplicationException { @@ -129,13 +142,15 @@ public void write(OutputStream os) throws IOException, WebApplicationException { jg.close(); } }; + performance.append("|parse:" + (System.currentTimeMillis() - start)); Response r = Response.ok(stream).build(); - audit.info(user + "|" + uriInfo.getPath() + "|GET|OK|" + r.getStatus() + "|returns " + response.getHits().getTotalHits()+ " properties"); + audit.fine(user + "|" + uriInfo.getPath() + "|GET|OK|" + performance.toString() + "|total:" + + (System.currentTimeMillis() - totalStart) + "|" + r.getStatus() + + "|returns " + response.getHits().getTotalHits()+ " properties"); return r; } catch (Exception e) { return handleException(user, "GET", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -150,7 +165,7 @@ public void write(OutputStream os) throws IOException, WebApplicationException { @PUT @Consumes("application/json") public Response create(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); ObjectMapper mapper = new ObjectMapper(); @@ -164,8 +179,8 @@ public Response create(List data) throws IOException { .source(mapper.writeValueAsBytes(property))) ); } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, bulkResponse.buildFailureMessage()); @@ -177,7 +192,6 @@ public Response create(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -194,7 +208,7 @@ public Response create(List data) throws IOException { @POST @Consumes("application/json") public Response update(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); ObjectMapper mapper = new ObjectMapper(); @@ -208,24 +222,23 @@ public Response update(List data) throws IOException { .source(mapper.writeValueAsBytes(property))) ); if (property.getChannels() != null) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", property.getName()); param.put("owner", property.getOwner()); param.put("value", property.getValue()); + HashMap params = new HashMap<>(); + params.put("prop", param); for (XmlChannel channel : property.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) - .script("removeProperty = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == property.name) { removeProperty = xmlProp} }; " - + "ctx._source.tags.remove(removeProperty);" - + "ctx._source.tags.add(property)") - .addScriptParam("property", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop.name);" + + "ctx._source.tags.add(params.prop);", + params))); } } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -243,7 +256,6 @@ public Response update(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), "POST", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } /** @@ -261,11 +273,11 @@ public Response update(List data) throws IOException { @Produces("application/json") public Response read(@PathParam("propName") String prop) { MultivaluedMap parameters = uriInfo.getQueryParameters(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; XmlProperty result = null; try { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); if (response.isExists()) { ObjectMapper mapper = new ObjectMapper(); result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); @@ -276,9 +288,9 @@ public Response read(@PathParam("propName") String prop) { //TODO iterator or scrolling needed if (parameters.containsKey("withChannels")) { final SearchResponse channelResult = client.prepareSearch("channelfinder") - .setQuery(matchQuery("properties.name", prop.trim())).setSize(10000).execute() - .actionGet(); - List channels = new ArrayList(); + .setQuery(matchQuery("properties.name", prop.trim())) + .setSize(10000).get(); + List channels = new ArrayList<>(); if (channelResult != null) { for (SearchHit hit : channelResult.getHits()) { channels.add(mapper.readValue(hit.source(), XmlChannel.class)); @@ -296,7 +308,6 @@ public Response read(@PathParam("propName") String prop) { } catch (Exception e) { return handleException(user, "GET", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -322,7 +333,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .entity("Specified property name '"+prop+"' and payload property name '"+data.getName()+"' do not match").build(); } long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -331,26 +342,27 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { .startObject().field("name", data.getName()).field("owner", data.getOwner()).endObject()); UpdateRequest updateRequest = new UpdateRequest("properties", "property", prop).doc(jsonBuilder() .startObject().field("name", data.getName()).field("owner", data.getOwner()).endObject()) - .upsert(indexRequest).refresh(true); + .upsert(indexRequest); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("properties.name", prop)).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("properties.name", prop)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).refresh(true) - .script("removeProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " - + "ctx._source.properties.remove(removeProp);") - .addScriptParam("prop", prop)); + String channelName = hit.getId(); + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop);", + Collections.singletonMap("prop", prop)))); } } if (data.getChannels() != null) { for (XmlChannel channel : data.getChannels()) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", data.getOwner()); String value = ChannelUtil.getProperty(channel, data.getName()).getValue(); @@ -359,14 +371,16 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { "Invalid property value (missing or null or empty string) for '"+data.getName()+"'"); } param.put("value", ChannelUtil.getProperty(channel, data.getName()).getValue()); + HashMap params = new HashMap<>(); + params.put("prop", param); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) - .script("ctx._source.properties.add(prop)") - .addScriptParam("prop", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.add(params.prop)", + params))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -377,7 +391,7 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { bulkResponse.buildFailureMessage()); } } else { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); ObjectMapper mapper = new ObjectMapper(); XmlProperty result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); Response r; @@ -392,7 +406,6 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -411,14 +424,14 @@ public Response create(@PathParam("propName") String prop, XmlProperty data) { @Path("{propName : " + propertyNameRegex + "}") @Consumes("application/json") public Response update(@PathParam("propName") String prop, XmlProperty data) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); if(data.getName() == null || data.getName().isEmpty()){ handleException(um.getUserName(), "POST", Status.BAD_REQUEST, "payload data has invalid/incorrect property name " + data.getName()); } try { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); if(!response.isExists()){ return handleException(um.getUserName(), "POST", Response.Status.NOT_FOUND, "A property named '"+prop+"' does not exist"); } @@ -440,22 +453,22 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { .endObject()); if(!original.getOwner().equals(data.getOwner())){ - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", propOwner); + HashMap params = new HashMap<>(); + params.put("prop", param); SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("properties.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("properties.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .refresh(true) - .script("origProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == prop.name) { origProp = xmlProp} }; " - + "ctx._source.properties.remove(origProp);" - + "origProp.owner = prop.owner;" - + "ctx._source.properties.add(origProp)") - .addScriptParam("prop", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.stream().filter(item -> item.name == params.prop.name)" + + ".forEach(item -> item.owner = params.prop.owner);", + params))); } } bulkRequest.add(updateRequest); @@ -470,18 +483,17 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { "Invalid property value (missing or null or empty string) for '"+data.getName()+"'"); } param.put("value", value); + HashMap params = new HashMap<>(); + params.put("prop", param); bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) - .script("removeProp = new Object();" - + "for (property in ctx._source.properties) " - + "{ if (property.name == prop.name) { removeProp = property} }; " - + "ctx._source.properties.remove(removeProp);" - + "ctx._source.properties.add(prop)") - .addScriptParam("prop", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop.name);" + + "ctx._source.properties.add(params.prop);", + params))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -499,15 +511,16 @@ public Response update(@PathParam("propName") String prop, XmlProperty data) { } catch (Exception e) { return handleException(um.getUserName(), "POST", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } private Response renameProperty(UserManager um, Client client, XmlProperty original, XmlProperty data) { try { SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("properties.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("properties.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); List channelNames = new ArrayList(); for (SearchHit hit : queryResponse.getHits()) { channelNames.add(hit.getId()); @@ -521,25 +534,23 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi .field("name", data.getName()).field("owner", data.getOwner()).endObject()).upsert(indexRequest); bulkRequest.add(updateRequest); if (!channelNames.isEmpty()) { - HashMap originalParam = new HashMap(); + HashMap originalParam = new HashMap<>(); originalParam.put("name", original.getName()); - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); + HashMap params = new HashMap<>(); + params.put("originalProp", originalParam); + params.put("newProp", param); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .refresh(true) - .script("origProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == originalProp.name) { origProp = xmlProp} }; " - + "ctx._source.properties.remove(origProp);" - + "origProp.name = newProp.name;" - + "ctx._source.properties.add(origProp)") - .addScriptParam("originalProp", originalParam) - .addScriptParam("newProp", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.stream().filter(item -> item.name == params.originalProp.name)" + + ".forEach(item -> item.name = params.newProp.name);", + params))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -571,34 +582,35 @@ private Response renameProperty(UserManager um, Client client, XmlProperty origi @DELETE @Path("{propName : " + propertyNameRegex + "}") public Response remove(@PathParam("propName") String prop) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { BulkRequestBuilder bulkRequest = client.prepareBulk(); bulkRequest.add(new DeleteRequest("properties", "property", prop)); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("properties.name", prop)).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("properties.name", prop)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).refresh(true) - .script("removeProp = new Object();" - + "for (xmlProp in ctx._source.properties) " - + "{ if (xmlProp.name == prop) { removeProp = xmlProp} }; " - + "ctx._source.properties.remove(removeProp);") - .addScriptParam("prop", prop)); + String channelName = hit.getId(); + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop);", + Collections.singletonMap("prop", prop)))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); throw new Exception(); } else { DeleteResponse deleteResponse = bulkResponse.getItems()[0].getResponse(); - if (deleteResponse.isFound()) { + if (deleteResponse.getResult() == DocWriteResponse.Result.DELETED) { Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); return r; @@ -610,7 +622,6 @@ public Response remove(@PathParam("propName") String prop) { } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -627,7 +638,7 @@ public Response remove(@PathParam("propName") String prop) { @Path("{propName}/{chName}") @Consumes({ "application/xml", "application/json" }) public Response addSingle(@PathParam("propName") String prop, @PathParam("chName") String chan, XmlProperty data) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); if (data.getValue() == null || data.getValue().isEmpty()) { return Response.status(Response.Status.BAD_REQUEST) @@ -640,25 +651,23 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); XmlProperty result = null; try { - GetResponse response = client.prepareGet("properties", "property", prop).execute().actionGet(); + GetResponse response = client.prepareGet("properties", "property", prop).get(); ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlChannel.class, MyMixInForXmlChannels.class); result = mapper.readValue(response.getSourceAsBytes(), XmlProperty.class); if (result != null) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("value", data.getValue()); // ignores the provided user and matches the one present in the properties index param.put("owner", result.getOwner()); + HashMap params = new HashMap<>(); + params.put("prop", param); - UpdateResponse updateResponse = client.update(new UpdateRequest("channelfinder", "channel", chan) - .refresh(true) - .script("removeProps = new java.util.ArrayList(); " - + "for (property in ctx._source.properties) " - + "{ if (property.name == prop.name) { removeProps.add(property)} }; " - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}; " - + "ctx._source.properties.add(prop)") - .addScriptParam("prop", param)) + client.update(new UpdateRequest("channelfinder", "channel", chan) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop.name);" + + "ctx._source.properties.add(params.prop)",params))) .actionGet(); Response r = Response.ok().build(); return r; @@ -666,13 +675,12 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName return handleException(um.getUserName(), "PUT", Status.BAD_REQUEST, "Property " +prop+ " does not exist "); } - } catch (DocumentMissingException e) { + } catch (DocumentMissingException | RemoteTransportException e) { return handleException(um.getUserName(), "PUT", Response.Status.BAD_REQUEST, - "Channels specified in property update do not exist" + e.getDetailedMessage()); + "Channels specified in property update do not exist" + e.getMessage()); } catch (Exception e) { return handleException(um.getUserName(), "PUT", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -687,16 +695,16 @@ public Response addSingle(@PathParam("propName") String prop, @PathParam("chName @DELETE @Path("{propName}/{chName}") public Response removeSingle(@PathParam("propName") String prop, @PathParam("chName") String chan) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { if(client.prepareGet("properties", "property", prop).get().isExists()){ - UpdateResponse updateResponse = client.update(new UpdateRequest("channelfinder", "channel", chan) - .script(" removeProps = new java.util.ArrayList();" + "for (property in ctx._source.properties)" - + "{ if (property.name == prop) { removeProps.add(property)} };" - + "for (removeProp in removeProps) {ctx._source.properties.remove(removeProp)}") - .addScriptParam("prop", prop).refresh(true)).actionGet(); + client.update(new UpdateRequest("channelfinder", "channel", chan) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.properties.removeIf(item -> item.name == params.prop);", + Collections.singletonMap("prop", prop)))) + .actionGet(); Response r = Response.ok().build(); return r; } else { @@ -707,7 +715,6 @@ public Response removeSingle(@PathParam("propName") String prop, @PathParam("chN } catch (Exception e) { return handleException(um.getUserName(), "DELETE", Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java b/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java index 8bbe73f..830fc83 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/ResponseCorsFilter.java @@ -1,5 +1,16 @@ package gov.bnl.channelfinder; +/* + * #%L + * ChannelFinder Directory Service + * %% + * Copyright (C) 2010 - 2016 Brookhaven National Laboratory / National Synchrotron Light Source II + * %% + * Copyright (C) 2010 - 2012 Brookhaven National Laboratory + * All rights reserved. Use is subject to license terms. + * #L% + */ + import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerResponseContext; import javax.ws.rs.container.ContainerResponseFilter; diff --git a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java index 78821c0..c98ff35 100644 --- a/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java +++ b/channelfinder/src/main/java/gov/bnl/channelfinder/TagsResource.java @@ -13,7 +13,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; -import static gov.bnl.channelfinder.ElasticSearchClient.getNewClient; import java.io.IOException; import java.io.OutputStream; @@ -49,15 +48,16 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.core.JsonEncoding; @@ -69,6 +69,13 @@ import com.google.common.collect.Collections2; import gov.bnl.channelfinder.ChannelsResource.OnlyXmlTag; +import java.util.Collections; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.transport.RemoteTransportException; /** * Top level Jersey HTTP methods for the .../tags URL @@ -99,7 +106,10 @@ public TagsResource() { @GET @Produces({MediaType.APPLICATION_JSON}) public Response list() { - Client client = getNewClient(); + StringBuffer performance = new StringBuffer(); + long start = System.currentTimeMillis(); + long totalStart = System.currentTimeMillis(); + Client client = ElasticSearchClient.getSearchClient(); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; final ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlTag.class, OnlyXmlTag.class); @@ -115,11 +125,16 @@ public Response list() { } } - final SearchResponse response = client.prepareSearch("tags") + performance.append("|prepare:" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); + SearchRequestBuilder builder = client.prepareSearch("tags") .setTypes("tag") - .setQuery(new MatchAllQueryBuilder()) - .setSize(size) - .execute().actionGet(); + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(size); + builder.addSort(SortBuilders.fieldSort("name.keyword")); + final SearchResponse response = builder.get(); + performance.append("|query:("+response.getHits().getTotalHits()+")" + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); StreamingOutput stream = new StreamingOutput(){ @Override public void write(OutputStream os) throws IOException, WebApplicationException { @@ -135,13 +150,15 @@ public void write(OutputStream os) throws IOException, WebApplicationException { jg.close(); } }; + performance.append("|parse:" + (System.currentTimeMillis() - start)); Response r = Response.ok(stream).build(); - log.fine(user + "|" + uriInfo.getPath() + "|GET|OK|" + r.getStatus() + response.getTook() + "|returns " + response.getHits().getTotalHits() + " tags"); + audit.fine(user + "|" + uriInfo.getPath() + "|GET|OK|" + performance.toString() + "|total:" + + (System.currentTimeMillis() - totalStart) + "|" + r.getStatus() + + "|returns " + response.getHits().getTotalHits()+ " tags"); return r; } catch (Exception e) { return handleException(user,Response.Status.INTERNAL_SERVER_ERROR , e); } finally { - client.close(); } } @@ -160,12 +177,12 @@ public void write(OutputStream os) throws IOException, WebApplicationException { public Response read(@PathParam("tagName") String tag) { MultivaluedMap parameters = uriInfo.getQueryParameters(); long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : ""; XmlTag result = null; try { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); if (response.isExists()) { ObjectMapper mapper = new ObjectMapper(); result = mapper.readValue(response.getSourceAsBytes(), XmlTag.class); @@ -176,7 +193,7 @@ public Response read(@PathParam("tagName") String tag) { if (parameters.containsKey("withChannels")) { // TODO iterator or scrolling needed final SearchResponse channelResult = client.prepareSearch("channelfinder") - .setQuery(matchQuery("tags.name", tag.trim())).setSize(10000).execute().actionGet(); + .setQuery(matchQuery("tags.name", tag.trim())).setSize(10000).get(); List channels = new ArrayList(); if (channelResult != null) { for (SearchHit hit : channelResult.getHits()) { @@ -199,7 +216,6 @@ public Response read(@PathParam("tagName") String tag) { } catch (IOException e) { return handleException(user, Response.Status.INTERNAL_SERVER_ERROR , e); } finally { - client.close(); } } @@ -218,7 +234,7 @@ public Response read(@PathParam("tagName") String tag) { @Consumes({MediaType.APPLICATION_JSON}) public Response create(@PathParam("tagName") String tag, XmlTag data) { long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); @@ -232,12 +248,14 @@ public Response create(@PathParam("tagName") String tag, XmlTag data) { .upsert(indexRequest); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("tags.name", tag)).addField("name").setSize(10000).execute() - .actionGet(); - + .setQuery(QueryBuilders.matchQuery("tags.name", tag)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); + Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { - existingChannels.add(hit.field("name").getValue().toString()); + existingChannels.add(hit.getId()); } Set newChannels = new HashSet(); @@ -250,29 +268,32 @@ public String apply(XmlChannel channel) { })); } - Set remove = new HashSet(existingChannels); + Set remove = new HashSet<>(existingChannels); remove.removeAll(newChannels); - Set add = new HashSet(newChannels); + Set add = new HashSet<>(newChannels); add.removeAll(existingChannels); - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", data.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); for (String ch : remove) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).refresh(true) - .script("removeTag = new Object();" + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);") - .addScriptParam("tag", param)); + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);", + params))); } for (String ch : add) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).refresh(true) - .script("ctx._source.tags.add(tag)").addScriptParam("tag", param)); + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.add(params.tag);", + params))); } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -283,7 +304,7 @@ public String apply(XmlChannel channel) { bulkResponse.buildFailureMessage()); } } else { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); ObjectMapper mapper = new ObjectMapper(); XmlTag result = mapper.readValue(response.getSourceAsBytes(), XmlTag.class); Response r; @@ -304,7 +325,6 @@ public String apply(XmlChannel channel) { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -321,7 +341,7 @@ public String apply(XmlChannel channel) { @Consumes({MediaType.APPLICATION_JSON}) public Response createTags(List data) { long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); @@ -335,14 +355,17 @@ public Response createTags(List data) { .field("name", xmlTag.getName()).field("owner", xmlTag.getOwner()).endObject()).upsert(indexRequest); bulkRequest.add(updateRequest); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("tags.name", xmlTag.getName())).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("tags.name", xmlTag.getName())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); Set existingChannels = new HashSet(); for (SearchHit hit : qbResult.getHits()) { - existingChannels.add(hit.field("name").getValue().toString()); + existingChannels.add(hit.getId()); } - Set newChannels = new HashSet(); + Set newChannels = new HashSet<>(); if (xmlTag.getChannels() != null) { newChannels.addAll( Collections2.transform(xmlTag.getChannels(), new Function() { @@ -362,23 +385,24 @@ public String apply(XmlChannel channel) { HashMap param = new HashMap(); param.put("name", xmlTag.getName()); param.put("owner", xmlTag.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); for (String ch : remove) { - bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch).refresh(true) - .script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);") - .addScriptParam("tag", param)); + bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);", + params))); } for (String ch : add) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", ch) - .script("ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.add(params.tag);", + params))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -394,7 +418,6 @@ public String apply(XmlChannel channel) { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -416,12 +439,12 @@ public String apply(XmlChannel channel) { @Consumes({"application/json"}) public Response update(@PathParam("tagName") String tag, XmlTag data) { long start = System.currentTimeMillis(); - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); audit.info("client initialization: "+ (System.currentTimeMillis() - start)); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); if(!response.isExists()){ return handleException(um.getUserName(), Response.Status.NOT_FOUND, "A tag named '"+tag+"' does not exist"); } @@ -443,36 +466,34 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { HashMap param = new HashMap(); param.put("name", data.getName()); param.put("owner", tagOwner); + HashMap params = new HashMap<>(); + params.put("tag", param); if(!original.getOwner().equals(data.getOwner())){ SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("tags.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("tags.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); for (SearchHit hit : queryResponse.getHits()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", hit.getId()) - .refresh(true) - .script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);", + params))); } } bulkRequest.add(updateRequest); if (data.getChannels() != null) { for (XmlChannel channel : data.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) - .script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);", + params))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -491,7 +512,6 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { } catch (Exception e) { return handleException(um.getUserName() , Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -509,8 +529,10 @@ public Response update(@PathParam("tagName") String tag, XmlTag data) { private Response renameTag(UserManager um, Client client, XmlTag original, XmlTag data) { try { SearchResponse queryResponse = client.prepareSearch("channelfinder") - .setQuery(wildcardQuery("tags.name", original.getName().trim())).addFields("name").setSize(10000).execute() - .actionGet(); + .setQuery(wildcardQuery("tags.name", original.getName().trim())) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); List channelNames = new ArrayList(); for (SearchHit hit : queryResponse.getHits()) { channelNames.add(hit.getId()); @@ -526,25 +548,24 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa .field("name", data.getName()).field("owner", data.getOwner()).endObject()).upsert(indexRequest); bulkRequest.add(updateRequest); if (!channelNames.isEmpty()) { - HashMap originalParam = new HashMap(); + HashMap originalParam = new HashMap<>(); originalParam.put("name", original.getName()); - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", data.getName()); param.put("owner", tagOwner); + HashMap params = new HashMap<>(); + params.put("originalTag", originalParam); + params.put("tag", param); for (String channel : channelNames) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel) - .refresh(true) - .script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == originalTag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("originalTag", originalParam) - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.originalTag.name);" + + "ctx._source.tags.add(params.tag);", + params))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -577,7 +598,7 @@ private Response renameTag(UserManager um, Client client, XmlTag original, XmlTa @POST @Consumes({MediaType.APPLICATION_JSON}) public Response updateTags(List data) throws IOException { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { @@ -589,23 +610,22 @@ public Response updateTags(List data) throws IOException { .field("owner", tag.getOwner()) .endObject())); if (tag.getChannels() != null) { - HashMap param = new HashMap(); + HashMap param = new HashMap<>(); param.put("name", tag.getName()); param.put("owner", tag.getOwner()); + HashMap params = new HashMap<>(); + params.put("tag", param); for (XmlChannel channel : tag.getChannels()) { bulkRequest.add(new UpdateRequest("channelfinder", "channel", channel.getName()) - .refresh(true) - .script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag.name) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)); + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);", + params))); } } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); if (bulkResponse.buildFailureMessage().contains("DocumentMissingException")) { @@ -624,7 +644,6 @@ public Response updateTags(List data) throws IOException { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR , e); } finally { - client.close(); } } @@ -638,34 +657,35 @@ public Response updateTags(List data) throws IOException { @DELETE @Path("{tagName: "+tagNameRegex+"}") public Response remove(@PathParam("tagName") String tag) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { BulkRequestBuilder bulkRequest = client.prepareBulk(); bulkRequest.add(new DeleteRequest("tags", "tag", tag)); SearchResponse qbResult = client.prepareSearch("channelfinder") - .setQuery(QueryBuilders.matchQuery("tags.name", tag)).addField("name").setSize(10000).execute().actionGet(); + .setQuery(QueryBuilders.matchQuery("tags.name", tag)) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setFetchSource(new String[]{"name"}, null) + .setSize(10000).get(); if (qbResult != null) { for (SearchHit hit : qbResult.getHits()) { - String channelName = hit.field("name").getValue().toString(); - bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName).refresh(true) - .script("removeTag = new Object();" - + "for (xmltag in ctx._source.tags) " - + "{ if (xmltag.name == tag) { removeTag = xmltag} }; " - + "ctx._source.tags.remove(removeTag);") - .addScriptParam("tag", tag)); + String channelName = hit.getId(); + bulkRequest.add(new UpdateRequest("channelfinder", "channel", channelName) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag);", + Collections.singletonMap("tag", tag)))); } } - bulkRequest.setRefresh(true); - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = bulkRequest.get(); if (bulkResponse.hasFailures()) { audit.severe(bulkResponse.buildFailureMessage()); return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, bulkResponse.buildFailureMessage()); } else { DeleteResponse deleteResponse = bulkResponse.getItems()[0].getResponse(); - if (deleteResponse.isFound()) { + if (deleteResponse.getResult() == DocWriteResponse.Result.DELETED) { Response r = Response.ok().build(); audit.info(um.getUserName() + "|" + uriInfo.getPath() + "|DELETE|OK|" + r.getStatus()); return r; @@ -677,7 +697,6 @@ public Response remove(@PathParam("tagName") String tag) { } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -701,30 +720,28 @@ public Response remove(@PathParam("tagName") String tag) { @Path("{tagName}/{chName}") @Consumes({"application/xml", "application/json"}) public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") String chan, XmlTag data) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); XmlTag result = null; try { - GetResponse response = client.prepareGet("tags", "tag", tag).execute().actionGet(); + GetResponse response = client.prepareGet("tags", "tag", tag).get(); ObjectMapper mapper = new ObjectMapper(); mapper.addMixIn(XmlChannel.class, MyMixInForXmlChannels.class); result = mapper.readValue(response.getSourceAsBytes(), XmlTag.class); if (result != null) { // if(um.userHasAdminRole() || um.userIsInGroup(result.getOwner())){ - if (validateTag(result, data)) { - HashMap param = new HashMap(); + if (validateTag(result, data)) { + HashMap param = new HashMap<>(); param.put("name", result.getName()); param.put("owner", result.getOwner()); - UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan).refresh(true) - .refresh(true) - .script("removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " - + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)};" - + "ctx._source.tags.add(tag)") - .addScriptParam("tag", param)) + HashMap params = new HashMap<>(); + params.put("tag", param); + client.update(new UpdateRequest("channelfinder", "channel", chan) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag.name);" + + "ctx._source.tags.add(params.tag);",params))) .actionGet(); Response r = Response.ok().build(); return r; @@ -741,12 +758,11 @@ public Response addSingle(@PathParam("tagName") String tag, @PathParam("chName") }else{ return Response.status(Status.BAD_REQUEST).entity(tag + " Does not exist").build(); } - } catch (DocumentMissingException e) { - return Response.status(Status.BAD_REQUEST).entity("Channels specified in tag update do not exist"+e.getDetailedMessage()).build(); + } catch (DocumentMissingException | RemoteTransportException e) { + return Response.status(Status.BAD_REQUEST).entity("Channels specified in tag update do not exist"+e.getMessage()).build(); } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } @@ -772,17 +788,16 @@ private boolean validateTag(XmlTag existing, XmlTag request) { @DELETE @Path("{tagName}/{chName}") public Response removeSingle(@PathParam("tagName") final String tag, @PathParam("chName") String chan) { - Client client = getNewClient(); + Client client = ElasticSearchClient.getSearchClient(); UserManager um = UserManager.getInstance(); um.setUser(securityContext.getUserPrincipal(), securityContext.isUserInRole("Administrator")); try { if (client.prepareGet("tags", "tag", tag).get().isExists()) { UpdateResponse updateResponse = client - .update(new UpdateRequest("channelfinder", "channel", chan).refresh(true) - .script(" removeTags = new java.util.ArrayList();" + "for (tag in ctx._source.tags) " - + "{ if (tag.name == tag.name) { removeTags.add(tag)} }; " - + "for (removeTag in removeTags) {ctx._source.tags.remove(removeTag)}") - .addScriptParam("tagName", tag)).actionGet(); + .update(new UpdateRequest("channelfinder", "channel", chan) + .script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, + "ctx._source.tags.removeIf(item -> item.name == params.tag);", + Collections.singletonMap("tag", tag)))).actionGet(); Response r = Response.ok().build(); return r; } else { @@ -793,7 +808,6 @@ public Response removeSingle(@PathParam("tagName") final String tag, @PathParam( } catch (Exception e) { return handleException(um.getUserName(), Response.Status.INTERNAL_SERVER_ERROR, e); } finally { - client.close(); } } diff --git a/channelfinder/src/main/resources/mapping_definitions.sh b/channelfinder/src/main/resources/mapping_definitions.sh index fee1465..2f2a3c1 100644 --- a/channelfinder/src/main/resources/mapping_definitions.sh +++ b/channelfinder/src/main/resources/mapping_definitions.sh @@ -15,6 +15,7 @@ #Create the Index + curl -XPUT 'http://localhost:9200/tags' #Set the mapping curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' @@ -22,59 +23,87 @@ curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' "tag" : { "properties" : { "name" : { - "type" : "string" + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "owner" : { - "type" : "string" + "type" : "text" } } } }' + curl -XPUT 'http://localhost:9200/properties' curl -XPUT 'http://localhost:9200/properties/_mapping/property' -d' { "property" : { "properties" : { "name" : { - "type" : "string" + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "owner" : { - "type" : "string" + "type" : "text" } } } }' + curl -XPUT 'http://localhost:9200/channelfinder' curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' { "channel" : { "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" }, "script" : { - "type" : "string" + "type" : "text" }, "properties" : { "type" : "nested", "include_in_parent" : true, "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string" + "type" : "text" }, "value" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword", + "null_value": "null" + } + }, "analyzer" : "whitespace" } } @@ -84,11 +113,16 @@ curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' "include_in_parent" : true, "properties" : { "name" : { - "type" : "string", + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, "analyzer" : "whitespace" }, "owner" : { - "type" : "string", + "type" : "text", "analyzer" : "whitespace" } } diff --git a/channelfinder/src/test/serverresources/config/domain.xml b/channelfinder/src/test/serverresources/config/domain.xml index 7d314ef..34f9a4d 100644 --- a/channelfinder/src/test/serverresources/config/domain.xml +++ b/channelfinder/src/test/serverresources/config/domain.xml @@ -2,23 +2,23 @@ #%L ChannelFinder Directory Service %% - Copyright (C) 2010 - 2012 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH + Copyright (C) 2010 - 2017 Brookhaven National Laboratory / National Synchrotron Light Source II %% Copyright (C) 2010 - 2012 Brookhaven National Laboratory All rights reserved. Use is subject to license terms. #L% --> - + - + - + @@ -32,7 +32,7 @@ - + @@ -54,7 +54,7 @@ - + @@ -62,7 +62,7 @@ - + @@ -105,9 +105,6 @@ - - - @@ -120,8 +117,18 @@ - + + + + + + + + + + + @@ -250,7 +257,7 @@ - + @@ -298,9 +305,6 @@ - - - @@ -391,7 +395,6 @@ - @@ -439,7 +442,7 @@ - + @@ -466,9 +469,10 @@ - - - + + + + diff --git a/channelfinder/src/test/serverresources/config/logging.properties b/channelfinder/src/test/serverresources/config/logging.properties index 53ca421..82ad9ce 100644 --- a/channelfinder/src/test/serverresources/config/logging.properties +++ b/channelfinder/src/test/serverresources/config/logging.properties @@ -1,3 +1,13 @@ +### +# #%L +# ChannelFinder Directory Service +# %% +# Copyright (C) 2010 - 2016 Brookhaven National Laboratory / National Synchrotron Light Source II +# %% +# Copyright (C) 2010 - 2012 Brookhaven National Laboratory +# All rights reserved. Use is subject to license terms. +# #L% +### # # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. # diff --git a/root/debian/changelog b/root/debian/changelog new file mode 100644 index 0000000..33a779c --- /dev/null +++ b/root/debian/changelog @@ -0,0 +1,64 @@ +channelfinder (1.0.2) unstable; urgency=medium + + * Non-maintainer upload. + * update settings + + -- Eric Berryman Tue, 27 Jun 2017 16:15:20 -0400 + +channelfinder (1.0.1) unstable; urgency=medium + + * Non-maintainer upload. + * update to elastic 5.4, payara, and systemd + + -- Eric Berryman Mon, 19 Jun 2017 11:27:23 -0400 + +channelfinder (1.0.0-5) unstable; urgency=medium + + * added JACCUserManager + + -- Eric Berryman Tue, 06 Sep 2016 16:51:26 -0400 + +channelfinder (1.0.0-4) unstable; urgency=medium + + * release + + -- Eric Berryman Wed, 11 May 2016 12:38:06 -0400 + +channelfinder (1.0.0-3) unstable; urgency=medium + + * dpkg init script fixes + + -- Eric Berryman Tue, 10 May 2016 14:19:25 -0400 + +channelfinder (1.0.0-2) unstable; urgency=medium + + * release + + -- Eric Berryman Tue, 10 May 2016 11:12:23 -0400 + +channelfinder (1.0.0-1) unstable; urgency=medium + + * release + + -- Eric Berryman Tue, 10 May 2016 11:09:27 -0400 + +channelfinder (1.0.0-0.2) unstable; urgency=medium + + * Non-maintainer upload. + * release + + -- Eric Berryman Tue, 10 May 2016 11:06:39 -0400 + +channelfinder (1.0.0-0.1) unstable; urgency=medium + + * Non-maintainer upload. + * release + + -- Eric Berryman Tue, 10 May 2016 10:53:33 -0400 + +channelfinder (1.0.0-0) unstable; urgency=low + + * initial + + -- Eric Berryman Mon, 09 May 2016 09:34:33 -0400 + diff --git a/root/debian/channelfinder.install b/root/debian/channelfinder.install new file mode 100644 index 0000000..9c20b61 --- /dev/null +++ b/root/debian/channelfinder.install @@ -0,0 +1,2 @@ +ChannelFinder.jar usr/share/channelfinder +etc diff --git a/root/debian/channelfinder.service b/root/debian/channelfinder.service new file mode 100644 index 0000000..1a0be23 --- /dev/null +++ b/root/debian/channelfinder.service @@ -0,0 +1,18 @@ +[Unit] +Description=ChannelFinder Service +After=network.target remote-fs.target + +[Service] +user=channelfinder +EnvironmentFile=/etc/default/channelfinder +ExecStartPre=/bin/mkdir -p /tmp/channelfinder +ExecStartPre=/usr/bin/unzip -o /usr/share/channelfinder/ChannelFinder.jar MICRO-INF/deploy/ChannelFinder.war -d /tmp/channelfinder +ExecStartPre=/usr/bin/jar -uf /tmp/channelfinder/MICRO-INF/deploy/ChannelFinder.war -C /etc/channelfinder WEB-INF/classes/elasticsearch.yml +ExecStartPre=/usr/bin/jar -uf /usr/share/channelfinder/ChannelFinder.jar -C /tmp/channelfinder MICRO-INF/deploy/ChannelFinder.war +ExecStart=/usr/bin/java -Xmx2G -jar /usr/share/channelfinder/ChannelFinder.jar --sslport ${SERVICESSLPORT} --nocluster true --port ${SERVICEPORT} --prebootcommandfile /etc/channelfinder/asadmin_commands --logtofile /var/log/channelfinder/log.txt +ExecStop=/bin/kill -SIGINT $MAINPID +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/root/debian/compat b/root/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/root/debian/compat @@ -0,0 +1 @@ +9 diff --git a/root/debian/control b/root/debian/control new file mode 100644 index 0000000..19557d2 --- /dev/null +++ b/root/debian/control @@ -0,0 +1,12 @@ +Source: channelfinder +Section: java +Priority: optional +Maintainer: Eric Berryman +Build-Depends: debhelper (>= 9), dh-systemd (>=1.5), +Standards-Version: 3.8.0 +Homepage: https://channelfinder.github.io/ + +Package: channelfinder +Depends: oracle-java8-jdk, unzip +Architecture: amd64 +Description: Directory service for EPICS diff --git a/root/debian/copyright b/root/debian/copyright new file mode 100644 index 0000000..e7e5376 --- /dev/null +++ b/root/debian/copyright @@ -0,0 +1,19 @@ +This package was debianized by Eric Berryman on +Mon, 09 May 2016 09:37:22 -0400. + + +Upstream Authors: + + Ralph Lange, Kunal Shroff + +Copyright: + + +License: + + +On Debian systems, the complete text of the GNU General +Public License can be found in `/usr/share/common-licenses/GPL'. + +The Debian packaging is (C) 2011, Eric Berryman and +is licensed under the MIT, see above. diff --git a/root/debian/dirs b/root/debian/dirs new file mode 100644 index 0000000..4936bd6 --- /dev/null +++ b/root/debian/dirs @@ -0,0 +1 @@ +/var/log/channelfinder diff --git a/root/debian/rules b/root/debian/rules new file mode 100755 index 0000000..953471c --- /dev/null +++ b/root/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f + + +%: + dh $@ --with systemd + diff --git a/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml b/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml new file mode 100644 index 0000000..cb3d72e --- /dev/null +++ b/root/etc/channelfinder/WEB-INF/classes/elasticsearch.yml @@ -0,0 +1,40 @@ +############################## Network And HTTP ############################### + +# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens +# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). + +# Set the bind address specifically (IPv4 or IPv6): +# +#network.bind_host: 192.168.0.1 + +# Set the address other nodes will use to communicate with this node. If not +# set, it is automatically derived. It must point to an actual IP address. +# +#network.publish_host: 192.168.0.1 + +# Set both 'bind_host' and 'publish_host': +# +network.host: 127.0.0.1 + +# Set a custom port for the node to node communication (9300 by default): +# +transport.tcp.port: 9300 + +# Enable compression for all communication between nodes (disabled by default): +# +#transport.tcp.compress: true + +# Set a custom port to listen for HTTP traffic: +# +#http.port: 9200 + +# Set a custom allowed content length: +# +#http.max_content_length: 100mb + +# Disable HTTP completely: +# +#http.enabled: false + diff --git a/root/etc/channelfinder/asadmin_commands b/root/etc/channelfinder/asadmin_commands new file mode 100644 index 0000000..6d732d8 --- /dev/null +++ b/root/etc/channelfinder/asadmin_commands @@ -0,0 +1,6 @@ +create-auth-realm --classname com.sun.enterprise.security.auth.realm.file.FileRealm --property file=${com.sun.aas.instanceRoot}/keyfile:jaas-context=fileRealm channelfinder + +create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property Value=gov.bnl.channelfinder.JACCUserManager channelfinder/userManager + +set-log-levels gov.bnl.channelfinder.ChannelsResource=FINEST +set-log-levels gov.bnl.channelfinder.ChannelsResource.audit=FINEST diff --git a/root/etc/channelfinder/glassfish_config_create b/root/etc/channelfinder/glassfish_config_create new file mode 100644 index 0000000..f9e4427 --- /dev/null +++ b/root/etc/channelfinder/glassfish_config_create @@ -0,0 +1,12 @@ +#!/bin/bash + +exec sh -ac '. /etc/default/channelfinder + +${ASADMIN} create-auth-realm --classname ${CLASSNAME} --property "jaas-context=${JAAS_CONTEXT}:directory=${DIRECTORY}:base-dn=${BASE_DN}:group-base-dn=${GROUP_BASE_DN}:group-search-filter=${GROUP_SEARCH_FILTER}:java.naming.referral=${JAVA_NAMING_REFERRAL}:search-filter=${SEARCH_FILTER}:search-bind-password=${SEARCH_BIND_PASSWORD}:search-bind-dn=${SEARCH_BIND_DN}" channelfinder + +${ASADMIN} create-custom-resource --restype javax.naming.directory.Directory --factoryclass com.sun.jndi.ldap.LdapCtxFactory --property "URL=${DIRECTORY}/${BASE_DN}:javax.naming.security.principal=${SEARCH_BIND_DN}:javax.naming.security.credentials=${SEARCH_BIND_PASSWORD}:java.naming.referral=${JAVA_NAMING_REFERRAL}" channelfinder/ldapManagerConnection + +${ASADMIN} create-custom-resource --restype java.lang.String --factoryclass org.glassfish.resources.custom.factory.PrimitivesAndStringFactory --property "Value=${USER_MANAGER}" channelfinder/userManager + +. /etc/channelfinder/mapping_definitions.sh +' diff --git a/root/etc/channelfinder/glassfish_config_remove b/root/etc/channelfinder/glassfish_config_remove new file mode 100644 index 0000000..9fffe6b --- /dev/null +++ b/root/etc/channelfinder/glassfish_config_remove @@ -0,0 +1,8 @@ +#!/bin/bash + +exec sh -ac '. /etc/default/channelfinder + +${ASADMIN} delete-auth-realm channelfinder +${ASADMIN} delete-custom-resource channelfinder/ldapManagerConnection +${ASADMIN} delete-custom-resource channelfinder/userManager +' diff --git a/root/etc/channelfinder/mapping_definitions.sh b/root/etc/channelfinder/mapping_definitions.sh new file mode 100644 index 0000000..2a68496 --- /dev/null +++ b/root/etc/channelfinder/mapping_definitions.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +### +# #%L +# ChannelFinder Directory Service +# %% +# Copyright (C) 2010 - 2016 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH +# %% +# Copyright (C) 2010 - 2012 Brookhaven National Laboratory +# All rights reserved. Use is subject to license terms. +# #L% +### + +# The mapping definition for the Indexes associated with the channelfinder v2 + + +#Create the Index +curl -XPUT 'http://localhost:9200/tags' +#Set the mapping +curl -XPUT 'http://localhost:9200/tags/_mapping/tag' -d' +{ + "tag" : { + "properties" : { + "name" : { + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "owner" : { + "type" : "text" + } + } + } +}' + +curl -XPUT 'http://localhost:9200/properties' +curl -XPUT 'http://localhost:9200/properties/_mapping/property' -d' +{ + "property" : { + "properties" : { + "name" : { + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "owner" : { + "type" : "text" + } + } + } +}' + +curl -XPUT 'http://localhost:9200/channelfinder' +curl -XPUT 'http://localhost:9200/channelfinder/_mapping/channel' -d' +{ + "channel" : { + "properties" : { + "name" : { + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, + "analyzer" : "whitespace" + }, + "owner" : { + "type" : "text", + "analyzer" : "whitespace" + }, + "script" : { + "type" : "text" + }, + "properties" : { + "type" : "nested", + "include_in_parent" : true, + "properties" : { + "name" : { + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, + "analyzer" : "whitespace" + }, + "owner" : { + "type" : "text" + }, + "value" : { + "type" : "text", + "analyzer" : "whitespace" + } + } + }, + "tags" : { + "type" : "nested", + "include_in_parent" : true, + "properties" : { + "name" : { + "type" : "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, + "analyzer" : "whitespace" + }, + "owner" : { + "type" : "text", + "analyzer" : "whitespace" + } + } + } + } + } +}' diff --git a/root/etc/default/channelfinder b/root/etc/default/channelfinder new file mode 100644 index 0000000..0fc42b9 --- /dev/null +++ b/root/etc/default/channelfinder @@ -0,0 +1,21 @@ +#Channelfinder defaults + +SERVICEPORT=8080 +SERVICESSLPORT=8181 + +GF_HOME=/home/glassfish/glassfish4/glassfish +ASADMIN=$GF_HOME/bin/asadmin + +ELASTIC_HOST="http\://localhost\:9200" + +CLASSNAME="com.sun.enterprise.security.auth.realm.ldap.LDAPRealm" +JAAS_CONTEXT="ldapRealm" +DIRECTORY="ldap\://localhost\:389" +BASE_DN="dc\=cf-test,dc\=local" +GROUP_BASE_DN="dc\=cf-test,dc\=local" +GROUP_SEARCH_FILTER="(&(objectClass\=group)(member\=%d))" +JAVA_NAMING_REFERRAL="follow" +SEARCH_FILTER="(&(objectClass\=user)(sAMAccountName\=%s))" +SEARCH_BIND_PASSWORD="1234" +SEARCH_BIND_DN="cn\=channelfinder,dc\=cf-test,dc\=local" +USER_MANAGER="gov.bnl.channelfinder.LDAPUserManager" diff --git a/test/testcf.py b/test/testcf.py index f8ae10c..4b77cd5 100644 --- a/test/testcf.py +++ b/test/testcf.py @@ -7,6 +7,7 @@ import sys, os import json from _testConf import _testConf +import time base_url = os.getenv("BASEURL") if base_url is None: @@ -213,6 +214,7 @@ def ordered(obj): return obj def doGetJSON(self, conn, g_url, g_result, g_resp): + time.sleep(1) response = conn_none.request_get(g_url, headers=copy(jsonheader)) self.assertEqual(`g_resp`, response[u'headers']['status'], 'unexpected return code for get operation - expected ' + `g_resp` + ', received ' + response[u'headers']['status'] + ', message body:\n' + response[u'body']) @@ -883,7 +885,7 @@ def test_AuthorizedAsAdminNewChannel(self): # Adding tag to non-existing channel def test_AuthorizedAsTagNonexChannel(self): - doPutAndFailMessageJSON(self, conn_tag, self.T1, T1_C3, 404, "DocumentMissingException[[channelfinder][-1] [channel][C3]") + doPutAndFailMessageJSON(self, conn_tag, self.T1, T1_C3, 404, "DocumentMissingException[[channel][C3]") # Payload and URL names do not match def test_AuthorizedAsTagLcPayload(self): @@ -1004,7 +1006,7 @@ def test_AuthorizedAsTagNonexTag(self): # Non-existing channel def test_AuthorizedAsTagNonexChannel(self): - doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channelfinder][-1] [channel][C4]") + doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channel][C4]") def tearDown(self): response = conn_admin.request_delete(self.T1, headers=copy(jsonheader)) @@ -1312,7 +1314,7 @@ def test_AuthorizedAsAdminNewChannel(self): # Adding property to non existing channel def test_AuthorizedAsPropNonexChannel(self): - doPutAndFailMessageJSON(self, conn_prop, self.P1, P1_C3, 404, "DocumentMissingException[[channelfinder][-1] [channel][C3]: document missing") + doPutAndFailMessageJSON(self, conn_prop, self.P1, P1_C3, 404, "DocumentMissingException[[channel][C3]: document missing") # Payload and URL names do not match def test_AuthorizedAsPropLcPayload(self): @@ -1460,7 +1462,7 @@ def test_AuthorizedAsPropNonexTag(self): # Non-existing channel def test_AuthorizedAsPropNonexChannel(self): - doPostAndFailMessageJSON(self, conn_prop, self.P2, P2_C4, 404, "DocumentMissingException[[channelfinder][-1] [channel][C4]: document missing") + doPostAndFailMessageJSON(self, conn_prop, self.P2, P2_C4, 404, "DocumentMissingException[[channel][C4]: document missing") def tearDown(self): response = conn_admin.request_delete(self.P1, headers=copy(jsonheader)) diff --git a/test/testcftags.py b/test/testcftags.py index 0b353f0..952a8a6 100644 --- a/test/testcftags.py +++ b/test/testcftags.py @@ -403,7 +403,7 @@ def test_AuthorizedAsTagNonexTag(self): # Non-existing channel def test_AuthorizedAsTagNonexChannel(self): - doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channelfinder][-1] [channel][C4]: document missing]") + doPostAndFailMessageJSON(self, conn_tag, self.T2, T2_C4, 404, "DocumentMissingException[[channel][C4]: document missing]") def tearDown(self): response = conn_admin.request_delete(self.T1, headers=copy(jsonheader))