From b20451c2bf8f204a0ec28ef337b78a1629130247 Mon Sep 17 00:00:00 2001 From: "haolin.song" Date: Wed, 12 Dec 2018 16:37:11 +0000 Subject: [PATCH 1/5] [TRAFODION-3248]provide more dcscheck option -m: memory status of dcsmaster -s: status of mxosrvrs -t: thread info of dcs ListenerWorker and ListenerService --- core/sqf/sql/scripts/dcscheck | 63 +++++++++++++++++-- .../trafodion/dcs/zookeeper/ZKShellTool.java | 61 ++++++++++++++++++ 2 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 dcs/src/main/java/org/trafodion/dcs/zookeeper/ZKShellTool.java diff --git a/core/sqf/sql/scripts/dcscheck b/core/sqf/sql/scripts/dcscheck index 914697c2cb..a4d3952e1a 100755 --- a/core/sqf/sql/scripts/dcscheck +++ b/core/sqf/sql/scripts/dcscheck @@ -20,6 +20,9 @@ dcsznode=/$USER/dcs/master jpscmd=$JAVA_HOME/bin/jps +JAVA=$JAVA_HOME/bin/java +jstatcmd=$JAVA_HOME/bin/jstat +jstackcmd=$JAVA_HOME/bin/jstack cfg_mxo_cnt=0 act_mxo_cnt=0 down_mxo_cnt=0 @@ -30,7 +33,7 @@ cfg_dcsserver_cnt=0 actual_dcsserver_cnt=0 down_dcsserver_cnt=0 activeMaster=localhost - +OPT_VALUE=$1 if [ -e $SQ_PDSH ]; then L_PDSH="$SQ_PDSH $MY_NODES " else @@ -171,11 +174,6 @@ if ( [ $sq_stat == 0 ] || [ $sq_stat == 1 ] ); then fi echo fi - echo -e "Process\t\tConfigured\tActual\t\tDown" - echo -e "---------\t----------\t------\t\t----" - echo -e "DcsMaster\t$cfg_dcsmaster_cnt\t\t$actual_dcsmaster_cnt\t\t$down_dcsmaster_cnt" - echo -e "DcsServer\t$cfg_dcsserver_cnt\t\t$actual_dcsserver_cnt\t\t$down_dcsserver_cnt" - echo -e "mxosrvr\t\t$cfg_mxo_cnt\t\t$act_mxo_cnt\t\t$down_mxo_cnt\n" else echo "DCS is not installed. Please install and configure DCS..." exit 1 @@ -183,5 +181,58 @@ if ( [ $sq_stat == 0 ] || [ $sq_stat == 1 ] ); then else echo "Trafodion is not started or is not operational..." echo + exit 1 +fi + +if [ -z "$OPT_VALUE" ]; then + + echo -e "Process\t\tConfigured\tActual\t\tDown" + echo -e "---------\t----------\t------\t\t----" + echo -e "DcsMaster\t$cfg_dcsmaster_cnt\t\t$actual_dcsmaster_cnt\t\t$down_dcsmaster_cnt" + echo -e "DcsServer\t$cfg_dcsserver_cnt\t\t$actual_dcsserver_cnt\t\t$down_dcsserver_cnt" + echo -e "mxosrvr\t\t$cfg_mxo_cnt\t\t$act_mxo_cnt\t\t$down_mxo_cnt\n" + exit 0 +else + echo + case $OPT_VALUE in + -m) + if [ -e $SQ_PDSH ]; then + STAT_CMD="$SQ_PDSH -w $activeMaster $jstatcmd -gc " + else + STAT_CMD=" $jstatcmd -gc " + fi + J_STATUS=`$STAT_CMD $activeDcsPid ` + EC=`echo $J_STATUS | sed -n 1p | awk '{ print $20 }'` + EU=`echo $J_STATUS | sed -n 1p | awk '{ print $21 }'` + OC=`echo $J_STATUS | sed -n 1p | awk '{ print $22 }'` + OU=`echo $J_STATUS | sed -n 1p | awk '{ print $23 }'` + PERCENT_EU=$(echo "scale=2;$EU/$EC "| bc) + PERCENT_OU=$(echo "scale=2;$OU/$OC "| bc) + + echo -e "Active DCSMaster VM status:" + echo -e "EC\tEU\tOC\tOU\tPERCENT_EU\tPERCENT_OU\t" + echo -e "$EC\t$EU\t$OC\t$OU\t$PERCENT_EU\t\t$PERCENT_OU\t" + #$STAT_CMD $activeDcsPid + ;; + + -t) + if [ -e $SQ_PDSH ]; then + STAT_CMD="$SQ_PDSH -w $activeMaster $jstackcmd " + else + STAT_CMD=" $jstackcmd " + fi + echo `$STAT_CMD $activeDcsPid \ + | sed -n '/Thread-6/, /ListenerWorker.java/p'` + + echo `$STAT_CMD $activeDcsPid \ + | sed -n '/Thread-5/, /ListenerService.java/p'` + #| awk '/java.lang.Thread.State/{print $2}' ` + ;; + -s) + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/target/classes org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null + ;; + *) + ;; +esac fi exit 0 diff --git a/dcs/src/main/java/org/trafodion/dcs/zookeeper/ZKShellTool.java b/dcs/src/main/java/org/trafodion/dcs/zookeeper/ZKShellTool.java new file mode 100644 index 0000000000..4f7f30c439 --- /dev/null +++ b/dcs/src/main/java/org/trafodion/dcs/zookeeper/ZKShellTool.java @@ -0,0 +1,61 @@ +/** +* @@@ START COPYRIGHT @@@ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +* @@@ END COPYRIGHT @@@ +*/ +package org.trafodion.dcs.zookeeper; + +import java.util.Properties; +import java.util.Map.Entry; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.trafodion.dcs.util.DcsConfiguration; +import java.sql.Timestamp; +/** + * Tool for reading ZooKeeper servers from dcs XML configuration and producing + * a line-by-line list for use by bash scripts. + */ +public class ZKShellTool { + /** + * Run the tool. + * @param args Command line arguments. + */ + public static void main(String args[]) throws Exception{ + String username = System.getProperty("user.name"); + System.out.println("User: " + username); + ZkClient zkc = new ZkClient(); + zkc.connect(); + + String registeredPath = new String("/"+username + "/dcs/servers/registered"); + List zks = zkc.getChildren(registeredPath, null); + + System.out.println("State\t\tNode\tProcessId\tProcessName\tIpAddress\tPort\tClientName\tClientApp\tClientIp\tClientPort"); + for (String nodeName: zks){ + String nodePath = registeredPath + "/" + nodeName; + String currentData = new String(zkc.getData(nodePath, false, null)); + String[] node_status = currentData.split(":"); + for(int i = 0; i < node_status.length; i++){ + if (i != 1 && i != 2) + System.out.print(node_status[i]+"\t"); + } + System.out.println(); + } + } +} From d569ea1172d7539351fd107a808701b4555611b0 Mon Sep 17 00:00:00 2001 From: "haolin.song" Date: Wed, 19 Dec 2018 10:41:19 +0000 Subject: [PATCH 2/5] modify the way to get zkshell when in cluster environment --- core/sqf/sql/scripts/dcscheck | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/sqf/sql/scripts/dcscheck b/core/sqf/sql/scripts/dcscheck index a4d3952e1a..3279f31679 100755 --- a/core/sqf/sql/scripts/dcscheck +++ b/core/sqf/sql/scripts/dcscheck @@ -211,7 +211,7 @@ else echo -e "Active DCSMaster VM status:" echo -e "EC\tEU\tOC\tOU\tPERCENT_EU\tPERCENT_OU\t" - echo -e "$EC\t$EU\t$OC\t$OU\t$PERCENT_EU\t\t$PERCENT_OU\t" + echo -e "$EC\t|$EU\t|$OC\t|$OU\t|$PERCENT_EU\t\t|$PERCENT_OU\t" #$STAT_CMD $activeDcsPid ;; @@ -229,7 +229,12 @@ else #| awk '/java.lang.Thread.State/{print $2}' ` ;; -s) - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/target/classes org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null + if [ -e $SQ_PDSH ]; then + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/"dcs-"$TRAFODION_VER".jar" org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null + else + echo "develop environment" + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/target/classes org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null + fi ;; *) ;; From b09c6976bc48b3de9b97c1a14c02bbf0afb2ed59 Mon Sep 17 00:00:00 2001 From: "haolin.song" Date: Sat, 22 Dec 2018 10:41:53 +0000 Subject: [PATCH 3/5] move null option to default section of case statement --- core/sqf/sql/scripts/dcscheck | 95 +++++++++++++++++------------------ 1 file changed, 45 insertions(+), 50 deletions(-) diff --git a/core/sqf/sql/scripts/dcscheck b/core/sqf/sql/scripts/dcscheck index 3279f31679..9b269ce56e 100755 --- a/core/sqf/sql/scripts/dcscheck +++ b/core/sqf/sql/scripts/dcscheck @@ -184,60 +184,55 @@ else exit 1 fi -if [ -z "$OPT_VALUE" ]; then - - echo -e "Process\t\tConfigured\tActual\t\tDown" - echo -e "---------\t----------\t------\t\t----" - echo -e "DcsMaster\t$cfg_dcsmaster_cnt\t\t$actual_dcsmaster_cnt\t\t$down_dcsmaster_cnt" - echo -e "DcsServer\t$cfg_dcsserver_cnt\t\t$actual_dcsserver_cnt\t\t$down_dcsserver_cnt" - echo -e "mxosrvr\t\t$cfg_mxo_cnt\t\t$act_mxo_cnt\t\t$down_mxo_cnt\n" - exit 0 -else - echo - case $OPT_VALUE in - -m) - if [ -e $SQ_PDSH ]; then - STAT_CMD="$SQ_PDSH -w $activeMaster $jstatcmd -gc " - else - STAT_CMD=" $jstatcmd -gc " - fi - J_STATUS=`$STAT_CMD $activeDcsPid ` - EC=`echo $J_STATUS | sed -n 1p | awk '{ print $20 }'` - EU=`echo $J_STATUS | sed -n 1p | awk '{ print $21 }'` - OC=`echo $J_STATUS | sed -n 1p | awk '{ print $22 }'` - OU=`echo $J_STATUS | sed -n 1p | awk '{ print $23 }'` - PERCENT_EU=$(echo "scale=2;$EU/$EC "| bc) - PERCENT_OU=$(echo "scale=2;$OU/$OC "| bc) - - echo -e "Active DCSMaster VM status:" - echo -e "EC\tEU\tOC\tOU\tPERCENT_EU\tPERCENT_OU\t" - echo -e "$EC\t|$EU\t|$OC\t|$OU\t|$PERCENT_EU\t\t|$PERCENT_OU\t" - #$STAT_CMD $activeDcsPid - ;; - - -t) - if [ -e $SQ_PDSH ]; then - STAT_CMD="$SQ_PDSH -w $activeMaster $jstackcmd " - else - STAT_CMD=" $jstackcmd " - fi - echo `$STAT_CMD $activeDcsPid \ +case $OPT_VALUE in + -m) + if [ -e $SQ_PDSH ]; then + STAT_CMD="$SQ_PDSH -w $activeMaster $jstatcmd -gc " + else + STAT_CMD=" $jstatcmd -gc " + fi + J_STATUS=`$STAT_CMD $activeDcsPid ` + EC=`echo $J_STATUS | sed -n 1p | awk '{ print $20 }'` + EU=`echo $J_STATUS | sed -n 1p | awk '{ print $21 }'` + OC=`echo $J_STATUS | sed -n 1p | awk '{ print $22 }'` + OU=`echo $J_STATUS | sed -n 1p | awk '{ print $23 }'` + PERCENT_EU=$(echo "scale=2;$EU/$EC "| bc) + PERCENT_OU=$(echo "scale=2;$OU/$OC "| bc) + + echo -e "Active DCSMaster VM status:" + echo -e "EC\tEU\tOC\tOU\tPERCENT_EU\tPERCENT_OU\t" + echo -e "$EC\t|$EU\t|$OC\t|$OU\t|$PERCENT_EU\t\t|$PERCENT_OU\t" + #$STAT_CMD $activeDcsPid + ;; + + -t) + if [ -e $SQ_PDSH ]; then + STAT_CMD="$SQ_PDSH -w $activeMaster $jstackcmd " + else + STAT_CMD=" $jstackcmd " + fi + echo `$STAT_CMD $activeDcsPid \ | sed -n '/Thread-6/, /ListenerWorker.java/p'` - echo `$STAT_CMD $activeDcsPid \ + echo `$STAT_CMD $activeDcsPid \ | sed -n '/Thread-5/, /ListenerService.java/p'` - #| awk '/java.lang.Thread.State/{print $2}' ` - ;; - -s) - if [ -e $SQ_PDSH ]; then - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/"dcs-"$TRAFODION_VER".jar" org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null - else - echo "develop environment" - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/target/classes org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null - fi - ;; + #| awk '/java.lang.Thread.State/{print $2}' ` + ;; + -s) + if [ -e $SQ_PDSH ]; then + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/"dcs-"$TRAFODION_VER".jar" org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null + else + echo "develop environment" + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/target/classes org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null + fi + ;; *) + echo -e "Process\t\tConfigured\tActual\t\tDown" + echo -e "---------\t----------\t------\t\t----" + echo -e "DcsMaster\t$cfg_dcsmaster_cnt\t\t$actual_dcsmaster_cnt\t\t$down_dcsmaster_cnt" + echo -e "DcsServer\t$cfg_dcsserver_cnt\t\t$actual_dcsserver_cnt\t\t$down_dcsserver_cnt" + echo -e "mxosrvr\t\t$cfg_mxo_cnt\t\t$act_mxo_cnt\t\t$down_mxo_cnt\n" + exit 0 ;; esac -fi exit 0 From 17e27d0b65e20e56d14866040e35e346b9a3a21a Mon Sep 17 00:00:00 2001 From: "haolin.song" Date: Thu, 10 Jan 2019 16:45:06 +0000 Subject: [PATCH 4/5] make the related code more clear --- core/sqf/sql/scripts/dcscheck | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/sqf/sql/scripts/dcscheck b/core/sqf/sql/scripts/dcscheck index 9b269ce56e..e945d0f113 100755 --- a/core/sqf/sql/scripts/dcscheck +++ b/core/sqf/sql/scripts/dcscheck @@ -219,12 +219,7 @@ case $OPT_VALUE in #| awk '/java.lang.Thread.State/{print $2}' ` ;; -s) - if [ -e $SQ_PDSH ]; then - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/"dcs-"$TRAFODION_VER".jar" org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null - else - echo "develop environment" - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/target/classes org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null - fi + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/"dcs-"$TRAFODION_VER".jar" org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null ;; *) echo -e "Process\t\tConfigured\tActual\t\tDown" From 2919c9654f4bbdee9e07adeb0766b75ea6ca9ca4 Mon Sep 17 00:00:00 2001 From: "haolin.song" Date: Sat, 19 Jan 2019 10:05:01 +0000 Subject: [PATCH 5/5] [TRAFODION-3248]remove debug info --- core/sqf/sql/scripts/dcscheck | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/sqf/sql/scripts/dcscheck b/core/sqf/sql/scripts/dcscheck index e945d0f113..788639bce1 100755 --- a/core/sqf/sql/scripts/dcscheck +++ b/core/sqf/sql/scripts/dcscheck @@ -202,7 +202,6 @@ case $OPT_VALUE in echo -e "Active DCSMaster VM status:" echo -e "EC\tEU\tOC\tOU\tPERCENT_EU\tPERCENT_OU\t" echo -e "$EC\t|$EU\t|$OC\t|$OU\t|$PERCENT_EU\t\t|$PERCENT_OU\t" - #$STAT_CMD $activeDcsPid ;; -t) @@ -216,7 +215,6 @@ case $OPT_VALUE in echo `$STAT_CMD $activeDcsPid \ | sed -n '/Thread-5/, /ListenerService.java/p'` - #| awk '/java.lang.Thread.State/{print $2}' ` ;; -s) "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" -classpath ${CLASSPATH}:${DCS_INSTALL_DIR}/"dcs-"$TRAFODION_VER".jar" org.trafodion.dcs.zookeeper.ZKShellTool 2>/dev/null