linux dump现场工具

发表于 | 分类于 | |

linux dump工具

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

#!/usr/bin/env bash

### use demo ###
# 1)upload dump.sh
# 2)dos2unix dump.sh;chmod +x dump.sh
# 3)usage: 
#   1. /data/sh/java/dump.sh /tmp/dump /usr/local/java/jdk1.8.0_05 23554
#   2. /data/sh/java/dump.sh /tmp/dump /usr/local/java/jdk1.8.0_05 23554 -F

### parameter description ###
# 1 ./tmp/dump is save dump file dir. 
# 2 /usr/local/java/jdk1.8.0_05 is java_home. 
# 3 23554 is java-pid. 
# 4 -F force jvm dum, optional.

declare SNAPSHOT_BASE_DIRECTORY=$1
declare JAVA_HOME=$2
declare PID=$3
declare JVM_FORCE_OPTS=$4

# remove the last '/' char
JAVA_HOME=${JAVA_HOME/%\//}
SNAPSHOT_BASE_DIRECTORY=${SNAPSHOT_BASE_DIRECTORY/%\//}

declare FULL_TIME_FORMAT="+%Y-%m-%d_%H_%M_%S_%N"
declare SHORT_TIME_FORMAT="+%H-%M-%S-%N"
declare TIMESTAMP=`date ""${FULL_TIME_FORMAT}"" `
declare RESULT_DIRECTORY=${SNAPSHOT_BASE_DIRECTORY}/${PID}/${TIMESTAMP}
declare RUN_LOG_PATH="${RESULT_DIRECTORY}/run.log"

run_log(){
    declare log_msg="$(date ""${FULL_TIME_FORMAT}""): $1"
    echo ${log_msg}
    echo ${log_msg} >>${RUN_LOG_PATH}
}

is_no_null(){
    if [ -z $1 ];then
	return 0;
    else
	return 1;
    fi
}

is_valid_number(){
    is_no_null $1
    if [ $? -eq 0 ];then
	return 0;
    fi

    if [ $1 -gt 0 ] 2>/dev/null ;then
	return 1;
    else
	return 0;
    fi
}

echo_blank_line(){
    echo 
}

valid_param(){
    if [ ! -d ${RESULT_DIRECTORY} ]; then
	mkdir -p ${RESULT_DIRECTORY}
	run_log "tip: Create RESULT_DIRECTORY=${RESULT_DIRECTORY}"
    fi
    run_log "tip: RESULT_DIRECTORY=${RESULT_DIRECTORY}"

    is_no_null ${SNAPSHOT_BASE_DIRECTORY}
    if [ $? -eq 0 ];then
	run_log "Param 1 SNAPSHOT_BASE_DIRECTORY is no exist and exit ,such as /tmp/dump"
	exit 1;
    else
	run_log "tip: SNAPSHOT_BASE_DIRECTORY=${SNAPSHOT_BASE_DIRECTORY}"
    fi

    if [ -d "${JAVA_HOME}" ];then
	run_log "tip: JAVA HOME: ${JAVA_HOME}"
    else
	run_log "Param2 JAVA_HOME is no exist and exit ,such as /usr/local/java/jdk1.8.0_05"
	exit 2;
    fi

    is_valid_number ${PID}
    if [ $? -eq 0 ];then
	run_log "Param3 PID is invalid and exit"
	exit 3;
    fi
}
valid_param
run_log "RESULT_DIRECTORY=${RESULT_DIRECTORY}"

machine_dump(){
    declare filename=$1;

    type $1 >/dev/null 2>&1 && {
	    run_log "Start $1 $2 dump"
	    declare timestamp=$(date ""${SHORT_TIME_FORMAT}"");
	    run_log "Execute $1 $2 >> ${RESULT_DIRECTORY}/machine_${filename}_${timestamp}.dump"
	    $1 $2 >> ${RESULT_DIRECTORY}/machine_${filename}_${timestamp}.dump
	    run_log "End $1 $2 dump"
	echo_blank_line
    }
}

machine_dump_pipeline(){
    declare filename=$1;

    type $2 >/dev/null 2>&1 && {
	run_log "Start $2 $3 | $4 dump"
	declare timestamp=$(date ""${SHORT_TIME_FORMAT}"");
	run_log "Execute $2 $3 | $4 >> ${RESULT_DIRECTORY}/machine_${filename}_${timestamp}.dump"
	$2 $3 | $4 >> ${RESULT_DIRECTORY}/machine_${filename}_${timestamp}.dump
	    run_log "End $2 $3 | $4 dump"
	echo_blank_line
    }
}

# 3 times interval 1s
declare machine_static_frequency="1 3"

# machine real time statistics information
echo_blank_line
machine_dump_pipeline "top-50-process" top "-b" "head -n 50" 
machine_dump free -glt
machine_dump vmstat "-t ${machine_static_frequency}"
machine_dump mpstat "-A ${machine_static_frequency}"
machine_dump iostat "${machine_static_frequency}"
machine_dump iotop "-o -b -n 3"
machine_dump netstat "-an"
machine_dump lsof "-p ${PID}"
# machine history statistics information
machine_dump sar -A

get_pid_user(){
    user_tip=`ps u -p $1 | tail -n 1 | awk '{print $1}'`
    is_valid_number ${user_tip}
    if [ $? -eq 0 ];then
	echo ${user_tip}
    else
	echo `cat /etc/passwd |grep x:${user_tip} | awk -F ':' '{print $1}'`
    fi
}

PID_USER=`get_pid_user ${PID}`

jvm_dump(){
    declare filename;
    if [ ! -z $3 ];then
	filename=$3
    else
	filename=$1
    fi
    run_log "Start $1 $2 dump"
    declare timestamp=$(date ""${SHORT_TIME_FORMAT}"");
    run_log "su -l ${PID_USER} -s /bin/bash -c \"${JAVA_HOME}/bin/$1 $2 \" >>  ${RESULT_DIRECTORY}/jvm_${filename}_${timestamp}.dump"
    su -l ${PID_USER} -s /bin/bash -c "${JAVA_HOME}/bin/$1 $2 " >> ${RESULT_DIRECTORY}/jvm_${filename}_${timestamp}.dump
    run_log "End $1 $2 dump"
    echo_blank_line
}

# 8 times interval 1s
declare jvm_static_frequency="1000 8"
# jvm real time statistics information
jvm_dump jinfo "${PID}"
jvm_dump jstat "-gcutil ${PID} ${jvm_static_frequency}" "jstat-gcutil"

machine_dump_pipeline "top-100-high-thread" top "-H -b -n 1 -p  ${PID}" "head -n 100"
chown ${PID_USER}:${PID_USER} ${RESULT_DIRECTORY}
# topH have must together with jstack
jvm_dump jstack "${JVM_FORCE_OPTS} ${PID}"

# dump 
jvm_head_dump(){
    run_log "Start jmap  $1 dump"
    declare timestamp=$(date ""${SHORT_TIME_FORMAT}"");
    run_log "su -l ${PID_USER} -s /bin/bash -c \"${JAVA_HOME}/bin/jmap ${JVM_FORCE_OPTS} -dump:format=b,file=${RESULT_DIRECTORY}/jvm_jmap_${timestamp}.hprof $1\""
    #su -l ${PID_USER} -s /bin/bash -c "${JAVA_HOME}/bin/jmap ${JVM_FORCE_OPTS} -dump:format=b,file=${RESULT_DIRECTORY}/jvm_jmap_${timestamp}.hprof $1" 
    ${JAVA_HOME}/bin/jmap ${JVM_FORCE_OPTS} -dump:format=b,file=${RESULT_DIRECTORY}/jvm_jmap_${timestamp}.hprof $1
    run_log "End jmap $1 dump"
    echo_blank_line
}
jvm_head_dump ${PID}

# statistics all kinds of tcp status
cat ${RESULT_DIRECTORY}/machine_netstat*.dump | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}' > ${RESULT_DIRECTORY}/machine_netstat_status_statistics.dump