JavaScript must be enabled in order for you to see "WP Copy Data Protect" effect. However, it seems JavaScript is either disabled or not supported by your browser. To see full result of "WP Copy Data Protector", enable JavaScript by changing your browser options, then try again.

Execute the kdump crash manually under the RHEL 7.3


Kerker, I don’t post a rubbish for a long time. Wish everyone have a happy Chinese New Year before making this article. I want to record how to execute the kdump manually under the RHEL 7.3 because this item need to be tested through the method of local disk and NFS when execute the OSV certification. In fact, this behaiver is be executed when your system was be crashed, and you can analysis the related information about system before system crashed. How do we do the kdump manualy? Let’s keep reading:

1) Check the related packages be installed or not
#rpm -qa | grep -i kexec-tools
kexec-tools-2.0.7-38.el7.x86_64
#rpm -qa | grep -i kernel-debuginfo | wc -l

2) Check the related parameter that’s be added in the configuration of boot
#cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet"
GRUB_DISABLE_RECOVERY="true"
#cat /boot/grub2/grub.cfg
#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub2-mkconfig using templates
# from /etc/grub.d and settings from /etc/default/grub
#
### BEGIN /etc/grub.d/00_header ###
set pager=1
if [ -s $prefix/grubenv ]; then
load_env
fi
if [ "${next_entry}" ] ; then
set default="${next_entry}"
set next_entry=
save_env next_entry
set boot_once=true
else
set default="${saved_entry}"
fi
if [ x"${feature_menuentry_id}" = xy ]; then
menuentry_id_option="--id"
else
menuentry_id_option=""
fi
export menuentry_id_option
if [ "${prev_saved_entry}" ]; then
set saved_entry="${prev_saved_entry}"
save_env saved_entry
set prev_saved_entry=
save_env prev_saved_entry
set boot_once=true
fi
function savedefault {
if [ -z "${boot_once}" ]; then
saved_entry="${chosen}"
save_env saved_entry
fi
}
function load_video {
if [ x$feature_all_video_module = xy ]; then
insmod all_video
else
insmod efi_gop
insmod efi_uga
insmod ieee1275_fb
insmod vbe
insmod vga
insmod video_bochs
insmod video_cirrus
fi
}
terminal_output console
if [ x$feature_timeout_style = xy ] ; then
set timeout_style=menu
set timeout=5
# Fallback normal timeout code in case the timeout_style feature is
# unavailable.
else
set timeout=5
fi
### END /etc/grub.d/00_header ###
### BEGIN /etc/grub.d/00_tuned ###
set tuned_params=""
### END /etc/grub.d/00_tuned ###
### BEGIN /etc/grub.d/01_users ###
if [ -f ${prefix}/user.cfg ]; then
source ${prefix}/user.cfg
if [ -n ${GRUB2_PASSWORD} ]; then
set superusers="root"
export superusers
password_pbkdf2 root ${GRUB2_PASSWORD}
fi
fi
### END /etc/grub.d/01_users ###
### BEGIN /etc/grub.d/10_linux ###
menuentry 'Red Hat Enterprise Linux Server (3.10.0-327.el7.x86_64) 7.2 (Maipo)' --class red --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-327.el7.x86_64-advanced-c31ce45b-b92d-4bb3-9f64-ffae78aacbf7' {
load_video
set gfxpayload=keep
insmod gzio
insmod part_msdos
insmod xfs
set root='hd0,msdos1'
if [ x$feature_platform_search_hint = xy ]; then
search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1' 6f64e4b3-76d3-4d0c-b84e-857efed23d2d
else
search --no-floppy --fs-uuid --set=root 6f64e4b3-76d3-4d0c-b84e-857efed23d2d
fi
linux16 /vmlinuz-3.10.0-327.el7.x86_64 root=/dev/mapper/rhel-root ro crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet LANG=en_US.UTF-8
initrd16 /initramfs-3.10.0-327.el7.x86_64.img
}
menuentry 'Red Hat Enterprise Linux Server (0-rescue-8c82c285ad354366b389cdd924d22eed) 7.2 (Maipo)' --class red --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-8c82c285ad354366b389cdd924d22eed-advanced-c31ce45b-b92d-4bb3-9f64-ffae78aacbf7' {
load_video
insmod gzio
insmod part_msdos
insmod xfs
set root='hd0,msdos1'
if [ x$feature_platform_search_hint = xy ]; then
search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1' 6f64e4b3-76d3-4d0c-b84e-857efed23d2d
else
search --no-floppy --fs-uuid --set=root 6f64e4b3-76d3-4d0c-b84e-857efed23d2d
fi
linux16 /vmlinuz-0-rescue-8c82c285ad354366b389cdd924d22eed root=/dev/mapper/rhel-root ro crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet
initrd16 /initramfs-0-rescue-8c82c285ad354366b389cdd924d22eed.img
}
### END /etc/grub.d/10_linux ###
### BEGIN /etc/grub.d/20_linux_xen ###
### END /etc/grub.d/20_linux_xen ###
### BEGIN /etc/grub.d/20_ppc_terminfo ###
### END /etc/grub.d/20_ppc_terminfo ###
### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###
### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries. Simply type the
# menu entries you want to add after this comment. Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###
### BEGIN /etc/grub.d/41_custom ###
if [ -f ${config_directory}/custom.cfg ]; then
source ${config_directory}/custom.cfg
elif [ -z "${config_directory}" -a -f $prefix/custom.cfg ]; then
source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###
#cat /proc/cmdline
BOOT_IMAGE=/vmlinuz-3.10.0-327.el7.x86_64 root=/dev/mapper/rhel-root ro crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet LANG=en_US.UTF-8

3) Check the sample configuration about /etc/kdump.conf
#cat /etc/kdump.conf
# Configures where to put the kdump /proc/vmcore files
#
# This file contains a series of commands to perform (in order) when a
# kernel crash has happened and the kdump kernel has been loaded. Directives in
# this file are only applicable to the kdump initramfs, and have no effect if
# the root filesystem is mounted and the normal init scripts are processed
#
# Currently only one dump target and path may be configured at once
# if the configured dump target fails, the default action will be preformed
# the default action may be configured with the default directive below. If the
# configured dump target succedes
#
# Basics commands supported are:
# raw - Will dd /proc/vmcore into .
# Use persistent device names for partition devices,
# such as /dev/vg/.
#
# nfs - Will mount fs and copy /proc/vmcore to
# /var/crash/%HOST-%DATE/, supports DNS.
#
# ssh - Will scp /proc/vmcore to
# :/var/crash/%HOST-%DATE/, supports DNS
# NOTE: make sure user has necessary write
# permissions on server
#
# sshkey - Will use the sshkey to do ssh dump
# Specifies the path of the ssh key you want to use
# when do ssh dump, the default value is
# /root/.ssh/kdump_id_rsa.
#
# - Will mount -t /mnt and copy
# /proc/vmcore to /mnt/var/crash/%DATE/.
# NOTE: can be a device node, label or uuid.
# It's recommended to use persistent device names
# such as /dev/vg/.
# Otherwise it's suggested to use label or uuid.
#
# path - "path" represents the file system path in which
# vmcore will be saved. If a dump target is specified
# in kdump.conf, then "path" is relative to the
# specified dump target. Interpretation of path
# changes a bit if user has not specified a dump
# target explicitly in kdump.conf. In this case,
# "path" represents the absolute path from root.
# And dump target and adjusted path are arrived
# at automatically depending on what's mounted
# in the current system.
# Ignored for raw device dumps. If unset, will
# default to /var/crash.
#
# core_collector
# - This allows you to specify the command to copy
# the vmcore. You could use the dump filtering
# program makedumpfile, the default one, to retrieve
# your core, which on some arches can drastically
# reduce core file size. See /sbin/makedumpfile --help
# for a list of options. Note that the -i and -g
# options are not needed here, as the initrd will
# automatically be populated with a config file
# appropriate for the running kernel.
# Default core_collector for raw/ssh dump is:
# "makedumpfile -F -l --message-level 1 -d 31".
# Default core_collector for other targets is:
# "makedumpfile -l --message-level 1 -d 31".
# For core_collector format details please refer to
# kexec-kdump-howto.txt or kdump.conf manpage.
#
# kdump_post
# - This directive allows you to run a specified
# executable just after the memory dump process
# terminates. The exit status from the dump process
# is fed to the kdump_post executable, which can be
# used to trigger different actions for success or
# failure.
#
# kdump_pre
# - works just like the kdump_post directive, but instead
# of running after the dump process, runs immediately
# before. Exit status of this binary is interpreted
# as follows:
# 0 - continue with dump process as usual
# non 0 - reboot the system
#
# extra_bins
# - This directive allows you to specify additional
# binaries or shell scripts you'd like to include in
# your kdump initrd. Generally only useful in
# conjunction with a kdump_post binary or script that
# relies on other binaries or scripts.
#
# extra_modules
# - This directive allows you to specify extra kernel
# modules that you want to be loaded in the kdump
# initrd, typically used to set up access to
# non-boot-path dump targets that might otherwise
# not be accessible in the kdump environment. Multiple
# modules can be listed, separated by a space, and any
# dependent modules will automatically be included.
#
# default
# - Action to preform in case dumping to intended target
# fails. If no default action is specified, "reboot"
# is assumed default.
# reboot: If the default action is reboot simply reboot
# the system and loose the core that you are
# trying to retrieve.
# halt: If the default action is halt, then simply
# halt the system after attempting to capture
# a vmcore, regardless of success or failure.
# poweroff: The system will be powered down
# shell: If the default action is shell, then drop to
# an shell session inside the initramfs from
# where you can try to record the core manually.
# Exiting this shell reboots the system.
# Note: kdump uses bash as the default shell.
# dump_to_rootfs: If non-root dump target is specified,
# the default action can be set as dump_to_rootfs.
# That means when dump to target fails, dump vmcore
# to rootfs from initramfs context and reboot.
#
# force_rebuild <0 | 1>
# - By default, kdump initrd only will be rebuilt when
# necessary. Specify 1 to force rebuilding kdump
# initrd every time when kdump service starts.
#
#override_resettable <0 | 1>
# - Usually a unresettable block device can't be dump target.
# Specifying 1 means though block target is unresettable, user
# understand this situation and want to try dumping. By default,
# it's set to 0, means not to try a destined failure.
#
# dracut_args
# - Pass extra dracut options when rebuilding kdump
# initrd.
#
# fence_kdump_args
# - Command line arguments for fence_kdump_send (it can contain
# all valid arguments except hosts to send notification to).
#
# fence_kdump_nodes
# - List of cluster node(s) separated by space to send fence_kdump
# notification to (this option is mandatory to enable fence_kdump).
#
#raw /dev/vg/lv_kdump
#ext4 /dev/vg/lv_kdump
#ext4 LABEL=/boot
#ext4 UUID=03138356-5e61-4ab3-b58e-27507ac41937
#nfs my.server.com:/export/tmp
#ssh user@my.server.com
#sshkey /root/.ssh/kdump_id_rsa
path /var/crash
core_collector makedumpfile -l --message-level 1 -d 31
#core_collector scp
#kdump_post /var/crash/scripts/kdump-post.sh
#kdump_pre /var/crash/scripts/kdump-pre.sh
#extra_bins /usr/bin/lftp
#extra_modules gfs2
#default shell
#force_rebuild 1
#dracut_args --omit-drivers "cfg80211 snd" --add-drivers "ext2 ext3"
#fence_kdump_args -p 7410 -f auto -c 0 -i 10
#fence_kdump_nodes node1 node2
#service kdump restart
#service kdump status
Redirecting to /bin/systemctl status kdump.service
kdump.service - Crash recovery kernel arming
Loaded: loaded (/usr/lib/systemd/system/kdump.service; enabled; vendor preset: enabled)
Active: active (exited) since Tue 2017-01-31 09:22:48 CST; 4min 44s ago
Process: 1258 ExecStart=/usr/bin/kdumpctl start (code=exited, status=0/SUCCESS)
Main PID: 1258 (code=exited, status=0/SUCCESS)
CGroup: /system.slice/kdump.service
Jan 31 09:22:47 ftp.sit.com dracut[3896]: -rw-r--r-- 1 root root 774 Oct 2 2015 usr/share/zoneinfo/A...Taipei
Jan 31 09:22:47 ftp.sit.com dracut[3896]: drwxr-xr-x 3 root root 0 Jan 31 09:22 var
Jan 31 09:22:47 ftp.sit.com dracut[3896]: drwxr-xr-x 3 root root 0 Jan 31 09:22 var/lib
Jan 31 09:22:47 ftp.sit.com dracut[3896]: drwxr-xr-x 2 root root 0 Jan 31 09:22 var/lib/lldpad
Jan 31 09:22:47 ftp.sit.com dracut[3896]: lrwxrwxrwx 1 root root 11 Jan 31 09:22 var/lock -> ../run/lock
Jan 31 09:22:47 ftp.sit.com dracut[3896]: lrwxrwxrwx 1 root root 6 Jan 31 09:22 var/run -> ../run
Jan 31 09:22:47 ftp.sit.com dracut[3896]: ========================================================================
Jan 31 09:22:48 ftp.sit.com kdumpctl[1258]: kexec: loaded kdump kernel
Jan 31 09:22:48 ftp.sit.com kdumpctl[1258]: Starting kdump: [OK]
Jan 31 09:22:48 ftp.sit.com systemd[1]: Started Crash recovery kernel arming.
Hint: Some lines were ellipsized, use -l to show in full.

4) Start to execute the kdump crash manually through the files from the /proc
#echo 1 > /proc/sys/kernel/sysrq
#echo c > /proc/sysrq-trigger

5) Review the files about the vmcore and the related files
#cd /var/crash/ && ls -al
total 4
drwxr-xr-x. 4 root root 78 Jan 31 09:32 .
drwxr-xr-x. 24 root root 4096 Jan 31 2017 ..
drwxr-xr-x 2 root root 42 Jan 31 09:30 127.0.0.1-2017-01-31-09:30:30
drwxr-xr-x 2 root root 42 Jan 31 09:32 127.0.0.1-2017-01-31-09:32:26
#cd 127.0.0.1-2017-01-31-09\:32\:26/ && ls -al
total 104948
drwxr-xr-x 2 root root 42 Jan 31 09:32 .
drwxr-xr-x. 4 root root 78 Jan 31 09:32 ..
-rw------- 1 root root 107327804 Jan 31 09:32 vmcore
-rw-r--r-- 1 root root 132648 Jan 31 09:32 vmcore-dmesg.txt
#cat vmcore-dmesg.txt | head -n 8
[ 0.000000] Initializing cgroup subsys cpuset
[ 0.000000] Initializing cgroup subsys cpu
[ 0.000000] Initializing cgroup subsys cpuacct
[ 0.000000] Linux version 3.10.0-327.el7.x86_64 (mockbuild@x86-034.build.eng.bos.redhat.com) (gcc version 4.8.3 20140911 (Red Hat 4.8.3-9) (GCC) ) #1 SMP Thu Oct 29 17:29:29 EDT 2015
[ 0.000000] Command line: BOOT_IMAGE=/vmlinuz-3.10.0-327.el7.x86_64 root=/dev/mapper/rhel-root ro crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet LANG=en_US.UTF-8
[ 0.000000] Disabled fast string operations
[ 0.000000] e820: BIOS-provided physical RAM map:
[ 0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009e7ff] usable

6) Review the related information before system crash through the file from the kdump
#crash /var/crash/127.0.0.1-2017-01-31-09\:32\:26/vmcore /usr/lib/debug/lib/modules/`uname –r`/vmlinux
crash>
crash> ps
PID PPID CPU TASK ST %MEM VSZ RSS COMM
0 0 0 ffffffff81a8d020 RU 0.0 0 0 [swapper]
1 0 0 ffff88013e7db500 IN 0.0 19356 1544 init
2 0 0 ffff88013e7daaa0 IN 0.0 0 0 [kthreadd]
3 2 0 ffff88013e7da040 IN 0.0 0 0 [migration/0]
4 2 0 ffff88013e7e9540 IN 0.0 0 0 [ksoftirqd/0]
7 2 0 ffff88013dc19500 IN 0.0 0 0 [events/0]
crash> swap
FILENAME TYPE SIZE USED PCT PRIORITY
/dm-1 PARTITION 2064376k 0k 0% -1
crash> ipcs
SHMID_KERNEL KEY SHMID UID PERMS BYTES NATTCH STATUS
(none allocated)
SEM_ARRAY KEY SEMID UID PERMS NSEMS
ffff8801394c0990 00000000 0 0 600 1
ffff880138f09bd0 00000000 65537 0 600 1
MSG_QUEUE KEY MSQID UID PERMS USED-BYTES MESSAGES
(none allocated)
crash> irq -s
CPU0
0: 149 IO-APIC-edge timer
1: 453 IO-APIC-edge i8042
7: 0 IO-APIC-edge parport0
8: 0 IO-APIC-edge rtc0
9: 0 IO-APIC-fasteoi acpi
12: 111 IO-APIC-edge i8042
14: 108 IO-APIC-edge ata_piix
crash> vm
PID: 5210 TASK: ffff8801396f6aa0 CPU: 0 COMMAND: "bash"
MM PGD RSS TOTAL_VM
ffff88013975d880 ffff88013a0c5000 1808k 108340k
VMA START END FLAGS FILE
ffff88013a0c4ed0 400000 4d4000 8001875 /bin/bash
ffff88013cd63210 3804800000 3804820000 8000875 /lib64/ld-2.12.so
ffff880138cf8ed0 3804c00000 3804c02000 8000075 /lib64/libdl-2.12.so
crash> files
PID: 5210 TASK: ffff8801396f6aa0 CPU: 0 COMMAND: "bash"
ROOT: / CWD: /root
FD FILE DENTRY INODE TYPE PATH
0 ffff88013cf76d40 ffff88013a836480 ffff880139b70d48 CHR /tty1
1 ffff88013c4a5d80 ffff88013c90a440 ffff880135992308 REG /proc/sysrq-trigger
255 ffff88013cf76d40 ffff88013a836480 ffff880139b70d48 CHR /tty1
..
crash> sys
KERNEL: /usr/lib/debug/lib/modules/2.6.32-431.5.1.el6.x86_64/vmlinux
DUMPFILE: /var/crash/127.0.0.1-2014-03-26-12:24:39/vmcore [PARTIAL DUMP]
CPUS: 1
DATE: Wed Mar 26 12:24:36 2014
UPTIME: 00:01:32
LOAD AVERAGE: 0.17, 0.09, 0.03
TASKS: 159
NODENAME: elserver1.abc.com
RELEASE: 2.6.32-431.5.1.el6.x86_64
VERSION: #1 SMP Fri Jan 10 14:46:43 EST 2014
MACHINE: x86_64 (2132 Mhz)
MEMORY: 4 GB
PANIC: "Oops: 0002 [#1] SMP " (check log for details)

◎、Above content is about executing the kdump crash manually under the RHEL 7.3, and I have referred this article from thegeekstuff and archlinux to do it, and you can google it for more detail usage about the kdump. That’s all for today, see you next time!

Leave a Comment


NOTE - You can use these HTML tags and attributes:
<a href="" title=""> <abbr title=""> <acronym title=""> <b> <blockquote cite=""> <cite> <code> <del datetime=""> <em> <i> <q cite=""> <strike> <strong>

 
This site is protected by WP-CopyRightPro