From ba863d0be4cbfbea938b10e49fb6ff459ac9ec20 Mon Sep 17 00:00:00 2001 From: Tony Nguyen Date: Tue, 22 May 2018 11:51:46 -0600 Subject: [PATCH] Profiling for perf tests Stack profiling is quite useful and Linux ZFS test suite does not current collect that data. Linux perf is a common tool for this purpose though the perf record data file can be quite large. With this change, Linux ZFS perf tests capture perf record data if perf is installed on the system and PERF_DO_PROFILING environment variable is set. Reviewed by: John Wren Kennedy Reviewed by: Matthew Ahrens Signed-off-by: Tony Nguyen External-issue: LX-971 Closes #7549 --- tests/zfs-tests/include/commands.cfg | 1 + tests/zfs-tests/tests/perf/regression/random_reads.ksh | 9 ++++++--- .../zfs-tests/tests/perf/regression/random_readwrite.ksh | 5 ++++- tests/zfs-tests/tests/perf/regression/random_writes.ksh | 5 ++++- .../zfs-tests/tests/perf/regression/sequential_reads.ksh | 6 +++++- .../perf/regression/sequential_reads_arc_cached.ksh | 6 +++++- .../regression/sequential_reads_arc_cached_clone.ksh | 6 +++++- .../perf/regression/sequential_reads_dbuf_cached.ksh | 6 +++++- .../tests/perf/regression/sequential_writes.ksh | 5 ++++- 9 files changed, 39 insertions(+), 10 deletions(-) diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 2a6c3b13d..50eb6bd6b 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -84,6 +84,7 @@ export SYSTEM_FILES='arp openssl parted pax + perf pgrep ping pkill diff --git a/tests/zfs-tests/tests/perf/regression/random_reads.ksh b/tests/zfs-tests/tests/perf/regression/random_reads.ksh index 5e91fe0a7..5bf269a85 100755 --- a/tests/zfs-tests/tests/perf/regression/random_reads.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_reads.ksh @@ -85,9 +85,12 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then - export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" - "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" - "iostat") + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" + "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" + "iostat" "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat") diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh index 446a440ec..e6d6e3a11 100755 --- a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh @@ -85,9 +85,12 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" - "iostat") + "iostat" "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat") diff --git a/tests/zfs-tests/tests/perf/regression/random_writes.ksh b/tests/zfs-tests/tests/perf/regression/random_writes.ksh index e724aefe2..d85a3d98a 100755 --- a/tests/zfs-tests/tests/perf/regression/random_writes.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_writes.ksh @@ -77,9 +77,12 @@ fi lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" - "iostat") + "iostat" "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat") diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh index e60dbb181..a9c62fe5a 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh @@ -85,9 +85,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1" - "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat") + "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat" + "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat" diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh index 5445f814e..6622ac973 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh @@ -75,9 +75,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1" - "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat") + "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat" + "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat" diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh index 5d7175b1a..9ed0e4792 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh @@ -91,9 +91,13 @@ export TESTFS=$PERFPOOL/$TESTCLONE lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1" - "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat") + "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat" + "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat" diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh index 5d029280e..edb7a96c9 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh @@ -77,9 +77,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1" - "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat") + "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat" + "$perf_record_cmd" "perf") else export collect_scripts=("kstat zfs:0 1" "kstat" "vmstat -T d 1" "vmstat" "mpstat -T d 1" "mpstat" "iostat -T d -xcnz 1" "iostat" diff --git a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh index d5fb89020..01ab80d4a 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh @@ -77,9 +77,12 @@ fi lun_list=$(pool_to_lun_list $PERFPOOL) log_note "Collecting backend IO stats with lun list $lun_list" if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" - "iostat") + "iostat" "$perf_record_cmd" "perf") else export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")