diff --git a/tests/runfiles/perf-regression.run b/tests/runfiles/perf-regression.run index d10ff8836..ec081040d 100644 --- a/tests/runfiles/perf-regression.run +++ b/tests/runfiles/perf-regression.run @@ -27,6 +27,7 @@ tags = ['perf'] [tests/perf/regression] tests = ['sequential_writes', 'sequential_reads', 'sequential_reads_arc_cached', 'sequential_reads_arc_cached_clone', 'sequential_reads_dbuf_cached', - 'random_reads', 'random_writes', 'random_readwrite', 'random_writes_zil'] + 'random_reads', 'random_writes', 'random_readwrite', 'random_writes_zil', + 'random_readwrite_fixed'] post = tags = ['perf', 'regression'] diff --git a/tests/zfs-tests/tests/perf/fio/Makefile.am b/tests/zfs-tests/tests/perf/fio/Makefile.am index c4a2a4c3f..619251682 100644 --- a/tests/zfs-tests/tests/perf/fio/Makefile.am +++ b/tests/zfs-tests/tests/perf/fio/Makefile.am @@ -3,6 +3,7 @@ dist_pkgdata_DATA = \ mkfiles.fio \ random_reads.fio \ random_readwrite.fio \ + random_readwrite_fixed.fio \ random_writes.fio \ sequential_reads.fio \ sequential_writes.fio diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio new file mode 100644 index 000000000..ed4495553 --- /dev/null +++ b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio @@ -0,0 +1,35 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Delphix. All rights reserved. +# + +[global] +filename_format=file$jobnum +nrfiles=16 +group_reporting=1 +fallocate=0 +overwrite=0 +thread=1 +rw=randrw +rwmixread=70 +time_based=1 +directory=${DIRECTORY} +runtime=${RUNTIME} +bs=${BLOCKSIZE} +ioengine=psync +sync=${SYNC_TYPE} +numjobs=${NUMJOBS} +buffer_compress_percentage=66 +buffer_compress_chunk=4096 + +[job] diff --git a/tests/zfs-tests/tests/perf/regression/Makefile.am b/tests/zfs-tests/tests/perf/regression/Makefile.am index 4f045880f..124e0936a 100644 --- a/tests/zfs-tests/tests/perf/regression/Makefile.am +++ b/tests/zfs-tests/tests/perf/regression/Makefile.am @@ -2,6 +2,7 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf/regression dist_pkgdata_SCRIPTS = \ random_reads.ksh \ random_readwrite.ksh \ + random_readwrite_fixed.ksh \ random_writes.ksh \ random_writes_zil.ksh \ sequential_reads_arc_cached_clone.ksh \ diff --git a/tests/zfs-tests/tests/perf/regression/random_reads.ksh b/tests/zfs-tests/tests/perf/regression/random_reads.ksh index f4e333696..079a53680 100755 --- a/tests/zfs-tests/tests/perf/regression/random_reads.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_reads.ksh @@ -62,7 +62,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'} - export PERF_IOSIZES=${PERF_IOSIZES:-'8k'} + export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'} elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} @@ -89,17 +89,17 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh index 00dd07025..5090d1998 100755 --- a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh @@ -89,17 +89,17 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh new file mode 100755 index 000000000..38c0669f6 --- /dev/null +++ b/tests/zfs-tests/tests/perf/regression/random_readwrite_fixed.ksh @@ -0,0 +1,99 @@ +#!/usr/bin/ksh +# file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Delphix. All rights reserved. +# + +# +# Description: +# Trigger fio runs using the random_readwrite_fixed job file. The number of runs and +# data collected is determined by the PERF_* variables. See do_fio_run for +# details about these variables. +# +# The files to read and write from are created prior to the first fio run, +# and used for all fio runs. The ARC is cleared with `zinject -a` prior to +# each run so reads will go to disk. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/perf/perf.shlib + +function cleanup +{ + # kill fio and iostat + pkill fio + pkill iostat + recreate_perf_pool +} + +trap "log_fail \"Measure IO stats during random read write load\"" SIGTERM +log_onexit cleanup + +recreate_perf_pool +populate_perf_filesystems + +# Aim to fill the pool to 50% capacity while accounting for a 3x compressratio. +export TOTAL_SIZE=$(($(get_prop avail $PERFPOOL) * 3 / 2)) + +# Variables for use by fio. +if [[ -n $PERF_REGRESSION_WEEKLY ]]; then + export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} + export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'} + export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'} + export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} + export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'} + export PERF_IOSIZES='8k 64k' +elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then + export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} + export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} + export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'} + export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} + export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'} + export PERF_IOSIZES='8k' +fi + +# Layout the files to be used by the readwrite tests. Create as many files +# as the largest number of threads. An fio run with fewer threads will use +# a subset of the available files. +export NUMJOBS=$(get_max $PERF_NTHREADS) +export FILE_SIZE=$((TOTAL_SIZE / NUMJOBS)) +export DIRECTORY=$(get_directory) +log_must fio $FIO_SCRIPTS/mkfiles.fio + +# Set up the scripts and output files that will log performance data. +lun_list=$(pool_to_lun_list $PERFPOOL) +log_note "Collecting backend IO stats with lun list $lun_list" +if is_linux; then + typeset perf_record_cmd="perf record -F 99 -a -g -q \ + -o /dev/stdout -- sleep ${PERF_RUNTIME}" + + export collect_scripts=( + "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" + "vmstat -t 1" "vmstat" + "mpstat -P ALL 1" "mpstat" + "iostat -tdxyz 1" "iostat" + "$perf_record_cmd" "perf" + ) +else + export collect_scripts=( + "kstat zfs:0 1" "kstat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" + "dtrace -Cs $PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" + "dtrace -s $PERF_SCRIPTS/profile.d" "profile" + ) +fi + +log_note "Random reads and writes with $PERF_RUNTYPE settings" +do_fio_run random_readwrite_fixed.fio false true +log_pass "Measure IO stats during random read and write load" diff --git a/tests/zfs-tests/tests/perf/regression/random_writes.ksh b/tests/zfs-tests/tests/perf/regression/random_writes.ksh index c84f96506..3101ac993 100755 --- a/tests/zfs-tests/tests/perf/regression/random_writes.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_writes.ksh @@ -61,7 +61,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'} - export PERF_IOSIZES=${PERF_IOSIZES:-'8k'} + export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'} elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} @@ -80,17 +80,17 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi diff --git a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh index 022021daa..0f3256351 100755 --- a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh +++ b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh @@ -68,9 +68,9 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh index 93c109e73..37e219627 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh @@ -62,7 +62,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'} - export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'} + export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'} elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} @@ -90,18 +90,18 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" - "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "vmstat -t 1" "vmstat" + "mpstat -P ALL 1" "mpstat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh index 5ef95c0e0..9528d97d6 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh @@ -49,10 +49,10 @@ export TOTAL_SIZE=$(($(get_max_arc_size) / 2)) if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'} - export PERF_NTHREADS=${PERF_NTHREADS:-'16 64'} + export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'} - export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'} + export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'} elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} @@ -80,18 +80,18 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh index 60f5d750d..f2747640e 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh @@ -55,10 +55,10 @@ export TOTAL_SIZE=$(($(get_max_arc_size) / 2)) if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'} - export PERF_NTHREADS=${PERF_NTHREADS:-'16 64'} + export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'} - export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'} + export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 128k'} elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} @@ -106,18 +106,18 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh index d49da6057..884856c2f 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh @@ -84,9 +84,9 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" - "vmstat 1" "vmstat" - "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "vmstat -t 1" "vmstat" + "mpstat -P ALL 1" "mpstat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else diff --git a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh index e6d9ec2d5..8ea4f8419 100755 --- a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh +++ b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh @@ -61,7 +61,7 @@ if [[ -n $PERF_REGRESSION_WEEKLY ]]; then export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'} export PERF_NTHREADS_PER_FS=${PERF_NTHREADS_PER_FS:-'0'} export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'} - export PERF_IOSIZES=${PERF_IOSIZES:-'8k 128k 1m'} + export PERF_IOSIZES=${PERF_IOSIZES:-'8k 64k 256k'} elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY} export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'} @@ -80,17 +80,17 @@ if is_linux; then export collect_scripts=( "zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat" - "vmstat 1" "vmstat" + "vmstat -t 1" "vmstat" "mpstat -P ALL 1" "mpstat" - "iostat -dxyz 1" "iostat" + "iostat -tdxyz 1" "iostat" "$perf_record_cmd" "perf" ) else export collect_scripts=( "$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io" - "vmstat 1" "vmstat" - "mpstat 1" "mpstat" - "iostat -xcnz 1" "iostat" + "vmstat -T d 1" "vmstat" + "mpstat -T d 1" "mpstat" + "iostat -T d -xcnz 1" "iostat" ) fi