etc/systemd/zfs-mount-generator: rewrite in C

A plain rewrite of the shell version, and generates identical
units, save for replacing some empty lines with nothing, having fewer
meaningless spaces in After=s and different spacing in the lock scripts,
for a clean git diff -w

This is a gain of anywhere from 0m0.336s vs 0m0.022s (15.27x)
to 0m0.202s vs 0m0.006s (33.67x), depending on the hardware,
a.k.a. from "absolutely unusable" to "perfectly fine"

This also properly deals with canmount=noauto units across multiple
pools

See PR for detailed timings (of an early version) and diffs

Reviewed-by: Antonio Russo <aerusso@aerusso.net>
Reviewed-by: Richard Laager <rlaager@wiktel.com>
Reviewed-by: InsanePrawn <insane.prawny@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Issue #11915
Closes #11917

Makefile.am

@@ -6,7 +6,7 @@ SUBDIRS += rpm
 endif
 
 if CONFIG_USER
-SUBDIRS += etc man scripts lib tests cmd contrib
+SUBDIRS += man scripts lib tests cmd etc contrib
 if BUILD_LINUX
 SUBDIRS += udev
 endif

etc/systemd/system-generators/Makefile.am

@@ -1,6 +1,14 @@
-include $(top_srcdir)/config/Substfiles.am
+include $(top_srcdir)/config/Rules.am
 
-systemdgenerator_SCRIPTS = \
+systemdgenerator_PROGRAMS = \
 	zfs-mount-generator
 
-SUBSTFILES += $(systemdgenerator_SCRIPTS)
+zfs_mount_generator_SOURCES = \
+	zfs-mount-generator.c
+
+zfs_mount_generator_LDADD = \
+	$(abs_top_builddir)/lib/libzfs/libzfs.la
+
+zfs_mount_generator_LDFLAGS = -pthread
+
+include $(top_srcdir)/config/CppCheck.am

etc/systemd/system-generators/zfs-mount-generator.in

@@ -1,474 +0,0 @@
-#!/bin/sh
-
-# zfs-mount-generator - generates systemd mount units for zfs
-# Copyright (c) 2017 Antonio Russo <antonio.e.russo@gmail.com>
-# Copyright (c) 2020 InsanePrawn <insane.prawny@gmail.com>
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-set -e
-
-FSLIST="@sysconfdir@/zfs/zfs-list.cache"
-
-[ -d "${FSLIST}" ] || exit 0
-[ "$(echo "${FSLIST}"/*)" = "${FSLIST}/*" ] && exit 0
-
-do_fail() {
-  printf 'zfs-mount-generator: %s\n' "$*" > /dev/kmsg
-  exit 1
-}
-
-# test if $1 is in space-separated list $2
-is_known() {
-  query="$1"
-  IFS=' '
-  for element in $2 ; do
-    if [ "$query" = "$element" ] ; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-# create dependency on unit file $1
-# of type $2, i.e. "wants" or "requires"
-# in the target units from space-separated list $3
-create_dependencies() {
-  unitfile="$1"
-  suffix="$2"
-  IFS=' '
-  for target in $3 ; do
-    target_dir="${dest_norm}/${target}.${suffix}/"
-    mkdir -p "${target_dir}"
-    ln -s "../${unitfile}" "${target_dir}"
-  done
-}
-
-# see systemd.generator
-if [ $# -eq 0 ] ; then
-  dest_norm="/tmp"
-elif [ $# -eq 3 ] ; then
-  dest_norm="${1}"
-else
-  do_fail "zero or three arguments required"
-fi
-
-pools=$(zpool list -H -o name || true)
-
-# All needed information about each ZFS is available from
-# zfs list -H -t filesystem -o <properties>
-# cached in $FSLIST, and each line is processed by the following function:
-# See the list below for the properties and their order
-
-process_line() {
-
-  # zfs list -H -o name,...
-  # fields are tab separated
-  IFS="$(printf '\t')"
-  # shellcheck disable=SC2086
-  set -- $1
-
-  dataset="${1}"
-  pool="${dataset%%/*}"
-  p_mountpoint="${2}"
-  p_canmount="${3}"
-  p_atime="${4}"
-  p_relatime="${5}"
-  p_devices="${6}"
-  p_exec="${7}"
-  p_readonly="${8}"
-  p_setuid="${9}"
-  p_nbmand="${10}"
-  p_encroot="${11}"
-  p_keyloc="${12}"
-  p_systemd_requires="${13}"
-  p_systemd_requiresmountsfor="${14}"
-  p_systemd_before="${15}"
-  p_systemd_after="${16}"
-  p_systemd_wantedby="${17}"
-  p_systemd_requiredby="${18}"
-  p_systemd_nofail="${19}"
-  p_systemd_ignore="${20}"
-
-  # Minimal pre-requisites to mount a ZFS dataset
-  # By ordering before zfs-mount.service, we avoid race conditions.
-  after="zfs-import.target"
-  before="zfs-mount.service"
-  wants="zfs-import.target"
-  requires=""
-  requiredmounts=""
-  bindsto=""
-  wantedby=""
-  requiredby=""
-  noauto="off"
-
-  # If the pool is already imported, zfs-import.target is not needed.  This
-  # avoids a dependency loop on root-on-ZFS systems:
-  # systemd-random-seed.service After (via RequiresMountsFor) var-lib.mount
-  # After zfs-import.target After zfs-import-{cache,scan}.service After
-  # cryptsetup.service After systemd-random-seed.service.
-  #
-  # Pools are newline-separated and may contain spaces in their names.
-  # There is no better portable way to set IFS to just a newline.  Using
-  # $(printf '\n') doesn't work because $(...) strips trailing newlines.
-  IFS="
-"
-  for p in $pools ; do
-    if [ "$p" = "$pool" ] ; then
-      after=""
-      wants=""
-      break
-    fi
-  done
-
-  if [ -n "${p_systemd_after}" ] && \
-      [ "${p_systemd_after}" != "-" ] ; then
-    after="${p_systemd_after} ${after}"
-  fi
-
-  if [ -n "${p_systemd_before}" ] && \
-      [ "${p_systemd_before}" != "-" ] ; then
-    before="${p_systemd_before} ${before}"
-  fi
-
-  if [ -n "${p_systemd_requires}" ] && \
-      [ "${p_systemd_requires}" != "-" ] ; then
-    requires="Requires=${p_systemd_requires}"
-  fi
-
-  if [ -n "${p_systemd_requiresmountsfor}" ] && \
-      [ "${p_systemd_requiresmountsfor}" != "-" ] ; then
-    requiredmounts="RequiresMountsFor=${p_systemd_requiresmountsfor}"
-  fi
-
-  # Handle encryption
-  if [ -n "${p_encroot}" ] &&
-      [ "${p_encroot}" != "-" ] ; then
-    keyloadunit="zfs-load-key-$(systemd-escape "${p_encroot}").service"
-    if [ "${p_encroot}" = "${dataset}" ] ; then
-      keymountdep=""
-      if [ "${p_keyloc%%://*}" = "file" ] ; then
-        if [ -n "${requiredmounts}" ] ; then
-          keymountdep="${requiredmounts} '${p_keyloc#file://}'"
-        else
-          keymountdep="RequiresMountsFor='${p_keyloc#file://}'"
-        fi
-        keyloadscript="@sbindir@/zfs load-key \"${dataset}\""
-      elif [ "${p_keyloc}" = "prompt" ] ; then
-        keyloadscript="\
-count=0;\
-while [ \$\$count -lt 3 ];do\
-  systemd-ask-password --id=\"zfs:${dataset}\"\
-    \"Enter passphrase for ${dataset}:\"|\
-    @sbindir@/zfs load-key \"${dataset}\" && exit 0;\
-  count=\$\$((count + 1));\
-done;\
-exit 1"
-      else
-        printf 'zfs-mount-generator: (%s) invalid keylocation\n' \
-          "${dataset}" >/dev/kmsg
-      fi
-      keyloadcmd="\
-/bin/sh -c '\
-set -eu;\
-keystatus=\"\$\$(@sbindir@/zfs get -H -o value keystatus \"${dataset}\")\";\
-[ \"\$\$keystatus\" = \"unavailable\" ] || exit 0;\
-${keyloadscript}'"
-      keyunloadcmd="\
-/bin/sh -c '\
-set -eu;\
-keystatus=\"\$\$(@sbindir@/zfs get -H -o value keystatus \"${dataset}\")\";\
-[ \"\$\$keystatus\" = \"available\" ] || exit 0;\
-@sbindir@/zfs unload-key \"${dataset}\"'"
-
-
-
-      # Generate the key-load .service unit
-      #
-      # Note: It is tempting to use a `<<EOF` style here-document for this, but
-      #   bash requires a writable /tmp or $TMPDIR for that. This is not always
-      #   available early during boot.
-      #
-      echo \
-"# Automatically generated by zfs-mount-generator
-
-[Unit]
-Description=Load ZFS key for ${dataset}
-SourcePath=${cachefile}
-Documentation=man:zfs-mount-generator(8)
-DefaultDependencies=no
-Wants=${wants}
-After=${after}
-${requires}
-${keymountdep}
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-# This avoids a dependency loop involving systemd-journald.socket if this
-# dataset is a parent of the root filesystem.
-StandardOutput=null
-StandardError=null
-ExecStart=${keyloadcmd}
-ExecStop=${keyunloadcmd}"   > "${dest_norm}/${keyloadunit}"
-    fi
-    # Update the dependencies for the mount file to want the
-    # key-loading unit.
-    wants="${wants}"
-    bindsto="BindsTo=${keyloadunit}"
-    after="${after} ${keyloadunit}"
-  fi
-
-  # Prepare the .mount unit
-
-  # skip generation of the mount unit if org.openzfs.systemd:ignore is "on"
-  if [ -n "${p_systemd_ignore}" ] ; then
-    if [ "${p_systemd_ignore}" = "on" ] ; then
-      return
-    elif [ "${p_systemd_ignore}" = "-" ] \
-      || [ "${p_systemd_ignore}" = "off" ] ; then
-      : # This is OK
-    else
-      do_fail "invalid org.openzfs.systemd:ignore for ${dataset}"
-    fi
-  fi
-
-  # Check for canmount=off .
-  if [ "${p_canmount}" = "off" ] ; then
-    return
-  elif [ "${p_canmount}" = "noauto" ] ; then
-    noauto="on"
-  elif [ "${p_canmount}" = "on" ] ; then
-    : # This is OK
-  else
-    do_fail "invalid canmount for ${dataset}"
-  fi
-
-  # Check for legacy and blank mountpoints.
-  if [ "${p_mountpoint}" = "legacy" ] ; then
-    return
-  elif [ "${p_mountpoint}" = "none" ] ; then
-    return
-  elif [ "${p_mountpoint%"${p_mountpoint#?}"}" != "/" ] ; then
-    do_fail "invalid mountpoint for ${dataset}"
-  fi
-
-  # Escape the mountpoint per systemd policy.
-  mountfile="$(systemd-escape --path --suffix=mount "${p_mountpoint}")"
-
-  # Parse options
-  # see lib/libzfs/libzfs_mount.c:zfs_add_options
-  opts=""
-
-  # atime
-  if [ "${p_atime}" = on ] ; then
-    # relatime
-    if [ "${p_relatime}" = on ] ; then
-      opts="${opts},atime,relatime"
-    elif [ "${p_relatime}" = off ] ; then
-      opts="${opts},atime,strictatime"
-    else
-      printf 'zfs-mount-generator: (%s) invalid relatime\n' \
-        "${dataset}" >/dev/kmsg
-    fi
-  elif [ "${p_atime}" = off ] ; then
-    opts="${opts},noatime"
-  else
-    printf 'zfs-mount-generator: (%s) invalid atime\n' \
-      "${dataset}" >/dev/kmsg
-  fi
-
-  # devices
-  if [ "${p_devices}" = on ] ; then
-    opts="${opts},dev"
-  elif [ "${p_devices}" = off ] ; then
-    opts="${opts},nodev"
-  else
-    printf 'zfs-mount-generator: (%s) invalid devices\n' \
-      "${dataset}" >/dev/kmsg
-  fi
-
-  # exec
-  if [ "${p_exec}" = on ] ; then
-    opts="${opts},exec"
-  elif [ "${p_exec}" = off ] ; then
-    opts="${opts},noexec"
-  else
-    printf 'zfs-mount-generator: (%s) invalid exec\n' \
-      "${dataset}" >/dev/kmsg
-  fi
-
-  # readonly
-  if [ "${p_readonly}" = on ] ; then
-    opts="${opts},ro"
-  elif [ "${p_readonly}" = off ] ; then
-    opts="${opts},rw"
-  else
-    printf 'zfs-mount-generator: (%s) invalid readonly\n' \
-      "${dataset}" >/dev/kmsg
-  fi
-
-  # setuid
-  if [ "${p_setuid}" = on ] ; then
-    opts="${opts},suid"
-  elif [ "${p_setuid}" = off ] ; then
-    opts="${opts},nosuid"
-  else
-    printf 'zfs-mount-generator: (%s) invalid setuid\n' \
-      "${dataset}" >/dev/kmsg
-  fi
-
-  # nbmand
-  if [ "${p_nbmand}" = on ]  ; then
-    opts="${opts},mand"
-  elif [ "${p_nbmand}" = off ] ; then
-    opts="${opts},nomand"
-  else
-    printf 'zfs-mount-generator: (%s) invalid nbmand\n' \
-      "${dataset}" >/dev/kmsg
-  fi
-
-  if [ -n "${p_systemd_wantedby}" ] && \
-      [ "${p_systemd_wantedby}" != "-" ] ; then
-    noauto="on"
-    if [ "${p_systemd_wantedby}" = "none" ] ; then
-      wantedby=""
-    else
-      wantedby="${p_systemd_wantedby}"
-      before="${before} ${wantedby}"
-    fi
-  fi
-
-  if [ -n "${p_systemd_requiredby}" ] && \
-      [ "${p_systemd_requiredby}" != "-" ] ; then
-    noauto="on"
-    if [ "${p_systemd_requiredby}" = "none" ] ; then
-      requiredby=""
-    else
-      requiredby="${p_systemd_requiredby}"
-      before="${before} ${requiredby}"
-    fi
-  fi
-
-  # For datasets with canmount=on, a dependency is created for
-  # local-fs.target by default. To avoid regressions, this dependency
-  # is reduced to "wants" rather than "requires" when nofail is not "off".
-  # **THIS MAY CHANGE**
-  # noauto=on disables this behavior completely.
-  if [ "${noauto}" != "on" ] ; then
-    if [ "${p_systemd_nofail}" = "off" ] ; then
-      requiredby="local-fs.target"
-      before="${before} local-fs.target"
-    else
-      wantedby="local-fs.target"
-      if [ "${p_systemd_nofail}" != "on" ] ; then
-        before="${before} local-fs.target"
-      fi
-    fi
-  fi
-
-  # Handle existing files:
-  # 1.  We never overwrite existing files, although we may delete
-  #     files if we're sure they were created by us. (see 5.)
-  # 2.  We handle files differently based on canmount. Units with canmount=on
-  #     always have precedence over noauto. This is enforced by the sort pipe
-  #     in the loop around this function.
-  #     It is important to use $p_canmount and not $noauto here, since we
-  #     sort by canmount while other properties also modify $noauto, e.g.
-  #     org.openzfs.systemd:wanted-by.
-  # 3.  If no unit file exists for a noauto dataset, we create one.
-  #     Additionally, we use $noauto_files to track the unit file names
-  #     (which are the systemd-escaped mountpoints) of all (exclusively)
-  #     noauto datasets that had a file created.
-  # 4.  If the file to be created is found in the tracking variable,
-  #     we do NOT create it.
-  # 5.  If a file exists for a noauto dataset, we check whether the file
-  #     name is in the variable. If it is, we have multiple noauto datasets
-  #     for the same mountpoint. In such cases, we remove the file for safety.
-  #     To avoid further noauto datasets creating a file for this path again,
-  #     we leave the file name in the tracking variable.
-  if [ -e "${dest_norm}/${mountfile}" ] ; then
-    if is_known "$mountfile" "$noauto_files" ; then
-      # if it's in $noauto_files, we must be noauto too. See 2.
-      printf 'zfs-mount-generator: removing duplicate noauto %s\n' \
-        "${mountfile}" >/dev/kmsg
-      # See 5.
-      rm "${dest_norm}/${mountfile}"
-    else
-      # don't log for canmount=noauto
-      if [  "${p_canmount}" = "on" ] ; then
-        printf 'zfs-mount-generator: %s already exists. Skipping.\n' \
-          "${mountfile}" >/dev/kmsg
-      fi
-    fi
-    # file exists; Skip current dataset.
-    return
-  else
-    if is_known "${mountfile}" "${noauto_files}" ; then
-      # See 4.
-      return
-    elif [ "${p_canmount}" = "noauto" ] ; then
-      noauto_files="${mountfile} ${noauto_files}"
-    fi
-  fi
-
-  # Create the .mount unit file.
-  #
-  # (Do not use `<<EOF`-style here-documents for this, see warning above)
-  #
-  echo \
-"# Automatically generated by zfs-mount-generator
-
-[Unit]
-SourcePath=${cachefile}
-Documentation=man:zfs-mount-generator(8)
-
-Before=${before}
-After=${after}
-Wants=${wants}
-${bindsto}
-${requires}
-${requiredmounts}
-
-[Mount]
-Where=${p_mountpoint}
-What=${dataset}
-Type=zfs
-Options=defaults${opts},zfsutil" > "${dest_norm}/${mountfile}"
-
-  # Finally, create the appropriate dependencies
-  create_dependencies "${mountfile}" "wants" "$wantedby"
-  create_dependencies "${mountfile}" "requires" "$requiredby"
-
-}
-
-for cachefile in "${FSLIST}/"* ; do
-  # Disable glob expansion to protect against special characters when parsing.
-  set -f
-  # Sort cachefile's lines by canmount, "on" before "noauto"
-  # and feed each line into process_line
-  sort -t "$(printf '\t')" -k 3 -r "${cachefile}" | \
-  ( # subshell is necessary for `sort|while read` and $noauto_files
-    noauto_files=""
-    while read -r fs ; do
-      process_line "${fs}"
-    done
-  )
-done

man/man8/zfs-mount-generator.8.in

@@ -22,7 +22,7 @@
 .\" OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 .\" WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-.TH ZFS-MOUNT-GENERATOR 8 "Aug 24, 2020" OpenZFS
+.TH ZFS-MOUNT-GENERATOR 8 "Apr 19, 2021" OpenZFS
 
 .SH "NAME"
 zfs\-mount\-generator \- generates systemd mount units for ZFS
@@ -83,19 +83,13 @@ zfs list -H -o name,mountpoint,canmount,atime,relatime,devices,exec,readonly,set
 .RE
 .PP
 for datasets that should be mounted by systemd, should be kept
-separate from the pool, at
+separate from the pool at
-.PP
+.RI @sysconfdir@/zfs/zfs-list.cache/ POOLNAME .
-.RS 4
-.RI @sysconfdir@/zfs/zfs-list.cache/ POOLNAME
-.
-.RE
 .PP
 The cache file, if writeable, will be kept synchronized with the pool
-state by the ZEDLET
+state by the
-.PP
+.I history_event-zfs-list-cacher.sh
-.RS 4
+ZEDLET.
-history_event-zfs-list-cacher.sh .
-.RE
 .PP
 .sp
 .SS PROPERTIES
@@ -189,6 +183,17 @@ See also
 .BR systemd.mount (5)
 
 .PP
+.SH ENVIRONMENT
+The
+.BR $ZFS_DEBUG
+environment variable, which can either be 0 (default),
+1 (print summary accounting information at the end),
+or at least 2 (print accounting information for each subprocess as it finishes).
+
+If not present, /proc/cmdline is additionally checked for
+.BR debug ,
+in which case the debug level is set to 2.
+
 .SH EXAMPLE
 To begin, enable tracking for the pool:
 .PP
@@ -221,7 +226,7 @@ This forces an update to the stale cache file.
 To test the generator output, run
 .PP
 .RS 4
-@systemdgeneratordir@/zfs-mount-generator /tmp/zfs-mount-generator . .
+@systemdgeneratordir@/zfs-mount-generator /tmp/zfs-mount-generator
 .RE
 .PP
 This will generate units and dependencies in